diff --git a/Cargo.lock b/Cargo.lock index 347e10f30..48958fdad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4737,6 +4737,7 @@ dependencies = [ "iana-time-zone", "image", "log", + "postgres", "rusqlite", "rustgis-analysis", "rustgis-data", diff --git a/crates/data/src/source/duckdb.rs b/crates/data/src/source/duckdb.rs index 71f13bda3..62cc6c577 100644 --- a/crates/data/src/source/duckdb.rs +++ b/crates/data/src/source/duckdb.rs @@ -112,8 +112,10 @@ impl Database { match self { Database::File(p) => p.file_stem().and_then(|s| s.to_str()).unwrap_or("database").to_string(), Database::Postgres(d) => { - let get = |k: &str| d.split_whitespace().find_map(|kv| kv.strip_prefix(&format!("{k}=")).map(str::to_string)); - get("dbname") + // Quoted values (`dbname='my gis'`) as libpq reads them. + super::pgconn::keyword_pairs(d) + .into_iter() + .find_map(|(k, v)| (k == "dbname").then_some(v)) .unwrap_or_else(|| d.rsplit('/').next().unwrap_or("postgres").split('?').next().unwrap_or("postgres").to_string()) } Database::Mysql(a) => super::myconn::database_name(a).unwrap_or_else(|| "mysql".into()), @@ -940,6 +942,9 @@ mod tests { assert_eq!(d, Database::Postgres("host=db.example dbname=gis user=me".into())); assert_eq!(d.label(), "postgres:host=db.example dbname=gis user=me"); assert_eq!(d.short(), "gis"); + // Quoted values (as `pg.connect` writes them) are named without their quotes, whole. + assert_eq!(Database::Postgres("host='h' dbname='my gis' user='o\\'neil'".into()).short(), "my gis"); + assert_eq!(Database::Postgres("host='h' dbname='gis'".into()).short(), "gis"); assert_eq!(password(Some(&d)).as_deref(), Some("pw1")); assert_eq!(Database::parse(&d.label()), d); let u = Database::parse("postgresql://me:pw2@db.example:5433/osm?sslmode=require"); diff --git a/crates/data/src/source/postgis.rs b/crates/data/src/source/postgis.rs index bd20c0784..a6f202485 100644 --- a/crates/data/src/source/postgis.rs +++ b/crates/data/src/source/postgis.rs @@ -28,6 +28,10 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// Tables with an estimated row count above this aren't counted exactly when opened. const EXACT_COUNT_BELOW: f64 = 2_000_000.0; +/// The most rows a layer loaded to be edited holds ([`PostgisSource::read_editable`]); a table of +/// several geometry types loads each as its own layer. +pub const MAX_EDITABLE: usize = 100_000; + /// How long counting or measuring a table when it opens may take (one never analyzed has no /// estimate, and could be large): past it the count or extent is left unknown. const PROBE_TIMEOUT: &str = "10s"; @@ -153,6 +157,8 @@ enum Tls { pub struct Conn { config: postgres::Config, tls: Tls, + /// A connection that saves edits ([`connect_writable`]); every other one is read only. + writable: bool, client: Mutex>, } @@ -163,7 +169,7 @@ impl Conn { let mut guard = self.client.lock().unwrap_or_else(|e| e.into_inner()); for attempt in 0..2 { if guard.as_ref().is_none_or(postgres::Client::is_closed) { - *guard = Some(open_client(&self.config, &self.tls)?); + *guard = Some(open_client(&self.config, &self.tls, self.writable)?); } let client = guard.as_mut().ok_or_else(|| DataError::Format("not connected".into()))?; let r = f(client); @@ -178,6 +184,28 @@ impl Conn { Err(DataError::Format("not connected".into())) } + /// Runs `f` once, without the reconnect of [`Conn::with`]: a write that failed when the connection + /// dropped may have been committed, so it must not run again on a new one. + fn once(&self, f: impl FnOnce(&mut postgres::Client) -> std::result::Result) -> Result { + let mut guard = self.client.lock().unwrap_or_else(|e| e.into_inner()); + if guard.as_ref().is_none_or(postgres::Client::is_closed) { + *guard = Some(open_client(&self.config, &self.tls, self.writable)?); + } + let client = guard.as_mut().ok_or_else(|| DataError::Format("not connected".into()))?; + let r = f(client); + let dropped = client.is_closed(); + if dropped { + *guard = None; + } + match r { + Ok(v) => Ok(v), + Err(_) if dropped => { + Err(DataError::Format("the connection dropped during the save: it may have been saved, so check the table".into())) + } + Err(e) => Err(redacted(&self.config, err(e))), + } + } + /// Runs `f` in a read-only transaction seeing one snapshot of the database (`REPEATABLE /// READ`), its statements timing out after `timeout` when given. fn snapshot( @@ -198,8 +226,8 @@ impl Conn { } /// Connects, with the password taken out of any error: a server or the client may echo what it was given. -fn open_client(config: &postgres::Config, tls: &Tls) -> Result { - connect_client(config, tls).map_err(|e| redacted(config, e)) +fn open_client(config: &postgres::Config, tls: &Tls, writable: bool) -> Result { + connect_client(config, tls, writable).map_err(|e| redacted(config, e)) } fn redacted(config: &postgres::Config, error: DataError) -> DataError { @@ -209,7 +237,7 @@ fn redacted(config: &postgres::Config, error: DataError) -> DataError { } } -fn connect_client(config: &postgres::Config, tls: &Tls) -> Result { +fn connect_client(config: &postgres::Config, tls: &Tls, writable: bool) -> Result { let mut client = match tls { Tls::Disable => config.connect(postgres::NoTls), Tls::Unverified | Tls::Verified(_) => config.connect(tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config(tls)?)), @@ -223,11 +251,8 @@ fn connect_client(config: &postgres::Config, tls: &Tls) -> Result, Weak); +/// A live connection: its connection string, the password it was opened with, and whether it +/// writes. +type Live = (String, Option, bool, Weak); /// Live connections, so layers of one server share one. fn live() -> &'static Mutex> { @@ -314,6 +340,16 @@ fn live() -> &'static Mutex> { /// database), libpq's `PGPASSWORD` and password file serve too; a project opening never hands /// them to the server it names. pub fn connect(dsn: &str, ask: bool) -> Result> { + connect_as(dsn, ask, false) +} + +/// Connects to a database to save edits: as [`connect`], but the session isn't read only. Never +/// shared with the connections that read layers, which stay read only. +pub fn connect_writable(dsn: &str, ask: bool) -> Result> { + connect_as(dsn, ask, true) +} + +fn connect_as(dsn: &str, ask: bool, writable: bool) -> Result> { let (mut config, tls) = config(dsn)?; let password = super::duckdb::remembered_password(dsn).or_else(|| { if !ask { @@ -323,8 +359,8 @@ pub fn connect(dsn: &str, ask: bool) -> Result> { }); // A connection opened with another password (one given since) isn't shared. let find = |live: &mut Vec| { - live.retain(|(_, _, w)| w.strong_count() > 0); - live.iter().find(|(d, p, _)| d == dsn && *p == password).and_then(|(_, _, w)| w.upgrade()) + live.retain(|(_, _, _, w)| w.strong_count() > 0); + live.iter().find(|(d, p, wr, _)| d == dsn && *p == password && *wr == writable).and_then(|(_, _, _, w)| w.upgrade()) }; if let Some(c) = find(&mut live().lock().unwrap_or_else(|e| e.into_inner())) { return Ok(c); @@ -333,14 +369,14 @@ pub fn connect(dsn: &str, ask: bool) -> Result> { config.password(p); } // Connecting can take as long as the timeout: other connections don't wait for it. - let client = open_client(&config, &tls)?; + let client = open_client(&config, &tls, writable)?; let mut live = live().lock().unwrap_or_else(|e| e.into_inner()); // Another layer may have connected meanwhile; this one's client then just closes. if let Some(c) = find(&mut live) { return Ok(c); } - let conn = Arc::new(Conn { config, tls, client: Mutex::new(Some(client)) }); - live.push((dsn.to_string(), password, Arc::downgrade(&conn))); + let conn = Arc::new(Conn { config, tls, writable, client: Mutex::new(Some(client)) }); + live.push((dsn.to_string(), password, writable, Arc::downgrade(&conn))); Ok(conn) } @@ -635,6 +671,19 @@ pub fn families(conn: &Conn, table: &Table) -> Result> { // --------------------------------------------------------------------------------------------- // The source +/// A table's rows read to be edited ([`PostgisSource::read_editable`]). +#[derive(Debug, Default)] +pub struct Editable { + /// The rows, in key order; each feature's id is its key. + pub features: Vec, + /// Each row's key and version (`xmin`), as read. + pub versions: Vec<(u64, i64)>, + /// The rows the condition keeps, of which `features` holds the first. + pub matched: u64, + /// Whether this connection's user may insert, update and delete the table's rows. + pub writable: bool, +} + /// One geometry column of a table (one family of it, for a column of mixed types), read live. pub struct PostgisSource { info: SourceInfo, @@ -648,6 +697,9 @@ pub struct PostgisSource { raw_srid: i32, /// The geometry, in its own coordinates. native: String, + /// The table's single-column integer primary key, which gives the ids: rows can be told + /// apart to be edited only with one. + key: Option, /// The expression of each column read: the id, the geometry (as WKB, longitude/latitude), /// then the fields. select: Vec, @@ -765,6 +817,7 @@ impl PostgisSource { } let mut select = vec![id, wkb]; select.extend(casts); + let key = pk; let family = family.or(table.declared_family()); let reproject = match srid { @@ -790,6 +843,7 @@ impl PostgisSource { srid, raw_srid, native, + key, select, reproject, }; @@ -799,6 +853,90 @@ impl PostgisSource { Ok(src) } + /// The table's single-column integer primary key, if it has one. + pub fn key(&self) -> Option<&str> { + self.key.as_deref() + } + + /// The SRID of the geometry read (0 when it says none). + pub fn srid(&self) -> i32 { + self.srid + } + + /// The table and geometry column read. + pub fn table(&self) -> &Table { + &self.table + } + + /// Reads the rows to edit: at most `limit` of them (and [`MAX_EDITABLE`]) in key order, those + /// `condition` (an SQL expression on the table's columns) keeps, each with its version + /// (`xmin`), which saving checks the database against. One snapshot, read only: the + /// condition can't change anything. Needs the table's integer primary key. + pub fn read_editable(&self, condition: Option<&str>, limit: usize) -> Result { + let key = self.key.as_deref().ok_or_else(|| { + DataError::Format(format!( + "{}: no single-column integer primary key, so its rows can't be told apart to edit", + self.table.label() + )) + })?; + let limit = limit.clamp(1, MAX_EDITABLE); + let from = self.table.sql_from(); + let mut filter = self.filter(false); + if let Some(c) = condition.map(str::trim).filter(|c| !c.is_empty()) { + if c.contains(';') { + return Err(DataError::Format("the condition: one SQL expression, without ';'".into())); + } + filter = format!("{filter} AND ({c})"); + } + let version = self.select.len(); + let mut cols: Vec = self.select.iter().enumerate().map(|(i, e)| format!("{e} AS c{i}")).collect(); + cols.push(format!("xmin::text::int8 AS c{version}")); + let sql = format!("SELECT {} FROM {from} WHERE {filter} ORDER BY {} LIMIT {limit}", cols.join(", "), ident(key)); + let rel = format!("{}.{}", ident(&self.table.schema), ident(&self.table.name)); + let (matched, rows, writable) = self.conn.snapshot(None, |t| { + let n: i64 = t.query_one(format!("SELECT count(*) FROM {from} WHERE {filter}").as_str(), &[])?.try_get(0)?; + let writable: bool = t + .query_one( + "SELECT has_table_privilege($1::text::regclass, 'INSERT') AND has_table_privilege($1::text::regclass, 'UPDATE') AND has_table_privilege($1::text::regclass, 'DELETE')", + &[&rel], + )? + .try_get(0)?; + Ok((u64::try_from(n).unwrap_or(0), t.query(sql.as_str(), &[])?, writable)) + })?; + let mut out = Editable { features: Vec::with_capacity(rows.len()), versions: Vec::with_capacity(rows.len()), matched, writable }; + for r in &rows { + let (Some(f), Some(v)) = (self.feature(r), r.try_get::<_, Option>(version).ok().flatten()) else { continue }; + out.versions.push((f.id, v)); + out.features.push(f); + } + Ok(out) + } + + /// A row read as a feature: the id, the geometry (WKB) and the fields, in the order of + /// `select`. `None` without an id or a geometry RustGIS reads. + fn feature(&self, r: &postgres::Row) -> Option { + let id = r.try_get::<_, Option>(0).ok().flatten()?; + let geometry = r.try_get::<_, Option>>(1).ok().flatten().and_then(|b| wkb::decode(&b))?; + let attrs = self + .info + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let i = i + 2; + match f.ty { + FieldType::Integer => r.try_get::<_, Option>(i).ok().flatten().map(Value::Int), + FieldType::Double => r.try_get::<_, Option>(i).ok().flatten().map(Value::Float), + FieldType::Boolean => r.try_get::<_, Option>(i).ok().flatten().map(Value::Bool), + FieldType::Text => r.try_get::<_, Option>(i).ok().flatten().map(Value::Text), + } + .unwrap_or(Value::Null) + }) + .collect(); + // A negative key keeps its bits: still one id per row, and the same on every read. + Some(Feature { id: id.cast_unsigned(), geometry: Some(geometry), attrs }) + } + /// The table's spatial index on the column, if it has one (a view has none of its own). fn has_index(&self) -> bool { let rel = format!("{}.{}", ident(&self.table.schema), ident(&self.table.name)); @@ -940,6 +1078,315 @@ impl PostgisSource { } } +// --------------------------------------------------------------------------------------------- +// Saving edits + +/// Edits to save to a table ([`PostgisSource::save`]), by its key. +#[derive(Debug, Default)] +pub struct Edits { + /// Rows changed: each as it is now, with the version it was read at. + pub updates: Vec<(Feature, i64)>, + /// Rows deleted: their keys and the versions they were read at. + pub deletes: Vec<(u64, i64)>, + /// New rows. Each gets its key field's value when that is a whole number, else the table's + /// default key when it has one, else its feature id. + pub inserts: Vec, +} + +impl Edits { + pub fn is_empty(&self) -> bool { + self.updates.is_empty() && self.deletes.is_empty() && self.inserts.is_empty() + } +} + +/// Why edits weren't saved: a row in the database isn't as it was read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Conflict { + /// A row changed in the database since it was read. + Changed(u64), + /// A row deleted from the database since it was read. + Deleted(u64), + /// A new row's key is already taken. + Exists(u64), +} + +impl std::fmt::Display for Conflict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Keys keep their sign (feature ids hold their bits). + match self { + Conflict::Changed(k) => write!(f, "row {} was changed in the database since it was read", k.cast_signed()), + Conflict::Deleted(k) => write!(f, "row {} was deleted from the database since it was read", k.cast_signed()), + Conflict::Exists(k) => write!(f, "a row with key {} is already in the database", k.cast_signed()), + } + } +} + +/// What saving edits did. +#[derive(Debug, Default)] +pub struct Saved { + /// Each new row's feature id and the key it was saved with. + pub inserted: Vec<(u64, u64)>, + pub updated: usize, + /// Rows edited only in what the table doesn't hold, left as they are. + pub unchanged: usize, + pub deleted: usize, + /// The version of each row inserted or updated, as saved. + pub versions: Vec<(u64, i64)>, + /// The layer's fields the table has no column for, which weren't saved. + pub skipped_fields: Vec, +} + +/// A column a value is written to: the layer's field, its type in the table, and the field of +/// the rows as read that holds its value. +struct Target { + field: usize, + name: String, + ty: String, + read: usize, +} + +/// A parameter of a statement. +type Param = Box; + +fn refs(params: &[Param]) -> Vec<&(dyn ToSql + Sync)> { + params.iter().map(|p| p.as_ref() as &(dyn ToSql + Sync)).collect() +} + +/// A value as text for the server to cast to its column's type (`NULL` for none). +fn text_value(v: &Value) -> Option { + match v { + Value::Null => None, + Value::Int(i) => Some(i.to_string()), + Value::Float(x) if x.is_nan() => Some("NaN".into()), + Value::Float(x) if x.is_infinite() => Some(if *x > 0.0 { "Infinity" } else { "-Infinity" }.into()), + Value::Float(x) => Some(x.to_string()), + Value::Text(t) => Some(t.clone()), + Value::Bool(b) => Some(b.to_string()), + } +} + +impl PostgisSource { + /// The geometry of parameter `$n` (WKB in longitude/latitude) as the column holds it: in its + /// coordinate system, its single or multi type, and its dimensions. + fn geometry_in(&self, n: usize, dims: i32) -> String { + let t = &self.table; + let mut e = format!("ST_SetSRID(ST_GeomFromWKB(${n}::bytea), 4326)"); + if !t.geography { + if self.srid != 0 && self.srid != 4326 { + e = format!("ST_Transform({e}, {})", self.srid); + } + // A column without a coordinate system of its own: the rows' (or none). + if self.raw_srid != self.srid || self.srid == 0 { + e = format!("ST_SetSRID({e}, {})", self.raw_srid); + } + } + let kind = t.kind.to_ascii_uppercase(); + let has_m = kind.ends_with('M') && !kind.ends_with("ZM") && dims == 3; + let base = kind.trim_end_matches(['Z', 'M']); + // RustGIS holds lines and polygons as multi-part: one part goes in a single-type column as + // that type, more fail there as the server says. A column of any type keeps what it holds. + if base.starts_with("MULTI") { + e = format!("ST_Multi({e})"); + } else if base != "GEOMETRYCOLLECTION" && base != "GEOMETRY" { + e = format!("(SELECT CASE WHEN ST_NumGeometries(g) = 1 THEN ST_GeometryN(g, 1) ELSE g END FROM (SELECT {e} AS g) s)"); + } + e = match dims { + 4 => format!("ST_Force4D({e})"), + 3 if has_m => format!("ST_Force3DM({e})"), + 3 => format!("ST_Force3DZ({e})"), + _ => e, + }; + if t.geography { format!("({e})::geography") } else { e } + } + + /// Saves edits to the table in one transaction: every row changed or deleted is locked and + /// must be at the version it was read at, and every new row's key free; otherwise nothing is + /// saved and the conflicts are the answer. Only the columns whose values changed are + /// written (a value read through a cast, such as a `numeric` as a number, is left as it is + /// unless it changed). `fields` are the layer's, matched to the table's columns by name. + /// Needs a connection from [`connect_writable`]. + pub fn save(&self, edits: &Edits, fields: &[Field]) -> Result>> { + let key = self.key.as_deref().ok_or_else(|| { + DataError::Format(format!( + "{}: no single-column integer primary key, so its rows can't be told apart to save", + self.table.label() + )) + })?; + if !self.conn.writable { + return Err(DataError::Format("this connection only reads".into())); + } + let k = ident(key); + let from = self.table.sql_from(); + let rel = format!("{}.{}", ident(&self.table.schema), ident(&self.table.name)); + let column = ident(&self.table.column); + let key_field = fields.iter().position(|f| f.name == key); + self.conn.once(|c| { + let mut t = c.build_transaction().isolation_level(postgres::IsolationLevel::ReadCommitted).start()?; + // The table's columns: their types, whether they are computed, and whether they have + // a default. + let columns: std::collections::HashMap = t + .query( + "SELECT a.attname::text, format_type(a.atttypid, a.atttypmod), \ + COALESCE(to_jsonb(a) ->> 'attgenerated', '') <> '', \ + a.atthasdef OR COALESCE(to_jsonb(a) ->> 'attidentity', '') <> '' \ + FROM pg_attribute a WHERE a.attrelid = $1::text::regclass AND a.attnum > 0 AND NOT a.attisdropped", + &[&rel], + )? + .iter() + .filter_map(|r| Some((r.try_get(0).ok()?, (r.try_get(1).ok()?, r.try_get(2).ok()?, r.try_get(3).ok()?)))) + .collect(); + let key_default = columns.get(key).is_some_and(|c| c.2); + let mut saved = Saved::default(); + let mut targets = Vec::new(); + for (i, f) in fields.iter().enumerate() { + if f.name == key || f.name == self.table.column { + continue; + } + match (columns.get(&f.name), self.info.fields.iter().position(|g| g.name == f.name)) { + (Some((ty, false, _)), Some(read)) => targets.push(Target { field: i, name: f.name.clone(), ty: ty.clone(), read }), + // Computed by the database. + (Some((_, true, _)), _) => {} + _ => saved.skipped_fields.push(f.name.clone()), + } + } + let dims: i32 = t + .query_opt( + "SELECT coord_dimension FROM geometry_columns WHERE f_table_schema = $1 AND f_table_name = $2 AND f_geometry_column = $3 \ + UNION ALL SELECT coord_dimension FROM geography_columns WHERE f_table_schema = $1 AND f_table_name = $2 AND f_geography_column = $3 \ + LIMIT 1", + &[&self.table.schema, &self.table.name, &self.table.column], + )? + .and_then(|r| r.try_get(0).ok()) + .unwrap_or(2); + + // The rows read: locked, and still as they were. + let read: Vec = + edits.updates.iter().map(|(f, _)| f.id.cast_signed()).chain(edits.deletes.iter().map(|(k, _)| k.cast_signed())).collect(); + let now: std::collections::HashMap = t + .query(format!("SELECT {k}::int8, xmin::text::int8 FROM {from} WHERE {k} = ANY($1::int8[]) FOR UPDATE").as_str(), &[&read])? + .iter() + .filter_map(|r| Some((r.try_get(0).ok()?, r.try_get(1).ok()?))) + .collect(); + let mut conflicts = Vec::new(); + for (id, version) in edits.updates.iter().map(|(f, v)| (f.id, *v)).chain(edits.deletes.iter().copied()) { + match now.get(&id.cast_signed()) { + None => conflicts.push(Conflict::Deleted(id)), + Some(v) if *v != version => conflicts.push(Conflict::Changed(id)), + Some(_) => {} + } + } + // New rows' keys: free, and each once. + let new_key = |f: &Feature| -> Option { + match key_field.and_then(|i| f.attrs.get(i)) { + Some(Value::Int(v)) => Some(*v), + _ if key_default => None, + _ => Some(f.id.cast_signed()), + } + }; + let given: Vec = edits.inserts.iter().filter_map(new_key).collect(); + let mut seen = std::collections::HashSet::new(); + for g in &given { + if !seen.insert(*g) { + conflicts.push(Conflict::Exists(g.cast_unsigned())); + } + } + for r in t.query(format!("SELECT {k}::int8 FROM {from} WHERE {k} = ANY($1::int8[])").as_str(), &[&given])? { + if let Ok(g) = r.try_get::<_, i64>(0) { + conflicts.push(Conflict::Exists(g.cast_unsigned())); + } + } + if !conflicts.is_empty() { + // Dropped unsaved: rolled back, the locks let go. + return Ok(Err(conflicts)); + } + + // Updates: the columns whose values differ from the rows as they are (as they were + // read, since their versions match). + let mut written: Vec = Vec::new(); + let ids: Vec = edits.updates.iter().map(|(f, _)| f.id.cast_signed()).collect(); + let cols: Vec = self.select.iter().enumerate().map(|(i, e)| format!("{e} AS c{i}")).collect(); + let current: std::collections::HashMap = t + .query(format!("SELECT {} FROM {from} WHERE {k} = ANY($1::int8[])", cols.join(", ")).as_str(), &[&ids])? + .iter() + .filter_map(|r| self.feature(r)) + .map(|f| (f.id, f)) + .collect(); + for (f, _) in &edits.updates { + let was = current.get(&f.id); + let mut sets = Vec::new(); + let mut params: Vec = Vec::new(); + for tg in &targets { + let value = f.attrs.get(tg.field).unwrap_or(&Value::Null); + if was.and_then(|w| w.attrs.get(tg.read)) == Some(value) { + continue; + } + params.push(Box::new(text_value(value))); + sets.push(format!("{} = CAST(${}::text AS {})", ident(&tg.name), params.len(), tg.ty)); + } + if let Some(g) = &f.geometry + && was.and_then(|w| w.geometry.as_ref()) != Some(g) + { + params.push(Box::new(wkb::encode(g))); + sets.push(format!("{column} = {}", self.geometry_in(params.len(), dims))); + } + if sets.is_empty() { + saved.unchanged += 1; + continue; + } + params.push(Box::new(f.id.cast_signed())); + let sql = format!("UPDATE {from} SET {} WHERE {k} = ${}::int8", sets.join(", "), params.len()); + t.execute(sql.as_str(), &refs(¶ms))?; + saved.updated += 1; + written.push(f.id.cast_signed()); + } + + // New rows: their values, the defaults for the rest. + for f in &edits.inserts { + let (mut names, mut values, mut params): (Vec, Vec, Vec) = Default::default(); + if let Some(v) = new_key(f) { + params.push(Box::new(v)); + names.push(k.clone()); + values.push(format!("${}::int8", params.len())); + } + for tg in &targets { + let value = f.attrs.get(tg.field).unwrap_or(&Value::Null); + if *value == Value::Null { + continue; + } + params.push(Box::new(text_value(value))); + names.push(ident(&tg.name)); + values.push(format!("CAST(${}::text AS {})", params.len(), tg.ty)); + } + if let Some(g) = &f.geometry { + params.push(Box::new(wkb::encode(g))); + names.push(column.clone()); + values.push(self.geometry_in(params.len(), dims)); + } + let sql = if names.is_empty() { + format!("INSERT INTO {from} DEFAULT VALUES RETURNING {k}::int8") + } else { + format!("INSERT INTO {from} ({}) VALUES ({}) RETURNING {k}::int8", names.join(", "), values.join(", ")) + }; + let id: i64 = t.query_one(sql.as_str(), &refs(¶ms))?.try_get(0)?; + saved.inserted.push((f.id, id.cast_unsigned())); + written.push(id); + } + + let gone: Vec = edits.deletes.iter().map(|(k, _)| k.cast_signed()).collect(); + if !gone.is_empty() { + saved.deleted = usize::try_from(t.execute(format!("DELETE FROM {from} WHERE {k} = ANY($1::int8[])").as_str(), &[&gone])?).unwrap_or(0); + } + saved.versions = t + .query(format!("SELECT {k}::int8, xmin::text::int8 FROM {from} WHERE {k} = ANY($1::int8[])").as_str(), &[&written])? + .iter() + .filter_map(|r| Some((r.try_get::<_, i64>(0).ok()?.cast_unsigned(), r.try_get(1).ok()?))) + .collect(); + t.commit()?; + Ok(Ok(saved)) + }) + } +} + fn row_extent(r: &postgres::Row) -> Option { let v: Vec = (0..4).filter_map(|i| r.try_get::<_, Option>(i).ok().flatten()).collect(); match v.as_slice() { @@ -988,35 +1435,18 @@ impl FeatureSource for PostgisSource { })?; out.matched = matched; for r in &rows { - let id = r.try_get::<_, Option>(0).ok().flatten(); - let geom = r.try_get::<_, Option>>(1).ok().flatten().and_then(|b| wkb::decode(&b)); - let (Some(id), Some(geometry)) = (id, geom) else { + let Some(feature) = self.feature(r) else { out.skipped += 1; continue; }; // The index matched the request's box in the table's coordinates, which holds more // than the request when they are projected. - if !in_request(&geometry, req) { + if let Some(geometry) = &feature.geometry + && !in_request(geometry, req) + { continue; } - let attrs = self - .info - .fields - .iter() - .enumerate() - .map(|(i, f)| { - let i = i + 2; - match f.ty { - FieldType::Integer => r.try_get::<_, Option>(i).ok().flatten().map(Value::Int), - FieldType::Double => r.try_get::<_, Option>(i).ok().flatten().map(Value::Float), - FieldType::Boolean => r.try_get::<_, Option>(i).ok().flatten().map(Value::Bool), - FieldType::Text => r.try_get::<_, Option>(i).ok().flatten().map(Value::Text), - } - .unwrap_or(Value::Null) - }) - .collect(); - // A negative key keeps its bits: still one id per row, and the same on every read. - out.features.push(Feature { id: id.cast_unsigned(), geometry: Some(geometry), attrs }); + out.features.push(feature); } out.complete = step <= 1 && (rows.len() as u64) >= out.matched; Ok(out) @@ -1038,6 +1468,43 @@ mod tests { assert_eq!(family_of("GEOMETRYCOLLECTION"), None); } + #[test] + fn unconstrained_geometry_keeps_its_parts() { + let source = |kind: &str| PostgisSource { + info: SourceInfo { + name: "t".into(), + format: "PostGIS", + geometry_type: GeometryType::Polygon, + fields: Vec::new(), + extent: Extent::EMPTY, + count: None, + crs: "EPSG:4326".into(), + indexed: false, + notes: Vec::new(), + }, + conn: Arc::new(Conn { config: postgres::Config::new(), tls: Tls::Disable, writable: false, client: Mutex::new(None) }), + table: Table { + schema: "public".into(), + name: "t".into(), + column: "geom".into(), + srid: 0, + kind: kind.into(), + geography: false, + several: false, + }, + family: None, + srid: 0, + raw_srid: 0, + native: String::new(), + key: None, + select: Vec::new(), + reproject: None, + }; + // A typed column takes one part as its type; an unconstrained one keeps the geometry as it is. + assert!(source("POLYGON").geometry_in(1, 2).contains("ST_NumGeometries")); + assert!(!source("GEOMETRY").geometry_in(1, 2).contains("ST_NumGeometries")); + } + #[test] fn table_names() { let t = |schema: &str, several| Table { diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 286607539..8dee7c482 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -39,3 +39,7 @@ tiny-skia = { workspace = true } # Time slider animations as GIFs (exported by the desktop app only). [target.'cfg(not(target_arch = "wasm32"))'.dependencies] image = { workspace = true, features = ["gif"] } + +# The live PostGIS tests make their own tables, and change rows as someone else would. +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +postgres = { workspace = true } diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 18fe434b5..9c2cda25d 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -27,6 +27,7 @@ pub mod layout; pub mod navigate; pub mod nldi; pub mod notebook; +pub mod pg; pub mod plugins; pub mod portals; pub mod presentation; @@ -98,6 +99,7 @@ pub fn all() -> Vec { v.extend(history::commands()); v.extend(share::commands()); v.extend(sql::commands()); + v.extend(pg::commands()); v.extend(plugins::commands()); v.extend(sharing::commands()); v.extend(stac::commands()); diff --git a/crates/engine/src/cmd/pg.rs b/crates/engine/src/cmd/pg.rs new file mode 100644 index 000000000..931599011 --- /dev/null +++ b/crates/engine/src/cmd/pg.rs @@ -0,0 +1,465 @@ +//! PostGIS: connect to a PostgreSQL database with PostGIS, list its tables and add them as layers +//! that can be edited (desktop only). Home › Add Data › Database… (Load for editing) and +//! Edit › Manage Edits run these. +//! +//! The connection is the session's (`Session::postgis`), never the project's, and never with its +//! password, which is remembered for the session only. A table added with `pg.addLayer` is loaded +//! whole, its feature ids its integer primary key, and the layer keeps each row's version +//! (`xmin`) as read, so saving can tell a row someone else changed since from one to overwrite. +//! +//! `pg.saveEdits` writes what changed in such a layer back to its table in one transaction, and +//! saves nothing when a row changed or deleted there was changed or deleted by someone else +//! since, or a new row's key is taken. `pg.discardEdits` loads the rows again as they are now. + +use serde_json::Value; +#[cfg(not(target_arch = "wasm32"))] +use serde_json::json; + +#[cfg(not(target_arch = "wasm32"))] +use super::{f, ps}; +use crate::{CmdResult, CommandSpec as C, Session}; + +/// Why a command needs a connection first. +const NOT_CONNECTED: &str = "connect to a PostGIS database first (Home › Add Data › Database…, Load for editing)"; + +/// `sslmode`s a connection may name, as libpq's. +#[cfg(not(target_arch = "wasm32"))] +const SSL_MODES: &[&str] = &["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]; + +/// The most tables `pg.tables` lists. +#[cfg(not(target_arch = "wasm32"))] +const MAX_LISTED: usize = 5_000; + +pub fn commands() -> Vec { + vec![ + C::new("pg.connect", "Connect PostGIS", "Home › Add › Add Data", connect).view().params( + r#"{"connection"?: "host=… port=… dbname=… user=…, or a postgresql:// address (instead of the parts below)", "host"?: string, "port"?: number, "database"?: string, "user"?: string, "password"?: "kept for this session only, never saved", "sslmode"?: "disable"|"prefer"|"require"|"verify-ca"|"verify-full"}"#, + ), + C::new("pg.disconnect", "Disconnect PostGIS", "Home › Add › Add Data", disconnect).view().when(need_connection), + C::new("pg.status", "PostGIS Connection", "Home › Add › Add Data", status).view(), + C::new("pg.tables", "List PostGIS Tables", "Home › Add › Add Data", tables).view().when(need_connection).params(r#"{"schema"?: string}"#), + C::new("pg.pendingEdits", "PostGIS Edits", "Edit › Manage Edits", pending_edits).view().params(r#"{"layer"?: "a layer added with pg.addLayer (default: the active one)"}"#), + C::new("pg.saveEdits", "Save to PostGIS", "Edit › Manage Edits", save_edits) + .params(r#"{"layer"?: "a layer added with pg.addLayer (default: the active one)"}"#), + C::new("pg.discardEdits", "Discard PostGIS Edits", "Edit › Manage Edits", discard_edits) + .params(r#"{"layer"?: "a layer added with pg.addLayer (default: the active one)"}"#), + C::new("pg.addLayer", "Add PostGIS Layer", "Home › Add › Add Data", add_layer).when(need_connection).params( + r#"{"table": "table, schema.table, or table:column for one of several geometry columns", "where"?: "an SQL condition on the table's columns", "limit"?: "the most rows of each layer (default and most: 100000)", "name"?: string, "crs"?: "EPSG:… of a geometry that doesn't say"}"#, + ), + ] +} + +pub(crate) fn need_connection(s: &Session) -> Option<&'static str> { + s.postgis.is_none().then_some(NOT_CONNECTED) +} + +/// The layer `p` names (or the active one), which must have been added from PostGIS: its id, +/// name and table. +fn postgis_layer(s: &Session, p: &Value) -> Result<(u32, String, rustgis_map::source::PostgisEdit), String> { + let id = s.layer_id(p)?; + let l = s.map().layer(id).ok_or("no such layer")?; + let edit = l + .postgis + .clone() + .ok_or_else(|| format!("{}: not a PostGIS layer loaded for editing (Home › Add Data › Database…, Load for editing)", l.name))?; + Ok((id, l.name.clone(), edit)) +} + +#[cfg(not(target_arch = "wasm32"))] +/// The rows of `features` that are loaded rows, with their versions and what they hold now: +/// the new baseline after a save (`saved`: the versions written) or a load. +fn versions_of( + features: &[rustgis_data::Feature], + old: &rustgis_map::source::Versions, + saved: &std::collections::HashMap, +) -> rustgis_map::source::Versions { + use rustgis_map::source::{RowVersion, row_hash}; + let mut seen = std::collections::HashSet::new(); + features + .iter() + .filter_map(|f| { + let version = saved.get(&f.id).copied().or_else(|| old.get(f.id).map(|r| r.version))?; + seen.insert(f.id).then(|| (f.id, RowVersion { version, hash: row_hash(f) })) + }) + .collect() +} + +fn pending_edits(s: &mut Session, p: &Value) -> CmdResult { + let (id, name, edit) = postgis_layer(s, p)?; + let l = s.map().layer(id).ok_or("no such layer")?; + let c = edit.changes(&l.dataset.features); + Ok(serde_json::json!({ + "layer": name, + "table": edit.origin.table, + "inserts": c.inserts.len(), + "updates": c.updates.len(), + "deletes": c.deletes.len(), + "writable": edit.writable, + })) +} + +fn save_edits(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("PostGIS databases are edited in the desktop app".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::{duckdb::Database, postgis}; + let (id, name, edit) = postgis_layer(s, p)?; + let l = s.map().layer(id).ok_or("no such layer")?; + let changes = edit.changes(&l.dataset.features); + let fields = l.dataset.fields.clone(); + if changes.is_empty() { + s.notify(format!("{name}: no edits to save"), false); + return Ok(json!({"saved": false, "inserted": 0, "updated": 0, "deleted": 0})); + } + let table = edit.origin.table.clone(); + // Undone past a save: the database holds those edits already, and its new rows would be + // inserted again. + if s.pg_saved_over.iter().any(|v| v.shares(&edit.versions)) { + return Err(format!( + "{name}: the layer is back before an edit saved to {table} (undone), so nothing was saved. Discard Edits loads the rows as they are now" + )); + } + if !edit.writable { + return Err(format!("{name}: the database's user can't insert, update or delete the rows of {table}")); + } + let Database::Postgres(dsn) = Database::parse(&edit.origin.database) else { + return Err(format!("{name}: not a PostgreSQL connection")); + }; + let label = Database::Postgres(dsn.clone()).short(); + let err = |e: rustgis_data::DataError| format!("{label}: {e}"); + // The user's own credentials (PGPASSWORD, the password file) go only to the database + // they connected to this session, never to one a project file names. + let ask = s.postgis.as_deref() == Some(dsn.as_str()); + let conn = postgis::connect_writable(&dsn, ask) + .map_err(|e| format!("{label}: {e} (connect again: Home › Add Data › Database…, Load for editing)"))?; + let t = postgis::find_table(&conn, &table).map_err(err)?; + let src = postgis::PostgisSource::open(conn, t, edit.origin.family, edit.origin.crs.as_deref()).map_err(err)?; + let rows = changes.insert_rows.clone(); + let edits = postgis::Edits { updates: changes.updates, deletes: changes.deletes, inserts: changes.inserts }; + let saved = match src.save(&edits, &fields).map_err(err)? { + Ok(saved) => saved, + Err(conflicts) => { + let mut said: Vec = conflicts.iter().take(20).map(ToString::to_string).collect(); + if conflicts.len() > 20 { + said.push(format!("{} more", conflicts.len() - 20)); + } + return Err(format!( + "{name}: nothing was saved to {table}, {} conflicts: {}. Discard Edits loads the rows as they are now", + conflicts.len(), + said.join("; ") + )); + } + }; + // Saved: from here on nothing fails, so the layer matches the database. New features + // take the keys they were saved with. + let key_field = fields.iter().position(|f| f.name == edit.key); + let versions: std::collections::HashMap = saved.versions.iter().copied().collect(); + if let Some(l) = s.map_mut().layer_mut(id) { + for (row, (_, key)) in rows.iter().zip(&saved.inserted) { + if let Some(f) = l.dataset.features.get_mut(*row) { + f.id = *key; + if let Some(a) = key_field.and_then(|i| f.attrs.get_mut(i)) { + *a = rustgis_data::Value::Int(key.cast_signed()); + } + } + } + let baseline = versions_of(&l.dataset.features, &edit.versions, &versions); + if let Some(pg) = l.postgis.as_mut() { + pg.versions = baseline; + } + } + // The versions saved over, while an undo step can bring them back. + s.pg_saved_over.retain(rustgis_map::source::Versions::held_elsewhere); + s.pg_saved_over.push(edit.versions); + s.notify( + format!("{name}: saved to {table}: {} new, {} changed, {} deleted", saved.inserted.len(), saved.updated, saved.deleted), + false, + ); + if !saved.skipped_fields.is_empty() { + s.notify(format!("{name}: {table} has no column for {}, so it wasn't saved", saved.skipped_fields.join(", ")), false); + } + Ok(json!({ + "saved": true, + "inserted": saved.inserted.len(), + "updated": saved.updated, + "deleted": saved.deleted, + "unchanged": saved.unchanged, + "keys": saved.inserted.iter().map(|(_, k)| k.cast_signed()).collect::>(), + "skippedFields": saved.skipped_fields, + })) + } +} + +fn discard_edits(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("PostGIS databases are read in the desktop app".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::{FeatureSource, duckdb::Database, postgis}; + let (id, name, edit) = postgis_layer(s, p)?; + let Database::Postgres(dsn) = Database::parse(&edit.origin.database) else { + return Err(format!("{name}: not a PostgreSQL connection")); + }; + let label = Database::Postgres(dsn.clone()).short(); + let err = |e: rustgis_data::DataError| format!("{label}: {e}"); + let ask = s.postgis.as_deref() == Some(dsn.as_str()); + let conn = postgis::connect(&dsn, ask) + .map_err(|e| format!("{label}: {e} (connect again: Home › Add Data › Database…, Load for editing)"))?; + let t = postgis::find_table(&conn, &edit.origin.table).map_err(err)?; + let src = postgis::PostgisSource::open(conn, t, edit.origin.family, edit.origin.crs.as_deref()).map_err(err)?; + let limit = if edit.limit == 0 { postgis::MAX_EDITABLE } else { edit.limit }; + let editable = src.read_editable(edit.condition.as_deref(), limit).map_err(err)?; + let read: std::collections::HashMap = editable.versions.iter().copied().collect(); + let n = editable.features.len(); + let l = s.map_mut().layer_mut(id).ok_or("no such layer")?; + l.dataset.fields = src.info().fields.clone(); + l.dataset.features = editable.features.into(); + l.selection.clear(); + l.touch_geometry(); + let baseline = versions_of(&l.dataset.features, &Default::default(), &read); + if let Some(pg) = l.postgis.as_mut() { + pg.versions = baseline; + pg.writable = editable.writable; + } + s.notify(format!("{name}: edits discarded; {n} rows loaded again from {}", edit.origin.table), false); + Ok(json!({"features": n, "matched": editable.matched})) + } +} + +fn status(s: &mut Session, _: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = s; + Ok(Value::Null) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::duckdb::Database; + Ok(match &s.postgis { + Some(dsn) => { + let db = Database::Postgres(dsn.clone()); + json!({"connected": true, "connection": db.label(), "database": db.short()}) + } + None => json!({"connected": false}), + }) + } +} + +fn disconnect(s: &mut Session, _: &Value) -> CmdResult { + let was = s.postgis.take(); + Ok(Value::Bool(was.is_some())) +} + +/// The connection the parameters give, without its password (which is remembered for the +/// session): a whole connection string, or its parts. +#[cfg(not(target_arch = "wasm32"))] +fn connection_param(p: &Value) -> Result { + use rustgis_data::source::duckdb::{Database, remember_password}; + let text = |k: &str| -> Option { + match p.get(k)? { + Value::String(v) => Some(v.trim().to_string()).filter(|v| !v.is_empty()), + Value::Number(n) => Some(n.to_string()), + _ => None, + } + }; + let spec = match text("connection") { + // A bare keyword string is a PostgreSQL one (as Add From Database takes it). + Some(c) if c.contains('=') && !c.contains("://") && !c.to_ascii_lowercase().starts_with("postgres:") => format!("postgres:{c}"), + Some(c) => c, + None => { + if text("host").is_none() { + return Err("give the host, or a whole connection string".into()); + } + if let Some(m) = text("sslmode") + && !SSL_MODES.contains(&m.to_ascii_lowercase().as_str()) + { + return Err(format!("sslmode '{}': one of {}", clip(&m), SSL_MODES.join(", "))); + } + let quote = |v: &str| format!("'{}'", v.replace('\\', "\\\\").replace('\'', "\\'")); + let parts: Vec = [("host", "host"), ("port", "port"), ("dbname", "database"), ("user", "user"), ("sslmode", "sslmode")] + .iter() + .filter_map(|(key, param)| text(param).map(|v| format!("{key}={}", quote(&v)))) + .collect(); + format!("postgres:{}", parts.join(" ")) + } + }; + let Database::Postgres(dsn) = Database::parse(&spec) else { + return Err("not a PostgreSQL connection: host=… dbname=… user=…, or a postgresql:// address".into()); + }; + if let Some(pw) = ps(p, "password").filter(|pw| !pw.is_empty()) { + remember_password(&dsn, pw); + } + Ok(dsn) +} + +/// Text from a parameter, shortened for a message. +#[cfg(not(target_arch = "wasm32"))] +fn clip(s: &str) -> String { + s.chars().take(40).collect() +} + +fn connect(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("PostGIS databases are read in the desktop app".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::{duckdb::Database, postgis}; + let dsn = connection_param(p)?; + let db = Database::Postgres(dsn.clone()); + let label = db.short(); + let conn = postgis::connect(&dsn, true).map_err(|e| format!("{label}: {e}"))?; + // Listing the tables checks the database has PostGIS. + let tables = postgis::tables(&conn).map_err(|e| format!("{label}: {e}"))?; + s.postgis = Some(dsn); + s.notify(format!("Connected to {label}: {} tables with geometry", tables.len()), false); + Ok(json!({"connected": true, "connection": db.label(), "database": label, "tables": tables.len()})) + } +} + +fn tables(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("PostGIS databases are read in the desktop app".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::{duckdb::Database, postgis}; + let dsn = s.postgis.clone().ok_or(NOT_CONNECTED)?; + let label = Database::Postgres(dsn.clone()).short(); + let conn = postgis::connect(&dsn, true).map_err(|e| format!("{label}: {e}"))?; + let all = postgis::tables(&conn).map_err(|e| format!("{label}: {e}"))?; + let schema = ps(p, "schema").map(str::trim).filter(|v| !v.is_empty()); + let listed: Vec = all + .iter() + .filter(|t| schema.is_none_or(|v| t.schema.eq_ignore_ascii_case(v))) + .take(MAX_LISTED) + .map(|t| { + json!({"name": t.label(), "schema": t.schema, "table": t.name, "column": t.column, "type": t.kind, "srid": t.srid, "geography": t.geography}) + }) + .collect(); + Ok(json!({"database": label, "tables": listed})) + } +} + +fn add_layer(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("PostGIS databases are read in the desktop app".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::{FeatureSource, duckdb::Database, postgis}; + use rustgis_map::source::{PostgisEdit, PostgisOrigin}; + let dsn = s.postgis.clone().ok_or(NOT_CONNECTED)?; + let name = ps(p, "table").map(str::trim).filter(|t| !t.is_empty()).ok_or("table is required")?; + let db = Database::Postgres(dsn.clone()); + let label = db.short(); + let err = |e: rustgis_data::DataError| format!("{label}: {e}"); + let crs = ps(p, "crs").map(str::trim).filter(|c| !c.is_empty()); + let condition = ps(p, "where"); + let limit = f(p, "limit").map_or(postgis::MAX_EDITABLE, |n| n.clamp(1.0, postgis::MAX_EDITABLE as f64) as usize); + let conn = postgis::connect(&dsn, true).map_err(err)?; + let table = postgis::find_table(&conn, name).map_err(err)?; + let families = postgis::families(&conn, &table).map_err(err)?; + if families.is_empty() { + return Err(format!("{label}: {}: the table has no rows with a geometry", table.label())); + } + let mixed = families.len() > 1; + let empty_before = s.map().layers.is_empty(); + let (mut ids, mut loaded, mut matched, mut writable) = (Vec::new(), 0u64, 0u64, true); + for family in families { + let src = postgis::PostgisSource::open(conn.clone(), table.clone(), Some(family), crs).map_err(err)?; + let key = src.key().map(str::to_string); + let editable = src.read_editable(condition, limit).map_err(err)?; + let layer_name = match ps(p, "name").map(str::trim).filter(|n| !n.is_empty() && !mixed) { + Some(n) => n.to_string(), + None if mixed => format!("{} ({})", table.label(), family.label()), + None => table.label(), + }; + let info = src.info(); + let mut ds = rustgis_data::Dataset::new(layer_name.clone(), info.geometry_type, info.fields.clone()); + ds.crs = u32::try_from(src.srid()).ok().filter(|&c| c != 0 && c != 4326).and_then(rustgis_data::raster::Crs::from_epsg); + let n = editable.features.len() as u64; + let read: std::collections::HashMap = editable.versions.iter().copied().collect(); + let versions = versions_of(&editable.features, &Default::default(), &read); + ds.features = editable.features.into(); + if n < editable.matched { + s.notify( + format!( + "{layer_name}: loaded {n} of {} rows; narrow them with a condition (where) to edit the others", + editable.matched + ), + false, + ); + } + if !editable.writable { + s.notify(format!("{layer_name}: this user can't insert, update or delete its rows, so edits can't be saved to it"), false); + } + let id = super::layer::add_dataset(s, ds); + if let Some(l) = s.map_mut().layer_mut(id) { + l.postgis = Some(PostgisEdit { + origin: PostgisOrigin { + database: db.label(), + table: table.label(), + family: Some(family), + crs: crs.map(str::to_string), + }, + key: key.unwrap_or_default(), + srid: src.srid(), + writable: editable.writable, + condition: condition.map(str::trim).filter(|c| !c.is_empty()).map(str::to_string), + limit, + versions, + }); + } + ids.push(id); + loaded += n; + matched += editable.matched; + writable &= editable.writable; + } + if empty_before { + let e = ids.iter().filter_map(|id| s.map().layer(*id)).fold(rustgis_geom::Extent::EMPTY, |e, l| e.union(&l.extent())); + if !e.is_empty() { + s.zoom_to(&e); + } + } + Ok(json!({"layers": ids, "features": loaded, "matched": matched, "writable": writable})) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use serde_json::json; + + #[test] + fn connections_are_built_from_their_parts_without_the_password() { + let dsn = super::connection_param(&json!({"host": "db.example", "port": 5433, "database": "my gis", "user": "o'neil", "password": "s3cret", "sslmode": "require"})) + .unwrap_or_else(|e| panic!("{e}")); + for part in ["host=db.example", "port=5433", "dbname='my gis'", r"user='o\'neil'", "sslmode=require"] { + assert!(dsn.contains(part), "{part} in {dsn}"); + } + assert!(!dsn.contains("s3cret"), "{dsn}"); + // A whole connection string wins over the parts, and loses its password. + let dsn = super::connection_param(&json!({"connection": "host=h dbname=g password=pw", "host": "other"})) + .unwrap_or_else(|e| panic!("{e}")); + assert_eq!(dsn, "host=h dbname=g"); + let dsn = super::connection_param(&json!({"connection": "postgresql://u:pw@h:5432/g"})).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(dsn, "postgresql://u@h:5432/g"); + // What isn't a PostgreSQL connection is refused before anything is sent. + for bad in [json!({}), json!({"host": " "}), json!({"host": "h", "sslmode": "bogus"}), json!({"connection": "mysql://u@h/db"})] { + assert!(super::connection_param(&bad).is_err(), "{bad}"); + } + } +} diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 77273a727..9ec383239 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -287,6 +287,10 @@ const NEVER: &[&str] = &[ "s3.disconnect", "earthdata.login", "earthdata.logout", + "pg.connect", + "pg.disconnect", + // Writes to a database: the user's to do, asked each time. + "pg.saveEdits", ]; /// Commands that write files without naming one in their parameters. @@ -309,6 +313,9 @@ const DATABASES: &[&str] = &[ "sql.schema", "raster.zarrInfo", "layer.addZarr", + "pg.tables", + "pg.addLayer", + "pg.discardEdits", "zarr.series", "zarr.reduce", "cube.trend", diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c71bae0f4..dc5c1a7c1 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -535,6 +535,14 @@ pub struct Session { /// Edits per layer id since the project was last saved (Edit › Status). A failed command's /// are rolled back; undo doesn't subtract them, since they count edits made, not the net change. pub edit_counts: std::collections::HashMap, + /// The PostGIS database that layers loaded for editing work with (`pg.connect`): its + /// connection, never with its password. Kept for the session, not in the project. + pub postgis: Option, + /// The PostGIS layers' row versions that `pg.saveEdits` saved over: a layer back on one + /// (undone, or restored by a failed command, past a save) would write its edits again. + /// Outside the project, so undo keeps it. + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + pub(crate) pg_saved_over: Vec, /// The feature cache fills itself after projection changes (Data › Feature Cache). pub auto_cache: bool, /// Mensuration results (Analyze › Mensuration › Results). @@ -687,6 +695,46 @@ fn edited_streamed(before: &Project, after: &Project) -> Option { None } +/// New features of a layer loaded from PostGIS never keep the key of a row it loaded (one +/// deleted since, whose id an editing tool can hand out again): saved, they would be that row, +/// changed. They take ids past every key instead. Layers whose rows were loaded or saved again by +/// the command (their versions replaced) are left alone. +fn fresh_postgis_ids(before: &Project, after: &mut Project) { + for (mb, ma) in before.maps.iter().zip(after.maps.iter_mut()) { + for la in ma.layers.iter_mut() { + let Some(pg) = &la.postgis else { continue }; + let Some(lb) = mb.layers.iter().find(|l| l.id == la.id) else { continue }; + let same_rows = lb.postgis.as_ref().is_some_and(|b| b.versions.shares(&pg.versions)); + if !same_rows || la.dataset.features.shares(&lb.dataset.features) { + continue; + } + let had: std::collections::HashSet = lb.dataset.features.iter().map(|f| f.id).collect(); + let reused: Vec = (0..la.dataset.features.len()) + .filter(|&i| la.dataset.features.get(i).is_some_and(|f| !had.contains(&f.id) && pg.versions.get(f.id).is_some())) + .collect(); + if reused.is_empty() { + continue; + } + // Past the largest key that isn't negative (a negative key's bits are the largest). + let mut next = la + .dataset + .features + .iter() + .map(|f| f.id) + .chain(pg.versions.iter().map(|(k, _)| k)) + .filter(|&k| k.cast_signed() >= 0) + .max() + .unwrap_or(0); + for i in reused { + next = next.saturating_add(1); + if let Some(f) = la.dataset.features.get_mut(i) { + f.id = next; + } + } + } + } +} + /// Why command `label` can't edit streamed layer `layer`, refused up front or undone after. fn streamed_refusal(label: &str, layer: &str) -> String { format!("{label}: {layer} is read in place from its file; Load All Features (layer.loadAll) to edit it") @@ -752,6 +800,8 @@ impl Session { topology_errors: Vec::new(), measurements: Vec::new(), auto_cache: false, + postgis: None, + pg_saved_over: Vec::new(), undo: Vec::new(), redo: Vec::new(), merge: None, @@ -998,6 +1048,7 @@ impl Session { // session's own flag says so, not the answer, which a plugin's command could forge. (Ok(_), Some(_)) if std::mem::take(&mut self.request_only) => {} (Ok(_), Some(mut snap)) => { + fresh_postgis_ids(&snap, &mut self.project); // An edit may change the dates the time slider's window filters on: bound layers // whose features or fields the command changed (features are shared with the // snapshot until written) take the window again. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4984ee366..9ca0f0aed 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -17509,6 +17509,261 @@ fn scans_have_no_coordinates_to_give_away() { let _ = std::fs::remove_dir_all(&dir); } +/// The PostGIS editing commands need a connection first, and a connection that isn't there fails +/// as an error that never shows its password, leaving the session unconnected. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn postgis_tab_needs_a_connection() { + let mut s = Session::new(Project::default()); + assert_eq!(s.run("pg.status", &json!({})).unwrap()["connected"], false); + for (id, p) in [("pg.tables", json!({})), ("pg.addLayer", json!({"table": "roads"})), ("pg.disconnect", json!({}))] { + let e = s.run(id, &p).unwrap_err(); + assert!(e.contains("connect to a PostGIS database first"), "{id}: {e}"); + } + let before = s.map().layers.len(); + let e = + s.run("pg.connect", &json!({"host": "127.0.0.1", "port": 1, "database": "gis", "user": "me", "password": "s3cret"})).unwrap_err(); + assert!(e.starts_with("gis: ") && !e.contains("s3cret"), "{e}"); + let e = s.run("pg.connect", &json!({"connection": "postgresql://me:s3cret@127.0.0.1:1/gis"})).unwrap_err(); + assert!(!e.contains("s3cret"), "{e}"); + assert!(s.postgis.is_none()); + assert_eq!(s.map().layers.len(), before); +} + +/// A live PostGIS database through the PostGIS editing commands: `RUSTGIS_TEST_POSTGIS` names one +/// (as for `postgis_live_end_to_end`) with a table that has an integer primary key. +#[cfg(not(target_arch = "wasm32"))] +#[test] +#[ignore] +fn postgis_tab_live() { + let Ok(dsn) = std::env::var("RUSTGIS_TEST_POSTGIS") else { return }; + let mut s = Session::new(Project::default()); + let dsn = dsn.trim_start_matches("postgres:").to_string(); + let r = s.run("pg.connect", &json!({"connection": dsn})).unwrap(); + assert_eq!(r["connected"], true); + assert_eq!(s.run("pg.status", &json!({})).unwrap()["connected"], true); + let listed = s.run("pg.tables", &json!({})).unwrap(); + let names: Vec = listed["tables"].as_array().unwrap().iter().filter_map(|t| t["name"].as_str()).map(str::to_string).collect(); + assert!(!names.is_empty()); + // The first table with an integer key loads whole, every row with its version. + let r = names.iter().find_map(|n| s.run("pg.addLayer", &json!({"table": n, "limit": 500})).ok()).expect("a table with an integer key"); + let id = r["layers"][0].as_u64().unwrap() as u32; + let l = s.map().layer(id).unwrap(); + assert!(!l.is_streamed() && !l.dataset.features.is_empty()); + let pg = l.postgis.as_ref().unwrap(); + assert_eq!(pg.versions.len(), l.dataset.features.len()); + assert!(l.dataset.features.iter().all(|f| pg.versions.get(f.id).is_some())); + // Saved with the project (never with a password), and read back. + let text = serde_json::to_string(l).unwrap(); + assert!(text.contains("\"versions\"") && !text.contains("password"), "{}", &text[..text.len().min(400)]); + // A condition no row meets loads nothing, and a bad one is the server's error. + let table = l.postgis.as_ref().unwrap().origin.table.clone(); + assert!(s.run("pg.addLayer", &json!({"table": table, "where": "1 = 1; DROP TABLE x"})).unwrap_err().contains("without ';'")); + assert!(s.run("pg.addLayer", &json!({"table": table, "where": "no_such_column > 1"})).is_err()); + s.run("pg.disconnect", &json!({})).unwrap(); + assert!(s.postgis.is_none()); +} + +/// A layer loaded from PostGIS knows what changed in it, and saving refuses what it can't do +/// before it reaches a database: no edits, a user who can't write, a layer not from PostGIS. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn postgis_edits_are_counted_and_checked() { + use rustgis_data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value}; + use rustgis_map::source::{PostgisEdit, PostgisOrigin, RowVersion, row_hash}; + let mut s = Session::new(Project::default()); + let mut ds = + Dataset::new("roads", GeometryType::Point, vec![Field::new("id", FieldType::Integer), Field::new("name", FieldType::Text)]); + for (i, n) in ["a", "b", "c"].iter().enumerate() { + ds.push_next(Some(Geometry::Point(Coord { x: i as f64, y: 0.0 })), vec![Value::Int(i as i64 + 1), Value::Text(n.to_string())]); + } + let versions = ds.features.iter().map(|f| (f.id, RowVersion { version: 7, hash: row_hash(f) })).collect(); + let id = crate::cmd::layer::add_dataset(&mut s, ds); + s.map_mut().layer_mut(id).unwrap().postgis = Some(PostgisEdit { + origin: PostgisOrigin { + database: "postgres:host=127.0.0.1 port=1 dbname=gis".into(), + table: "roads".into(), + family: None, + crs: None, + }, + key: "id".into(), + srid: 4326, + writable: false, + condition: None, + limit: 0, + versions, + }); + let p = json!({"layer": "roads"}); + let r = s.run("pg.pendingEdits", &p).unwrap(); + assert_eq!((r["inserts"].as_u64(), r["updates"].as_u64(), r["deletes"].as_u64()), (Some(0), Some(0), Some(0)), "{r}"); + assert_eq!(s.run("pg.saveEdits", &p).unwrap()["saved"], false); + // An edit, a deletion and a new feature, counted as such. + s.map_mut().editing.settings.enabled = true; + s.run("edit.setValue", &json!({"layer": "roads", "row": 0, "field": "name", "value": "aa"})).unwrap(); + s.run( + "edit.addFeature", + &json!({"layer": "roads", "geometry": {"type": "Point", "coordinates": [5, 5]}, "attributes": {"name": "new"}}), + ) + .unwrap(); + { + let l = s.map_mut().layer_mut(id).unwrap(); + l.dataset.features.remove(1); + l.touch_geometry(); + } + let r = s.run("pg.pendingEdits", &p).unwrap(); + assert_eq!((r["inserts"].as_u64(), r["updates"].as_u64(), r["deletes"].as_u64()), (Some(1), Some(1), Some(1)), "{r}"); + // A user who can't write saves nothing; nor does a database that isn't there. + let e = s.run("pg.saveEdits", &p).unwrap_err(); + assert!(e.contains("can't insert, update or delete"), "{e}"); + s.map_mut().layer_mut(id).unwrap().postgis.as_mut().unwrap().writable = true; + let e = s.run("pg.saveEdits", &p).unwrap_err(); + assert!(e.starts_with("gis: "), "{e}"); + assert!(s.run("pg.discardEdits", &p).is_err()); + // The edits are still there to save. + assert_eq!(s.run("pg.pendingEdits", &p).unwrap()["updates"], 1); + // Back on rows a save wrote over (undone past it): its edits, new rows too, aren't written + // again, and no database is asked. + let read = s.map().layer(id).unwrap().postgis.as_ref().unwrap().versions.clone(); + s.pg_saved_over.push(read.clone()); + let e = s.run("pg.saveEdits", &p).unwrap_err(); + assert!(e.contains("back before an edit saved to roads"), "{e}"); + // Rows read again are a baseline of their own. + s.map_mut().layer_mut(id).unwrap().postgis.as_mut().unwrap().versions = read.iter().collect(); + let e = s.run("pg.saveEdits", &p).unwrap_err(); + assert!(e.starts_with("gis: "), "{e}"); + s.pg_saved_over.clear(); + // A layer not from PostGIS has no edits to save there. + crate::cmd::layer::add_dataset(&mut s, Dataset::new("plain", GeometryType::Point, Vec::new())); + let e = s.run("pg.saveEdits", &json!({"layer": "plain"})).unwrap_err(); + assert!(e.contains("not a PostGIS layer loaded for editing"), "{e}"); +} + +/// Edits saved to a live PostGIS database and read back, and every conflict refused: +/// `RUSTGIS_TEST_POSTGIS` names a database (`postgresql://…`) where the test may make and drop +/// two tables of its own. +#[cfg(not(target_arch = "wasm32"))] +#[test] +#[ignore] +fn postgis_edits_live() { + /// Drops the test's tables, however the test ends. + struct Cleanup(String, String, String); + impl Drop for Cleanup { + fn drop(&mut self) { + if let Ok(mut c) = postgres::Client::connect(&self.0, postgres::NoTls) { + let _ = c.batch_execute(&format!("DROP TABLE IF EXISTS public.{}; DROP TABLE IF EXISTS public.{}", self.1, self.2)); + } + } + } + let Ok(url) = std::env::var("RUSTGIS_TEST_POSTGIS") else { return }; + let url = url.trim_start_matches("postgres:").to_string(); + let mut db = postgres::Client::connect(&url, postgres::NoTls).unwrap(); + let serial = format!("rustgis_edit_test_{}", std::process::id()); + let keyed = format!("{serial}_k"); + let _cleanup = Cleanup(url.clone(), serial.clone(), keyed.clone()); + db.batch_execute(&format!( + "CREATE TABLE public.{serial} (id serial PRIMARY KEY, name text, pop numeric(12,3), geom geometry(Polygon, 3857)); + INSERT INTO public.{serial} (name, pop, geom) VALUES + ('a', 12345.678, ST_Transform(ST_MakeEnvelope(10, 10, 10.1, 10.1, 4326), 3857)), + ('b', 1.001, ST_Transform(ST_MakeEnvelope(11, 10, 11.1, 10.1, 4326), 3857)), + ('c', 2.002, ST_Transform(ST_MakeEnvelope(12, 10, 12.1, 10.1, 4326), 3857)); + CREATE TABLE public.{keyed} (id int PRIMARY KEY, name text, geom geometry(Point, 4326)); + INSERT INTO public.{keyed} VALUES (1, 'one', ST_SetSRID(ST_MakePoint(1, 1), 4326)), (2, 'two', ST_SetSRID(ST_MakePoint(2, 2), 4326));" + )) + .unwrap(); + let name_of = |db: &mut postgres::Client, id: i32| -> Option { + db.query_opt(format!("SELECT name FROM public.{serial} WHERE id = $1").as_str(), &[&id]).unwrap().map(|r| r.get(0)) + }; + + let mut s = Session::new(Project::default()); + s.run("pg.connect", &json!({"connection": url})).unwrap(); + let r = s.run("pg.addLayer", &json!({"table": format!("public.{serial}"), "name": "T"})).unwrap(); + let id = r["layers"][0].as_u64().unwrap() as u32; + s.map_mut().editing.settings.enabled = true; + let row_of = |s: &Session, key: u64| s.map().layer(id).unwrap().dataset.features.iter().position(|f| f.id == key).unwrap(); + + // Rename row 1, delete row 3, add a polygon: saved in one go. + let r1 = row_of(&s, 1); + s.run("edit.setValue", &json!({"layer": "T", "row": r1, "field": "name", "value": "renamed"})).unwrap(); + let r3 = row_of(&s, 3); + s.map_mut().layer_mut(id).unwrap().selection = vec![r3]; + s.active_layer = Some(id); + s.run("edit.delete", &json!({})).unwrap(); + s.run( + "edit.addFeature", + &json!({"layer": "T", "geometry": {"type": "Polygon", "coordinates": [[[13, 10], [13.1, 10], [13.1, 10.1], [13, 10.1], [13, 10]]]}, "attributes": {"name": "new", "pop": 4.5}}), + ) + .unwrap(); + let r = s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap(); + assert_eq!((r["inserted"].as_u64(), r["updated"].as_u64(), r["deleted"].as_u64()), (Some(1), Some(1), Some(1)), "{r}"); + let new_key = r["keys"][0].as_i64().unwrap() as i32; + // Undone, the save isn't made again (its new row inserted twice); redone, it is the layer + // as saved. + assert!(s.undo()); + let e = s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap_err(); + assert!(e.contains("back before an edit saved"), "{e}"); + assert!(s.redo()); + assert_eq!(s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap()["saved"], false); + assert_eq!(name_of(&mut db, 1).as_deref(), Some("renamed")); + assert_eq!(name_of(&mut db, 3), None); + let row = db + .query_one( + format!( + "SELECT name, pop::text, ST_GeometryType(geom), ST_SRID(geom), round(ST_XMin(ST_Transform(geom, 4326))::numeric, 6)::text FROM public.{serial} WHERE id = $1" + ) + .as_str(), + &[&new_key], + ) + .unwrap(); + assert_eq!( + (row.get::<_, String>(0), row.get::<_, String>(1), row.get::<_, String>(2), row.get::<_, i32>(3), row.get::<_, String>(4)), + ("new".into(), "4.500".into(), "ST_Polygon".into(), 3857, "13.000000".into()) + ); + // The numeric of a row whose name changed is as it was, to the last digit. + let pop: String = db.query_one(format!("SELECT pop::text FROM public.{serial} WHERE id = 1").as_str(), &[]).unwrap().get(0); + assert_eq!(pop, "12345.678"); + // The new feature took its key; nothing is left to save. + assert!(s.map().layer(id).unwrap().dataset.features.iter().any(|f| f.id == new_key as u64)); + let r = s.run("pg.pendingEdits", &json!({"layer": "T"})).unwrap(); + assert_eq!((r["inserts"].as_u64(), r["updates"].as_u64(), r["deletes"].as_u64()), (Some(0), Some(0), Some(0)), "{r}"); + + // Someone else renames row 1 while it is edited here: nothing is saved. + db.execute(format!("UPDATE public.{serial} SET name = 'theirs' WHERE id = 1").as_str(), &[]).unwrap(); + let r1 = row_of(&s, 1); + s.run("edit.setValue", &json!({"layer": "T", "row": r1, "field": "name", "value": "mine"})).unwrap(); + let e = s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap_err(); + assert!(e.contains("row 1 was changed in the database"), "{e}"); + assert_eq!(name_of(&mut db, 1).as_deref(), Some("theirs")); + // Discarded: the rows as they are now; and a row someone else changed that isn't edited here + // is no conflict. + s.run("pg.discardEdits", &json!({"layer": "T"})).unwrap(); + db.execute(format!("UPDATE public.{serial} SET name = 'theirs too' WHERE id = 2").as_str(), &[]).unwrap(); + let r1 = row_of(&s, 1); + assert_eq!(s.map().layer(id).unwrap().dataset.value(r1, 1), &rustgis_data::Value::Text("theirs".into())); + s.run("edit.setValue", &json!({"layer": "T", "row": r1, "field": "name", "value": "mine"})).unwrap(); + assert_eq!(s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap()["updated"], 1); + assert_eq!((name_of(&mut db, 1).as_deref(), name_of(&mut db, 2).as_deref()), (Some("mine"), Some("theirs too"))); + // A row someone else deleted can't be saved. + db.execute(format!("DELETE FROM public.{serial} WHERE id = 1").as_str(), &[]).unwrap(); + let r1 = row_of(&s, 1); + s.run("edit.setValue", &json!({"layer": "T", "row": r1, "field": "name", "value": "gone"})).unwrap(); + let e = s.run("pg.saveEdits", &json!({"layer": "T"})).unwrap_err(); + assert!(e.contains("row 1 was deleted from the database"), "{e}"); + + // A table without a default key: a new row's key is its key field, which must be free. + let r = s.run("pg.addLayer", &json!({"table": format!("public.{keyed}"), "name": "K"})).unwrap(); + let kid = r["layers"][0].as_u64().unwrap() as u32; + let point = |id: i64, name: &str| json!({"layer": "K", "geometry": {"type": "Point", "coordinates": [3, 3]}, "attributes": {"id": id, "name": name}}); + s.run("edit.addFeature", &point(2, "taken")).unwrap(); + let e = s.run("pg.saveEdits", &json!({"layer": "K"})).unwrap_err(); + assert!(e.contains("a row with key 2 is already in the database"), "{e}"); + let rows = s.map().layer(kid).unwrap().dataset.features.len(); + s.map_mut().layer_mut(kid).unwrap().dataset.features.truncate(rows - 1); + s.run("edit.addFeature", &point(7, "seven")).unwrap(); + assert_eq!(s.run("pg.saveEdits", &json!({"layer": "K"})).unwrap()["keys"][0], 7); + let n: i64 = db.query_one(format!("SELECT count(*) FROM public.{keyed} WHERE id = 7 AND name = 'seven'").as_str(), &[]).unwrap().get(0); + assert_eq!(n, 1); +} + #[test] fn raster_tools_read_projected_zone_files_without_reprojecting_twice() { use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index c16a2ccac..7bb69179a 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -176,6 +176,10 @@ pub struct Layer { /// Set for streamed layers, whose `dataset` holds the features of the last view read. #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, + /// Set for a layer loaded whole from a PostGIS table to be edited: the table its edits are + /// saved to, and the rows' versions as loaded (desktop only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub postgis: Option, /// Shared, like the features: an undo snapshot or other copy of the layer doesn't copy it. #[serde(skip)] projected: std::sync::Arc>>, @@ -320,6 +324,7 @@ impl Layer { design: design::Design::default(), raster: None, source: None, + postgis: None, projected: Default::default(), projected_extents: Default::default(), projected_vertices: 0, diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 2cfb866ab..fabc1ce83 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -126,6 +126,165 @@ pub struct PostgisOrigin { pub crs: Option, } +/// A PostGIS table a layer was loaded from whole to be edited: what saving its edits writes to +/// (desktop only). The layer's feature ids are the table's key. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PostgisEdit { + /// The connection (never with its password), the table and the family loaded. + pub origin: PostgisOrigin, + /// The table's integer primary key column. + pub key: String, + /// The SRID of the table's geometry. + pub srid: i32, + /// Whether the user it was loaded as may insert, update and delete rows. + pub writable: bool, + /// The SQL condition the rows were loaded with, if any (loaded again with it when edits + /// are discarded). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub condition: Option, + /// The most rows loaded. + #[serde(default)] + pub limit: usize, + /// Each loaded row's version and what it held, by its key, as read: saving checks the + /// database against them, so a row changed or deleted by someone else since is a conflict, + /// not overwritten. + #[serde(default)] + pub versions: Versions, +} + +/// How a loaded row is told apart from the same row edited: a hash of its geometry and +/// attributes (FNV-1a over their JSON, the same in every build). +pub fn row_hash(f: &rustgis_data::Feature) -> u64 { + /// FNV-1a over the bytes written to it, kept as they come (no buffer per row). + struct Fnv(u64); + impl std::io::Write for Fnv { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 = buf.iter().fold(self.0, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut h = Fnv(0xcbf2_9ce4_8422_2325); + // As before, a row that can't be written hashes as no bytes. + if serde_json::to_writer(&mut h, &(&f.geometry, &f.attrs)).is_err() { + return 0xcbf2_9ce4_8422_2325; + } + h.0 +} + +/// A loaded row: its version (PostgreSQL's `xmin`) and [`row_hash`] as read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RowVersion { + pub version: i64, + pub hash: u64, +} + +/// Loaded rows by key, shared until changed like a dataset's features: an undo snapshot copies +/// a pointer, not the map. +#[derive(Clone, Default, PartialEq)] +pub struct Versions(Arc>); + +impl Versions { + /// The row with key `id`, if it was loaded. + pub fn get(&self, id: u64) -> Option { + self.0.get(&id).copied() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Whether two are the same rows in memory (one cloned from the other). + pub fn shares(&self, other: &Versions) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + /// Whether anything but this holds the same rows (a layer, an undo step). + pub fn held_elsewhere(&self) -> bool { + Arc::strong_count(&self.0) > 1 + } + + /// The keys and rows, in key order. + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter().map(|(k, v)| (*k, *v)) + } +} + +impl FromIterator<(u64, RowVersion)> for Versions { + fn from_iter>(it: I) -> Versions { + Versions(Arc::new(it.into_iter().collect())) + } +} + +impl std::fmt::Debug for Versions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Versions({} rows)", self.0.len()) + } +} + +/// Saved as `[[key, version, hash], …]` (JSON object keys would be text). +impl Serialize for Versions { + fn serialize(&self, s: S) -> Result { + s.collect_seq(self.0.iter().map(|(k, r)| (k, r.version, r.hash))) + } +} + +impl<'de> Deserialize<'de> for Versions { + fn deserialize>(d: D) -> Result { + Vec::<(u64, i64, u64)>::deserialize(d).map(|v| v.into_iter().map(|(k, version, hash)| (k, RowVersion { version, hash })).collect()) + } +} + +/// What changed in a layer loaded from PostGIS since it was loaded (or last saved), by +/// comparing its features with the rows as read. +#[derive(Debug, Default)] +pub struct PostgisChanges { + /// Features of loaded rows that differ from them, with the rows' versions. + pub updates: Vec<(rustgis_data::Feature, i64)>, + /// Loaded rows no feature holds any more: their keys and versions. + pub deletes: Vec<(u64, i64)>, + /// Features of no loaded row. + pub inserts: Vec, + /// Where each of `inserts` is among the features. + pub insert_rows: Vec, +} + +impl PostgisChanges { + pub fn is_empty(&self) -> bool { + self.updates.is_empty() && self.deletes.is_empty() && self.inserts.is_empty() + } +} + +impl PostgisEdit { + /// What changed in `features` since the rows were read. A feature whose id is a loaded row's + /// is that row (once: a copy with the same id is new). + pub fn changes(&self, features: &[rustgis_data::Feature]) -> PostgisChanges { + let mut out = PostgisChanges::default(); + let mut seen = std::collections::HashSet::new(); + for (i, f) in features.iter().enumerate() { + match self.versions.get(f.id) { + Some(row) if seen.insert(f.id) => { + if row_hash(f) != row.hash { + out.updates.push((f.clone(), row.version)); + } + } + _ => { + out.inserts.push(f.clone()); + out.insert_rows.push(i); + } + } + } + out.deletes = self.versions.iter().filter(|(k, _)| !seen.contains(k)).map(|(k, r)| (k, r.version)).collect(); + out + } +} + /// A MySQL or MariaDB table a layer reads live. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct MysqlOrigin { @@ -885,3 +1044,76 @@ pub fn set_waker(f: impl Fn() + Send + Sync + 'static) { pub fn generation() -> u64 { GENERATION.load(Ordering::Relaxed) } + +#[cfg(test)] +mod tests { + #[test] + fn postgis_edits_keep_their_row_versions() { + let edit = super::PostgisEdit { + origin: super::PostgisOrigin { database: "postgres:host=h dbname=g".into(), table: "roads".into(), family: None, crs: None }, + key: "gid".into(), + srid: 3857, + writable: true, + condition: Some("lanes > 2".into()), + limit: 100, + versions: [(1, 700), (2, 701), (u64::MAX, 9)] + .into_iter() + .map(|(k, version)| (k, super::RowVersion { version, hash: k ^ 5 })) + .collect(), + }; + let text = serde_json::to_string(&edit).unwrap_or_default(); + assert!(text.contains("\"versions\":[[1,700,4],[2,701,7],"), "{text}"); + let back: super::PostgisEdit = serde_json::from_str(&text).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(back, edit); + assert_eq!((back.versions.get(2).map(|r| r.version), back.versions.get(3), back.versions.len()), (Some(701), None, 3)); + // A copy (an undo snapshot) shares the versions. + let copy = edit.clone(); + assert!(std::sync::Arc::ptr_eq(©.versions.0, &edit.versions.0)); + } + + /// The hash is FNV-1a over the row's JSON, the same as the projects saved with it hold. + #[test] + fn row_hashes_are_fnv_over_the_rows_json() { + use rustgis_data::{Coord, Feature, Geometry, Value}; + let f = Feature { + id: 3, + geometry: Some(Geometry::Point(Coord { x: 1.5, y: -2.0 })), + attrs: vec![Value::Text("Main St".into()), Value::Int(4), Value::Null], + }; + let bytes = serde_json::to_vec(&(&f.geometry, &f.attrs)).unwrap(); + let fnv = bytes.iter().fold(0xcbf2_9ce4_8422_2325_u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x0100_0000_01b3)); + assert_eq!(super::row_hash(&f), fnv); + // And the value itself: a change in how rows are written would show every saved row edited. + assert_eq!(super::row_hash(&f), 8_547_063_913_914_077_080); + } + + #[test] + fn changes_are_what_differs_from_the_rows_as_read() { + use rustgis_data::{Coord, Feature, Geometry, Value}; + let row = |id: u64, name: &str| Feature { + id, + geometry: Some(Geometry::Point(Coord { x: id as f64, y: 1.0 })), + attrs: vec![Value::Text(name.into())], + }; + let loaded = [row(1, "a"), row(2, "b"), row(3, "c")]; + let edit = super::PostgisEdit { + origin: super::PostgisOrigin { database: "postgres:dbname=g".into(), table: "t".into(), family: None, crs: None }, + key: "id".into(), + srid: 4326, + writable: true, + condition: None, + limit: 0, + versions: loaded.iter().map(|f| (f.id, super::RowVersion { version: f.id as i64 * 10, hash: super::row_hash(f) })).collect(), + }; + assert!(edit.changes(&loaded).is_empty()); + // Row 1 as it was, row 2 renamed, row 3 deleted, a new feature and a copy of row 1. + let now = [row(1, "a"), row(2, "bee"), row(9, "new"), row(1, "copy")]; + let c = edit.changes(&now); + assert_eq!(c.updates.iter().map(|(f, v)| (f.id, *v)).collect::>(), vec![(2, 20)]); + assert_eq!(c.deletes, vec![(3, 30)]); + assert_eq!(c.inserts.iter().map(|f| f.id).collect::>(), vec![9, 1]); + assert_eq!(c.insert_rows, vec![2, 3]); + // The hash is the same however often it is taken. + assert_eq!(super::row_hash(&now[1]), super::row_hash(&row(2, "bee"))); + } +} diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 99c83758d..25cd698c3 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -708,6 +708,10 @@ fn search_action(id: &str) -> Option { "connection.resolveS3" => Command("ui.s3Connections"), "layer.addSql" => Pane(RightPane::Sql), "layer.addDatabase" => Dialog("addDatabase"), + // Add From Database with Load for editing ticked. + "pg.connect" | "pg.addLayer" => Dialog("pgAddLayer"), + "pg.saveEdits" => Dialog("pgSave"), + "pg.discardEdits" => Dialog("pgDiscard"), "layer.addZarr" => Command("ui.addZarr"), "layer.addLayerFile" => OpenItem("layer"), "layer.save" => SaveItem("layer"), @@ -813,6 +817,7 @@ fn search_platform_disabled(id: &str, web: bool) -> Option<&'static str> { match id { "project.addFolder" => Some("folder connections need the desktop app"), "layer.addDatabase" => Some("databases are read through DuckDB, a program the browser can't run"), + "pg.connect" | "pg.addLayer" | "pg.saveEdits" | "pg.discardEdits" => Some("PostGIS tables are edited in the desktop app"), _ => None, } } @@ -1196,7 +1201,7 @@ mod tests { #[test] fn desktop_only_search_actions_are_disabled_only_on_the_web() { - for id in ["project.addFolder", "layer.addDatabase"] { + for id in ["project.addFolder", "layer.addDatabase", "pg.addLayer", "pg.saveEdits"] { assert!(search_platform_disabled(id, true).is_some(), "{id} must explain its web restriction"); assert_eq!(search_platform_disabled(id, false), None, "{id} stays available on desktop"); } diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index e0ae8ce8d..c72301c6e 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -18,10 +18,16 @@ pub struct Dialogs { pub repair: Option, pub open: Option<&'static str>, pub layer: String, + /// The PostGIS layer whose edits were last counted (Save or Discard PostGIS Edits). + pub counted: String, pub text: String, pub a: String, pub b: String, pub c: String, + pub d: String, + pub e: String, + /// Add From Database: load the PostgreSQL table whole, to be edited and saved back. + pub edit: bool, pub mode: &'static str, pub flag: bool, pub error: Option, @@ -83,6 +89,10 @@ impl Dialogs { "addUrl" => "addUrl", "addService" => "addService", "addDatabase" => "addDatabase", + // Add From Database with Load for editing ticked. + "pgAddLayer" => "addDatabase", + "pgSave" => "pgSave", + "pgDiscard" => "pgDiscard", "chooseConnection" => "chooseConnection", "s3" => "s3", "zarr" => "zarr", @@ -98,10 +108,13 @@ impl Dialogs { if name == "exportLayer" && active.is_some_and(|l| l.is_raster()) { return Err("Export Features needs a vector layer (Data › Export Raster writes a raster)".into()); } + if matches!(name, "pgSave" | "pgDiscard") && !s.map().layers.iter().any(|l| l.postgis.is_some()) { + return Err("the map has no PostGIS layer loaded for editing (Home › Add Data › Database…, Load for editing)".into()); + } if name == "selectByAttributes" && s.map().layers.is_empty() { return Err("the map has no layers".into()); } - *self = Dialogs { open: Some(name), mode: "new", reset_collapse: true, ..Dialogs::default() }; + *self = Dialogs { open: Some(name), mode: "new", edit: which == "pgAddLayer", reset_collapse: true, ..Dialogs::default() }; self.layer = active.or_else(|| s.map().layers.first()).map(|l| l.name.clone()).unwrap_or_default(); match name { "defineProjection" => { @@ -137,6 +150,11 @@ impl Dialogs { // A file name can't hold a folder separator, which a project name may. "saveProject" => self.text = format!("{}.rgis", s.project.name.replace(['/', '\\'], "-")), "exportLayer" => self.mode = "geojson", + // The active layer when it is from PostGIS, else the first that is. + "pgSave" | "pgDiscard" => { + let pg = |l: &&rustgis_engine::map::Layer| l.postgis.is_some(); + self.layer = active.filter(pg).or_else(|| s.map().layers.iter().find(pg)).map(|l| l.name.clone()).unwrap_or_default(); + } // A table of addresses is what's usually geocoded: picked by its id (`flag` says the // choice is a table), since names can repeat. "geocode" => { @@ -184,6 +202,8 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { "addUrl" => "Add Data From URL", "addService" => "Add From Service", "addDatabase" => "Add From Database", + "pgSave" => "Save to PostGIS", + "pgDiscard" => "Discard PostGIS Edits", "newConnection" => "New Connection", "editConnection" => "Edit Connection", "chooseConnection" => "Choose Connection", @@ -325,6 +345,8 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { "addUrl" => close = add_url(app, ui), "addService" => close = add_service(app, ui), "addDatabase" => close = add_database(app, ui), + "pgSave" => close = pg_save(app, ui), + "pgDiscard" => close = pg_discard(app, ui), "newConnection" | "editConnection" => close = crate::connectionui::dialog(app, ui), "chooseConnection" => close = crate::connectionui::chooser(app, ui), "s3" => crate::s3ui::dialog(app, ui), @@ -1028,6 +1050,8 @@ fn add_database(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { let connections = crate::sourcespane::connections_snapshot(app); let typed = database(&app.dialogs.a); let current = crate::connectionui::picked(&connections, app.dialogs.connection.as_deref(), &typed); + #[cfg(not(target_arch = "wasm32"))] + let postgres = rustgis_engine::connections::database_kind(&typed) == Some("postgres"); let mut pick = None; let mut list = false; egui::Grid::new("database").num_columns(2).spacing(vec2(10.0, 8.0)).show(ui, |ui| { @@ -1071,7 +1095,45 @@ fn add_database(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { ui.add(egui::TextEdit::singleline(&mut app.dialogs.text).hint_text(tl!("of a geometry that doesn't say")).desired_width(280.0)) .labelled_by(input_label.id); ui.end_row(); + // A PostgreSQL table can instead be loaded whole, to be edited and saved back to it. + #[cfg(not(target_arch = "wasm32"))] + if postgres { + ui.label(""); + ui.checkbox(&mut app.dialogs.edit, tl!("Load for editing")).on_hover_text(tl!( + "Load the table whole (up to 100,000 rows) so its edits can be saved back to it with Edit › Save to PostGIS. It needs a single-column integer primary key." + )); + ui.end_row(); + if app.dialogs.edit { + let input_label = ui.label(tl!("Condition")); + ui.add( + egui::TextEdit::singleline(&mut app.dialogs.d) + .hint_text(tl!("optional, such as population > 10000")) + .desired_width(280.0), + ) + .labelled_by(input_label.id); + ui.end_row(); + let input_label = ui.label(tl!("Most rows")); + ui.add(egui::TextEdit::singleline(&mut app.dialogs.e).hint_text("100000").desired_width(280.0)).labelled_by(input_label.id); + ui.end_row(); + } + } }); + // The connection that layers loaded for editing save through, until RustGIS closes. + #[cfg(not(target_arch = "wasm32"))] + if let Some(dsn) = app.session.postgis.clone() { + ui.horizontal(|ui| { + let short = rustgis_engine::data::source::duckdb::Database::Postgres(dsn).short(); + ui.label(egui::RichText::new(crate::i18n::fmt("Connected for editing: {db}", &[("db", &short)])).weak()); + if ui + .button(tl!("Disconnect")) + .on_hover_text(tl!("Close the connection that PostGIS layers loaded for editing save through.")) + .clicked() + && let Err(e) = app.run("pg.disconnect", Value::Null) + { + app.dialogs.error = Some(e); + } + }); + } // The tables listed and the one chosen belong to the address they were listed for. if app.dialogs.listed.as_deref().is_some_and(|l| l != database(&app.dialogs.a)) { app.dialogs.listed = None; @@ -1118,7 +1180,11 @@ fn add_database(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { } }); buttons(app, ui, "Add", |app| { - let mut p = json!({"database": database(&app.dialogs.a), "password": app.dialogs.b}); + let address = database(&app.dialogs.a); + if app.dialogs.edit && rustgis_engine::connections::database_kind(&address) == Some("postgres") { + return add_for_editing(app, address); + } + let mut p = json!({"database": address, "password": app.dialogs.b}); for (key, v) in [("layer", &app.dialogs.c), ("crs", &app.dialogs.text)] { if !v.trim().is_empty() { p[key] = json!(v.trim()); @@ -1133,6 +1199,92 @@ fn add_database(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { }) } +/// Add From Database with Load for editing: connects to the PostgreSQL database for editing +/// (`pg.connect`) and loads the chosen table whole (`pg.addLayer`). +fn add_for_editing(app: &mut RustGisApp, address: String) -> Result<(), String> { + let d = &app.dialogs; + let table = d.c.trim(); + if table.is_empty() { + return Err(tl!("Choose a table to load for editing").to_string()); + } + let mut p = json!({"table": table}); + for (key, v) in [("where", &d.d), ("crs", &d.text)] { + if !v.trim().is_empty() { + p[key] = json!(v.trim()); + } + } + if !d.e.trim().is_empty() { + let n: u64 = d.e.trim().replace([',', '.'], "").parse().map_err(|_| tl!("Most rows: a whole number").to_string())?; + p["limit"] = json!(n); + } + let connect = json!({"connection": address, "password": d.b}); + app.run("pg.connect", connect)?; + // The password is the session's to keep, not the dialog's. + app.dialogs.b.clear(); + app.run("pg.addLayer", p).map(|_| ()) +} + +/// A PostGIS layer to save or discard the edits of, and what changed in it: its table and how +/// many rows are new, changed and deleted (counted again when another layer is chosen). +fn pg_layer_edits(app: &mut RustGisApp, ui: &mut egui::Ui) { + let layers: Vec = app.session.map().layers.iter().filter(|l| l.postgis.is_some()).map(|l| l.name.clone()).collect(); + if app.dialogs.counted != app.dialogs.layer { + app.dialogs.counted = app.dialogs.layer.clone(); + app.dialogs.list = match app.run("pg.pendingEdits", json!({"layer": app.dialogs.layer})) { + Ok(v) => vec![ + v["table"].as_str().unwrap_or_default().to_string(), + v["inserts"].to_string(), + v["updates"].to_string(), + v["deletes"].to_string(), + ], + Err(e) => { + app.dialogs.error = Some(e); + Vec::new() + } + }; + } + egui::Grid::new("pg_edits").num_columns(2).spacing(vec2(10.0, 8.0)).show(ui, |ui| { + ui.label(tl!("Layer")); + egui::ComboBox::from_id_salt("pg_edit_layer").selected_text(app.dialogs.layer.clone()).width(260.0).show_ui(ui, |ui| { + for l in layers { + ui.selectable_value(&mut app.dialogs.layer, l.clone(), l); + } + }); + ui.end_row(); + let shown = app.dialogs.list.clone(); + for (label, value) in [ + (tl!("Table"), shown.first()), + (tl!("New rows"), shown.get(1)), + (tl!("Changed rows"), shown.get(2)), + (tl!("Deleted rows"), shown.get(3)), + ] { + ui.label(label); + ui.label(egui::RichText::new(value.cloned().unwrap_or_default()).strong()); + ui.end_row(); + } + }); +} + +/// Save PostGIS Edits: a PostGIS layer's edits written to its table in one transaction, or none +/// of them when a row changed there since. +fn pg_save(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { + ui.label(egui::RichText::new(tl!("Saves the layer's edits to its table in one transaction. When a row you changed or deleted was changed or deleted in the database since it was loaded, or a new row's key is taken, nothing is saved and the rows are listed.")).weak()); + pg_layer_edits(app, ui); + buttons(app, ui, "Save", |app| { + let r = app.run("pg.saveEdits", json!({"layer": app.dialogs.layer})).map(|_| ()); + // Counted again, whatever happened. + app.dialogs.counted.clear(); + r + }) +} + +/// Discard PostGIS Edits: a PostGIS layer's rows loaded again as they are in the database. +fn pg_discard(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { + ui.label(egui::RichText::new(tl!("Loads the layer's rows again as they are in the database now. Its edits that weren't saved are dropped (Undo brings them back).")).weak()); + pg_layer_edits(app, ui); + buttons(app, ui, "Discard", |app| app.run("pg.discardEdits", json!({"layer": app.dialogs.layer})).map(|_| ())) +} + /// Add Tile Layer: an XYZ/TMS/WMS template, vector tiles or a MapLibre style by address. fn add_tiles(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { use rustgis_engine::map::TileKind; diff --git a/crates/ui-egui/src/i18n/id.tsv b/crates/ui-egui/src/i18n/id.tsv index 9cfad847f..279843c36 100644 --- a/crates/ui-egui/src/i18n/id.tsv +++ b/crates/ui-egui/src/i18n/id.tsv @@ -3561,3 +3561,18 @@ Zoom: the map's zoom level, as web maps count it. Bearing: the direction the top ⌘+Enter ⌘+Enter ⌘K ⌘K ▶ Simulate ▶ Simulasikan +Condition Kondisi +optional, such as population > 10000 opsional, misalnya population > 10000 +Most rows Baris maksimum +Most rows: a whole number Baris maksimum: bilangan bulat +{n} tables with geometry {n} tabel dengan geometri +Save a PostGIS layer's edits to its table, unless rows changed there since. Simpan edit layer PostGIS ke tabelnya, kecuali ada baris yang sudah berubah di sana. +Load a PostGIS layer's rows again as they are in the database, dropping its edits. Muat ulang baris layer PostGIS sesuai isi database, dan buang editnya. +Discard PostGIS Edits Batalkan Edit PostGIS +New rows Baris baru +Changed rows Baris berubah +Deleted rows Baris terhapus +Saves the layer's edits to its table in one transaction. When a row you changed or deleted was changed or deleted in the database since it was loaded, or a new row's key is taken, nothing is saved and the rows are listed. Menyimpan edit layer ke tabelnya dalam satu transaksi. Jika baris yang Anda ubah atau hapus sudah diubah atau dihapus di database sejak dimuat, atau key baris baru sudah dipakai, tidak ada yang disimpan dan baris-baris itu ditampilkan. +Loads the layer's rows again as they are in the database now. Its edits that weren't saved are dropped (Undo brings them back). Memuat ulang baris layer sesuai isi database saat ini. Edit yang belum disimpan dibuang (Undo bisa mengembalikannya). +Connect PostGIS Hubungkan PostGIS +Add PostGIS Layer Tambah Layer PostGIS diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 033702c1e..0752c3362 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -3846,6 +3846,8 @@ const DEFAULTS_TO_ACTIVE_LAYER: &[&str] = &[ "share.webLayer", "share.webStyle", "share.locator", + "pg.saveEdits", + "pg.discardEdits", "zarr.reduce", "cube.trend", "cube.reduce", diff --git a/crates/ui-egui/src/ribbon.rs b/crates/ui-egui/src/ribbon.rs index 4e02bd2aa..6ff40de05 100644 --- a/crates/ui-egui/src/ribbon.rs +++ b/crates/ui-egui/src/ribbon.rs @@ -2347,6 +2347,33 @@ fn edit_tab(app: &mut RustGisApp, ui: &mut Ui) { small(ui, app, "redo", "Redo", "Redo the change you undid.", "edit.redo", Value::Null, false); small(ui, app, "history", "Status", "Edits since the project was saved.", "ui.pane.editing", Value::Null, pane); }); + // A layer loaded from PostGIS for editing saves its edits to its table, not the project + // (desktop only: the browser doesn't reach databases). + #[cfg(not(target_arch = "wasm32"))] + if app.session.map().layers.iter().any(|l| l.postgis.is_some()) { + col(ui, |ui| { + small( + ui, + app, + "save", + "Save to PostGIS…", + "Save a PostGIS layer's edits to its table, unless rows changed there since.", + "ui.dialog", + json!({"name": "pgSave"}), + false, + ); + small( + ui, + app, + "update", + "Discard PostGIS Edits…", + "Load a PostGIS layer's rows again as they are in the database, dropping its edits.", + "ui.dialog", + json!({"name": "pgDiscard"}), + false, + ); + }); + } col(ui, |ui| { small(ui, app, "settings", "Settings", "Editing settings.", "ui.pane.editing", Value::Null, pane); small( diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index e8c001e08..adbaf7822 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -217,6 +217,57 @@ A query can read any file the app can and fetch any web address, and a database server, so the [AI Assistant](assistant.md) asks before it runs a query or reads a database, and plugins need both the files and network permissions. +### Editing PostGIS tables + +On the desktop, a PostgreSQL table with PostGIS can be loaded to be edited and its edits saved +back to it. In **Home › Add › Add Data › Database…**, give a PostgreSQL connection, choose a table +(**List Tables**) and tick **Load for editing**. + +- The connection (`pg.connect`) lasts until RustGIS closes, or until **Disconnect** in the same + form (`pg.disconnect`). It isn't saved with the project, and neither is the password. Without a + password, `PGPASSWORD` or libpq's password file serve. +- The table (`pg.addLayer`; `pg.tables` lists those with a geometry or geography column) loads + whole, up to 100,000 rows a layer, in the order of its key. + A table whose geometries are of several types loads each type as its own layer, each up to that many. + A **condition** (an SQL expression on its columns, such as `population > 10000`) loads only the + rows it keeps. The layer is an ordinary layer in memory, so every editing tool works on it. + +- **Edit › Manage Edits › Save to PostGIS…** (`pg.saveEdits`), there while a layer is loaded for + editing, shows how many of the layer's rows are new, changed and deleted, and writes them to its + table in one transaction. **Discard PostGIS Edits…** (`pg.discardEdits`) + loads the rows again as they are in the database, dropping the edits (Undo brings them back). + +A table needs a single-column integer primary key to be added this way. The key becomes each +feature's id, and the layer keeps each row's version (PostgreSQL's `xmin`) and what it held as it +was read. Saving locks the rows you changed or deleted and checks them first: when one was changed +or deleted in the database since you loaded it, or a new row's key is already taken, **nothing is +saved**, and the message lists those rows. Discard PostGIS Edits then loads them as they are now. A row +someone else changed that you didn't edit is no conflict, and stays as they left it. + +Undo doesn't reach into the database: undoing a save brings back the layer as it was before, but +the rows stay saved. Saving that layer again is refused, since it would write the same edits +twice (its new rows as more new rows); Redo, or Discard PostGIS Edits, brings the layer back in step. + +Only what you changed is written: the columns whose values differ, and the geometry when it moved +(converted back to the table's coordinate system, single or multi type and dimensions). A new row +takes the value of its key field, else the table's default key (`serial`, identity), else its +feature id, and its feature takes the key it was saved with. A new row's empty fields take their +column's default, NULL when it has none. Fields the table has no column for +aren't saved, and the message names them. The layer says when the database's user can't insert, +update or delete the table's rows; saving is then refused. + +```sh +rustgis-cli run --cmd 'pg.connect={"host":"localhost","database":"gis","user":"postgres"}' \ + --cmd 'pg.addLayer={"table":"public.roads","where":"lanes > 2","name":"Roads"}' \ + --cmd 'edit.enable={"on":true}' \ + --cmd 'edit.setValue={"layer":"Roads","row":0,"field":"name","value":"Main St"}' \ + --cmd 'pg.saveEdits={"layer":"Roads"}' +``` + +Connecting and adding a layer read the database, so the [AI Assistant](assistant.md) asks +before it connects or adds a table, and asks every time before it saves edits. Plugins can't +connect or save at all. + ## MySQL and MariaDB **Home › Add › Add Data › Database…** (`layer.addDatabase`) also connects to a MySQL (8 or later) or diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 928eb18a5..1ed4b28c0 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -112,7 +112,7 @@ ribbon, arrows at its ends scroll it. | **Data** | Layer Templates, Data Source (Repair Source); for a feature layer: Definition Query, Table, Selection, Data Design, Archiving, Relationship and Export; for a raster layer: Information and Export; then Offline and Feature Cache | | **Style** | For the layer selected in Layers. A feature layer: Visibility Range, Effects, Drawing, Drawing Effects, Extrusion for polygons (with Faces and Light and Shadow once on) and Layer. A raster layer: Visibility Range, Effects, Compare, Raster Effects, Rendering, Enhancement and Layer. Both: Styles | | **Labeling** | For the feature layer selected in Layers: Labels (Label, Properties), Label Class (Class, Field), Text Symbol (Size, Color, Bold, Halo), Label Placement and Label Range. Whatever is selected: Map Labeling (Lock, Pause, View Unplaced, Convert) | -| **Edit** | Manage Edits (Edit, Save, Undo, Redo, Status, Settings, Templates, Topology), Snapping, Features (Create, Delete, Edit Vertices, Modify), Tools, Elevation, Corrections, Graphics, Graphics and Text, Clipboard | +| **Edit** | Manage Edits (Edit, Save, Undo, Redo, Status, Save to PostGIS and Discard PostGIS Edits on the desktop while a [PostGIS layer is loaded for editing](data.md#editing-postgis-tables), Settings, Templates, Topology), Snapping, Features (Create, Delete, Edit Vertices, Modify), Tools, Elevation, Corrections, Graphics, Graphics and Text, Clipboard | | **Analyze** | Geoprocessing (History, Tools, Model Builder, Script, SQL, Environments), Tasks (the Tasks pane; Task: Record Task, New Task From History, Import Task, run a task), AI (AI Assistant), Portal (Feature Analysis, Raster Analysis), Workflows, Raster (Raster Functions, Function Editor), Tools (the tool gallery), Selection, Notebook, the imagery groups (Ortho Mapping, Reality Mapping, Alignment, Image Analysis, Image Classification, Imagery Sharing, Mensuration, Imagery Tools) | | **View** | Display (Graticule, Dark Mode, GPU Rendering), View (Globe, Projection, Global; Local on the desktop), Effects, Scene, Windows, Navigation, Link, Workspace, Animation, Time (Time Slider), Profile (Profile; Time Series on the desktop) | | **Publish** | While a layout is open, its groups first (Page, Map Frames, Map Surrounds, Shapes, Text, Export); then Report, Presentation, Output (Export Map, Export Features, Print Map, Capture To Clipboard, Report, Presentation, Export To 3D), Package, Share As, Layout, Save As, Thumbnail |