diff --git a/apps/rustgis-python/src/lib.rs b/apps/rustgis-python/src/lib.rs index 648862772..ad09da547 100644 --- a/apps/rustgis-python/src/lib.rs +++ b/apps/rustgis-python/src/lib.rs @@ -177,7 +177,7 @@ impl Engine { } /// The project as `.rgis` JSON text, as `project.saveAs` writes it (what `project.openText` - /// opens). + /// opens), without its project log. fn project_json(&self, py: Python<'_>) -> PyResult { let result = self.with_session(py, |session| serde_json::to_string(&session.project).map_err(|e| e.to_string())); self.flush(py); diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 2f79bf0b6..1eb567827 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -560,10 +560,18 @@ pub(crate) fn temp_beside(path: &Path) -> std::io::Result { /// Writes and syncs a new file (`create_new`, so an existing file or symlink is never followed); /// removes it on failure. fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_new_with(path, |w| w.write_all(bytes)) +} + +/// [`write_new`] with the content written by `write`, through a buffer, as it goes. +fn write_new_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { use std::io::Write; - let mut f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?; + let f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?; let res = (|| { - f.write_all(bytes)?; + let mut w = std::io::BufWriter::new(&f); + write(&mut w)?; + w.flush()?; + drop(w); f.sync_all() })(); // Close before removing a failed write, including on Windows. @@ -577,8 +585,14 @@ fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { /// Writes through a temporary file that is synced and renamed into place, so an interrupted /// write never truncates an existing file. The temporary file is removed on failure. pub fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_atomic_with(path, |w| w.write_all(bytes)) +} + +/// [`write_atomic`] with the content written by `write` as it goes, so a long text needn't be +/// held whole first. A failure in `write` leaves an existing file as it was. +pub fn write_atomic_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> { let tmp = temp_beside(path)?; - write_new(&tmp, bytes)?; + write_new_with(&tmp, write)?; let res = std::fs::rename(&tmp, path); if res.is_err() { let _ = std::fs::remove_file(&tmp); @@ -2305,6 +2319,16 @@ mod tests { assert_eq!(std::fs::read(&p).unwrap(), b"two"); assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "no temp files left behind"); assert!(write_atomic(Path::new("/"), b"x").is_err()); + // Written as it goes: a failure part way leaves the file as it was, and no temp file. + let failed = write_atomic_with(&p, |w| { + w.write_all(b"half")?; + Err(std::io::Error::other("stopped")) + }); + assert!(failed.is_err()); + assert_eq!(std::fs::read(&p).unwrap(), b"two"); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "no temp files left behind"); + write_atomic_with(&p, |w| w.write_all(b"th").and_then(|()| w.write_all(b"ree"))).unwrap(); + assert_eq!(std::fs::read(&p).unwrap(), b"three"); std::fs::remove_dir_all(&dir).ok(); } diff --git a/crates/engine/src/cmd/gp.rs b/crates/engine/src/cmd/gp.rs index 1c1b03863..f72c120b9 100644 --- a/crates/engine/src/cmd/gp.rs +++ b/crates/engine/src/cmd/gp.rs @@ -5,6 +5,7 @@ use rustgis_data::Dataset; use serde_json::{Value, json}; use super::{need_layers, ps}; +use crate::project_log::now_ms; use crate::{CmdResult, CommandSpec as C, GpRecord, Session}; pub fn commands() -> Vec { @@ -701,10 +702,6 @@ impl Inputs for MapInputs<'_> { } } -fn now_ms() -> u64 { - web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) -} - fn run(s: &mut Session, p: &Value) -> CmdResult { let tool = ps(p, "tool").ok_or("tool is required")?.to_string(); let params = p.get("params").cloned().unwrap_or(json!({})); diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 157c3ec80..3a22bd151 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -112,7 +112,7 @@ pub fn tool_of(r: &GpRecord) -> Option { /// `password`, an `api_token`; a plain `key` when it reads as one or sits beside a web address, /// as `sources::holds_secret` judges it), left empty. An object's keys too: the file may put /// anything there. -fn redacted(v: &Value) -> Value { +pub(crate) fn redacted(v: &Value) -> Value { use crate::sources::looks_like_a_key; use rustgis_data::source::credentials::secret_key; match v { @@ -139,7 +139,7 @@ fn redacted(v: &Value) -> Value { /// name addresses among its words; a single word may be one. Any text may still be a connection /// in a form `redact` doesn't read (libpq's words alone, GDAL's `MYSQL:db,user=…,password=…`, /// ODBC's `UID=…;PWD=…`): its secret-named values go too. -fn redacted_text(t: &str) -> String { +pub(crate) fn redacted_text(t: &str) -> String { let r = if connection(t) || !t.contains(char::is_whitespace) { crate::sources::redact(t) } else { crate::sources::redact_text(t) }; // After `redact` too: a prefix it knows may come in a form it doesn't (GDAL's `MYSQL:`). query_pairs_emptied(&private_keys_removed(&r)) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 95b35e4bd..039ad9297 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -523,7 +523,7 @@ fn import_styles(s: &mut Session, p: &Value) -> CmdResult { /// The project without its features (layer schemas, symbology, layouts and styles stay). fn template(p: &Project) -> Project { - let mut t = p.clone(); + let mut t = p.own_copy(); t.gp_history.clear(); for m in &mut t.maps { for l in &mut m.layers { diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 55825e5de..07e97c4a1 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -290,6 +290,9 @@ const NEVER: &[&str] = &[ "s3.disconnect", "earthdata.login", "earthdata.logout", + // The project log records what was done (#797): a plugin never empties it or turns it off. + "project.clearLog", + "project.logCommands", "pg.connect", "pg.disconnect", // Writes to a database: the user's to do, asked each time. @@ -297,15 +300,16 @@ const NEVER: &[&str] = &[ "gpkg.saveEdits", ]; -/// Commands that write files without naming one in their parameters. -const WRITES_FILES: &[&str] = &["project.save", "project.saveAs"]; +/// Commands that write files without naming one in their parameters, or read what the user's +/// commands named (the project log's paths, addresses and SQL). +const REACHES_FILES: &[&str] = &["project.save", "project.saveAs", "project.log"]; /// Commands that read a file a layer names, not their parameters. const READS_FILES: &[&str] = &["gpkg.discardEdits"]; /// Parameter keys that name a file, folder or address. (`data` and `text` carry content, which /// plugins may pass: `layer.addDataBytes`, `map.import` from text.) -const PLACE_KEYS: &[&str] = &["path", "folder", "dir", "file", "url", "uri", "source", "output", "connection", "database"]; +pub(crate) const PLACE_KEYS: &[&str] = &["path", "folder", "dir", "file", "url", "uri", "source", "output", "connection", "database"]; /// The command running an SQL query through DuckDB, which can read any file and web address. pub const SQL: &str = "layer.addSql"; @@ -383,7 +387,7 @@ pub fn reaches(spec: &C, params: &Value) -> Option { } let files = super::items::names_place(params, PLACE_KEYS, 0) || PLACE_KEYS.iter().any(|k| spec.params.contains(&format!("\"{k}\""))) - || WRITES_FILES.contains(&spec.id) + || REACHES_FILES.contains(&spec.id) || READS_FILES.contains(&spec.id) || ["share.", "offline.", "cache.", "notebook."].iter().any(|p| spec.id.starts_with(p)); files.then_some(Reach::Files) diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index 5afee10d7..741de6fd3 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -31,9 +31,166 @@ pub fn commands() -> Vec { .params(r#"{"tool": "explore"|"select"|"selectPolygon"|"selectLasso"|"selectRadius"|"zoomIn"|"measure"|"measureArea"|"createPoint"|"createLine"|"createPolygon"|"editVertices"|"graphicPoint"|"graphicText"|"graphicLine"|"graphicCurve"|"graphicFreehand"|"graphicPolygon"|"graphicLasso"|"graphicRectangle"|"graphicCircle"|"graphicEllipse"|"graphicPicture"}"#), C::new("info.project", "Project Info", "Project", info).view().params(r#"{"nameFilter"?: string, "includeFields"?: bool}"#), C::new("info.commands", "List Commands", "Help", list).view(), + // The project log (#797): read, written out, emptied, turned on or off. Clearing and + // switching change what is saved, but take no undo step: the log sits outside the undo + // history, so one would restore nothing of it. + C::new("project.log", "Project Log", "Project", log).view().params( + r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis"}], "count": matching, "total": in the log, "on": bool}"#, + ), + C::new("project.exportLog", "Export Project Log", "Project", export_log) + .view() + .params(r#"{"path": ".csv, .jsonl or .ndjson file", "format"?: "csv"|"jsonl" (for a path with no extension; one given must match the extension)}"#), + C::new("project.clearLog", "Clear Project Log", "Project", clear_log).outside_undo(), + C::new("project.logCommands", "Log Commands", "Project", log_commands).outside_undo().params(r#"{"on"?: bool (default: turn it the other way)}"#), ] } +/// `project.log`: the log's entries that match, newest last. +fn log(s: &mut Session, p: &Value) -> CmdResult { + // Milliseconds as a whole number or not (1.7e12), never below 0. + let num = |k: &str| -> Result, String> { + match p.get(k) { + None | Some(Value::Null) => Ok(None), + Some(v) => v + .as_f64() + .filter(|n| n.is_finite() && *n >= 0.0) + .map(|n| Some(n as u64)) + .ok_or_else(|| format!("{k} must be a number, 0 or more")), + } + }; + let (since, until) = (num("since")?, num("until")?); + let command = match p.get("command") { + None | Some(Value::Null) => None, + Some(v) => Some(v.as_str().ok_or("command must be a command id, or the start of one")?.trim()).filter(|c| !c.is_empty()), + }; + let failed = match p.get("failed") { + None | Some(Value::Null) => None, + Some(v) => Some(v.as_bool().ok_or("failed must be true or false")?), + }; + // At most a thousand (some 4 MB at the most), so one answer can't fill an assistant's + // context or a script's memory; `count` still says how many match. + let limit = match p.get("limit") { + None | Some(Value::Null) => 100, + Some(v) => match v.as_f64().filter(|n| n.is_finite() && *n >= 1.0 && n.fract() == 0.0) { + Some(n) => n.min(1_000.0) as usize, + None => return Err("limit must be a whole number, 1 or more".into()), + }, + }; + let keep = |e: &crate::project::LogEntry| { + since.is_none_or(|t| e.at >= t) + && until.is_none_or(|t| e.at <= t) + && command.is_none_or(|c| e.command == c || (c.ends_with('.') && e.command.starts_with(c))) + && failed.is_none_or(|f| e.ok != f) + }; + let (shown, count, total) = s.project.log.select(keep, limit); + Ok(json!({"entries": shown, "count": count, "total": total, "on": s.project.log.on()})) +} + +/// A CSV cell: quoted, and kept from being read as a formula where it starts as one does (`=`, +/// `+`, `-`, `@`, or a tab or carriage return before them), as spreadsheets would. +fn csv_cell(t: &str) -> String { + let lead = if t.starts_with(['=', '+', '-', '@', '\t', '\r']) { "'" } else { "" }; + format!("\"{lead}{}\"", t.replace('"', "\"\"")) +} + +/// `project.exportLog`: the log written out, as CSV (one row an entry, the parameters as a JSON +/// column) or JSON Lines (one entry a line). +fn export_log(s: &mut Session, p: &Value) -> CmdResult { + let path = PathBuf::from(ps(p, "path").ok_or("path is required")?); + let kind = |f: &str| match f { + "csv" => Some("csv"), + "jsonl" | "ndjson" => Some("jsonl"), + _ => None, + }; + // The path's extension, when it has one, is the format's whatever `format` says: the log + // never goes over a project, a JSON document or any other kind of file. + let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase); + let by_ext = + match ext.as_deref() { + None => None, + Some(e) => Some(kind(e).ok_or_else(|| { + format!("the log is written to a .csv or .jsonl file, not a .{e} one (JSON Lines isn't one JSON document)") + })?), + }; + let asked = ps(p, "format").map(str::to_ascii_lowercase); + let format = match (asked.as_deref(), by_ext) { + (Some(f), by_ext) => { + let f = kind(f).ok_or("format is csv or jsonl")?; + if by_ext.is_some_and(|e| e != f) { + return Err(format!("format {f} doesn't match the path's .{}", ext.as_deref().unwrap_or_default())); + } + f + } + (None, Some(e)) => e, + (None, None) => return Err("give a .csv or .jsonl path, or a format (csv or jsonl)".into()), + }; + // Written entry by entry, a few hundred copied at a time under the log's lock and written + // after it is let go: a long log isn't held as text, and an autosave reading the log doesn't + // wait on the disk. A failure part way leaves an existing file as it was. + let mut n = 0usize; + rustgis_data::io::write_atomic_with(&path, |w| { + if format == "csv" { + w.write_all(b"time_utc,at,command,ok,millis,error,params\n")?; + } + loop { + let part = s.project.log.chunk(n, 256); + if part.is_empty() { + return Ok(()); + } + for e in &part { + write_entry(w, e, format)?; + } + n += part.len(); + } + }) + .map_err(|e| format!("{}: {e}", path.display()))?; + s.notify(format!("Exported the project log ({n} entries) to {}", path.display()), false); + Ok(json!({"path": path.display().to_string(), "entries": n, "format": format})) +} + +/// Entry `e` as a CSV row or a JSON line, written to `w`. +fn write_entry(w: &mut dyn std::io::Write, e: &crate::project::LogEntry, format: &str) -> std::io::Result<()> { + if format == "csv" { + let (date, time) = rustgis_map::graphics::civil(e.at / 1000); + writeln!( + w, + "{date} {time}:{:02},{},{},{},{},{},{}", + (e.at / 1000) % 60, + e.at, + csv_cell(&e.command), + e.ok, + e.millis, + csv_cell(e.error.as_deref().unwrap_or("")), + csv_cell(&e.params.to_string()) + ) + } else { + serde_json::to_writer(&mut *w, e).map_err(std::io::Error::other)?; + w.write_all(b"\n") + } +} + +/// `project.clearLog`: empties the log, which then holds this command's own entry, so a cleared +/// log says it was cleared. +fn clear_log(s: &mut Session, _: &Value) -> CmdResult { + let n = s.project.log.clear(); + // Unsaved whatever undo does: no undo step brings the log back. + s.mark_unsaved(); + Ok(json!({"cleared": n})) +} + +/// `project.logCommands`: turns the log on or off; off, nothing more goes in (what is there stays). +fn log_commands(s: &mut Session, p: &Value) -> CmdResult { + let on = match p.get("on") { + None | Some(Value::Null) => !s.project.log.on(), + Some(v) => v.as_bool().ok_or("on must be true or false")?, + }; + if on != s.project.log.on() { + s.project.log.set_on(on); + s.mark_unsaved(); + } + Ok(json!({"on": on})) +} + /// Replaces the project, clearing history and per-project state. pub(crate) fn reset_to(s: &mut Session, p: Project) { reset(s, p); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ed1a67b29..dc700c1e8 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -12,6 +12,7 @@ pub mod model; #[cfg(not(target_arch = "wasm32"))] pub mod prefs; pub mod project; +mod project_log; pub mod s3_connections; pub mod s3_keys; #[cfg(test)] @@ -48,8 +49,11 @@ pub struct CommandSpec { pub shortcut: &'static str, /// Parameter description for scripts and agents. pub params: &'static str, - /// Whether it changes the project (and so takes an undo snapshot). + /// Whether it changes the project (and so takes an undo snapshot, unless `undoable` is off). pub mutates: bool, + /// Whether its change is an undo step: off for one outside the undo history, which marks the + /// project as changed itself ([`CommandSpec::outside_undo`]). + pub undoable: bool, /// `None` = enabled; `Some(reason)` = disabled. pub enabled: fn(&Session) -> Option<&'static str>, pub run: fn(&mut Session, &Value) -> CmdResult, @@ -72,7 +76,7 @@ impl CommandSpec { } pub const fn new(id: &'static str, label: &'static str, location: &'static str, run: fn(&mut Session, &Value) -> CmdResult) -> Self { - CommandSpec { id, label, location, shortcut: "", params: "{}", mutates: true, enabled: always, run } + CommandSpec { id, label, location, shortcut: "", params: "{}", mutates: true, undoable: true, enabled: always, run } } pub const fn key(mut self, s: &'static str) -> Self { self.shortcut = s; @@ -87,6 +91,12 @@ impl CommandSpec { self.mutates = false; self } + /// Changes the project, but outside its undo history (the project log, #797): no undo step, + /// which would restore nothing; the command marks the project as changed itself. + pub const fn outside_undo(mut self) -> Self { + self.undoable = false; + self + } pub const fn when(mut self, f: fn(&Session) -> Option<&'static str>) -> Self { self.enabled = f; self @@ -976,8 +986,55 @@ impl Session { } /// Runs a command by id. Mutating commands snapshot the project for undo; an error or a - /// panic restores it. + /// panic restores it. A command run by itself (not as a part of another) that changes the + /// project, or reaches a file or a service, goes into the project's log (#797). pub fn run(&mut self, id: &str, params: &Value) -> CmdResult { + if self.depth > 0 { + return self.run_command(id, params); + } + // Undo and redo are logged with the step they take back or bring back. + let step = match id { + "edit.undo" => self.undo_label(), + "edit.redo" => self.redo_label(), + _ => None, + }; + // A step that joins the last one (a held arrow key nudging an element: `merge_undo`) is + // one entry too, as it is one step to undo. + let joins = self.merge.is_some() && self.merge == self.merged && !self.undo.is_empty(); + let (at, start) = (project_log::now_ms(), web_time::Instant::now()); + let res = self.run_command(id, params); + // A plugin's command waiting for its fetches answers `pending` and runs again when they + // land: logged then, once. (Another command's `pending`, a download the web app makes, is + // logged as it is asked for: nothing runs it again.) + let pending = + id.starts_with(rustgis_plugins::PREFIX) && res.as_ref().is_ok_and(|v| v.get("pending").and_then(Value::as_bool) == Some(true)); + // With logging off, only its own switching and clearing go in: nothing else is clipped + // or redacted for nothing. + let wanted = self.project.log.on() || matches!(id, "project.logCommands" | "project.clearLog"); + // Undo and redo go in when they take a step back or bring one back: pressed with nothing + // to undo, they change nothing, and each press would be an entry. + let stepped = !matches!(id, "edit.undo" | "edit.redo") || res.is_ok(); + if !pending && wanted && stepped { + // Outside `guarded`'s catch: logging is best-effort, so a panic in it leaves the + // command's result as it was, without an entry, and says so in the app's log. + let logged = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if project_log::is_logged(self, id, params) { + let params = match step { + Some(label) => serde_json::json!({"step": label}), + None => project_log::kept_params(params), + }; + let e = project_log::entry(id, params, &res, at, start.elapsed()); + if joins && res.is_ok() { self.project.log.join_last(e) } else { self.project.log.push(e) } + } + })); + if logged.is_err() { + log::warn!("the project log has no entry for {id}: writing it failed unexpectedly"); + } + } + res + } + + fn run_command(&mut self, id: &str, params: &Value) -> CmdResult { // The merge key asked for ([`Session::merge_undo`]) goes to this command if it runs, and is // gone if it's refused first, so it can't merge a later command into the last step. let merge = if self.depth == 0 { self.merge.take() } else { None }; @@ -1028,7 +1085,7 @@ impl Session { return Err(e); } self.merge = merge; - let res = self.guarded(label, mutates, |s| run(s, params)); + let res = self.guarded(label, mutates && spec.undoable, |s| run(s, params)); if id == "layer.addDownload" && res.is_ok() && let Some(token) = params.get("geolibreLayer").and_then(|v| v.get("token")).and_then(serde_json::Value::as_str) diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 9b3b52a2f..27aa5f277 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -10,6 +10,8 @@ use rustgis_map::Map; use serde::{Deserialize, Serialize}; use crate::GpRecord; +use crate::project_log::with_log; +pub use crate::project_log::{LOG_ERROR_CHARS, LOG_LIMIT, LOG_LIST_ITEMS, LOG_PARAMS_BYTES, LogEntry, ProjectLog}; /// A map export cannot keep runtime-only download reservations, even in a larger project. pub(crate) fn ensure_map_imports_complete(map: &Map) -> Result<(), String> { @@ -121,6 +123,8 @@ fn identity(text: &str) -> (usize, u64) { (text.len(), KEY.get_or_init(RandomState::new).hash_one(text.as_bytes())) } +/// A project. A clone shares its log ([`ProjectLog`]), as the undo snapshots that are clones of +/// it must; a copy that is a project of its own (a template) is made with [`Project::own_copy`]. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Project { #[serde(default = "format")] @@ -188,6 +192,13 @@ pub struct Project { pub shared: Vec, #[serde(default)] pub data_stores: Vec, + /// Every command run on the project (#797), outside its undo history: see [`ProjectLog`]. + /// Read from the file, and written only by [`Project::saved_text`] and the autosaved + /// [`Project::recovery_text`] (left out while it is empty and on, so a project that never + /// logged saves as before): the project's own JSON (undo checks, comparisons, packages) + /// doesn't carry it. + #[serde(default, skip_serializing)] + pub log: ProjectLog, } /// A standalone table: a dataset without geometry, under an id that stays with it. Names can @@ -431,6 +442,14 @@ pub(crate) fn resolve_file_paths<'a>(dir: &str, paths: impl IntoIterator Project { + let mut p = self.clone(); + p.log = ProjectLog::default(); + p + } + fn file_paths_mut(&mut self) -> Vec<(&mut String, bool)> { let mut paths = Vec::new(); for map in &mut self.maps { @@ -451,6 +470,7 @@ impl Project { fn resolve_paths(&mut self, dir: &str) { resolve_file_paths(dir, self.file_paths_mut()); + self.log.resolve_paths(dir); } pub fn new(name: impl Into) -> Self { @@ -480,6 +500,7 @@ impl Project { dl_models: Vec::new(), shared: Vec::new(), data_stores: Vec::new(), + log: ProjectLog::default(), } } @@ -773,7 +794,12 @@ impl Project { } else { Vec::new() }; - let text = serde_json::to_string(self).map_err(|e| e.to_string()); + let text = serde_json::to_string(self).map_err(|e| e.to_string()).and_then(|t| { + let bare = t.len(); + let t = with_log(t, &self.log, dir.and_then(Path::to_str))?; + log_fits(bare, t.len())?; + Ok(t) + }); if !originals.is_empty() { for ((p, _), original) in self.file_paths_mut().into_iter().zip(originals) { if let Some(original) = original { @@ -825,7 +851,8 @@ impl Project { map.tiles.retain(|tile| !tile.id.starts_with("geolibre-pending:")); } strip_addresses(self.addresses_mut()); - let text = serde_json::to_string(&self).map_err(|e| e.to_string())?; + // With its log (#797), its paths as the open project holds them. + let text = serde_json::to_string(&self).map_err(|e| e.to_string()).and_then(|t| with_log(t, &self.log, None))?; if text.len() as u64 > MAX_BYTES { return Err("the project is too large to keep a recovery copy".into()); } @@ -858,8 +885,11 @@ impl Project { } /// Reads imported JSON (including package contents) without restoring a saved file's secrets. + /// A template or a package starts a log of its own (#797), whatever log its file holds. pub(crate) fn from_import_json(text: &str, dir: Option<&Path>) -> Result { - Self::read_json(text, false, dir) + let mut p = Self::read_json(text, false, dir)?; + p.log = ProjectLog::default(); + Ok(p) } fn read_json(text: &str, restore: bool, dir: Option<&Path>) -> Result { @@ -879,6 +909,8 @@ impl Project { crate::s3_connections::clear_bindings(p.addresses_mut()); if let Some(d) = dir.and_then(Path::to_str) { p.resolve_paths(d); + } else { + p.log.drop_relative(); } p.normalize(); Ok(p) @@ -923,3 +955,17 @@ fn unique_names<'a>(names: impl Iterator) { } } } + +/// Refuses a project text that its log (#797) takes past [`MAX_BYTES`], which would save but not +/// open again; `bare` is the text's length without the log. A project over it by itself is not +/// the log's doing, and saves as before. +pub(crate) fn log_fits(bare: usize, with_log: usize) -> Result<(), String> { + if bare as u64 > MAX_BYTES || with_log as u64 <= MAX_BYTES { + return Ok(()); + } + Err(format!( + "with its log the project comes to {} MB, more than a project file opens with ({} MB): export the log (project.exportLog), then clear it (project.clearLog)", + with_log.div_ceil(1 << 20), + MAX_BYTES >> 20 + )) +} diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs new file mode 100644 index 000000000..612749b34 --- /dev/null +++ b/crates/engine/src/project_log.rs @@ -0,0 +1,903 @@ +//! The project log (#797): the log a project keeps ([`ProjectLog`]), which commands go in, what +//! of them is kept, and how its paths are written portable into a project file and read back. + +use std::sync::{Mutex, MutexGuard}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{CmdResult, Session}; + +/// Now, in milliseconds since 1970 (UTC). +pub(crate) fn now_ms() -> u64 { + web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) +} + +/// Whether command `id`, run by itself with `params`, goes into the log: one that changes the +/// project (all but those built with `.view()`; a plugin's when its manifest asks for the Edit +/// permission), undo and redo, a save, the log's own clearing and switching, and a command whose +/// parameters name a file, folder, address or connection (an export), as the plugin permission +/// gate reads them. Reading and looking around (listing, describing, moving the view) is left +/// out: agents, scripts and plugins call those hundreds of times, and they would push the changes +/// out of the log. +pub(crate) fn is_logged(s: &Session, id: &str, params: &Value) -> bool { + // Making or opening a project from text names no place: it is logged as the open from a file is. + if matches!( + id, + "edit.undo" + | "edit.redo" + | "project.save" + | "project.clearLog" + | "project.logCommands" + | "project.new" + | "project.sample" + | "project.openText" + ) { + return true; + } + let mutates = if id.starts_with(rustgis_plugins::PREFIX) { + rustgis_plugins::command(id).is_some_and(|c| c.mutates()) + } else { + match s.registry.get(id) { + Some(spec) => spec.mutates, + None => return false, + } + }; + mutates || crate::cmd::items::names_place(params, crate::cmd::plugins::PLACE_KEYS, 0) +} + +/// `params` as the log keeps them: a document given as text (a map, a project, a capabilities +/// answer, GeoJSON) as its size only, since one may hold secrets no redaction finds in it (an S3 +/// source's token inside a map's JSON); the rest clipped (so a long script costs little), then +/// without secrets (as History's Copy as code takes them out). A secret cut short is still taken +/// out: what is left of it follows its name. Within [`LOG_PARAMS_BYTES`] of JSON in three tries, +/// each giving up more: texts at that bound and lists at their first [`LOG_LIST_ITEMS`]; then +/// texts at 512 characters; then the first keys with their short values only. +pub(crate) fn kept_params(params: &Value) -> Value { + let v = redacted(&clip(params, LOG_PARAMS_BYTES, LOG_LIST_ITEMS, 0)); + if serde_json::to_vec(&v).map_or(0, |b| b.len()) <= LOG_PARAMS_BYTES { + return v; + } + // Texts shorter; lists as they are (already cut, their "… N items in all" kept). + let clipped = clip(&v, 512, usize::MAX, 0); + if serde_json::to_vec(&clipped).map_or(0, |b| b.len()) <= LOG_PARAMS_BYTES { + return clipped; + } + // Still too large (many keys, long lists): its first keys with their short values only (a + // number, a flag, a text of up to 64 characters with no brace), the rest as "…", as many as + // fit within the bound. Nothing in it is cut JSON, so it reads back as it was written. + let short = |v: &Value| match v { + Value::String(t) if t.chars().nth(64).is_none() && !t.contains('{') => v.clone(), + Value::Number(_) | Value::Bool(_) | Value::Null => v.clone(), + _ => Value::String("…".into()), + }; + match clipped { + Value::Object(o) => { + // Each key and value is short, but forty of them can still add up (escaped or + // multi-byte text): each pair counts its bytes, a colon and a comma. Room is kept for + // the note of how many keys there were, when some are left out. + let mut room = LOG_PARAMS_BYTES.saturating_sub(2 + 48); + let mut out = serde_json::Map::new(); + for (k, v) in o.iter().take(40) { + let (k, v): (String, Value) = (k.chars().take(64).collect(), short(v)); + // Two keys alike in their first 64 characters: the first stays. + if out.contains_key(&k) { + continue; + } + let size = serde_json::to_vec(&k) + .map_or(usize::MAX, |b| b.len()) + .saturating_add(serde_json::to_vec(&v).map_or(usize::MAX, |b| b.len())) + .saturating_add(2); + if size > room { + break; + } + room -= size; + out.insert(k, v); + } + if out.len() < o.len() { + out.insert("…".into(), Value::String(format!("{} keys in all", o.len()))); + } + Value::Object(out) + } + other => short(&other), + } +} + +/// An entry read from a file, kept as a new one is: its command clipped, its parameters and error +/// clipped and without secrets (a file may hold what RustGIS never wrote, and a plugin may read +/// the log). +pub(crate) fn bounded(mut e: LogEntry) -> LogEntry { + if e.command.chars().nth(200).is_some() { + e.command = e.command.chars().take(200).collect(); + } + // One that fits, as written, keeps its clipped lists' counts: only the document rule applies, + // then its secrets are taken out. + let fits = serde_json::to_vec(&e.params).is_ok_and(|b| b.len() <= LOG_PARAMS_BYTES); + e.params = if fits { redacted(&clip(&e.params, usize::MAX, usize::MAX, 0)) } else { kept_params(&e.params) }; + e.error = e.error.as_deref().map(kept_error); + // Pointers into the entry, as many as its texts at most. + e.relative.truncate(1_000); + e.relative.retain(|p| p.len() <= 1_024); + e +} + +/// An error as the log keeps it: clipped to [`LOG_ERROR_CHARS`] before its secrets are taken out, +/// so a long one costs little (a secret cut short still follows its name). +fn kept_error(e: &str) -> String { + let e = if e.chars().nth(LOG_ERROR_CHARS).is_some() { + format!("{}…", e.chars().take(LOG_ERROR_CHARS).collect::()) + } else { + e.to_string() + }; + crate::cmd::history::redacted_text(&sql_secrets_removed(&e)) +} + +/// `v` without secrets: as History's Copy as code takes them out (`history::redacted`), and, +/// first, inside the JSON a text holds (a script's command lines: `s3.connect {"secret": …}`), +/// whose objects are redacted in place. +fn redacted(v: &Value) -> Value { + crate::cmd::history::redacted(&embedded_redacted(v, 0)) +} + +fn embedded_redacted(v: &Value, depth: usize) -> Value { + match v { + Value::String(t) => { + let t = if t.contains('{') { json_in_text_redacted(t) } else { t.clone() }; + Value::String(sql_secrets_removed(&t)) + } + Value::Array(a) if depth < 8 => Value::Array(a.iter().map(|x| embedded_redacted(x, depth + 1)).collect()), + Value::Object(o) if depth < 8 => Value::Object(o.iter().map(|(k, x)| (k.clone(), embedded_redacted(x, depth + 1))).collect()), + other => other.clone(), + } +} + +/// `t` with each JSON object it holds (from a `{` to where the object ends) redacted. Texts are +/// clipped before this, so trying each `{` stays cheap; the secret names are looked for once. +pub(crate) fn json_in_text_redacted(t: &str) -> String { + let mut out = String::with_capacity(t.len()); + let last_secret = last_secret_name(t); + let mut rest = t; + while let Some(at) = rest.find('{') { + // Where this `{` is in `t`: a secret's name after it is one at a later offset. + let brace = t.len() - rest.len() + at; + let (before, from) = rest.split_at(at); + out.push_str(before); + let mut objects = serde_json::Deserializer::from_str(from).into_iter::(); + match objects.next() { + Some(Ok(v @ Value::Object(_))) => { + let end = objects.byte_offset(); + out.push_str(&serde_json::to_string(&crate::cmd::history::redacted(&v)).unwrap_or_default()); + rest = from.get(end..).unwrap_or(""); + } + // Cut short by the clip, or an object that doesn't parse (a trailing comma, single + // quotes) though a key starts it, or whose keys aren't quoted but one is a secret's + // (`{secretAccessKey: …}`): what follows its `{` may hold a secret, so none of it + // stays. Text such as `{name}` or `{x:.2f}` stays as it is. + Some(Err(e)) + if e.is_eof() + || from.get(1..).is_some_and(|r| r.trim_start().starts_with(['"', '\''])) + || last_secret.is_some_and(|s| s > brace) => + { + out.push('…'); + return out; + } + _ => { + out.push('{'); + rest = from.get(1..).unwrap_or(""); + } + } + } + out.push_str(rest); + out +} + +/// `t` with each SQL `CREATE … SECRET`'s options (`CREATE SECRET s3 (TYPE s3, KEY_ID 'a', +/// SECRET 'b')`, DuckDB's form, `OR REPLACE` and `PERSISTENT` too) as `(…)`: their values are +/// written as plain words, which no name or address form finds. With no `)`, the rest goes. +pub(crate) fn sql_secrets_removed(t: &str) -> String { + // ASCII lowercase keeps every byte where it was. + let lower = t.to_ascii_lowercase(); + let mut out = String::with_capacity(t.len()); + let mut from = 0; + while let Some(at) = lower.get(from..).and_then(|r| r.find("create")).map(|i| i + from) { + let after = at + "create".len(); + // `create` as a word of its own (not `recreate` or `created`). + let word = lower.get(..at).and_then(|b| b.chars().next_back()).is_none_or(|c| !(c.is_alphanumeric() || c == '_')) + && lower.get(after..).is_some_and(|r| r.starts_with(char::is_whitespace)); + if !word { + out.push_str(t.get(from..after).unwrap_or("")); + from = after; + continue; + } + // The statement's head, to its `(`: `create [or replace] [persistent] secret [name] (`. + let Some(open) = lower.get(after..).and_then(|r| r.find(['(', ';'])).map(|i| i + after) else { break }; + let head = lower.get(after..open).unwrap_or(""); + // The object's kind comes first (`SECRET`, after `OR REPLACE`, `PERSISTENT` or + // `TEMPORARY`): a table named `secret` isn't one. + let kind = head.split_whitespace().find(|w| !matches!(*w, "or" | "replace" | "persistent" | "temporary" | "temp")); + if !lower.get(open..).is_some_and(|r| r.starts_with('(')) || kind != Some("secret") { + out.push_str(t.get(from..after).unwrap_or("")); + from = after; + continue; + } + out.push_str(t.get(from..open).unwrap_or("")); + out.push_str("(…"); + // Its `)`, quotes and nested brackets aside. + let (mut depth, mut quote, mut close) = (0usize, None, None); + for (i, c) in t.get(open..).unwrap_or("").char_indices() { + match (quote, c) { + (Some(q), c) if c == q => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(c), + (None, '(') => depth += 1, + (None, ')') => { + depth = depth.saturating_sub(1); + if depth == 0 { + close = Some(open + i); + break; + } + } + _ => {} + } + } + match close { + Some(c) => { + out.push(')'); + from = c + 1; + } + None => return out, + } + } + out.push_str(t.get(from..).unwrap_or("")); + out +} + +/// Where in `t` the last secret's name (`credentials::secret_key`) written as a key, quoted or +/// not, before a `:` or `=` (`secretAccessKey: "…"`, `"token": …`, `token = …`) starts: one pass +/// over the text. +fn last_secret_name(t: &str) -> Option { + let name = |c: char| c.is_alphanumeric() || matches!(c, '_' | '-' | '.'); + let mut last = None; + let mut rest = t; + while let Some(at) = rest.find(name) { + let from = rest.get(at..).unwrap_or(""); + let end = from.find(|c: char| !name(c)).unwrap_or(from.len()); + let (key, after) = from.split_at(end); + let separator = after.trim_start_matches(['"', '\'']).trim_start(); + if separator.starts_with([':', '=']) && rustgis_data::source::credentials::secret_key(key) { + last = Some(t.len() - from.len()); + } + rest = after; + } + last +} + +/// The longest text kept as it is that reads as a document (it starts with `{`, `[` or `<`). +const DOCUMENT_CHARS: usize = 200; + +/// Whether `t`, past [`DOCUMENT_CHARS`], is a file's contents in base64 (`layer.addDataBytes`'s +/// `data`, or a `data:` address): only base64's characters after any `data:…;base64,` head. A +/// secret's name can't be seen in it, so none of it is kept. +fn is_base64(t: &str) -> bool { + let body = t.strip_prefix("data:").and_then(|r| r.split_once(";base64,")).map_or(t, |(_, b)| b); + body.len() > DOCUMENT_CHARS + && body.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=' | b'-' | b'_' | b'\n' | b'\r')) +} + +/// `v` with each text cut to `chars` characters (a key to 200), a text that reads as a document +/// (JSON or XML past [`DOCUMENT_CHARS`]) or as a file's contents in base64 given as its size, and +/// each list cut to its first `items` (and how many there were); nesting past eight levels is +/// left out. +fn clip(v: &Value, chars: usize, items: usize, depth: usize) -> Value { + match v { + // A byte-order mark before it is still a document's start. + Value::String(t) + if t.trim_start_matches('\u{feff}').trim_start().starts_with(['{', '[', '<']) && t.chars().nth(DOCUMENT_CHARS).is_some() => + { + Value::String(format!("(a document of {} characters)", t.chars().count())) + } + Value::String(t) if is_base64(t) => Value::String(format!("(file contents of {} characters, base64)", t.len())), + Value::String(t) if t.chars().nth(chars).is_some() => Value::String(format!("{}…", t.chars().take(chars).collect::())), + Value::Array(a) if depth < 8 => { + let mut out: Vec = a.iter().take(items).map(|x| clip(x, chars, items, depth + 1)).collect(); + if a.len() > items { + out.push(Value::String(format!("… {} items in all", a.len()))); + } + Value::Array(out) + } + Value::Object(o) if depth < 8 => { + // Two keys alike in their first 200 characters: the first stays. A key cut short may + // hide a secret's name past the cut (redaction reads the names after this): its value + // goes. + let mut out = serde_json::Map::new(); + for (k, x) in o { + let value = || if k.chars().nth(200).is_some() { Value::String("…".into()) } else { clip(x, chars, items, depth + 1) }; + out.entry(k.chars().take(200).collect::()).or_insert_with(value); + } + Value::Object(out) + } + Value::Array(_) | Value::Object(_) => Value::String("…".into()), + other => other.clone(), + } +} + +/// The entry of command `id`, which started at `at` and took `took`; its error without secrets, +/// clipped to [`LOG_ERROR_CHARS`]. +pub(crate) fn entry(id: &str, params: Value, res: &CmdResult, at: u64, took: web_time::Duration) -> LogEntry { + let error = res.as_ref().err().map(|e| kept_error(e)); + LogEntry { + at, + command: id.chars().take(200).collect(), + params, + ok: res.is_ok(), + error, + millis: u64::try_from(took.as_millis()).unwrap_or(u64::MAX), + relative: Vec::new(), + } +} + +/// Most entries a project's log keeps; past it, the oldest go first. +pub const LOG_LIMIT: usize = 10_000; +/// The most entries of plugins' commands the log keeps, their oldest going first past it: a +/// plugin run many times can't push the user's own entries out of the log. +pub const LOG_PLUGIN_LIMIT: usize = 1_000; +/// Most bytes of JSON an entry's parameters keep (a long text is cut, a long list keeps its first +/// [`LOG_LIST_ITEMS`]), and most characters of its error. +pub const LOG_PARAMS_BYTES: usize = 4096; +pub const LOG_LIST_ITEMS: usize = 100; +pub const LOG_ERROR_CHARS: usize = 500; + +/// One command run on a project: when it started (UTC, milliseconds since 1970), its id, its +/// parameters (secrets taken out, large values clipped), whether it succeeded, its error, and how +/// long it took. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct LogEntry { + pub at: u64, + pub command: String, + #[serde(default)] + pub params: serde_json::Value, + pub ok: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(default)] + pub millis: u64, + /// In a saved file, what saving made portable: the parameters' texts it changed, by their + /// JSON pointers (`/params/path`, always starting with `/`), and each relative path it wrote + /// (`./My Data/a.tif`, always starting with `.`). Reading makes those very paths absolute + /// again, in those texts and in the error, and nothing else: a text or a word given as + /// relative (`./a.tif`, a name such as `./Maps`) reads back as it was run. Taken once read; + /// never shown. + #[serde(default, skip_serializing)] + pub relative: Vec, +} + +/// The longest saved log text kept for the next save ([`LogState::written`]). +const WRITTEN_KEPT_BYTES: usize = 1 << 20; + +/// A log as it is held, written by [`ProjectLog::write_json`] (its paths made portable) and read +/// back through its own `Deserialize`. +#[derive(Debug, Default)] +struct LogState { + /// Logging turned off (`project.logCommands`); on by default. + off: bool, + /// Oldest first; the oldest go first past [`LOG_LIMIT`], without moving the rest. + entries: std::collections::VecDeque, + /// How many of `entries` are plugins' commands', kept in step with them so a push doesn't + /// count them. + plugin_entries: usize, + /// Moved on by each change, so a text written earlier is known to be the log as it is. + generation: u64, + /// The text the last save wrote, with the generation it was written from and the folder it + /// was written for: a save or autosave with nothing logged since writes it as it is. Up to + /// [`WRITTEN_KEPT_BYTES`] only. + written: Option<(u64, Option, std::sync::Arc)>, +} + +/// Values a file's log can hold that aren't what it should be: read as nothing (an empty log, no +/// entries), so that nothing in the log keeps a project from opening. +macro_rules! anything_else_is_default { + () => { + fn visit_bool(self, _: bool) -> Result { + Ok(Default::default()) + } + fn visit_i64(self, _: i64) -> Result { + Ok(Default::default()) + } + fn visit_u64(self, _: u64) -> Result { + Ok(Default::default()) + } + fn visit_f64(self, _: f64) -> Result { + Ok(Default::default()) + } + fn visit_str(self, _: &str) -> Result { + Ok(Default::default()) + } + fn visit_unit(self) -> Result { + Ok(Default::default()) + } + fn visit_none(self) -> Result { + Ok(Default::default()) + } + }; +} + +impl<'de> Deserialize<'de> for LogState { + /// A file's log (a project file is hostile input), read so that nothing in it keeps the + /// project from opening: its entries one at a time ([`LogEntries`]); anything but an object + /// is an empty log. + fn deserialize>(d: D) -> Result { + struct State; + impl<'de> serde::de::Visitor<'de> for State { + type Value = LogState; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a project log") + } + fn visit_map>(self, mut map: A) -> Result { + let mut s = LogState::default(); + while let Some(k) = map.next_key::()? { + match k.as_str() { + "off" => s.off = map.next_value::()?.as_bool().unwrap_or(false), + "entries" => { + let entries = map.next_value_seed(LogEntries)?; + s.plugin_entries = entries.iter().filter(|e| e.command.starts_with(rustgis_plugins::PREFIX)).count(); + s.entries = entries; + } + _ => { + map.next_value::()?; + } + } + } + Ok(s) + } + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + Ok(LogState::default()) + } + anything_else_is_default!(); + } + d.deserialize_any(State) + } +} + +/// A file's log entries, each read in turn as a JSON value (the file is in memory already), +/// bounded and without secrets as a new entry is ([`bounded`]) as it comes, and only the last +/// [`LOG_LIMIT`] kept, of which plugins' commands' last [`LOG_PLUGIN_LIMIT`]. One that doesn't +/// read as an entry (a file edited by hand, another version's) is skipped. +struct LogEntries; + +/// `entries` without the oldest of plugins' commands' past [`LOG_PLUGIN_LIMIT`], in one pass: a +/// file's log is held to the cap a running log keeps. +fn cap_plugins(entries: &mut std::collections::VecDeque) { + let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX); + let mut over = entries.iter().filter(|e| plugin(e)).count().saturating_sub(LOG_PLUGIN_LIMIT); + entries.retain(|e| { + let drop = over > 0 && plugin(e); + over -= usize::from(drop); + !drop + }); +} + +impl<'de> serde::de::DeserializeSeed<'de> for LogEntries { + type Value = std::collections::VecDeque; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self) + } +} + +impl<'de> serde::de::Visitor<'de> for LogEntries { + type Value = std::collections::VecDeque; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a list of log entries") + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = std::collections::VecDeque::new(); + while let Some(v) = seq.next_element::()? { + let Ok(e) = serde_json::from_value::(v) else { continue }; + // Bounded at once, so what is held is at most the cap's worth of bounded entries, + // whatever the file holds. + out.push_back(bounded(e)); + if out.len() > LOG_LIMIT { + out.pop_front(); + } + } + cap_plugins(&mut out); + Ok(out) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + Ok(Default::default()) + } + anything_else_is_default!(); +} + +/// A project's log (#797): one entry for each command run on it, from any front end, kept in +/// the `.rgis`. It sits outside the undo history: the project's undo snapshots share it (a clone +/// is the same log), so undoing a command never takes its entry back, and a snapshot copies +/// nothing of it. A project opened, made new or from a template has a log of its own. It is read +/// in place, under its lock; only saving copies its entries, once ([`ProjectLog::write_json`]). +#[derive(Clone, Debug, Default)] +pub struct ProjectLog(std::sync::Arc>); + +impl ProjectLog { + fn state(&self) -> MutexGuard<'_, LogState> { + self.0.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// The state, to change: what a save wrote of it is no longer the log. + fn changing(&self) -> MutexGuard<'_, LogState> { + let mut s = self.state(); + s.generation = s.generation.wrapping_add(1); + s.written = None; + s + } + + /// No entries, and logging on: nothing for a project file to keep. + pub fn nothing_to_save(&self) -> bool { + let s = self.state(); + s.entries.is_empty() && !s.off + } + + pub fn on(&self) -> bool { + !self.state().off + } + + pub fn set_on(&self, on: bool) { + self.changing().off = !on; + } + + /// Adds `e` when logging is on, the oldest entries dropped past [`LOG_LIMIT`], and a plugin's + /// command's past [`LOG_PLUGIN_LIMIT`] the oldest of those; turning the log off + /// (`project.logCommands`), and clearing it while off, are recorded too, so the log says why + /// it stops or is empty. + pub fn push(&self, e: LogEntry) { + let mut s = self.changing(); + if s.off && !matches!(e.command.as_str(), "project.logCommands" | "project.clearLog") { + return; + } + let plugin = |e: &LogEntry| e.command.starts_with(rustgis_plugins::PREFIX); + if plugin(&e) + && s.plugin_entries >= LOG_PLUGIN_LIMIT + && let Some(oldest) = s.entries.iter().position(plugin) + { + s.entries.remove(oldest); + s.plugin_entries = s.plugin_entries.saturating_sub(1); + } + if plugin(&e) { + s.plugin_entries += 1; + } + s.entries.push_back(e); + while s.entries.len() > LOG_LIMIT { + if s.entries.pop_front().is_some_and(|x| plugin(&x)) { + s.plugin_entries = s.plugin_entries.saturating_sub(1); + } + } + } + + /// Adds `e` as part of the last entry when that is the same command, run well: a step merged + /// into the last undo step ([`crate::Session::merge_undo`]). The entry keeps when the first + /// started, takes the last one's parameters, and adds up how long they took. + pub fn join_last(&self, mut e: LogEntry) { + { + let mut s = self.changing(); + if let Some(last) = s.entries.back_mut().filter(|l| l.ok && l.command == e.command) { + e.at = last.at; + e.millis = last.millis.saturating_add(e.millis); + *last = e; + return; + } + } + self.push(e); + } + + /// Every entry, oldest first (a copy: for tests and small logs; [`ProjectLog::select`] and + /// [`ProjectLog::each`] read in place). + pub fn entries(&self) -> Vec { + self.state().entries.iter().cloned().collect() + } + + /// The last `limit` entries that `keep` keeps, oldest first, with how many it keeps and how + /// many there are: only those are copied. + pub fn select(&self, keep: impl Fn(&LogEntry) -> bool, limit: usize) -> (Vec, usize, usize) { + let s = self.state(); + let matching: Vec<&LogEntry> = s.entries.iter().filter(|e| keep(e)).collect(); + let tail = matching.get(matching.len().saturating_sub(limit)..).unwrap_or_default().iter().map(|e| (*e).clone()).collect(); + (tail, matching.len(), s.entries.len()) + } + + /// Up to `n` entries from the `from`th, oldest first: copied under the log's lock, so the + /// caller writes them after letting it go. + pub fn chunk(&self, from: usize, n: usize) -> Vec { + self.state().entries.iter().skip(from).take(n).cloned().collect() + } + + pub fn len(&self) -> usize { + self.state().entries.len() + } + + /// No entries (logging may still be off: see [`ProjectLog::nothing_to_save`]). + pub fn is_empty(&self) -> bool { + self.state().entries.is_empty() + } + + /// The paths a file's log was saved with relative to its folder (`./data/a.tif`), made + /// absolute again from `dir`, the folder it was read from, as the project's sources are; so + /// a later Save As elsewhere writes them relative to that folder. Only the texts the save + /// made portable (an entry's `relative`) are read as such: one given as relative stays so. + pub(crate) fn resolve_paths(&self, dir: &str) { + // The folder itself is written as `.`. + let word = |p: &str| { + if p == "." { + return Some(dir.to_string()); + } + (p.starts_with("./") || p.starts_with("../")).then(|| rustgis_map::paths::resolve(dir, p)).flatten() + }; + for e in &mut self.changing().entries { + let (pointers, wrote): (Vec, Vec) = + std::mem::take(&mut e.relative).into_iter().partition(|r| r.starts_with('/')); + // The paths the save wrote, each back to absolute, found whole (spaces and all). + let swaps: Vec<(String, String)> = wrote.into_iter().filter_map(|w| word(&w).map(|abs| (w, abs))).collect(); + if swaps.is_empty() { + continue; + } + for at in &pointers { + if let Some(serde_json::Value::String(t)) = at.strip_prefix("/params").and_then(|p| e.params.pointer_mut(p)) { + *t = swapped(t, &swaps); + } + } + if let Some(err) = e.error.as_mut() { + *err = swapped(err, &swaps); + } + } + } + + /// Forgets the relative markers of a log read without a folder (`openText`, a package): its + /// texts stay as they are, and a save writes them as they are, so a copy saved into a folder + /// isn't resolved against that folder later. + pub(crate) fn drop_relative(&self) { + for e in &mut self.changing().entries { + e.relative.clear(); + } + } + + /// Empties the log; how many entries it had. + pub fn clear(&self) -> usize { + let mut s = self.changing(); + s.plugin_entries = 0; + std::mem::take(&mut s.entries).len() + } + + /// The log as a project file keeps it, written at the end of `out`, each entry in turn; saved + /// into a folder (`dir`), its local paths made portable ([`make_portable`]). + fn write_json(&self, out: &mut String, dir: Option<&str>) -> Result<(), String> { + // Nothing logged since the last save into the same folder: its text as it was. + let written = { + let s = self.state(); + s.written.as_ref().filter(|(g, d, _)| *g == s.generation && d.as_deref() == dir).map(|(_, _, t)| t.clone()) + }; + if let Some(text) = written { + out.push_str(&text); + return Ok(()); + } + // Copied under the lock and written after it is let go, so a command logged meanwhile + // (the interface thread, while an autosave writes) doesn't wait on the writing. That copy + // is the only one: each entry is made portable in place and written into `out`. + let (off, entries, generation) = { + let s = self.state(); + (s.off, s.entries.clone(), s.generation) + }; + let start = out.len(); + out.push_str(if off { r#"{"off":true,"entries":["# } else { r#"{"entries":["# }); + for (i, mut e) in entries.into_iter().enumerate() { + if i > 0 { + out.push(','); + } + // Where its paths were made portable: those read back as such (an entry read from a + // file that wasn't resolved keeps its own). + let mut relative = std::mem::take(&mut e.relative); + if let Some(d) = dir { + let kept = relative.len(); + let (params, error) = (e.params.clone(), e.error.clone()); + portable_entry(&mut e, d, &mut relative, false); + // A text that already held, as a word, a relative path this save wrote would read + // back with that word made absolute too: the entry's paths are written as their + // file names instead, which read back as they are. + if collides(¶ms, error.as_deref(), relative.get(kept..).unwrap_or_default()) { + (e.params, e.error) = (params, error); + relative.truncate(kept); + portable_entry(&mut e, d, &mut relative, true); + } + } + let text = serde_json::to_string(&e).map_err(|e| e.to_string())?; + match text.strip_suffix('}').filter(|_| !relative.is_empty()) { + Some(open) => { + out.push_str(open); + out.push_str(",\"relative\":"); + out.push_str(&serde_json::to_string(&relative).map_err(|e| e.to_string())?); + out.push('}'); + } + None => out.push_str(&text), + } + } + out.push_str("]}"); + // Kept for the next save, unless something was logged while this one wrote, or the text + // is long enough (a log near its limit runs to tens of MB) that holding a second copy of + // it until the next change costs more than writing it again. + let mut s = self.state(); + if s.generation == generation && out.len() - start <= WRITTEN_KEPT_BYTES { + s.written = Some((generation, dir.map(str::to_string), out.get(start..).unwrap_or("").into())); + } + Ok(()) + } +} + +impl<'de> Deserialize<'de> for ProjectLog { + /// A file's log, its last [`LOG_LIMIT`] entries ([`LogEntries`]). + fn deserialize>(d: D) -> Result { + Ok(ProjectLog(std::sync::Arc::new(Mutex::new(LogState::deserialize(d)?)))) + } +} + +/// A project's saved JSON with its log (#797) as its last member, when there is one to keep. Saved +/// into a folder (`dir`), the local paths in the log's parameters and errors are written relative +/// to it, as the project's own sources are, or as their file names where they can't be, so a +/// project moved with its data keeps no trace of the folder it was in. +pub(crate) fn with_log(mut text: String, log: &ProjectLog, dir: Option<&str>) -> Result { + if log.nothing_to_save() || !text.ends_with('}') { + return Ok(text); + } + text.pop(); + text.push_str(",\"log\":"); + log.write_json(&mut text, dir)?; + text.push('}'); + Ok(text) +} + +/// `e`'s parameters and error with their local paths made portable, in place, each change noted +/// in `relative` ([`make_portable`]); as file names only with `names`. +fn portable_entry(e: &mut LogEntry, dir: &str, relative: &mut Vec, names: bool) { + let mut swaps = Vec::new(); + make_portable(&mut e.params, dir, "/params", 0, names, relative, &mut swaps); + if let Some(err) = e.error.as_mut() { + // The parameters' paths first, whole (a folder may have spaces in its name), then any + // other word that is one. + *err = portable_text(&swapped(err, &swaps), dir, names, relative); + } +} + +/// Whether reading back what a save noted (`noted`: the pointers of the parameters it changed +/// and the relative paths it wrote) would change more than it wrote: one of those paths was +/// already a word of a text read back as such (a changed parameter's, as `params` held it, or +/// the error). +fn collides(params: &serde_json::Value, error: Option<&str>, noted: &[String]) -> bool { + let (pointers, wrote): (Vec<&String>, Vec<&String>) = noted.iter().partition(|r| r.starts_with('/')); + let texts = pointers.iter().filter_map(|p| p.strip_prefix("/params").and_then(|p| params.pointer(p))?.as_str()).chain(error); + // `swapped` changes a text only where the path stands as a word of it. + texts.into_iter().any(|t| wrote.iter().any(|w| swapped(t, &[((*w).clone(), String::new())]) != t)) +} + +/// `v`'s texts with their local paths made portable ([`portable_text`]; as file names only with +/// `names`), in place; each one changed noted in `changed` by its JSON pointer, `at` being `v`'s, +/// and each whole-text path in `swaps`, as it was and as it is written. +fn make_portable( + v: &mut serde_json::Value, + dir: &str, + at: &str, + depth: usize, + names: bool, + changed: &mut Vec, + swaps: &mut Vec<(String, String)>, +) { + use serde_json::Value; + match v { + Value::String(t) => { + let portable = portable_text(t, dir, names, changed); + if portable != *t { + if one_path(t) { + swaps.push((t.clone(), portable.clone())); + } + *t = portable; + changed.push(at.to_string()); + } + } + Value::Array(a) if depth < 16 => { + for (i, x) in a.iter_mut().enumerate() { + make_portable(x, dir, &format!("{at}/{i}"), depth + 1, names, changed, swaps); + } + } + Value::Object(o) if depth < 16 => { + for (k, x) in o.iter_mut() { + make_portable(x, dir, &format!("{at}/{}", k.replace('~', "~0").replace('/', "~1")), depth + 1, names, changed, swaps); + } + } + _ => {} + } +} + +/// `t` with each text of `swaps` that is a word of its own (from the text's start or a space, +/// quote or bracket, to its end or a space, quote, bracket or punctuation) as its other half, +/// the longest first: a path with spaces in it, known whole. +fn swapped(t: &str, swaps: &[(String, String)]) -> String { + let mut swaps: Vec<&(String, String)> = swaps.iter().filter(|(from, _)| !from.is_empty()).collect(); + swaps.sort_by_key(|(from, _)| std::cmp::Reverse(from.len())); + let mut t = t.to_string(); + for (from, to) in swaps { + let mut out = String::with_capacity(t.len()); + let mut rest = t.as_str(); + while let Some(at) = rest.find(from.as_str()) { + out.push_str(rest.get(..at).unwrap_or("")); + let starts = out.chars().next_back().is_none_or(|c| c.is_whitespace() || matches!(c, '"' | '\'' | '`' | '(' | '[')); + rest = rest.get(at + from.len()..).unwrap_or(""); + let ends = rest.chars().next().is_none_or(|c| c.is_whitespace() || matches!(c, '"' | '\'' | '`' | ')' | ']' | ',' | ';' | ':')); + out.push_str(if starts && ends { to } else { from }); + } + out.push_str(rest); + t = out; + } + t +} + +/// `t` with each absolute local path in it (the whole text, or a word of it, quotes and trailing +/// punctuation aside) relative to `dir`, or its file name where it can't be or with `names`; +/// each relative path written noted in `wrote` (once), for reading to make absolute again. +fn portable_text(t: &str, dir: &str, names: bool, wrote: &mut Vec) -> String { + let made = std::cell::RefCell::new(Vec::new()); + let portable = |p: &str| { + let name = || last_name(p).to_string(); + if names { + return name(); + } + let r = rustgis_map::paths::relative_to(dir, p).filter(|r| rustgis_map::paths::is_local(r)).unwrap_or_else(name); + made.borrow_mut().push(r.clone()); + r + }; + let out = if one_path(t) { portable(t) } else { path_words(t, |p| rustgis_map::paths::is_absolute_local(p).then(|| portable(p))) }; + for r in made.into_inner() { + if (r == "." || r.starts_with("./") || r.starts_with("../")) && !wrote.contains(&r) { + wrote.push(r); + } + } + out +} + +/// The last part of path `p`, its file's name, or its folder's for one that ends in a separator +/// (`/data/My Data/`); the root as it is. +pub(crate) fn last_name(p: &str) -> &str { + p.trim_end_matches(['/', '\\']).rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or(p) +} + +/// Whether `t` is one absolute local path as a whole (spaces in its folders' names and all), not +/// a text that starts with one: no other word of it is a path, and no `: ` follows the first +/// (`/data/a.tif: not found`). The root alone (`/`) is no file's path. +fn one_path(t: &str) -> bool { + let path_word = |w: &str| { + let core = word_core(w); + core.get(1..).is_some_and(|r| r.contains(['/', '\\'])) && rustgis_map::paths::is_absolute_local(core) + }; + t.len() > 1 + && rustgis_map::paths::is_absolute_local(t) + && (!t.contains(char::is_whitespace) || (!t.contains(": ") && !t.split_whitespace().skip(1).any(path_word))) +} + +/// A word without the quotes, brackets and trailing `,`, `;` or `:` around it. +fn word_core(word: &str) -> &str { + let core = word.trim_end().trim_matches(|c: char| matches!(c, '"' | '\'' | '`' | '(' | ')' | '[' | ']' | ',' | ';')); + core.strip_suffix(':').unwrap_or(core) +} + +/// `t` with each word (quotes, brackets, a trailing `,`, `;` or `:` aside) replaced by what `f` +/// makes of it, where `f` gives something: how a path among a text's words is found when it is +/// made portable on saving. Only a word with a separator past its first character is tried +/// (`/data/a.tif`, `C:\a.tif`, `./a.tif`): a division's `/` or a `/home` among other words is +/// left as written. +fn path_words(t: &str, f: impl Fn(&str) -> Option) -> String { + t.split_inclusive(char::is_whitespace) + .map(|word| { + let core = word_core(word); + let path_like = core.get(1..).is_some_and(|r| r.contains(['/', '\\'])); + match path_like.then(|| f(core)).flatten() { + Some(new) => word.replacen(core, &new, 1), + None => word.to_string(), + } + }) + .collect() +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index bad084eaf..9681b9bf0 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1933,6 +1933,7 @@ fn hostile_params_never_panic() { id, "project.save" | "project.saveAs" + | "project.exportLog" | "share.exportMap" | "time.export" | "share.exportLayer" @@ -20427,6 +20428,458 @@ fn calculate_geometry_updates_ellipsoidal_lengths_in_one_undo_step() { assert!((out.value(0, length).as_f64().unwrap() - 111.31949079327356).abs() < 1e-6); } +/// A log read without its folder (`openText`) forgets the relative markers a save into a folder +/// left in it: saved again, its paths are written as they are, with no markers to resolve later. +#[test] +fn a_log_read_without_its_folder_forgets_its_relative_markers() { + let mut doc: Value = serde_json::from_str(&Project::new("Log").saved_text(None).unwrap().0).unwrap(); + let entry = json!({"at": 1, "command": "gp.run", "params": {"path": "./a.tif"}, "ok": true, "relative": ["/params/path"]}); + doc["log"] = json!({"entries": [entry]}); + let mut back = Project::from_json(&doc.to_string()).unwrap(); + assert_eq!(back.log.entries()[0].params, json!({"path": "./a.tif"})); + let again: Value = serde_json::from_str(&back.saved_text(None).unwrap().0).unwrap(); + assert_eq!(again["log"]["entries"][0]["params"], json!({"path": "./a.tif"})); + assert!(again["log"]["entries"][0].get("relative").is_none(), "{again}"); +} + +/// The project log (#797): a command run by itself that changes the project, or reaches a file +/// or a service, is logged once (a script's lines inside it), with its outcome; reading isn't. +/// Undo keeps the entries and is logged with the step it took back (not when there is none). +/// Secrets are taken out and large values clipped; the log is capped, saved with the project +/// and read back; it can be read by filter, written out, cleared and turned off. +#[test] +fn the_project_log_records_each_command_run_on_the_project() { + use crate::project::{LOG_LIMIT, LOG_PARAMS_BYTES, LogEntry}; + let mut s = world(); + // The sample project's making is its first entry: counted from after it. + assert_eq!(s.project.log.clear(), 1); + let commands = |s: &Session| s.project.log.entries().iter().map(|e| e.command.clone()).collect::>(); + s.run("project.rename", &json!({"name": "Atlas"})).unwrap(); + s.run("info.project", &Value::Null).unwrap(); + s.run("map.zoomIn", &Value::Null).unwrap(); + s.run("script.run", &json!({"script": "project.rename {\"name\": \"A\"}\nproject.rename {\"name\": \"B\"}"})).unwrap(); + assert_eq!(commands(&s), ["project.rename", "script.run"], "reading and moving the view aren't logged, nor a script's lines"); + let first = &s.project.log.entries()[0]; + assert!(first.ok && first.error.is_none() && first.at > 1_700_000_000_000, "{first:?}"); + assert_eq!(first.params, json!({"name": "Atlas"})); + // Undone: the entries stay, and the undo says what it took back. + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.project.name, "Atlas"); + assert_eq!(commands(&s), ["project.rename", "script.run", "edit.undo"]); + assert_eq!(s.project.log.entries()[2].params, json!({"step": "Run Script"})); + // A failure is logged with its error; a service's address without its token. + assert!(s.run("project.rename", &json!({"name": " "})).is_err()); + let failed = s.project.log.entries().last().cloned().unwrap(); + assert!(!failed.ok && failed.error.as_deref().is_some_and(|e| e.contains("name")), "{failed:?}"); + let _ = s.run("service.layers", &json!({"url": "https://x.org/wms?token=S3CRET&layers=a", "text": "not a capabilities document"})); + let service = s.project.log.entries().last().cloned().unwrap(); + assert_eq!(service.command, "service.layers"); + assert!(!service.params.to_string().contains("S3CRET"), "{service:?}"); + // A document given as text is kept as its size only: it may hold secrets no redaction finds. + let qgs = format!("{}", "rustgis-s3:TOKEN:s3://b/k ".repeat(20)); + let _ = s.run("project.importQgis", &json!({"text": qgs})); + let doc = s.project.log.entries().last().cloned().unwrap(); + assert_eq!(doc.params["text"], json!(format!("(a document of {} characters)", qgs.chars().count())), "{doc:?}"); + // So is one after a byte-order mark (a QGIS project read from a file that has one). + let marked = format!("\u{feff}{qgs}"); + let _ = s.run("project.importQgis", &json!({"text": marked})); + let doc = s.project.log.entries().last().cloned().unwrap(); + assert!(doc.params["text"].as_str().is_some_and(|t| t.starts_with("(a document of")), "{doc:?}"); + // A secret in the JSON a text holds (a script's command line) is taken out too. + let line = r#"s3.connect {"accessKeyId": "AKIA", "secretAccessKey": "S3CRETKEY", "region": "x"}"#; + let _ = s.run("script.run", &json!({"script": line})); + let script = s.project.log.entries().last().cloned().unwrap(); + assert!(!script.params.to_string().contains("S3CRETKEY") && script.params.to_string().contains("s3.connect"), "{script:?}"); + // Nor where the clip cuts the object short: what follows its `{` goes. + // The secret comes first and a long field after it, past the cut. + let long = format!(r#"s3.connect {{"secretAccessKey": "S3CRETKEY", "notes": "{}"}}"#, "x".repeat(5000)); + let _ = s.run("script.run", &json!({"script": long})); + let cut = s.project.log.entries().last().cloned().unwrap(); + assert!(!cut.params.to_string().contains("S3CRET"), "{}", cut.params); + // Nor in an object that doesn't parse (a trailing comma), while text such as `{name}` stays. + let broken = r#"s3.connect {"secretAccessKey": "S3CRETKEY", "region": "x",} # see {name}"#; + let _ = s.run("script.run", &json!({"script": broken})); + let kept = s.project.log.entries().last().cloned().unwrap().params.to_string(); + assert!(!kept.contains("S3CRET") && kept.contains("s3.connect"), "{kept}"); + let _ = s.run("script.run", &json!({"script": "# keep {name} as it is"})); + assert!(s.project.log.entries().last().unwrap().params.to_string().contains("{name}")); + // A long list keeps its count when its texts are cut again to fit. + let long: Vec = (0..5_000).map(|i| format!("{i}{}", "y".repeat(100))).collect(); + let _ = s.run("project.rename", &json!({"name": "Long", "tags": long})); + let tags = s.project.log.entries().last().unwrap().params.to_string(); + assert_eq!(s.project.log.entries().last().unwrap().params, json!({"name": "Long", "tags": "…"}), "{tags}"); + // Large values clipped. + s.run("project.rename", &json!({"name": "x".repeat(20_000), "note": vec![1; 5_000]})).ok(); + let big = s.project.log.entries().last().cloned().unwrap(); + assert!(serde_json::to_vec(&big.params).unwrap().len() <= LOG_PARAMS_BYTES, "{}", big.params.to_string().len()); + // Saved with the project and read back; a project that never logged saves without it. + let (text, _) = s.project.saved_text(None).unwrap(); + let back = Project::from_json(&text).unwrap(); + assert_eq!(back.log.entries(), s.project.log.entries()); + // So is the autosaved recovery copy. + let recovered = Project::from_json(&s.project.clone().recovery_text().unwrap()).unwrap(); + assert_eq!(recovered.log.entries(), s.project.log.entries()); + assert!(!Project::default().saved_text(None).unwrap().0.contains("\"log\"")); + // Read by filter, newest last. + let r = s.run("project.log", &json!({"command": "project.rename", "failed": true})).unwrap(); + assert_eq!((r["count"].as_u64(), r["entries"][0]["command"].as_str()), (Some(1), Some("project.rename"))); + assert_eq!(s.run("project.log", &json!({"limit": 2})).unwrap()["entries"].as_array().map(Vec::len), Some(2)); + assert!(s.run("project.log", &json!({"limit": 0})).is_err()); + assert!(s.run("project.log", &json!({"limit": 0.5})).unwrap_err().contains("whole number")); + // Steps merged into one undo step (a held arrow key) are one entry, with the last parameters. + let n = s.project.log.len(); + for name in ["Nudged 1", "Nudged 2", "Nudged 3"] { + s.merge_undo("nudge"); + s.run("project.rename", &json!({"name": name})).unwrap(); + } + assert_eq!(s.project.log.len(), n + 1); + assert_eq!(s.project.log.entries().last().unwrap().params["name"], json!("Nudged 3")); + // Times as decimals too; a negative time, or a filter that isn't true or false, is refused. + assert_eq!(s.run("project.log", &json!({"since": 1.7e12, "limit": 1})).unwrap()["entries"].as_array().map(Vec::len), Some(1)); + assert!(s.run("project.log", &json!({"since": -1})).is_err()); + assert!(s.run("project.log", &json!({"failed": "true"})).is_err()); + assert!(s.run("project.log", &json!({"command": 7})).is_err()); + // An error that starts as a spreadsheet formula is written as text in the CSV. + let formula = LogEntry { + at: 2, + command: "x".into(), + params: Value::Null, + ok: false, + error: Some("=HYPERLINK(1)".into()), + millis: 0, + relative: Vec::new(), + }; + s.project.log.push(formula); + // Written out as CSV and JSON Lines. + let dir = std::env::temp_dir().join(format!("rustgis-project-log-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let csv = dir.join("log.csv"); + s.run("project.exportLog", &json!({"path": csv.to_string_lossy()})).unwrap(); + let csv_text = std::fs::read_to_string(&csv).unwrap(); + assert!(csv_text.starts_with("time_utc,at,command,ok,millis,error,params\n") && csv_text.contains("\"edit.undo\""), "{csv_text}"); + assert!(csv_text.contains("\"'=HYPERLINK(1)\"") && !csv_text.contains(",\"=HYPERLINK"), "{csv_text}"); + let jsonl = dir.join("log.jsonl"); + s.run("project.exportLog", &json!({"path": jsonl.to_string_lossy()})).unwrap(); + let lines: Vec = std::fs::read_to_string(&jsonl).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len() + 1, s.project.log.len(), "every entry but the export's own"); + let _ = std::fs::remove_dir_all(&dir); + // Cleared, it says so; off, nothing goes in; on again, it does. Clearing and switching change + // the project (an assistant is told so) but take no undo step, which would restore nothing. + for id in ["project.clearLog", "project.logCommands"] { + let spec = s.registry.get(id).unwrap(); + assert!(spec.mutates && !spec.undoable, "{id}"); + } + let steps = s.undo.len(); + s.mark_saved(); + assert!(s.run("project.clearLog", &Value::Null).unwrap()["cleared"].as_u64().unwrap() > 5); + assert_eq!(commands(&s), ["project.clearLog"]); + assert!(s.dirty() && s.undo.len() == steps, "changed, with no undo step"); + s.run("project.logCommands", &json!({"on": false})).unwrap(); + s.run("project.rename", &json!({"name": "Quiet"})).unwrap(); + assert_eq!(commands(&s), ["project.clearLog", "project.logCommands"], "turned off, and said so"); + s.run("project.logCommands", &json!({"on": true})).unwrap(); + assert_eq!(commands(&s), ["project.clearLog", "project.logCommands", "project.logCommands"]); + // A .json path is refused: JSON Lines isn't one JSON document. A format given doesn't let + // the log go over another kind of file, nor disagree with the path's extension. + assert!(s.run("project.exportLog", &json!({"path": "log.json"})).is_err()); + for (path, format) in [("log.json", "jsonl"), ("map.rgis", "csv"), ("log.csv", "jsonl"), ("log", "xml")] { + let e = s.run("project.exportLog", &json!({"path": path, "format": format})).unwrap_err(); + assert!(e.contains("not a .") || e.contains("doesn't match") || e.contains("format is"), "{path}: {e}"); + } + assert!(s.run("project.exportLog", &json!({"path": "log"})).unwrap_err().contains("give a .csv")); + // Switching to what it already is changes nothing. + s.mark_saved(); + s.run("project.logCommands", &json!({"on": true})).unwrap(); + assert!(!s.dirty(), "no change, nothing to save"); + // Making a project, or opening one from text, starts its log. + let mut made = world(); + made.run("project.new", &json!({"name": "Fresh"})).unwrap(); + assert_eq!(made.project.log.entries().iter().map(|e| e.command.as_str()).collect::>(), ["project.new"]); + // Cleared while off, it still says so. + s.run("project.logCommands", &json!({"on": false})).unwrap(); + s.run("project.clearLog", &Value::Null).unwrap(); + assert_eq!(commands(&s), ["project.clearLog"]); + s.run("project.logCommands", &json!({"on": true})).unwrap(); + // Capped: the oldest go first. + for i in 0..LOG_LIMIT + 5 { + s.project.log.push(LogEntry { + at: i as u64, + command: "x".into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + relative: Vec::new(), + }); + } + assert_eq!(s.project.log.len(), LOG_LIMIT); + assert_eq!(s.project.log.entries()[0].at, 5); + // Undo or redo with nothing to take back changes nothing, and isn't logged. + s.clear_history(); + let n = s.project.log.len(); + assert!(s.run("edit.undo", &Value::Null).is_err() && s.run("edit.redo", &Value::Null).is_err()); + assert_eq!(s.project.log.len(), n, "nothing to undo or redo: not logged"); +} + +/// Saved into a folder, the log's local paths are written relative to it, as the project's +/// sources are, and read back absolute; a text given as relative reads back as it was run. A +/// save with nothing logged since reuses its text. A log that would take a project past what a +/// project file opens with is refused. +#[test] +fn the_project_log_writes_its_paths_relative_to_the_project_folder() { + use crate::project::LogEntry; + let mut p = Project::default(); + // Saved into a folder, its paths are written relative to it, as the project's sources are. + let folder = std::env::temp_dir().join(format!("rustgis-log-portable-{}", std::process::id())); + let inside = folder.join("My Data").join("a.tif"); + let error = Some(format!("{}: not found (see ./x)", inside.display())); + let params = json!({"path": inside.to_string_lossy(), "name": "./Maps and data", "sql": "SELECT a / b FROM t -- /home", "root": "/"}); + p.log.push(LogEntry { at: 1, command: "layer.addData".into(), params, ok: false, error, millis: 0, relative: Vec::new() }); + let (text, _) = p.saved_text(Some(&folder)).unwrap(); + let saved: Value = serde_json::from_str(&text).unwrap(); + let last = saved["log"]["entries"].as_array().unwrap().last().unwrap().clone(); + assert_eq!( + (last["params"]["path"].as_str(), last["error"].as_str()), + (Some("./My Data/a.tif"), Some("./My Data/a.tif: not found (see ./x)")) + ); + assert_eq!(p.log.entries().last().unwrap().params["path"], json!(inside.to_string_lossy()), "the open project keeps them"); + assert_eq!((last["params"]["sql"].as_str(), last["params"]["root"].as_str()), (Some("SELECT a / b FROM t -- /home"), Some("/"))); + // Read back from that folder, they are absolute again, so a Save As elsewhere keeps them right. + std::fs::create_dir_all(&folder).unwrap(); + let file = folder.join("p.rgis"); + std::fs::write(&file, &text).unwrap(); + let reopened = Project::load(&file).unwrap(); + let path = reopened.log.entries().last().unwrap().params["path"].as_str().unwrap().to_string(); + assert!(std::path::Path::new(&path).is_absolute() && path.ends_with("a.tif"), "{path}"); + let error = reopened.log.entries().last().unwrap().error.clone().unwrap(); + assert_eq!(error, format!("{}: not found (see ./x)", path), "the error names the path whole again, and ./x as given"); + // A text given as relative is read back as it was run, not against the project's folder. + assert_eq!(reopened.log.entries().last().unwrap().params["name"], json!("./Maps and data")); + assert!(reopened.log.entries().iter().all(|e| e.relative.is_empty())); + // A text that already holds, as a word, the relative form a path would be written as: the + // entry's paths are written as file names, and every text reads back as it was written. + let error = Some("not found; see ./My Data/a.tif".to_string()); + let params = json!({"path": inside.to_string_lossy()}); + p.log.push(LogEntry { at: 2, command: "layer.addData".into(), params, ok: false, error, millis: 0, relative: Vec::new() }); + let (text, _) = p.saved_text(Some(&folder)).unwrap(); + std::fs::write(&file, &text).unwrap(); + let last = Project::load(&file).unwrap().log.entries().last().unwrap().clone(); + assert_eq!((last.params["path"].as_str(), last.error.as_deref()), (Some("a.tif"), Some("not found; see ./My Data/a.tif")), "{text}"); + let _ = std::fs::remove_dir_all(&folder); + // A text that starts with a path but holds more is read word by word, not as one path. (Paths + // in the folder: one elsewhere is written relative too where it can be, `../../mnt/…` on + // Linux, or as its name where it can't, another drive on Windows.) + let (a, b) = (folder.join("x").join("a.tif"), folder.join("y").join("b.tif")); + let (a, b) = (a.display(), b.display()); + for (error, saved) in [ + (format!("{a}: cannot open {b}"), "./x/a.tif: cannot open ./y/b.tif"), + (format!("{a} and {b} are missing"), "./x/a.tif and ./y/b.tif are missing"), + ] { + let entry = LogEntry { + at: 3, + command: "x".into(), + params: Value::Null, + ok: false, + error: Some(error.clone()), + millis: 0, + relative: Vec::new(), + }; + p.log.push(entry); + let (text, _) = p.saved_text(Some(&folder)).unwrap(); + let back: Value = serde_json::from_str(&text).unwrap(); + assert_eq!(back["log"]["entries"].as_array().unwrap().last().unwrap()["error"], json!(saved), "{error}"); + } + // A save with nothing logged since writes the same text; one after a change, or into + // another folder, writes it again. + let (a, _) = p.saved_text(Some(&folder)).unwrap(); + let (b, _) = p.saved_text(Some(&folder)).unwrap(); + assert_eq!(a, b); + assert_ne!(p.saved_text(None).unwrap().0, a); + p.log.push(LogEntry { at: 4, command: "y".into(), params: Value::Null, ok: true, error: None, millis: 0, relative: Vec::new() }); + assert!(p.saved_text(Some(&folder)).unwrap().0.contains(r#""command":"y""#)); + // The project's own folder is written as `.`, and read back as that folder. + let entry = LogEntry { + at: 5, + command: "z".into(), + params: json!({"path": folder.to_string_lossy()}), + ok: true, + error: None, + millis: 0, + relative: Vec::new(), + }; + p.log.push(entry); + let (text, _) = p.saved_text(Some(&folder)).unwrap(); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::write(folder.join("p.rgis"), &text).unwrap(); + let reopened = Project::load(&folder.join("p.rgis")).unwrap(); + let path = reopened.log.entries().last().unwrap().params["path"].as_str().unwrap().to_string(); + assert!(std::path::Path::new(&path).is_absolute(), "{path}"); + let _ = std::fs::remove_dir_all(&folder); + // A log that would take a project past what a project file opens with is refused at saving; + // a project over it by itself is not the log's doing. + let max = crate::project::MAX_BYTES as usize; + assert!(crate::project::log_fits(max - 10, max).is_ok()); + assert!(crate::project::log_fits(max - 10, max + 1).unwrap_err().contains("project.clearLog")); + assert!(crate::project::log_fits(max + 1, max + 50).is_ok()); +} + +/// What the project log keeps of a command's parameters: bounded in bytes, keys and time, and +/// without secrets, those in JSON a text holds and in SQL `CREATE … SECRET` options included. +#[test] +fn the_project_log_keeps_parameters_bounded_and_without_secrets() { + // Many keys with short values still fit the bound. + let wide: serde_json::Map = (0..40).map(|i| (format!("{i:02}{}", "k".repeat(62)), json!("é".repeat(64)))).collect(); + let kept = crate::project_log::kept_params(&Value::Object(wide)); + assert!(serde_json::to_vec(&kept).unwrap().len() <= crate::project::LOG_PARAMS_BYTES, "{kept}"); + assert!(kept.as_object().is_some_and(|o| o.len() > 1 && o.len() < 40)); + assert_eq!(kept["…"], json!("40 keys in all")); + // Keys alike in their first 64 characters: the first stays, the other counted as left out. + let mut alike: serde_json::Map = (0..40).map(|i| (format!("{i:02}{}", "k".repeat(62)), json!("v".repeat(64)))).collect(); + // Sorts right after the first key, and is the same once cut to 64 characters. + alike.insert(format!("00{}a", "k".repeat(62)), json!(2)); + let kept = crate::project_log::kept_params(&Value::Object(alike)); + assert_eq!(kept["…"], json!("41 keys in all")); + assert_eq!(kept[format!("00{}", "k".repeat(62))], json!("v".repeat(64)), "the first stays"); + assert!(kept.as_object().is_some_and(|o| !o.values().any(|v| *v == json!(2))), "{kept}"); + // A crafted text costs its length, not its braces times its length: 200,000 braces are + // milliseconds once, and would be hours if each `{` scanned the rest (a bound far from both). + let crafted = "{1".repeat(200_000); + let started = std::time::Instant::now(); + assert_eq!(crate::project_log::json_in_text_redacted(&crafted).len(), crafted.len()); + assert!(started.elapsed() < std::time::Duration::from_secs(60), "{:?}", started.elapsed()); + // A key cut short keeps no value: a secret's name may be past the cut. + let long_key = format!("{}secretAccessKey", "x".repeat(195)); + let kept = crate::project_log::kept_params(&json!({long_key: "S3CRET"})); + assert!(!kept.to_string().contains("S3CRET"), "{kept}"); + // An SQL secret's options, written as plain words, go; the statement's shape stays. + let sql = "CREATE OR REPLACE SECRET s3 (TYPE s3, KEY_ID 'AKIA1', SECRET 'S3CRET'); SELECT 1 / 2"; + let kept = crate::project_log::kept_params(&json!({"sql": sql})); + assert_eq!(kept["sql"], json!("CREATE OR REPLACE SECRET s3 (…); SELECT 1 / 2")); + assert_eq!(crate::project_log::sql_secrets_removed("create table t (a int)"), "create table t (a int)"); + assert_eq!(crate::project_log::sql_secrets_removed("create table secret (a int)"), "create table secret (a int)"); + // A path written by its name keeps a folder's name, not the nothing after its last separator. + for (path, name) in [("/data/My Data/", "My Data"), ("C:\\data\\a.tif", "a.tif"), ("/data/a.tif", "a.tif"), ("/", "/")] { + assert_eq!(crate::project_log::last_name(path), name, "{path}"); + } + // A file's contents in base64 (`layer.addDataBytes`) are kept as their size: a secret's name + // can't be seen in them. A long text that isn't base64 is clipped as before. + let encoded = "eyJ0b2tlbiI6ICJTM0NSRVQifQ==".repeat(20); + for data in [encoded.clone(), format!("data:application/json;base64,{encoded}")] { + let kept = crate::project_log::kept_params(&json!({"fileName": "a.geojson", "data": data})); + assert_eq!(kept["data"], json!(format!("(file contents of {} characters, base64)", data.len())), "{kept}"); + } + let prose = crate::project_log::kept_params(&json!({"note": "word ".repeat(100)})); + assert!(prose["note"].as_str().is_some_and(|n| n.starts_with("word word")), "{prose}"); + // `create` inside another word starts no statement. + for t in ["recreate secret s (key 'a')", "created secret (key 'a')"] { + assert_eq!(crate::project_log::sql_secrets_removed(t), t); + } + assert_eq!(crate::project_log::sql_secrets_removed("CREATE PERSISTENT SECRET (TYPE s3, SECRET 'x')"), "CREATE PERSISTENT SECRET (…)"); + // An unquoted secret key in an object that doesn't parse goes, with the rest of the text; + // a format string's braces stay. + let line = crate::project_log::kept_params(&json!({"script": "s3.connect {secretAccessKey: \"S3CRET\"}\nprint(f\"{x:.2f}\")"})); + assert!(!line.to_string().contains("S3CRET"), "{line}"); + let fmt = crate::project_log::kept_params(&json!({"script": "print(f\"{x:.2f}\")"})); + assert_eq!(fmt["script"], json!("print(f\"{x:.2f}\")")); + // A quoted secret key in such an object goes too, its value with it, double or single quotes. + for line in [r#"s3.connect {a: "x", "secretAccessKey": "S3CRET",}"#, "s3.connect {a: 'x', 'secretAccessKey': 'S3CRET',}"] { + let kept = crate::project_log::kept_params(&json!({"script": line})); + assert!(!kept.to_string().contains("S3CRET") && kept.to_string().contains("s3.connect"), "{kept}"); + } +} + +/// Plugins can neither empty the project log nor turn it off, nor push the user's entries out of +/// it; writing the log out, or reading it, reaches a file. The log reads in chunks, as the +/// export writes it. +#[test] +fn plugins_can_neither_clear_the_project_log_nor_crowd_out_the_users_entries() { + use crate::project::LogEntry; + // Plugins' commands keep at most their own share of the log: the user's entries stay. + let log = crate::project::ProjectLog::default(); + let entry = |command: &str| LogEntry { + at: 1, + command: command.into(), + params: Value::Null, + ok: true, + error: None, + millis: 0, + relative: Vec::new(), + }; + log.push(entry("layer.rename")); + for _ in 0..crate::project_log::LOG_PLUGIN_LIMIT + 5 { + log.push(entry("plugin:p/flood")); + } + let kept = log.entries(); + assert_eq!((kept.len(), kept[0].command.as_str()), (crate::project_log::LOG_PLUGIN_LIMIT + 1, "layer.rename")); + // Read in chunks (as the export writes it), every entry comes once, in order. + let mut read = Vec::new(); + while let Some(part) = Some(log.chunk(read.len(), 256)).filter(|p| !p.is_empty()) { + read.extend(part); + } + assert_eq!(read.len(), kept.len()); + assert!(read.iter().zip(&kept).all(|(a, b)| a.command == b.command && a.at == b.at)); + // A plugin never empties the log or turns it off; writing it out is reaching a file. + assert!(crate::cmd::plugins::user_only("project.clearLog") && crate::cmd::plugins::user_only("project.logCommands")); + { + use crate::cmd::plugins::{Reach, reaches}; + let export = crate::cmd::all().into_iter().find(|c| c.id == "project.exportLog").unwrap(); + assert_eq!(reaches(&export, &json!({"path": "log.csv"})), Some(Reach::Files)); + // Reading it too: it names the files and addresses the user's commands used. + let read = crate::cmd::all().into_iter().find(|c| c.id == "project.log").unwrap(); + assert_eq!(reaches(&read, &json!({})), Some(Reach::Files)); + } +} + +/// A file's project log is read keeping its last entries, each bounded and without secrets, and +/// nothing in it keeps the project from opening. +#[test] +fn a_files_project_log_is_read_bounded_and_never_keeps_the_project_from_opening() { + use crate::project::{LOG_LIMIT, LOG_PARAMS_BYTES}; + // A file's log is read keeping its last entries, each bounded. + let entries: Vec = (0..LOG_LIMIT + 3) + .map(|i| json!({"at": i, "command": "x".repeat(if i == LOG_LIMIT + 2 { 1000 } else { 1 }), "ok": true, "params": {"a": "y".repeat(if i == LOG_LIMIT + 2 { 20_000 } else { 1 })}})) + .collect(); + let log: crate::project::ProjectLog = serde_json::from_value(json!({"entries": entries})).unwrap(); + let read = log.entries(); + assert_eq!((read.len(), read[0].at), (LOG_LIMIT, 3)); + let last = read.last().unwrap(); + assert!( + last.command.chars().count() == 200 && serde_json::to_vec(&last.params).unwrap().len() <= LOG_PARAMS_BYTES, + "{}", + last.command.len() + ); + // Nothing in a file's log keeps the project from opening: a bad entry is skipped, a log that + // isn't one is empty; a secret it holds is taken out as it is read. + let log: crate::project::ProjectLog = serde_json::from_value(json!({"entries": [7, {"at": "x"}, {"at": 1, "command": "a", "ok": true, "params": {"url": "https://x.org/a?token=S3CRET"}, "error": "https://x.org/b?token=S3CRET: refused"}]})).unwrap(); + let read = log.entries(); + assert_eq!(read.len(), 1); + assert!(!serde_json::to_string(&read).unwrap().contains("S3CRET"), "{read:?}"); + for odd in [json!(7), json!("x"), json!([1, 2]), json!({"entries": 7, "off": "yes"}), json!(null)] { + let log: crate::project::ProjectLog = serde_json::from_value(odd.clone()).unwrap(); + assert!(log.is_empty() && log.on(), "{odd}"); + } + let opened = Project::from_json(r#"{"name": "A", "maps": [], "active_map": 0, "log": {"entries": [{"at": "never"}]}}"#).unwrap(); + assert!(opened.log.is_empty()); + // A file's plugins' entries are held to their cap too, their oldest going first; the user's + // entries stay, in order. + let plugin_limit = crate::project_log::LOG_PLUGIN_LIMIT; + let entries: Vec = (0..plugin_limit + 10) + .map(|i| json!({"at": i, "command": if i % 2 == 0 { "plugin:p/a" } else { "layer.rename" }, "ok": true})) + .chain((0..plugin_limit).map(|i| json!({"at": 100_000 + i, "command": "plugin:p/b", "ok": true}))) + .collect(); + let log: crate::project::ProjectLog = serde_json::from_value(json!({"entries": entries})).unwrap(); + let read = log.entries(); + let plugins: Vec<&crate::project::LogEntry> = read.iter().filter(|e| e.command.starts_with("plugin:")).collect(); + assert_eq!(plugins.len(), plugin_limit); + assert!(plugins.iter().all(|e| e.command == "plugin:p/b"), "the newest kept"); + assert_eq!(read.iter().filter(|e| e.command == "layer.rename").count(), (plugin_limit + 10) / 2); + assert!(read.windows(2).all(|w| w[0].at < w[1].at), "in order"); + // A template or a package starts a log of its own, even when its file holds one. + let seeded = r#"{"name": "A", "maps": [], "active_map": 0, "log": {"entries": [{"at": 1, "command": "a", "ok": true}]}}"#; + assert_eq!(Project::from_json(seeded).unwrap().log.len(), 1); + assert!(Project::from_import_json(seeded, None).unwrap().log.is_empty()); +} + /// `raster.project` puts a raster on a grid of another coordinate system, cell size, origin and /// extent, or on another raster's grid: grids of the same cell and origin line up, a polygon /// layer's selection gives the extent, and bad parameters are refused before anything is made. diff --git a/crates/engine/tests/portable_projects.rs b/crates/engine/tests/portable_projects.rs index 74a64522f..1662ab6c2 100644 --- a/crates/engine/tests/portable_projects.rs +++ b/crates/engine/tests/portable_projects.rs @@ -215,7 +215,14 @@ fn missing_stream_can_be_repaired_undone_redone_and_saved() { assert_eq!(paths(&mut s), before); let text = std::fs::read_to_string(&project).unwrap(); assert!(text.contains("./data/towns.fgb")); - assert!(!text.contains("./data/cities.fgb")); + // The layers no longer link the missing file; the project's log (#797) still records the + // commands that once named it. + let mut saved: Value = serde_json::from_str(&text).unwrap(); + // Its paths are written relative to the project's folder too, and said to be. + let log = saved.as_object_mut().unwrap().remove("log").unwrap().to_string(); + assert_no_root(&log, &f.root); + assert!(log.contains("towns.fgb") && log.contains("\"relative\""), "{log}"); + assert!(!saved.to_string().contains("./data/cities.fgb")); let reopened = open(&project); assert_stream(&reopened, layers.cities, &replacement); assert_other_sources(&reopened, &layers, &f.root); diff --git a/docs/python.md b/docs/python.md index ffc849079..8f6cd18e9 100644 --- a/docs/python.md +++ b/docs/python.md @@ -283,8 +283,9 @@ last view in memory; run `layer.loadAll` on it before `to_geojson` or `to_geopan of them. Open an existing project with `Session(project="atlas.rgis")` or `s.open("atlas.rgis")`. -`s.project_json()` returns its JSON text; `s.run("project.openText", text=...)` opens project -JSON without an intermediate file. `to_png()` returns PNG bytes even when it also writes a +`s.project_json()` returns its JSON text, without its +[project log](user-guide/workspace.md#project-log); `s.run("project.openText", text=...)` opens +project JSON without an intermediate file. `to_png()` returns PNG bytes even when it also writes a file; omit the path to keep the image in memory. ## From source diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index 3f3f95060..5ee37bc41 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -36,6 +36,13 @@ Saved layer addresses omit passwords, access keys, tokens and signatures. The op keeps them, but after restarting RustGIS a protected source may need you to sign in to its connection again or add it again with credentials. +A saved `.rgis` also carries its [project log](#project-log), which is on by default: the paths, +SQL and addresses of the commands run on it. Passwords, keys and tokens are taken out where a +name or an address shows them, but not a secret typed as plain words (in SQL, a note or a +parameter with an unusual name). Before sharing a project, look at its log (`project.log`), or +clear it (`project.clearLog`) or turn it off for that project. A package (`.rgispkg`) or a +template carries no log. + ### Autosave and recovery The desktop app keeps a recovery copy of unsaved work. While the project has changes that aren't @@ -71,6 +78,60 @@ replace the first one's copy; it says so as it starts. Nor does RustGIS autosave folder can't lock files (some network or synced folders), where it can't tell a second copy apart. The browser app doesn't autosave yet; it warns before you leave a page with unsaved changes. +### Project log + +A project keeps a **log** of the commands run on it: what was done, when, with what, and whether +it worked, for reproducing an analysis, reviewing work or writing a methods section. Every front +end goes into it alike: the app, scripts, the AI Assistant, the command line, Python and MCP. + +- **What goes in:** each command that changes the project, and each that names a file, folder, + address or connection, reads included (an export, a download, a save, opening a file), plus + undo and redo, with the step they took back or brought back. Reading and looking around within + the project (listing layers, describing a tool, reading features, moving the view) is left + out. A command run by another (a script's lines, a model's steps) is inside its parent's + entry. Opening or making a project goes into the log of the project it opens or makes; one + that fails to open stays in the log of the project open. A plugin can't empty the log or turn + it off, and reading it (`project.log`) takes the file permission, since it names the files and + addresses the user's commands used. +- **Each entry:** `at` (when it started, in milliseconds since 1970, UTC; for a plugin's command + that waited for its downloads, when its last run started), `command`, `params`, `ok`, `error` + when it failed, and `millis`, how long it took. +- **Outside undo:** undoing a command doesn't take its entry back, and clearing the log or + turning it on or off is no undo step. Steps that undo takes back as one (a held arrow key + nudging an element) are one entry, with the last one's parameters. +- **No secrets:** the parameters lose passwords, keys and tokens as History's Copy as code does, and + so does an error's text; a document given as text (a map, a project), or a file's contents in + base64, is kept as its size only, and an SQL `CREATE SECRET`'s options as `(…)`. A secret written + as plain words, with no name or address form (a password typed into a note), isn't recognised: + clear the log before sharing a project that used one. Each entry's parameters are clipped to about + 4 KB of JSON (a long text is cut, a list keeps its first 100 items), its error to 500 characters + and a `…`, and the log keeps its last 10,000 entries, the oldest dropped first. Plugins' commands + take at most 1,000 of those places, also in a log read from a file: past that, a plugin's new + entry pushes out an older plugin entry, not one of the user's. +- **Saved with the project:** a shared `.rgis` carries its log (a package, `.rgispkg`, or a + template doesn't). Clear it first if that isn't wanted (`project.clearLog`, which leaves its + own entry), or turn logging off for a project (`project.logCommands {"on": false}`; what is + there stays). A project with no log saves as before. A save that the log would take past + the 512 MB a project file opens with is refused, saying so: export the log, then clear it. + Saved into a folder, the local paths in its parameters and errors are written relative to + it, as the project's sources are; a path with spaces in it that an error names is found whole + when it is one of the entry's parameters, and otherwise by its words. An entry whose text + already holds, as a word, the relative form one of its paths would take (`./data/a.tif`) has + its paths written as file names instead, so every text reads back as it was written. An + earlier RustGIS opens a project that has a log as it opens any other, ignoring the log, and + saves it without one. The project JSON that the web app sends to a page embedding it, and + `project_json()` in Python, leave the log out: a project kept that way and opened again + (`project.openText`) starts a log of its own. + +From scripts: `project.log {"since"?, "until"?, "command"?, "failed"?, "limit"?}` lists entries, +newest last (`command` is an id, or the start of one ending in a dot, `layer.`; `limit` is a whole +number, 100 by default and at most 1,000, with `count` saying how many match), and +`project.exportLog {"path": "log.csv"}` writes it out as CSV (one row an entry, the parameters as a +JSON column) or JSON Lines (`.jsonl` or `.ndjson`), as the path's extension says; any other +extension (a `.json`, a project) is refused, whatever `format` is given. The History pane still +keeps geoprocessing runs with their outputs and Copy as code; the log is a plain record and runs +nothing again. + ### GeoLibre projects **Open** on desktop or **Project › New › GeoLibre Project…** opens a `.geolibre` or legacy