Projects: a log of every command run on a project, with its time, parameters and outcome (#797, part 1)

apps/rustgis-python/src/lib.rs

@@ -90,7 +90,7 @@ impl Engine {
     }
 
     /// The project as `.rgis` JSON text, as `project.saveAs` writes it (what `project.openText`
-    /// opens).
+    /// opens), without its project log.
     fn project_json(&self) -> PyResult<String> {
         serde_json::to_string(&self.session.project).map_err(|e| RustGISError::new_err(e.to_string()))
     }

crates/data/src/io.rs

@@ -560,10 +560,18 @@ pub(crate) fn temp_beside(path: &Path) -> std::io::Result<std::path::PathBuf> {
 /// Writes and syncs a new file (`create_new`, so an existing file or symlink is never followed);
 /// removes it on failure.
 fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
+    write_new_with(path, |w| w.write_all(bytes))
+}
+
+/// [`write_new`] with the content written by `write`, through a buffer, as it goes.
+fn write_new_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> {
     use std::io::Write;
-    let mut f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?;
+    let f = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?;
     let res = (|| {
-        f.write_all(bytes)?;
+        let mut w = std::io::BufWriter::new(&f);
+        write(&mut w)?;
+        w.flush()?;
+        drop(w);
         f.sync_all()
     })();
     // Close before removing a failed write, including on Windows.
@@ -577,8 +585,14 @@ fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
 /// Writes through a temporary file that is synced and renamed into place, so an interrupted
 /// write never truncates an existing file. The temporary file is removed on failure.
 pub fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
+    write_atomic_with(path, |w| w.write_all(bytes))
+}
+
+/// [`write_atomic`] with the content written by `write` as it goes, so a long text needn't be
+/// held whole first. A failure in `write` leaves an existing file as it was.
+pub fn write_atomic_with(path: &Path, write: impl FnOnce(&mut dyn std::io::Write) -> std::io::Result<()>) -> std::io::Result<()> {
     let tmp = temp_beside(path)?;
-    write_new(&tmp, bytes)?;
+    write_new_with(&tmp, write)?;
     let res = std::fs::rename(&tmp, path);
     if res.is_err() {
         let _ = std::fs::remove_file(&tmp);
@@ -2305,6 +2319,16 @@ mod tests {
         assert_eq!(std::fs::read(&p).unwrap(), b"two");
         assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "no temp files left behind");
         assert!(write_atomic(Path::new("/"), b"x").is_err());
+        // Written as it goes: a failure part way leaves the file as it was, and no temp file.
+        let failed = write_atomic_with(&p, |w| {
+            w.write_all(b"half")?;
+            Err(std::io::Error::other("stopped"))
+        });
+        assert!(failed.is_err());
+        assert_eq!(std::fs::read(&p).unwrap(), b"two");
+        assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "no temp files left behind");
+        write_atomic_with(&p, |w| w.write_all(b"th").and_then(|()| w.write_all(b"ree"))).unwrap();
+        assert_eq!(std::fs::read(&p).unwrap(), b"three");
         std::fs::remove_dir_all(&dir).ok();
     }
 

crates/engine/src/cmd/gp.rs

@@ -5,6 +5,7 @@ use rustgis_data::Dataset;
 use serde_json::{Value, json};
 
 use super::{need_layers, ps};
+use crate::project_log::now_ms;
 use crate::{CmdResult, CommandSpec as C, GpRecord, Session};
 
 pub fn commands() -> Vec<C> {
@@ -644,10 +645,6 @@ impl Inputs for MapInputs<'_> {
     }
 }
 
-fn now_ms() -> u64 {
-    web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0)
-}
-
 fn run(s: &mut Session, p: &Value) -> CmdResult {
     let tool = ps(p, "tool").ok_or("tool is required")?.to_string();
     let params = p.get("params").cloned().unwrap_or(json!({}));

crates/engine/src/cmd/history.rs

@@ -112,7 +112,7 @@ pub fn tool_of(r: &GpRecord) -> Option<String> {
 /// `password`, an `api_token`; a plain `key` when it reads as one or sits beside a web address,
 /// as `sources::holds_secret` judges it), left empty. An object's keys too: the file may put
 /// anything there.
-fn redacted(v: &Value) -> Value {
+pub(crate) fn redacted(v: &Value) -> Value {
     use crate::sources::looks_like_a_key;
     use rustgis_data::source::credentials::secret_key;
     match v {
@@ -139,7 +139,7 @@ fn redacted(v: &Value) -> Value {
 /// name addresses among its words; a single word may be one. Any text may still be a connection
 /// in a form `redact` doesn't read (libpq's words alone, GDAL's `MYSQL:db,user=…,password=…`,
 /// ODBC's `UID=…;PWD=…`): its secret-named values go too.
-fn redacted_text(t: &str) -> String {
+pub(crate) fn redacted_text(t: &str) -> String {
     let r = if connection(t) || !t.contains(char::is_whitespace) { crate::sources::redact(t) } else { crate::sources::redact_text(t) };
     // After `redact` too: a prefix it knows may come in a form it doesn't (GDAL's `MYSQL:`).
     query_pairs_emptied(&private_keys_removed(&r))

crates/engine/src/cmd/items.rs

@@ -519,7 +519,7 @@ fn import_styles(s: &mut Session, p: &Value) -> CmdResult {
 
 /// The project without its features (layer schemas, symbology, layouts and styles stay).
 fn template(p: &Project) -> Project {
-    let mut t = p.clone();
+    let mut t = p.own_copy();
     t.gp_history.clear();
     for m in &mut t.maps {
         for l in &mut m.layers {

crates/engine/src/cmd/plugins.rs

@@ -287,18 +287,22 @@ const NEVER: &[&str] = &[
     "s3.disconnect",
     "earthdata.login",
     "earthdata.logout",
+    // The project log records what was done (#797): a plugin never empties it or turns it off.
+    "project.clearLog",
+    "project.logCommands",
     "pg.connect",
     "pg.disconnect",
     // Writes to a database: the user's to do, asked each time.
     "pg.saveEdits",
 ];
 
-/// Commands that write files without naming one in their parameters.
-const WRITES_FILES: &[&str] = &["project.save", "project.saveAs"];
+/// Commands that write files without naming one in their parameters, or read what the user's
+/// commands named (the project log's paths, addresses and SQL).
+const REACHES_FILES: &[&str] = &["project.save", "project.saveAs", "project.log"];
 
 /// Parameter keys that name a file, folder or address. (`data` and `text` carry content, which
 /// plugins may pass: `layer.addDataBytes`, `map.import` from text.)
-const PLACE_KEYS: &[&str] = &["path", "folder", "dir", "file", "url", "uri", "source", "output", "connection", "database"];
+pub(crate) const PLACE_KEYS: &[&str] = &["path", "folder", "dir", "file", "url", "uri", "source", "output", "connection", "database"];
 
 /// The command running an SQL query through DuckDB, which can read any file and web address.
 pub const SQL: &str = "layer.addSql";
@@ -369,7 +373,7 @@ pub fn reaches(spec: &C, params: &Value) -> Option<Reach> {
     }
     let files = super::items::names_place(params, PLACE_KEYS, 0)
         || PLACE_KEYS.iter().any(|k| spec.params.contains(&format!("\"{k}\"")))
-        || WRITES_FILES.contains(&spec.id)
+        || REACHES_FILES.contains(&spec.id)
         || ["share.", "offline.", "cache.", "notebook."].iter().any(|p| spec.id.starts_with(p));
     files.then_some(Reach::Files)
 }

crates/engine/src/cmd/project.rs

@@ -31,9 +31,166 @@ pub fn commands() -> Vec<C> {
             .params(r#"{"tool": "explore"|"select"|"selectPolygon"|"selectLasso"|"selectRadius"|"zoomIn"|"measure"|"measureArea"|"createPoint"|"createLine"|"createPolygon"|"editVertices"|"graphicPoint"|"graphicText"|"graphicLine"|"graphicCurve"|"graphicFreehand"|"graphicPolygon"|"graphicLasso"|"graphicRectangle"|"graphicCircle"|"graphicEllipse"|"graphicPicture"}"#),
         C::new("info.project", "Project Info", "Project", info).view().params(r#"{"nameFilter"?: string, "includeFields"?: bool}"#),
         C::new("info.commands", "List Commands", "Help", list).view(),
+        // The project log (#797): read, written out, emptied, turned on or off. Clearing and
+        // switching change what is saved, but take no undo step: the log sits outside the undo
+        // history, so one would restore nothing of it.
+        C::new("project.log", "Project Log", "Project", log).view().params(
+            r#"{"since"?: milliseconds since 1970 (UTC), "until"?: milliseconds, "command"?: "a command id, or the start of one (layer.)", "failed"?: bool (true: only those that failed; false: only those that succeeded), "limit"?: entries, a whole number (default 100, the newest; 1 to 1000)} → {"entries": [{"at", "command", "params", "ok", "error"?, "millis"}], "count": matching, "total": in the log, "on": bool}"#,
+        ),
+        C::new("project.exportLog", "Export Project Log", "Project", export_log)
+            .view()
+            .params(r#"{"path": ".csv, .jsonl or .ndjson file", "format"?: "csv"|"jsonl" (for a path with no extension; one given must match the extension)}"#),
+        C::new("project.clearLog", "Clear Project Log", "Project", clear_log).outside_undo(),
+        C::new("project.logCommands", "Log Commands", "Project", log_commands).outside_undo().params(r#"{"on"?: bool (default: turn it the other way)}"#),
     ]
 }
 
+/// `project.log`: the log's entries that match, newest last.
+fn log(s: &mut Session, p: &Value) -> CmdResult {
+    // Milliseconds as a whole number or not (1.7e12), never below 0.
+    let num = |k: &str| -> Result<Option<u64>, String> {
+        match p.get(k) {
+            None | Some(Value::Null) => Ok(None),
+            Some(v) => v
+                .as_f64()
+                .filter(|n| n.is_finite() && *n >= 0.0)
+                .map(|n| Some(n as u64))
+                .ok_or_else(|| format!("{k} must be a number, 0 or more")),
+        }
+    };
+    let (since, until) = (num("since")?, num("until")?);
+    let command = match p.get("command") {
+        None | Some(Value::Null) => None,
+        Some(v) => Some(v.as_str().ok_or("command must be a command id, or the start of one")?.trim()).filter(|c| !c.is_empty()),
+    };
+    let failed = match p.get("failed") {
+        None | Some(Value::Null) => None,
+        Some(v) => Some(v.as_bool().ok_or("failed must be true or false")?),
+    };
+    // At most a thousand (some 4 MB at the most), so one answer can't fill an assistant's
+    // context or a script's memory; `count` still says how many match.
+    let limit = match p.get("limit") {
+        None | Some(Value::Null) => 100,
+        Some(v) => match v.as_f64().filter(|n| n.is_finite() && *n >= 1.0 && n.fract() == 0.0) {
+            Some(n) => n.min(1_000.0) as usize,
+            None => return Err("limit must be a whole number, 1 or more".into()),
+        },
+    };
+    let keep = |e: &crate::project::LogEntry| {
+        since.is_none_or(|t| e.at >= t)
+            && until.is_none_or(|t| e.at <= t)
+            && command.is_none_or(|c| e.command == c || (c.ends_with('.') && e.command.starts_with(c)))
+            && failed.is_none_or(|f| e.ok != f)
+    };
+    let (shown, count, total) = s.project.log.select(keep, limit);
+    Ok(json!({"entries": shown, "count": count, "total": total, "on": s.project.log.on()}))
+}
+
+/// A CSV cell: quoted, and kept from being read as a formula where it starts as one does (`=`,
+/// `+`, `-`, `@`, or a tab or carriage return before them), as spreadsheets would.
+fn csv_cell(t: &str) -> String {
+    let lead = if t.starts_with(['=', '+', '-', '@', '\t', '\r']) { "'" } else { "" };
+    format!("\"{lead}{}\"", t.replace('"', "\"\""))
+}
+
+/// `project.exportLog`: the log written out, as CSV (one row an entry, the parameters as a JSON
+/// column) or JSON Lines (one entry a line).
+fn export_log(s: &mut Session, p: &Value) -> CmdResult {
+    let path = PathBuf::from(ps(p, "path").ok_or("path is required")?);
+    let kind = |f: &str| match f {
+        "csv" => Some("csv"),
+        "jsonl" | "ndjson" => Some("jsonl"),
+        _ => None,
+    };
+    // The path's extension, when it has one, is the format's whatever `format` says: the log
+    // never goes over a project, a JSON document or any other kind of file.
+    let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase);
+    let by_ext =
+        match ext.as_deref() {
+            None => None,
+            Some(e) => Some(kind(e).ok_or_else(|| {
+                format!("the log is written to a .csv or .jsonl file, not a .{e} one (JSON Lines isn't one JSON document)")
+            })?),
+        };
+    let asked = ps(p, "format").map(str::to_ascii_lowercase);
+    let format = match (asked.as_deref(), by_ext) {
+        (Some(f), by_ext) => {
+            let f = kind(f).ok_or("format is csv or jsonl")?;
+            if by_ext.is_some_and(|e| e != f) {
+                return Err(format!("format {f} doesn't match the path's .{}", ext.as_deref().unwrap_or_default()));
+            }
+            f
+        }
+        (None, Some(e)) => e,
+        (None, None) => return Err("give a .csv or .jsonl path, or a format (csv or jsonl)".into()),
+    };
+    // Written entry by entry, a few hundred copied at a time under the log's lock and written
+    // after it is let go: a long log isn't held as text, and an autosave reading the log doesn't
+    // wait on the disk. A failure part way leaves an existing file as it was.
+    let mut n = 0usize;
+    rustgis_data::io::write_atomic_with(&path, |w| {
+        if format == "csv" {
+            w.write_all(b"time_utc,at,command,ok,millis,error,params\n")?;
+        }
+        loop {
+            let part = s.project.log.chunk(n, 256);
+            if part.is_empty() {
+                return Ok(());
+            }
+            for e in &part {
+                write_entry(w, e, format)?;
+            }
+            n += part.len();
+        }
+    })
+    .map_err(|e| format!("{}: {e}", path.display()))?;
+    s.notify(format!("Exported the project log ({n} entries) to {}", path.display()), false);
+    Ok(json!({"path": path.display().to_string(), "entries": n, "format": format}))
+}
+
+/// Entry `e` as a CSV row or a JSON line, written to `w`.
+fn write_entry(w: &mut dyn std::io::Write, e: &crate::project::LogEntry, format: &str) -> std::io::Result<()> {
+    if format == "csv" {
+        let (date, time) = rustgis_map::graphics::civil(e.at / 1000);
+        writeln!(
+            w,
+            "{date} {time}:{:02},{},{},{},{},{},{}",
+            (e.at / 1000) % 60,
+            e.at,
+            csv_cell(&e.command),
+            e.ok,
+            e.millis,
+            csv_cell(e.error.as_deref().unwrap_or("")),
+            csv_cell(&e.params.to_string())
+        )
+    } else {
+        serde_json::to_writer(&mut *w, e).map_err(std::io::Error::other)?;
+        w.write_all(b"\n")
+    }
+}
+
+/// `project.clearLog`: empties the log, which then holds this command's own entry, so a cleared
+/// log says it was cleared.
+fn clear_log(s: &mut Session, _: &Value) -> CmdResult {
+    let n = s.project.log.clear();
+    // Unsaved whatever undo does: no undo step brings the log back.
+    s.mark_unsaved();
+    Ok(json!({"cleared": n}))
+}
+
+/// `project.logCommands`: turns the log on or off; off, nothing more goes in (what is there stays).
+fn log_commands(s: &mut Session, p: &Value) -> CmdResult {
+    let on = match p.get("on") {
+        None | Some(Value::Null) => !s.project.log.on(),
+        Some(v) => v.as_bool().ok_or("on must be true or false")?,
+    };
+    if on != s.project.log.on() {
+        s.project.log.set_on(on);
+        s.mark_unsaved();
+    }
+    Ok(json!({"on": on}))
+}
+
 /// Replaces the project, clearing history and per-project state.
 pub(crate) fn reset_to(s: &mut Session, p: Project) {
     reset(s, p);