From 5d1dbcce6e99710c8979af6e459b1b6158ebd0d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 02:49:12 +0700 Subject: [PATCH] feat(swat): SWAT+ project model and TxtInOut reader/writer (phase 6a) Two new crates for the SWAT+ editor: - rustgis-swat-model (L1): a SWAT+ project as its TxtInOut text files, kept losslessly (lines, per-line endings, UTF-8 or Latin-1), with token-level edits that keep columns aligned. - rustgis-swat-txtinout (L2): files from and to bytes (so it runs in the browser), file.cio's index and the input files it names, and each file's layout: tables, counted tables, and records owning the lines after them (soil layers, plant communities, schedules, decision tables, weather generator months, calibration conditions). Written from the format's documentation, not SWAT+'s code. A test reads the private test scenarios from SWAT_TESTDATA (skipped without it): all four (Osu_1hru, Ames_sub1, Batanghari, Pessel) come back byte for byte, and every line of every file has a place in its view. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L94WcvvLDssnAEZihGAwYo --- AGENTS.md | 2 + Cargo.lock | 11 + Cargo.toml | 2 + crates/swat-model/Cargo.toml | 11 + crates/swat-model/src/lib.rs | 68 +++++ crates/swat-model/src/text.rs | 226 ++++++++++++++ crates/swat-txtinout/Cargo.toml | 12 + crates/swat-txtinout/src/cio.rs | 106 +++++++ crates/swat-txtinout/src/lib.rs | 69 +++++ crates/swat-txtinout/src/view.rs | 388 +++++++++++++++++++++++++ crates/swat-txtinout/tests/projects.rs | 91 ++++++ xtask/src/main.rs | 2 + 12 files changed, 988 insertions(+) create mode 100644 crates/swat-model/Cargo.toml create mode 100644 crates/swat-model/src/lib.rs create mode 100644 crates/swat-model/src/text.rs create mode 100644 crates/swat-txtinout/Cargo.toml create mode 100644 crates/swat-txtinout/src/cio.rs create mode 100644 crates/swat-txtinout/src/lib.rs create mode 100644 crates/swat-txtinout/src/view.rs create mode 100644 crates/swat-txtinout/tests/projects.rs diff --git a/AGENTS.md b/AGENTS.md index a5ce7be66..6d8b8affc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,8 +17,10 @@ is a command, a thin egui front end, headless screenshots for UI work. |---|---|---| | L0 | `geom` | coordinates, extents, Web Mercator, geodesic measurement, scale | | L1 | `data` | features, typed attributes, GeoJSON/CSV/Shapefile/KML/GPX I/O, GeoPackage/GeoParquet/FlatGeobuf/LiDAR (LAS/LAZ)/PostGIS/MySQL providers (`source`; PostGIS through the pure-Rust `postgres` client, MySQL and MariaDB through the pure-Rust `mysql` client, desktop only; GeoPackages in the browser through the pure-Rust `sqlite` reader; GDAL formats and DuckDB queries through their programs), SQL where clauses, rasters (GeoTIFF/ASCII grid, Zarr stores, EPSG reprojection) | +| L1 | `swat-model` | a SWAT+ project as its TxtInOut files, kept losslessly (lines, endings, encoding), edited token by token so columns stay aligned | | L2 | `map`, `analysis` | layers, symbology, labels, camera, basemaps, web tile layers; geoprocessing tools | | L2 | `tiles` | web map tiles: XYZ/TMS/WMS rasters, Mapbox Vector Tiles, PMTiles and MBTiles archives, a MapLibre style subset; the tile cache and background fetching (see `docs/user-guide/maps.md` › Basemaps) | +| L2 | `swat-txtinout` | SWAT+ TxtInOut: files from and to bytes (so it works in the browser), `file.cio`'s inputs, each file's layout (tables, records owning lines: soil layers, schedules, decision tables); written from the format's documentation, not SWAT+'s code | | L3 | `render` | tiny-skia map renderer, label placement, PNG export, font discovery | | L3 | `plugins` | plugins: sandboxed WebAssembly modules (`wasmi`) adding commands and tools; `.rgplugin` packages, the plugin registry (see `docs/plugins.md`) | | L3 | `geolibre` | the GeoLibre/Whitebox tool suite as geoprocessing tools (engine feature `geolibre`): run by the optional `geolibre` program natively, by geolibre-rust's WASI runner in the browser | diff --git a/Cargo.lock b/Cargo.lock index c863b2551..8475dabd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4648,6 +4648,17 @@ dependencies = [ "web-time", ] +[[package]] +name = "rustgis-swat-model" +version = "0.3.0" + +[[package]] +name = "rustgis-swat-txtinout" +version = "0.3.0" +dependencies = [ + "rustgis-swat-model", +] + [[package]] name = "rustgis-tiles" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index d6db9b636..2ae2b1a38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ rustgis-geolibre = { path = "crates/geolibre" } rustgis-plugins = { path = "crates/plugins" } rustgis-tiles = { path = "crates/tiles" } rustgis-assistant = { path = "crates/assistant" } +rustgis-swat-model = { path = "crates/swat-model" } +rustgis-swat-txtinout = { path = "crates/swat-txtinout" } # foundation thiserror = "2" diff --git a/crates/swat-model/Cargo.toml b/crates/swat-model/Cargo.toml new file mode 100644 index 000000000..699257fc2 --- /dev/null +++ b/crates/swat-model/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rustgis-swat-model" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "RustGIS L1: a SWAT+ project as its TxtInOut text files, kept losslessly, with token-level edits" + +[lints] +workspace = true + +[dependencies] diff --git a/crates/swat-model/src/lib.rs b/crates/swat-model/src/lib.rs new file mode 100644 index 000000000..35aa9ebcf --- /dev/null +++ b/crates/swat-model/src/lib.rs @@ -0,0 +1,68 @@ +//! L1: a SWAT+ project for editing. +//! +//! A project is its TxtInOut files, each kept as the text it was read from ([`TextFile`]), so a +//! project that is read and written again is the same bytes, and an edit changes only the value +//! edited. SWAT+ reads its files as whitespace-separated values (Fortran list-directed input), +//! so a file is lines of tokens; [`TextFile::set_token`] replaces one while keeping the columns +//! aligned. What each file's lines mean (titles, headers, records, nested records) is read by +//! `rustgis-swat-txtinout`; this crate has no I/O. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::collections::BTreeMap; + +mod text; + +pub use text::{Encoding, TextFile, Token}; + +/// Why an edit was refused. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EditError { + /// No such file in the project. + NoFile(String), + /// No such line in the file. + NoLine(usize), + /// No such token on the line. + NoToken { line: usize, token: usize }, + /// A value must be one token: not empty, without whitespace. + NotAToken(String), +} + +impl std::fmt::Display for EditError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EditError::NoFile(n) => write!(f, "no file {n} in the project"), + EditError::NoLine(l) => write!(f, "no line {}", l + 1), + EditError::NoToken { line, token } => write!(f, "no value {} on line {}", token + 1, line + 1), + EditError::NotAToken(t) => write!(f, "{t:?} is not a single value"), + } + } +} + +impl std::error::Error for EditError {} + +/// A SWAT+ project: its files by name (as `file.cio` and the other files name them, relative +/// to the project folder, `/` separated). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Project { + pub files: BTreeMap, +} + +impl Project { + pub fn new() -> Self { + Self::default() + } + + /// The file named `name`. SWAT+ names are case-sensitive, as on Linux. + pub fn file(&self, name: &str) -> Option<&TextFile> { + self.files.get(name) + } + + pub fn file_mut(&mut self, name: &str) -> Option<&mut TextFile> { + self.files.get_mut(name) + } + + /// Replaces value `token` (0-based) of line `line` (0-based) of file `name`. + pub fn set_token(&mut self, name: &str, line: usize, token: usize, value: &str) -> Result<(), EditError> { + self.files.get_mut(name).ok_or_else(|| EditError::NoFile(name.to_string()))?.set_token(line, token, value) + } +} diff --git a/crates/swat-model/src/text.rs b/crates/swat-model/src/text.rs new file mode 100644 index 000000000..071cc450f --- /dev/null +++ b/crates/swat-model/src/text.rs @@ -0,0 +1,226 @@ +//! A text file kept as it was read: its lines, line endings and encoding. + +use crate::EditError; + +/// How a file's bytes became text, to write the same bytes back. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Encoding { + #[default] + Utf8, + /// Not valid UTF-8: each byte a character (Latin-1), as old Windows tools wrote. + Latin1, +} + +/// A value on a line: its byte range. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Token { + pub start: usize, + pub end: usize, +} + +/// A text file: its lines without their endings, which of them end in `\r\n` (files mixing +/// endings keep them), and whether the last line has an ending. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextFile { + lines: Vec, + /// The ending of new lines: the first line's. + crlf: bool, + /// Each line's ending is `\r\n`. + cr: Vec, + final_newline: bool, + pub encoding: Encoding, +} + +fn is_space(c: char) -> bool { + c == ' ' || c == '\t' +} + +impl TextFile { + /// The file of text `s` (its line endings found from the first line). + pub fn from_text(s: &str) -> Self { + let crlf = s.find('\n').is_some_and(|i| i > 0 && s.as_bytes().get(i - 1) == Some(&b'\r')); + let final_newline = s.ends_with('\n'); + let body = if final_newline { &s[..s.len() - 1] } else { s }; + let (mut lines, mut cr) = (Vec::new(), Vec::new()); + if !s.is_empty() { + let parts: Vec<&str> = body.split('\n').collect(); + let n = parts.len(); + for (k, l) in parts.into_iter().enumerate() { + // A last line without an ending keeps its `\r`. + let ended = k + 1 < n || final_newline; + match l.strip_suffix('\r').filter(|_| ended) { + Some(t) => { + lines.push(t.to_string()); + cr.push(true); + } + None => { + lines.push(l.to_string()); + cr.push(false); + } + } + } + } + TextFile { lines, crlf, cr, final_newline, encoding: Encoding::Utf8 } + } + + /// The text again, with the endings it was read with. + pub fn to_text(&self) -> String { + let mut out = String::with_capacity(self.lines.iter().map(|l| l.len() + 2).sum()); + let n = self.lines.len(); + for (k, l) in self.lines.iter().enumerate() { + out.push_str(l); + if k + 1 < n || self.final_newline { + out.push_str(if self.cr.get(k).copied().unwrap_or(self.crlf) { "\r\n" } else { "\n" }); + } + } + out + } + + pub fn len(&self) -> usize { + self.lines.len() + } + + pub fn is_empty(&self) -> bool { + self.lines.is_empty() + } + + pub fn line(&self, i: usize) -> Option<&str> { + self.lines.get(i).map(String::as_str) + } + + pub fn lines(&self) -> impl Iterator { + self.lines.iter().map(String::as_str) + } + + /// The values on line `i`, separated by spaces and tabs. + pub fn tokens(&self, i: usize) -> Vec { + let Some(l) = self.lines.get(i) else { return Vec::new() }; + let mut out = Vec::new(); + let mut start = None; + for (k, c) in l.char_indices() { + match (is_space(c), start) { + (false, None) => start = Some(k), + (true, Some(s)) => { + out.push(Token { start: s, end: k }); + start = None; + } + _ => {} + } + } + if let Some(s) = start { + out.push(Token { start: s, end: l.len() }); + } + out + } + + /// The text of value `k` of line `i`. + pub fn token(&self, i: usize, k: usize) -> Option<&str> { + let t = *self.tokens(i).get(k)?; + self.lines.get(i).and_then(|l| l.get(t.start..t.end)) + } + + /// The values of line `i` as text. + pub fn values(&self, i: usize) -> Vec<&str> { + let Some(l) = self.lines.get(i) else { return Vec::new() }; + self.tokens(i).into_iter().filter_map(|t| l.get(t.start..t.end)).collect() + } + + /// Replaces value `k` of line `i` with `value`, keeping the columns: a number keeps its right + /// edge (numbers are right-aligned), any other value its left edge; a longer value takes the + /// spaces beside it, keeping one, before it moves the rest of the line. + pub fn set_token(&mut self, i: usize, k: usize, value: &str) -> Result<(), EditError> { + if value.is_empty() || value.chars().any(|c| is_space(c) || c == '\n' || c == '\r') { + return Err(EditError::NotAToken(value.to_string())); + } + let toks = self.tokens(i); + let line = self.lines.get(i).ok_or(EditError::NoLine(i))?; + let t = *toks.get(k).ok_or(EditError::NoToken { line: i, token: k })?; + let old_len = t.end - t.start; + let numeric = line[t.start..t.end].parse::().is_ok(); + let n = value.len(); + let spaces = |s: &str| s.len() - s.trim_end_matches(is_space).len(); + let mut out = line[..t.start].to_string(); + if numeric { + if n <= old_len { + out.push_str(&" ".repeat(old_len - n)); + } else { + // Take the spaces before it, keeping one after a previous value. + let keep = usize::from(k > 0); + let take = (n - old_len).min(spaces(&out).saturating_sub(keep)); + out.truncate(out.len() - take); + } + out.push_str(value); + } else { + out.push_str(value); + if n < old_len { + out.push_str(&" ".repeat(old_len - n)); + } + } + // What follows: if the value grew past its old end, take the spaces after it, keeping + // one before a next value. + let rest = &line[t.end..]; + let lead = rest.len() - rest.trim_start_matches(is_space).len(); + let over = out.len().saturating_sub(t.end); + let keep = usize::from(k + 1 < toks.len()); + let take = over.min(lead.saturating_sub(keep)); + out.push_str(&rest[take..]); + self.lines[i] = out; + Ok(()) + } + + /// Replaces line `i`. + pub fn set_line(&mut self, i: usize, text: &str) -> Result<(), EditError> { + let l = self.lines.get_mut(i).ok_or(EditError::NoLine(i))?; + *l = text.replace(['\n', '\r'], " "); + Ok(()) + } + + /// Inserts a line before line `i` (at the end when `i` is the number of lines). + pub fn insert_line(&mut self, i: usize, text: &str) -> Result<(), EditError> { + if i > self.lines.len() { + return Err(EditError::NoLine(i)); + } + self.lines.insert(i, text.replace(['\n', '\r'], " ")); + self.cr.insert(i.min(self.cr.len()), self.crlf); + Ok(()) + } + + /// Removes line `i`. + pub fn remove_line(&mut self, i: usize) -> Result { + if i >= self.lines.len() { + return Err(EditError::NoLine(i)); + } + if i < self.cr.len() { + self.cr.remove(i); + } + Ok(self.lines.remove(i)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + for s in ["", "a\n", "a\r\nb\r\n", "a\nb", "t\n\n 1 2\n", "a\r\nb\nc\r\n", "a\r", "\r\n"] { + assert_eq!(TextFile::from_text(s).to_text(), s); + } + } + + #[test] + fn edit_keeps_columns() { + let mut f = TextFile::from_text("name a b\nhru001 0.050 1.000\n"); + f.set_token(1, 1, "0.12345").ok(); + assert_eq!(f.line(1), Some("hru001 0.12345 1.000")); + f.set_token(1, 0, "x").ok(); + assert_eq!(f.line(1), Some("x 0.12345 1.000")); + f.set_token(1, 2, "12345678901").ok(); + assert_eq!(f.line(1), Some("x 0.12345 12345678901")); + f.set_token(1, 0, "a_very_long_name").ok(); + assert_eq!(f.token(1, 0), Some("a_very_long_name")); + assert_eq!(f.values(1), vec!["a_very_long_name", "0.12345", "12345678901"]); + assert!(f.set_token(1, 3, "1").is_err()); + assert!(f.set_token(1, 0, "a b").is_err()); + } +} diff --git a/crates/swat-txtinout/Cargo.toml b/crates/swat-txtinout/Cargo.toml new file mode 100644 index 000000000..7159d244e --- /dev/null +++ b/crates/swat-txtinout/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "rustgis-swat-txtinout" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "RustGIS L2: SWAT+ TxtInOut files read and written losslessly, and their structure (tables, nested records) for editing" + +[lints] +workspace = true + +[dependencies] +rustgis-swat-model = { workspace = true } diff --git a/crates/swat-txtinout/src/cio.rs b/crates/swat-txtinout/src/cio.rs new file mode 100644 index 000000000..872b0bccb --- /dev/null +++ b/crates/swat-txtinout/src/cio.rs @@ -0,0 +1,106 @@ +//! `file.cio`: the scenario's index. Its first line is a title; each other line is a section +//! (`simulation`, `climate`, `connect`…) and the files of that section, `null` where there is +//! none. Weather files are named again in the climate section's lists (`pcp.cli`…), under the +//! folder of the `*_path` sections, and recall files in `recall.rec`. + +use rustgis_swat_model::{Project, TextFile}; + +/// A section of `file.cio`: its name, its line, and its files by position (`null` kept). +#[derive(Clone, Debug, PartialEq)] +pub struct Section { + pub name: String, + pub line: usize, + pub files: Vec, +} + +/// `file.cio` read. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Cio { + pub sections: Vec
, +} + +impl Cio { + pub fn parse(f: &TextFile) -> Cio { + let mut sections = Vec::new(); + for i in 1..f.len() { + let v = f.values(i); + let Some((name, files)) = v.split_first() else { continue }; + sections.push(Section { name: name.to_string(), line: i, files: files.iter().map(|s| s.to_string()).collect() }); + } + Cio { sections } + } + + pub fn section(&self, name: &str) -> Option<&Section> { + self.sections.iter().find(|s| s.name == name) + } + + /// The files it names, without `null`, in order. + pub fn files(&self) -> Vec { + self.sections.iter().filter(|s| !s.name.ends_with("_path")).flat_map(|s| s.files.iter()).filter(|f| !is_null(f)).cloned().collect() + } + + /// The folder a kind of weather file is in (`pcp` → `pcp_path`), `""` for the scenario's. + pub fn path(&self, kind: &str) -> String { + match self.section(&format!("{kind}_path")).and_then(|s| s.files.first()) { + Some(p) if !is_null(p) => { + let p = p.replace('\\', "/"); + if p.ends_with('/') { p } else { format!("{p}/") } + } + _ => String::new(), + } + } +} + +fn is_null(s: &str) -> bool { + s.eq_ignore_ascii_case("null") +} + +/// The climate lists and the kind of weather file each names. +const LISTS: [(&str, &str); 6] = + [("pcp.cli", "pcp"), ("tmp.cli", "tmp"), ("slr.cli", "slr"), ("hmd.cli", "hmd"), ("wnd.cli", "wnd"), ("pet.cli", "pet")]; + +/// Every input file the scenario names: `file.cio`'s, the weather files of its climate lists, +/// and `recall.rec`'s files. `file.cio` itself first. +pub fn inputs(p: &Project) -> Vec { + let Some(cio_file) = p.file("file.cio") else { return Vec::new() }; + let cio = Cio::parse(cio_file); + let mut out = vec!["file.cio".to_string()]; + for f in cio.files() { + if !out.contains(&f) { + out.push(f); + } + } + for (list, kind) in LISTS { + // The list as file.cio names it, or by its usual name. + let Some(lf) = p.file(list) else { continue }; + if !out.iter().any(|o| o == list) { + continue; + } + let dir = cio.path(kind); + for i in 2..lf.len() { + if let Some(name) = lf.token(i, 0) { + let n = format!("{dir}{name}"); + if !out.contains(&n) { + out.push(n); + } + } + } + } + if let Some(rec) = cio.section("recall").and_then(|s| s.files.first()).filter(|f| !is_null(f)).and_then(|n| p.file(n)) { + // id, name, type, file: the file is the fourth value. + for i in 2..rec.len() { + if let Some(name) = rec.token(i, 3) { + let n = name.to_string(); + if !out.contains(&n) { + out.push(n); + } + } + } + } + out +} + +/// The input files the scenario names but doesn't have. +pub fn missing(p: &Project) -> Vec { + inputs(p).into_iter().filter(|f| p.file(f).is_none()).collect() +} diff --git a/crates/swat-txtinout/src/lib.rs b/crates/swat-txtinout/src/lib.rs new file mode 100644 index 000000000..56e71ef0e --- /dev/null +++ b/crates/swat-txtinout/src/lib.rs @@ -0,0 +1,69 @@ +//! L2: SWAT+'s TxtInOut files. +//! +//! [`read`] makes a [`Project`] of a scenario's files (bytes, from a folder or an archive: the +//! caller reads them, so this works in the browser too) and [`write`] gives the same bytes back +//! for a file nobody edited. [`cio`] reads `file.cio`, the index of the scenario's files, and +//! [`view`] what a file's lines are: its title, headers and records, nested where a record owns +//! the lines after it (a soil's layers, a schedule's operations, a decision table's conditions). +//! +//! The formats are written from their description (the SWAT+ input documentation and the files +//! the SWAT+ Editor writes), not from SWAT+'s source. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod cio; +pub mod view; + +use rustgis_swat_model::{Encoding, Project, TextFile}; + +/// A file's bytes as text: UTF-8, or else one character a byte (Latin-1). +pub fn decode(bytes: &[u8]) -> TextFile { + match std::str::from_utf8(bytes) { + Ok(s) => TextFile::from_text(s), + Err(_) => { + let s: String = bytes.iter().map(|&b| char::from(b)).collect(); + let mut f = TextFile::from_text(&s); + f.encoding = Encoding::Latin1; + f + } + } +} + +/// A file's text as the bytes it was read from (characters outside Latin-1 as `?` in a Latin-1 +/// file). +pub fn encode(f: &TextFile) -> Vec { + let s = f.to_text(); + match f.encoding { + Encoding::Utf8 => s.into_bytes(), + Encoding::Latin1 => s.chars().map(|c| u8::try_from(u32::from(c)).unwrap_or(b'?')).collect(), + } +} + +/// A project of files given as (name, bytes): every file of the scenario folder, names relative +/// to it with `/` between folders. +pub fn read, B: AsRef<[u8]>>(files: impl IntoIterator) -> Project { + let mut p = Project::new(); + for (name, bytes) in files { + p.files.insert(name.into().replace('\\', "/"), decode(bytes.as_ref())); + } + p +} + +/// The project's files as (name, bytes). +pub fn write(p: &Project) -> Vec<(String, Vec)> { + p.files.iter().map(|(n, f)| (n.clone(), encode(f))).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bytes_round_trip() { + let files = [("a.txt", b"x\r\ny\r\n".to_vec()), ("b.txt", vec![b'n', 0xe9, b'\n']), ("c.txt", "ä\n".as_bytes().to_vec())]; + let p = read(files.iter().map(|(n, b)| (*n, b.clone()))); + let out = write(&p); + for ((n, b), (n2, b2)) in files.iter().zip(&out) { + assert_eq!((*n, b), (n2.as_str(), b2)); + } + } +} diff --git a/crates/swat-txtinout/src/view.rs b/crates/swat-txtinout/src/view.rs new file mode 100644 index 000000000..d59522a7a --- /dev/null +++ b/crates/swat-txtinout/src/view.rs @@ -0,0 +1,388 @@ +//! What a TxtInOut file's lines are. Most files are a table: a title, a line of column names and +//! a record a line. Some nest: a soil owns its layers' lines, a plant community its plants', a +//! schedule its automatic tables and operations, a decision table its conditions and actions, a +//! weather generator station its twelve months, a calibration change its conditions. A view +//! never fails: lines it can't place are [`Kind::Other`], and an edit is always made to tokens +//! of the file's lines, so the file stays as it was around it. + +use rustgis_swat_model::TextFile; + +/// How a file is laid out. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Layout { + /// `file.cio`: a section a line. + Cio, + /// `print.prt`: pairs of names and values, then the objects' table. + Print, + /// A title, column names, a record a line (most files, `*.con` with their outflows too). + Table, + /// A title, a count, column names, a record a line (`cal_parms.cal`). + CountedTable, + /// `calibration.cal`: as [`Layout::CountedTable`], a change owning its condition lines. + Calibration, + /// `soils.sol`: a soil owning its layers. + Soils, + /// `plant.ini`: a community owning its plants. + PlantIni, + /// `management.sch`: a schedule owning its automatic tables and its operations. + Schedules, + /// `*.dtl`: decision tables, each with its conditions and actions. + DecisionTables, + /// `weather-wgn.cli`: stations owning a header and twelve months. + Wgn, + /// A weather file: a title, the names and values of its station, a day (or step) a line. + Series, +} + +impl Layout { + /// The layout of a file by its name (folders and case aside). + pub fn of(name: &str) -> Layout { + let base = name.rsplit('/').next().unwrap_or(name).to_ascii_lowercase(); + let ext = base.rsplit_once('.').map(|(_, e)| e).unwrap_or(""); + match base.as_str() { + "file.cio" => Layout::Cio, + "print.prt" => Layout::Print, + "cal_parms.cal" => Layout::CountedTable, + "calibration.cal" => Layout::Calibration, + "soils.sol" => Layout::Soils, + "plant.ini" => Layout::PlantIni, + "management.sch" => Layout::Schedules, + "weather-wgn.cli" => Layout::Wgn, + _ => match ext { + "dtl" => Layout::DecisionTables, + "pcp" | "tmp" | "tem" | "slr" | "hmd" | "wnd" | "pet" => Layout::Series, + _ => Layout::Table, + }, + } + } +} + +/// What a line is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Title, + Count, + /// Column names. + Header, + /// Values, named by a header's columns. + Record, + Blank, + /// A line the layout doesn't expect. + Other, +} + +/// A line, and the lines it owns. +#[derive(Clone, Debug, PartialEq)] +pub struct Record { + pub line: usize, + pub kind: Kind, + /// The header naming a record's values: its line, and the column of its first value. + pub header: Option<(usize, usize)>, + pub children: Vec, +} + +impl Record { + fn new(line: usize, kind: Kind) -> Record { + Record { line, kind, header: None, children: Vec::new() } + } + + fn rec(line: usize, header: Option<(usize, usize)>) -> Record { + Record { line, kind: Kind::Record, header, children: Vec::new() } + } + + /// The record's name: its first value. + pub fn name<'a>(&self, f: &'a TextFile) -> Option<&'a str> { + f.token(self.line, 0) + } +} + +/// A file's lines as records. +#[derive(Clone, Debug, PartialEq)] +pub struct View { + pub layout: Layout, + pub records: Vec, +} + +impl View { + pub fn of(name: &str, f: &TextFile) -> View { + let layout = Layout::of(name); + let records = match layout { + Layout::Cio => cio(f), + Layout::Print => print(f), + Layout::Table => table(f, 1), + Layout::CountedTable => counted(f, false), + Layout::Calibration => counted(f, true), + Layout::Soils => owners(f, 7, |v| count(v, 1)), + Layout::PlantIni => owners(f, 3, |v| count(v, 1)), + Layout::Schedules => schedules(f), + Layout::DecisionTables => dtl(f), + Layout::Wgn => wgn(f), + Layout::Series => series(f), + }; + View { layout, records } + } + + /// Every record, children after their owner, in the order of the lines. + pub fn flat(&self) -> Vec<&Record> { + fn walk<'a>(r: &'a Record, out: &mut Vec<&'a Record>) { + out.push(r); + for c in &r.children { + walk(c, out); + } + } + let mut out = Vec::new(); + for r in &self.records { + walk(r, &mut out); + } + out + } +} + +/// The names of a record's values, one a value: the header's from its first column; past the +/// header's end, a repeated group (a connection's outflows: `obj_typ obj_id hyd_typ frac`) +/// numbered from 2, else `col`. +pub fn columns(f: &TextFile, r: &Record) -> Vec { + let n = f.values(r.line).len(); + let Some((h, first)) = r.header else { return (1..=n).map(|k| format!("col{k}")).collect() }; + let names: Vec<&str> = f.values(h).into_iter().skip(first).collect(); + let group = names.iter().position(|c| c.eq_ignore_ascii_case("obj_typ")).filter(|&g| names.len() - g == 4); + (0..n) + .map(|k| match (names.get(k), group) { + (Some(c), _) => c.to_string(), + (None, Some(g)) => { + let i = k - g; + format!("{}_{}", names[g + i % 4], i / 4 + 1) + } + (None, None) => format!("col{}", k + 1), + }) + .collect() +} + +fn count(v: &[&str], k: usize) -> usize { + v.get(k).and_then(|s| s.parse::().ok()).unwrap_or(0) +} + +fn blank(f: &TextFile, i: usize) -> bool { + f.line(i).is_none_or(|l| l.trim().is_empty()) +} + +/// Lines `i..` as records of the header at `h` (blank lines as blanks). +fn rows(f: &TextFile, from: usize, to: usize, header: Option<(usize, usize)>, out: &mut Vec) { + for i in from..to { + out.push(if blank(f, i) { Record::new(i, Kind::Blank) } else { Record::rec(i, header) }); + } +} + +fn table(f: &TextFile, header: usize) -> Vec { + let mut out = Vec::new(); + for i in 0..header.min(f.len()) { + out.push(Record::new(i, if i == 0 { Kind::Title } else { Kind::Count })); + } + if header < f.len() { + out.push(Record::new(header, Kind::Header)); + } + rows(f, header + 1, f.len(), Some((header, 0)), &mut out); + out +} + +fn cio(f: &TextFile) -> Vec { + let mut out = Vec::new(); + if !f.is_empty() { + out.push(Record::new(0, Kind::Title)); + } + rows(f, 1, f.len(), None, &mut out); + out +} + +fn print(f: &TextFile) -> Vec { + let mut out = Vec::new(); + if !f.is_empty() { + out.push(Record::new(0, Kind::Title)); + } + let mut i = 1; + while i < f.len() { + if blank(f, i) { + out.push(Record::new(i, Kind::Blank)); + i += 1; + } else if f.token(i, 0) == Some("objects") { + out.push(Record::new(i, Kind::Header)); + rows(f, i + 1, f.len(), Some((i, 0)), &mut out); + break; + } else { + out.push(Record::new(i, Kind::Header)); + if i + 1 < f.len() { + out.push(Record::rec(i + 1, Some((i, 0)))); + } + i += 2; + } + } + out +} + +fn counted(f: &TextFile, conditions: bool) -> Vec { + let mut out = Vec::new(); + for (i, k) in [(0, Kind::Title), (1, Kind::Count), (2, Kind::Header)] { + if i < f.len() { + out.push(Record::new(i, k)); + } + } + let h = Some((2, 0)); + let mut i = 3; + while i < f.len() { + if blank(f, i) { + out.push(Record::new(i, Kind::Blank)); + i += 1; + continue; + } + let mut r = Record::rec(i, h); + i += 1; + if conditions { + // A change's fourth value is its number of conditions: `var op value text` lines. + let n = count(&f.values(r.line), 3); + for _ in 0..n { + if i >= f.len() { + break; + } + r.children.push(Record::rec(i, None)); + i += 1; + } + } + out.push(r); + } + out +} + +/// A title, a header, and records owning the next `owned(values)` lines, named by the header +/// from column `split`. +fn owners(f: &TextFile, split: usize, owned: impl Fn(&[&str]) -> usize) -> Vec { + let mut out = Vec::new(); + for (i, k) in [(0, Kind::Title), (1, Kind::Header)] { + if i < f.len() { + out.push(Record::new(i, k)); + } + } + let mut i = 2; + while i < f.len() { + if blank(f, i) { + out.push(Record::new(i, Kind::Blank)); + i += 1; + continue; + } + let mut r = Record::rec(i, Some((1, 0))); + let n = owned(&f.values(i)); + i += 1; + for _ in 0..n { + if i >= f.len() { + break; + } + r.children.push(Record::rec(i, Some((1, split)))); + i += 1; + } + out.push(r); + } + out +} + +/// Schedules: a schedule's automatic tables come first, a name (and a plant) a line, with no +/// header of their own. +fn schedules(f: &TextFile) -> Vec { + let mut out = owners(f, 3, |v| count(v, 1) + count(v, 2)); + for r in out.iter_mut().filter(|r| r.kind == Kind::Record) { + let autos = count(&f.values(r.line), 2); + for c in r.children.iter_mut().take(autos) { + c.header = None; + } + } + out +} + +fn dtl(f: &TextFile) -> Vec { + let mut out = Vec::new(); + for (i, k) in [(0, Kind::Title), (1, Kind::Count)] { + if i < f.len() { + out.push(Record::new(i, k)); + } + } + let mut i = 2; + while i < f.len() { + if blank(f, i) { + out.push(Record::new(i, Kind::Blank)); + i += 1; + continue; + } + // "name conds alts acts", then the table's line. + let is_head = f.token(i, 0).is_some_and(|t| t.eq_ignore_ascii_case("name")); + if !is_head || i + 1 >= f.len() { + out.push(Record::new(i, Kind::Other)); + i += 1; + continue; + } + out.push(Record::new(i, Kind::Header)); + let mut t = Record::rec(i + 1, Some((i, 0))); + let v = f.values(i + 1); + let (conds, acts) = (count(&v, 1), count(&v, 3)); + i += 2; + for n in [conds, acts] { + if i >= f.len() { + break; + } + let h = i; + t.children.push(Record::new(h, Kind::Header)); + i += 1; + for _ in 0..n { + if i >= f.len() { + break; + } + t.children.push(Record::rec(i, Some((h, 0)))); + i += 1; + } + } + out.push(t); + } + out +} + +fn wgn(f: &TextFile) -> Vec { + let mut out = Vec::new(); + if !f.is_empty() { + out.push(Record::new(0, Kind::Title)); + } + let mut i = 1; + while i < f.len() { + if blank(f, i) { + out.push(Record::new(i, Kind::Blank)); + i += 1; + continue; + } + let mut s = Record::rec(i, None); + i += 1; + if i < f.len() { + let h = i; + s.children.push(Record::new(h, Kind::Header)); + i += 1; + for _ in 0..12 { + if i >= f.len() { + break; + } + s.children.push(Record::rec(i, Some((h, 0)))); + i += 1; + } + } + out.push(s); + } + out +} + +fn series(f: &TextFile) -> Vec { + let mut out = Vec::new(); + for (i, k) in [(0, Kind::Title), (1, Kind::Header)] { + if i < f.len() { + out.push(Record::new(i, k)); + } + } + if f.len() > 2 { + out.push(Record::rec(2, Some((1, 0)))); + } + rows(f, 3, f.len(), None, &mut out); + out +} diff --git a/crates/swat-txtinout/tests/projects.rs b/crates/swat-txtinout/tests/projects.rs new file mode 100644 index 000000000..e691d1757 --- /dev/null +++ b/crates/swat-txtinout/tests/projects.rs @@ -0,0 +1,91 @@ +//! The test scenarios (private data, not in this repository: read from the folder `SWAT_TESTDATA` +//! names, holding `Osu_1hru`, `Ames_sub1`, `Batanghari` and `Pessel`; skipped without it): every file comes back byte for byte, and every +//! line of every file has a place in its view. + +use rustgis_swat_model::TextFile; +use rustgis_swat_txtinout::view::{Kind, Layout, View, columns}; +use rustgis_swat_txtinout::{cio, read, write}; +use std::path::{Path, PathBuf}; + +/// What the nested layouts must find. +fn check(name: &str, fname: &str, f: &TextFile, v: &View) { + let first = |l: usize| f.token(l, 0).unwrap_or("").to_ascii_lowercase(); + let num = |l: usize| f.token(l, 0).is_some_and(|t| t.parse::().is_ok()); + for r in v.records.iter().filter(|r| r.kind == Kind::Record) { + let at = format!("{name}/{fname}:{}", r.line + 1); + match v.layout { + Layout::Soils => assert!(!r.children.is_empty() && r.children.iter().all(|c| num(c.line)), "{at}"), + Layout::PlantIni => assert!(r.children.iter().all(|c| !num(c.line)), "{at}"), + Layout::Wgn => assert!(r.children.len() == 13 && r.children[1..].iter().all(|c| num(c.line)), "{at}"), + Layout::DecisionTables => { + let heads: Vec = r.children.iter().filter(|c| c.kind == Kind::Header).map(|c| first(c.line)).collect(); + assert_eq!(heads, ["var", "act_typ"], "{at}"); + } + Layout::Schedules => { + let ops = f.token(r.line, 1).and_then(|t| t.parse::().ok()).unwrap_or(0); + let autos = r.children.len() - ops; + assert!(r.children[autos..].iter().all(|c| f.values(c.line).len() >= 4), "{at}"); + } + _ => {} + } + } +} + +fn root() -> PathBuf { + std::env::var_os("SWAT_TESTDATA").map(PathBuf::from).unwrap_or_default() +} + +fn files(dir: &Path) -> Vec<(String, Vec)> { + fn walk(base: &Path, dir: &Path, out: &mut Vec<(String, Vec)>) { + let Ok(rd) = std::fs::read_dir(dir) else { return }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + walk(base, &p, out); + } else if let (Ok(b), Ok(rel)) = (std::fs::read(&p), p.strip_prefix(base)) { + out.push((rel.to_string_lossy().replace('\\', "/"), b)); + } + } + } + let mut out = Vec::new(); + walk(dir, dir, &mut out); + out.sort(); + out +} + +#[test] +fn scenarios() { + let mut seen = 0; + for name in ["Osu_1hru", "Ames_sub1", "Batanghari", "Pessel"] { + let dir = root().join(name); + if !dir.join("file.cio").exists() { + continue; + } + seen += 1; + let orig = files(&dir); + let p = read(orig.clone()); + let back = write(&p); + let bad: Vec<&str> = orig.iter().zip(&back).filter(|(a, b)| a != b).map(|(a, _)| a.0.as_str()).collect(); + assert!(bad.is_empty() && back.len() == orig.len(), "{name}: not byte-identical {bad:?}"); + let missing = cio::missing(&p); + eprintln!("{name}: {} files, {} inputs, missing {:?}", orig.len(), cio::inputs(&p).len(), missing); + let mut others = Vec::new(); + for (fname, f) in &p.files { + let v = View::of(fname, f); + let flat = v.flat(); + let lines: Vec = flat.iter().map(|r| r.line).collect(); + assert_eq!(lines, (0..f.len()).collect::>(), "{name}/{fname}: lines"); + check(name, fname, f, &v); + for r in &flat { + if r.kind == Kind::Other { + others.push(format!("{fname}:{}", r.line + 1)); + } + if r.kind == Kind::Record && v.layout != Layout::Table { + let _ = columns(f, r); + } + } + } + assert!(others.is_empty(), "{name}: lines out of place {others:?}"); + } + eprintln!("{seen} scenarios"); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 81f8b5e95..c94d2de23 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -10,9 +10,11 @@ use std::process::{Command, ExitCode}; const LAYERS: &[(&str, u8)] = &[ ("geom", 0), ("data", 1), + ("swat-model", 1), ("map", 2), ("analysis", 2), ("tiles", 2), + ("swat-txtinout", 2), ("render", 3), ("geolibre", 3), ("plugins", 3),