diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 441acf50c..2f79bf0b6 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -648,7 +648,7 @@ pub fn read_geojson(text: &str, name: &str) -> Result<(Dataset, ReadReport)> { let mut features = Collector::default(); match stream_features(text, &mut features) { Ok(rest) => match rest.get("type").and_then(Json::as_str) { - Some("FeatureCollection") => Ok(features.finish(name)), + Some("FeatureCollection") => Ok(features.finish(name, &rest)), Some(_) => read_geojson_value(Json::Object(rest), name), None => Err(DataError::Format("GeoJSON: missing \"type\"".into())), }, @@ -657,6 +657,107 @@ pub fn read_geojson(text: &str, name: &str) -> Result<(Dataset, ReadReport)> { } } +/// The longest `crs` name looked at; a longer one is reported, not parsed. +const CRS_NAME_MAX: usize = 256; + +/// At most 80 characters of `s`, for a message. +fn clip(s: &str) -> String { + let mut out: String = s.chars().take(80).collect(); + if out.len() < s.len() { + out.push('…'); + } + out +} + +/// The coordinate system a FeatureCollection declares with the legacy (GeoJSON 2008) `crs` member, +/// `{"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::28992"}}`, with the transformer +/// to longitude/latitude: `None` for longitude/latitude (no member, `null`, `EPSG:4326`, `CRS84`) +/// and for a declaration that isn't understood, which is said in `messages`: coordinates are then +/// read as longitude/latitude, but never without saying so. +/// +/// Only named EPSG codes and CRS84 are honoured (not links, WKT or PROJ strings in the name, nor +/// codes RustGIS doesn't know). The coordinates are x/y (easting/northing, longitude/latitude) +/// whatever the axis order of the EPSG definition, as the legacy GeoJSON text says. +pub(crate) fn declared_crs(root: &Map, messages: &mut Vec) -> Option<(Crs, crate::raster::Transformer)> { + let decl = root.get("crs").filter(|v| !v.is_null())?; + match legacy_crs(decl) { + Ok(found) => found, + Err(why) => { + let shown = match decl.get("properties").and_then(|p| p.get("name")).and_then(Json::as_str) { + Some(name) => format!("\"{}\"", clip(name)), + None => match decl.get("type").and_then(Json::as_str) { + Some(kind) => format!("of type \"{}\"", clip(kind)), + None => "of an unknown shape".to_string(), + }, + }; + messages.push(format!("GeoJSON crs {shown}: {why}; coordinates are read as longitude/latitude")); + None + } + } +} + +fn legacy_crs(decl: &Json) -> std::result::Result, String> { + if decl.get("type").and_then(Json::as_str) != Some("name") { + return Err("only named coordinate systems are supported".into()); + } + let name = decl.get("properties").and_then(|p| p.get("name")).and_then(Json::as_str).ok_or("it has no properties.name")?; + // Only the forms below, anchored (not any text holding "EPSG"), and no other text for + // `Crs::parse`, which also takes WKT and PROJ strings (and registers a system for them). + if name.len() > CRS_NAME_MAX || !is_epsg_or_crs84_name(name) { + return Err("only EPSG codes and CRS84 are supported".into()); + } + let crs = Crs::parse(name)?; + if crs == Crs::LonLat { + return Ok(None); + } + let t = crs.transformer()?; + Ok(Some((crs, t))) +} + +/// Whether a `crs` name is `EPSG:N`, `urn:ogc:def:crs:EPSG::N`, +/// `http(s)://www.opengis.net/def/crs/EPSG//N`, or CRS84 as `CRS84`, `OGC:CRS84`, +/// `urn:ogc:def:crs:OGC::CRS84` or +/// `http(s)://www.opengis.net/def/crs/OGC//CRS84` (any case, whitespace around). +fn is_epsg_or_crs84_name(name: &str) -> bool { + let up = name.trim().to_ascii_uppercase(); + let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + let version = |s: &str| s.bytes().all(|b| b.is_ascii_digit() || b == b'.'); + // ``: a version (maybe empty), then what's after the last separator. + let tail = |rest: &str, sep: char| { + rest.split_once(sep).filter(|(v, last)| version(v) && !last.contains(sep)).map(|(_, last)| last.to_string()) + }; + fn url(rest: &str) -> Option<&str> { + rest.strip_prefix("HTTP://").or_else(|| rest.strip_prefix("HTTPS://")).and_then(|r| r.strip_prefix("WWW.OPENGIS.NET/DEF/CRS/")) + } + if let Some(code) = up.strip_prefix("EPSG:") { + return digits(code); + } + if let Some(rest) = up.strip_prefix("URN:OGC:DEF:CRS:EPSG:") { + return tail(rest, ':').is_some_and(|c| digits(&c)); + } + if let Some(rest) = up.strip_prefix("URN:OGC:DEF:CRS:OGC:") { + return tail(rest, ':').is_some_and(|c| c == "CRS84"); + } + if let Some(rest) = url(&up).and_then(|r| r.strip_prefix("EPSG/")) { + return tail(rest, '/').is_some_and(|c| digits(&c)); + } + if let Some(rest) = url(&up).and_then(|r| r.strip_prefix("OGC/")) { + return tail(rest, '/').is_some_and(|c| c == "CRS84"); + } + // The shorthands: `CRS84`, `OGC:CRS84`, `OGC::CRS84`, `OGC:1.3:CRS84`. + up == "CRS84" || up == "OGC:CRS84" || up.strip_prefix("OGC:").and_then(|rest| tail(rest, ':')).is_some_and(|c| c == "CRS84") +} + +/// A geometry in longitude/latitude from the coordinates `t` converts: `None` when a coordinate +/// can't be converted or comes out beyond longitude/latitude (a declared system that doesn't match +/// the coordinates). +pub(crate) fn geometry_to_lonlat(g: &Geometry, t: &crate::raster::Transformer) -> Option { + let mut ok = true; + let out = g.map_coords(&|c| t.to_lonlat(c).unwrap_or(Coord::new(f64::NAN, f64::NAN))); + out.for_each_coord(|c: Coord| ok &= c.is_finite() && c.x.abs() <= 180.5 && c.y.abs() <= 90.5); + ok.then_some(out) +} + /// What takes a FeatureCollection's features as they're read. pub(crate) trait FeatureSink { /// The `"features"` member starts (again, when a document repeats it: the last one wins). @@ -828,41 +929,73 @@ impl Collector { } } - fn finish(self, name: &str) -> (Dataset, ReadReport) { + /// The dataset; `root` is the document's top level (for its `crs`, see [`declared_crs`]). + fn finish(self, name: &str, root: &Map) -> (Dataset, ReadReport) { let mut report = ReadReport { skipped: self.skipped, ..ReadReport::default() }; let mut ds = Dataset::new(name, GeometryType::None, self.schema.fields()); ds.features.reserve(self.features.len()); - for (geom, values) in self.features { + if report.skipped > 0 { + report.messages.push(format!("{} features had unreadable geometry", report.skipped)); + } + let declared = declared_crs(root, &mut report.messages); + let mut unconverted = 0usize; + for (n, (geom, values)) in self.features.into_iter().enumerate() { + // A feature that can't be converted is dropped, and its id is not reused, as when the + // file is read in place. + let geom = match (&declared, geom) { + (Some((_, t)), Some(g)) => match geometry_to_lonlat(&g, t) { + Some(g) => Some(g), + None => { + unconverted += 1; + continue; + } + }, + (_, g) => g, + }; let mut attrs = vec![Value::Null; ds.fields.len()]; for (i, v) in values { if let (Some(slot), Some(field)) = (attrs.get_mut(i), ds.fields.get(i)) { *slot = coerce(v, field.ty); } } - ds.push_next(geom, attrs); + ds.features.push(crate::Feature { id: n as u64 + 1, geometry: geom, attrs }); } ds.geometry_type = dominant_type(&ds); - if report.skipped > 0 { - report.messages.push(format!("{} features had unreadable geometry", report.skipped)); + if let Some((crs, _)) = &declared { + ds.crs = Some(*crs); + if unconverted > 0 { + report.skipped += unconverted; + report.messages.push(format!( + "{unconverted} features couldn't be converted from {} to longitude/latitude and were skipped", + crs.label() + )); + } + // What couldn't be converted can't come back by defining the projection again. + ds.crs_incomplete = unconverted > 0; } (ds, report) } } -/// Reads a GeoJSON document already parsed: a FeatureCollection, Feature or bare geometry. +/// Reads a GeoJSON document already parsed: a FeatureCollection, Feature or bare geometry. Only a +/// FeatureCollection's `crs` is read (a Feature's or a geometry's isn't part of the legacy text). fn read_geojson_value(root: Json, name: &str) -> Result<(Dataset, ReadReport)> { let mut features = Collector::default(); + let mut top = Map::new(); match root.get("type").and_then(Json::as_str) { Some("FeatureCollection") => { for f in root.get("features").and_then(Json::as_array).into_iter().flatten() { features.add(f); } + if let Some(crs) = root.get("crs") { + top.insert("crs".to_string(), crs.clone()); + } } Some("Feature") => features.add(&root), Some(_) => features.add(&json!({"type": "Feature", "geometry": &root, "properties": {}})), None => return Err(DataError::Format("GeoJSON: missing \"type\"".into())), } - Ok(features.finish(name)) + Ok(features.finish(name, &top)) } fn json_type(v: &Json) -> Option { @@ -2345,6 +2478,22 @@ mod tests { for t in ["", "{", "[]", "{\"type\":5}", "{\"type\":\"FeatureCollection\",\"features\":7}"] { let _ = read_geojson(t, "x"); } + // A `crs` of any shape, anywhere in the document: read, never a panic. + let deep = "[".repeat(5000) + &"]".repeat(5000); + for crs in + ["null", "7", "-1e308", "\"\"", "[]", "{}", "{\"type\":\"name\"}", "{\"type\":\"name\",\"properties\":{\"name\":{}}}", &deep] + { + for t in [ + format!( + "{{\"crs\":{crs},\"type\":\"FeatureCollection\",\"features\":[{{\"type\":\"Feature\",\"geometry\":{{\"type\":\"Point\",\"coordinates\":[1,2]}}}}]}}" + ), + format!("{{\"type\":\"FeatureCollection\",\"features\":7,\"crs\":{crs}}}"), + format!("{{\"type\":\"Feature\",\"crs\":{crs},\"geometry\":null}}"), + format!("{{\"type\":\"Point\",\"coordinates\":[1,2],\"crs\":{crs}}}"), + ] { + let _ = read_geojson(&t, "x"); + } + } } #[test] diff --git a/crates/data/src/source/geojson.rs b/crates/data/src/source/geojson.rs index cf03c9f92..390cd6a18 100644 --- a/crates/data/src/source/geojson.rs +++ b/crates/data/src/source/geojson.rs @@ -11,8 +11,8 @@ use std::fs::File; use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; -use super::{Batch, FeatureSource, Request, SourceInfo, clamp_limit, stride}; -use crate::io::{FeatureSink, Schema, geojson_attrs, geojson_geometry}; +use super::{Batch, FeatureSource, Reproject, Request, SourceInfo, clamp_limit, stride}; +use crate::io::{FeatureSink, Schema, declared_crs, geojson_attrs, geojson_geometry, geometry_to_lonlat}; use crate::{DataError, Feature, GeometryType, Result}; use rustgis_geom::Extent; use serde_json::Value as Json; @@ -36,8 +36,17 @@ pub struct GeoJsonSource { info: SourceInfo, path: PathBuf, places: Vec, - /// Each feature's box, rounded outward (NaN for a feature without geometry). + /// Each feature's box in the file's own coordinates, rounded outward (NaN for a feature + /// without geometry). boxes: Vec<[f32; 4]>, + /// The coordinate system the file declares (its `crs`), converting what's read to + /// longitude/latitude. + reproject: Reproject, + /// The features (by place) the declared system can't convert: in no read, and not counted. + /// Sorted. + failed: Vec, + /// Features left out when it was opened for geometry that couldn't be read. + unreadable: usize, } /// The features of a file as it's scanned. @@ -76,11 +85,7 @@ impl Scan { Some(g) => { let e = g.extent(); self.extent = self.extent.union(&e); - if let Some(slot) = self.kinds.get_mut(match g.kind() { - GeometryType::Point => 0, - GeometryType::Polyline => 1, - _ => 2, - }) { + if let Some(slot) = self.kinds.get_mut(kind_slot(g.kind())) { *slot += 1; } outward(&e) @@ -92,6 +97,15 @@ impl Scan { } } +/// Where in [`Scan::kinds`] a kind of geometry is counted. +fn kind_slot(kind: GeometryType) -> usize { + match kind { + GeometryType::Point => 0, + GeometryType::Polyline => 1, + _ => 2, + } +} + /// An extent as `f32`s, rounded outward (NaN when it isn't finite, so no request meets it). fn outward(e: &Extent) -> [f32; 4] { if e.is_empty() || ![e.min_x, e.min_y, e.max_x, e.max_y].iter().all(|v| v.is_finite()) { @@ -139,6 +153,27 @@ impl GeoJsonSource { if scan.skipped > 0 { notes.push(format!("{} features had unreadable geometry", scan.skipped)); } + let reproject = Reproject::known(declared_crs(&rest, &mut notes)); + // The declaration may come after the features, so only now can each be tried: those the + // declared system can't convert are found here (and left out of every read, the count, + // the extent and the geometry type), as they are when the file is read whole. + let mut failed = Vec::new(); + let mut extent = scan.extent; + if let Some(t) = reproject.transformer() { + let pass = convert_pass(path, &scan.places, t)?; + extent = pass.extent; + for (kept, lost) in scan.kinds.iter_mut().zip(pass.dropped) { + *kept = kept.saturating_sub(lost); + } + if !pass.failed.is_empty() { + notes.push(format!( + "{} features couldn't be converted from {} to longitude/latitude and were skipped", + pass.failed.len(), + reproject.label() + )); + } + failed = pass.failed; + } // The kind most features have, as a layer read whole takes. let geometry_type = match scan.kinds.iter().enumerate().max_by_key(|(_, n)| **n) { Some((_, 0)) | None => GeometryType::None, @@ -151,25 +186,31 @@ impl GeoJsonSource { format: "GeoJSON", geometry_type, fields: scan.schema.fields(), - extent: scan.extent, - count: Some(scan.places.len() as u64), - crs: "EPSG:4326".into(), + extent, + count: Some((scan.places.len() - failed.len()) as u64), + crs: reproject.label(), indexed: false, notes, }; - Ok(GeoJsonSource { info, path: path.to_path_buf(), places: scan.places, boxes: scan.boxes }) + let unreadable = scan.skipped; + Ok(GeoJsonSource { info, path: path.to_path_buf(), places: scan.places, boxes: scan.boxes, reproject, failed, unreadable }) } /// The features a request matches, in file order: every one without an extent; otherwise - /// those whose boxes meet it (features without geometry never do). - fn matches(&self, req: &Request) -> impl Iterator + '_ { + /// those whose boxes meet it (features without geometry never do). The extent is in the + /// file's own coordinates. + fn matches(&self, extent: Option) -> impl Iterator + '_ { // Rounded to the nearest f32: never past a box rounded outward that meets the extent. - let e = req.extent.map(|e| (e.is_empty(), [e.min_x as f32, e.min_y as f32, e.max_x as f32, e.max_y as f32])); - self.boxes.iter().enumerate().filter_map(move |(i, b)| match e { - None => Some(i as u32), - Some((true, _)) => None, - Some((false, [x0, y0, x1, y1])) => (b[0] <= x1 && b[2] >= x0 && b[1] <= y1 && b[3] >= y0).then_some(i as u32), - }) + let e = extent.map(|e| (e.is_empty(), [e.min_x as f32, e.min_y as f32, e.max_x as f32, e.max_y as f32])); + self.boxes + .iter() + .enumerate() + .filter_map(move |(i, b)| match e { + None => Some(i as u32), + Some((true, _)) => None, + Some((false, [x0, y0, x1, y1])) => (b[0] <= x1 && b[2] >= x0 && b[1] <= y1 && b[3] >= y0).then_some(i as u32), + }) + .filter(|i| self.failed.binary_search(i).is_err()) } } @@ -181,40 +222,28 @@ impl FeatureSource for GeoJsonSource { &self.info } + fn lost_to_crs(&self) -> bool { + !self.failed.is_empty() + } + + fn skipped_at_open(&self) -> usize { + self.unreadable + self.failed.len() + } + fn read(&self, req: &Request) -> Result { let limit = clamp_limit(req.limit); // Counted first, then the sample taken: no list of every match. - let matched = self.matches(req).count(); + let extent = req.extent.map(|e| self.reproject.extent_from_lonlat(&e)); + let matched = self.matches(extent).count(); let step = stride(matched as u64, limit).max(1) as usize; let mut f = BufReader::with_capacity(BUFFER, File::open(&self.path)?); let mut pos = 0u64; let mut buf = Vec::new(); let mut features = Vec::with_capacity(matched.div_ceil(step).min(limit)); let mut skipped = 0; - for i in self.matches(req).step_by(step).take(limit) { + for i in self.matches(extent).step_by(step).take(limit) { let Some(place) = self.places.get(i as usize) else { continue }; - // Ahead and near: through the buffer; otherwise a seek. - let read = (|| -> std::io::Result<()> { - match i64::try_from(place.offset.wrapping_sub(pos)) { - Ok(0) => {} - Ok(d) if place.offset > pos && d < BUFFER as i64 => f.seek_relative(d)?, - _ => { - f.seek(SeekFrom::Start(place.offset))?; - } - } - buf.resize(place.len as usize, 0); - f.read_exact(&mut buf) - })(); - if read.is_err() { - // The file changed since it was opened: where it is now is unknown. - pos = u64::MAX; - skipped += 1; - continue; - } - pos = place.offset + u64::from(place.len); - // Past the comma and whitespace before the feature. - let text = buf.iter().position(|b| !b.is_ascii_whitespace() && *b != b',').and_then(|at| buf.get(at..)).unwrap_or_default(); - let Ok(json) = serde_json::from_slice::(text) else { + let Some(json) = read_place(&mut f, &mut pos, &mut buf, place) else { skipped += 1; continue; }; @@ -222,12 +251,83 @@ impl FeatureSource for GeoJsonSource { skipped += 1; continue; }; + // To longitude/latitude; a feature that can't be is skipped, as one that can't be read. + let geometry = match (self.reproject.transformer(), geometry) { + (Some(t), Some(g)) => match geometry_to_lonlat(&g, t) { + Some(g) => Some(g), + None => { + skipped += 1; + continue; + } + }, + (_, g) => g, + }; features.push(Feature { id: u64::from(i) + 1, geometry, attrs: geojson_attrs(&json, &self.info.fields) }); } Ok(Batch { features, matched: matched as u64, complete: step == 1, skipped }) } } +/// The feature at `place`, read through `f` (at `pos`, which is kept up to date): `None` when the +/// file has changed since it was opened. +fn read_place(f: &mut BufReader, pos: &mut u64, buf: &mut Vec, place: &Place) -> Option { + // Ahead and near: through the buffer; otherwise a seek. + let read = (|| -> std::io::Result<()> { + match i64::try_from(place.offset.wrapping_sub(*pos)) { + Ok(0) => {} + Ok(d) if place.offset > *pos && d < BUFFER as i64 => f.seek_relative(d)?, + _ => { + f.seek(SeekFrom::Start(place.offset))?; + } + } + buf.resize(place.len as usize, 0); + f.read_exact(buf) + })(); + if read.is_err() { + // Where the file is now is unknown. + *pos = u64::MAX; + return None; + } + *pos = place.offset + u64::from(place.len); + // Past the comma and whitespace before the feature. + let text = buf.iter().position(|b| !b.is_ascii_whitespace() && *b != b',').and_then(|at| buf.get(at..)).unwrap_or_default(); + serde_json::from_slice::(text).ok() +} + +/// What a pass over a file's features converts to longitude/latitude finds. +struct Pass { + /// Places of the features the transformer can't convert (see [`geometry_to_lonlat`]), sorted. + failed: Vec, + /// Points, lines and polygons among them (as [`Scan::kinds`]). + dropped: [usize; 3], + /// The extent, in longitude/latitude, of those it can. + extent: Extent, +} + +/// A second pass over the file, made only for a declared coordinate system. +fn convert_pass(path: &Path, places: &[Place], t: &crate::raster::Transformer) -> Result { + let mut f = BufReader::with_capacity(BUFFER, File::open(path)?); + let (mut pos, mut buf) = (0u64, Vec::new()); + let mut pass = Pass { failed: Vec::new(), dropped: [0; 3], extent: Extent::EMPTY }; + for (i, place) in places.iter().enumerate() { + // A feature that can't be read now is left to the reads, which skip it. + if let Some(json) = read_place(&mut f, &mut pos, &mut buf, place) + && let Ok(Some(g)) = geojson_geometry(&json) + { + match geometry_to_lonlat(&g, t) { + Some(g) => pass.extent = pass.extent.union(&g.extent()), + None => { + pass.failed.push(i as u32); + if let Some(slot) = pass.dropped.get_mut(kind_slot(g.kind())) { + *slot += 1; + } + } + } + } + } + Ok(pass) +} + /// A reader counting the bytes read through it, for features' places in the file. The JSON reader /// takes a byte at a time and looks at most one ahead, which it doesn't after a feature's last /// byte, so the count after one is where it ends. diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index 36301083a..d787d7250 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -212,6 +212,16 @@ pub trait FeatureSource: Send + Sync { Err(DataError::Format("this provider does not support chunk traversal".into())) } fn info(&self) -> &SourceInfo; + /// Whether features were left out of every read because the coordinate system the source + /// declares can't place them, so that defining its projection again can't bring them back. + fn lost_to_crs(&self) -> bool { + false + } + /// How many features the source left out when it was opened (and from every read), for a + /// whole read's `ReadReport::skipped` to count with what reads skip. + fn skipped_at_open(&self) -> usize { + 0 + } /// Reads the features a request asks for. A source read by its web address fails with /// [`remote::NEED`] until the bytes the read needs are fetched: run it with /// [`remote::drive`] (see [`FeatureSource::remote`]). @@ -451,6 +461,14 @@ impl FeatureSource for Fetching { self.0.info() } + fn lost_to_crs(&self) -> bool { + self.0.lost_to_crs() + } + + fn skipped_at_open(&self) -> usize { + self.0.skipped_at_open() + } + fn read(&self, req: &Request) -> Result { read_blocking(&self.0, req) } @@ -473,9 +491,12 @@ pub fn materialize(src: &dyn FeatureSource) -> Result<(Dataset, ReadReport)> { ds.crs = info.coordinate_system(); // A streamed Shapefile skips unconvertible shapes rather than retaining empty geometry. // Once materialized, it must still direct CRS repairs back to the original file. - ds.crs_incomplete = info.format == "Shapefile" && batch.skipped > 0; + ds.crs_incomplete = (info.format == "Shapefile" && batch.skipped > 0) || src.lost_to_crs(); ds.features = batch.features.into(); - let mut report = ReadReport { skipped: batch.skipped, messages: info.notes.iter().map(|n| format!("{}: {n}", info.name)).collect() }; + let mut report = ReadReport { + skipped: batch.skipped + src.skipped_at_open(), + messages: info.notes.iter().map(|n| format!("{}: {n}", info.name)).collect(), + }; if !batch.complete { report.messages.push(format!("{}: read a sample of {} of {} features", info.name, ds.features.len(), batch.matched)); } @@ -541,6 +562,21 @@ impl Reproject { Reproject { crs: if t.is_some() { crs } else { Crs::LonLat }, t } } + /// From a coordinate system already known to convert (`None` for longitude/latitude). + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn known(found: Option<(Crs, Transformer)>) -> Reproject { + match found { + Some((crs, t)) => Reproject { crs, t: Some(t) }, + None => Reproject { crs: Crs::LonLat, t: None }, + } + } + + /// What converts to longitude/latitude (`None` when coordinates are taken as they are). + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn transformer(&self) -> Option<&Transformer> { + self.t.as_ref() + } + /// Whether coordinates are taken as they are. #[cfg(not(target_arch = "wasm32"))] pub(crate) fn is_lonlat(&self) -> bool { diff --git a/crates/data/src/source/tests.rs b/crates/data/src/source/tests.rs index 224481872..a3d50c53e 100644 --- a/crates/data/src/source/tests.rs +++ b/crates/data/src/source/tests.rs @@ -1140,6 +1140,286 @@ fn geojson_in_place_matches_one_read_whole() { } } +/// A GeoJSON FeatureCollection of Arnhem's municipal outline's box and a point in RD New +/// (EPSG:28992), `crs` being what the file declares (left out for `None`). +fn rd_geojson(crs: Option<&str>) -> String { + let crs = crs.map(|c| format!(r#""crs":{c},"#)).unwrap_or_default(); + format!( + r#"{{"type":"FeatureCollection","name":"arnhem",{crs}"features":[ + {{"type":"Feature","properties":{{"n":"area"}},"geometry":{{"type":"Polygon","coordinates":[[[183548,438459],[196442,438459],[196442,454549],[183548,454549],[183548,438459]]]}}}}, + {{"type":"Feature","properties":{{"n":"point"}},"geometry":{{"type":"Point","coordinates":[188000,445000]}}}}]}}"# + ) +} + +fn named(name: &str) -> String { + format!(r#"{{"type":"name","properties":{{"name":"{name}"}}}}"#) +} + +/// What a GeoJSON text reads as through every path: whole, and in place scanned in memory and +/// through a buffer; the paths must agree. Returns the dataset, its messages and the source's +/// `info.crs`. +fn geojson_paths(file: &str, text: &str) -> (Dataset, Vec, String) { + let d = dir().join("geojson-crs"); + std::fs::create_dir_all(&d).unwrap(); + let p = d.join(file); + std::fs::write(&p, text).unwrap(); + let (whole, report) = crate::io::read_geojson(text, "x").unwrap(); + for in_memory in [crate::io::MAX_BYTES, 0] { + let src = geojson::GeoJsonSource::open_with(&p, "x", in_memory).unwrap(); + let (inplace, in_report) = materialize(&src).unwrap(); + assert_eq!(inplace.features, whole.features, "{file}"); + assert_eq!(inplace.crs, whole.crs, "{file}"); + assert_eq!(inplace.crs_incomplete, whole.crs_incomplete, "{file}"); + assert_eq!(inplace.geometry_type, whole.geometry_type, "{file}"); + assert_eq!(report.skipped, in_report.skipped, "{file}"); + let strip = |m: &Vec| m.iter().map(|m| m.trim_start_matches("x: ").to_string()).collect::>(); + assert_eq!(strip(&report.messages), strip(&in_report.messages), "{file}"); + // The extent a source says is that of the features it reads (those that convert). + assert_eq!(src.info().extent, whole.extent(), "{file}"); + assert_eq!(src.info().count, Some(whole.features.len() as u64), "{file}"); + assert_eq!(src.info().coordinate_system(), whole.crs, "{file}"); + } + let src = geojson::GeoJsonSource::open_with(&p, "x", crate::io::MAX_BYTES).unwrap(); + (whole, report.messages, src.info().crs.clone()) +} + +fn in_arnhem(e: &Extent) -> bool { + (5.8..6.0).contains(&e.min_x) && (5.8..6.0).contains(&e.max_x) && (51.9..52.1).contains(&e.min_y) && (51.9..52.1).contains(&e.max_y) +} + +#[test] +fn geojson_legacy_crs_is_honoured_by_both_paths() { + // The issue #930 declaration, and the other ways EPSG:28992 is named. + for name in [ + "urn:ogc:def:crs:EPSG::28992", + "urn:ogc:def:crs:EPSG:6.6:28992", + "EPSG:28992", + "epsg:28992", + "http://www.opengis.net/def/crs/EPSG/0/28992", + ] { + let (ds, messages, info_crs) = geojson_paths("rd.geojson", &rd_geojson(Some(&named(name)))); + assert!(in_arnhem(&ds.extent()), "{name}: {:?}", ds.extent()); + assert_eq!(ds.crs, Some(crate::raster::Crs::Epsg(28992)), "{name}"); + assert_eq!(info_crs, "EPSG:28992", "{name}"); + assert!(messages.is_empty(), "{name}: {messages:?}"); + assert!(!ds.crs_incomplete); + assert_eq!(ds.geometry_type, GeometryType::Polygon); + } + // Reads in place go by the layer's own coordinates: a window of longitude/latitude finds the + // features, one elsewhere finds none. + let p = dir().join("geojson-crs").join("rd.geojson"); + for in_memory in [crate::io::MAX_BYTES, 0] { + let src = geojson::GeoJsonSource::open_with(&p, "x", in_memory).unwrap(); + let at = |e: Extent| src.read(&Request { extent: Some(e), limit: 100 }).unwrap().features.len(); + assert_eq!(at(Extent::new(5.85, 51.95, 5.86, 51.96)), 1, "the polygon only"); + assert_eq!(at(Extent::new(5.95, 52.02, 5.96, 52.03)), 1, "the polygon only"); + assert_eq!(at(Extent::new(5.8, 51.9, 6.0, 52.1)), 2); + assert_eq!(at(Extent::new(0.0, 0.0, 1.0, 1.0)), 0); + assert_eq!(at(Extent::new(500000.0, 438459.0, 600000.0, 454549.0)), 0, "the file's coordinates aren't lon/lat"); + } +} + +#[test] +fn geojson_without_a_crs_or_in_lonlat_is_unchanged() { + let fc = |crs: &str| { + format!( + r#"{{"type":"FeatureCollection",{crs}"features":[{{"type":"Feature","properties":{{}},"geometry":{{"type":"Polygon","coordinates":[[[5.8,51.9],[6.0,51.9],[6.0,52.1],[5.8,52.1],[5.8,51.9]]]}}}}, + {{"type":"Feature","properties":{{}},"geometry":{{"type":"Point","coordinates":[-122.5,37.75]}}}}]}}"# + ) + }; + let wgs84 = [ + String::new(), + r#""crs":null,"#.to_string(), + format!(r#""crs":{},"#, named("urn:ogc:def:crs:OGC:1.3:CRS84")), + format!(r#""crs":{},"#, named("urn:ogc:def:crs:OGC::CRS84")), + format!(r#""crs":{},"#, named("OGC::CRS84")), + format!(r#""crs":{},"#, named("OGC:1.3:CRS84")), + format!(r#""crs":{},"#, named("OGC:CRS84")), + format!(r#""crs":{},"#, named("CRS84")), + format!(r#""crs":{},"#, named("http://www.opengis.net/def/crs/OGC/1.3/CRS84")), + format!(r#""crs":{},"#, named("EPSG:4326")), + format!(r#""crs":{},"#, named("urn:ogc:def:crs:EPSG::4326")), + ]; + let (base, _, _) = geojson_paths("plain.geojson", &fc("")); + assert_eq!(base.extent(), Extent::new(-122.5, 37.75, 6.0, 52.1)); + for crs in &wgs84 { + let (ds, messages, info_crs) = geojson_paths("wgs84.geojson", &fc(crs)); + assert_eq!(ds.features, base.features, "{crs}: not a coordinate moved"); + assert_eq!(ds.crs, None, "{crs}"); + assert_eq!(info_crs, "EPSG:4326", "{crs}"); + assert!(messages.is_empty(), "{crs}: {messages:?}"); + assert!(!ds.crs_incomplete); + } +} + +#[test] +fn geojson_geographic_crs_keeps_x_y_order() { + // ETRS89 (EPSG:4258) is latitude/longitude in EPSG, but a legacy GeoJSON crs still gives + // longitude, latitude: the point stays in the Netherlands, not at 52 E 5.9 N. + let text = r#"{"type":"FeatureCollection","crs":{"type":"name","properties":{"name":"urn:ogc:def:crs:EPSG::4258"}}, + "features":[{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[5.9,52.0]}}]}"#; + let (ds, messages, info_crs) = geojson_paths("etrs89.geojson", text); + assert!(messages.is_empty(), "{messages:?}"); + assert_eq!(ds.crs, Some(crate::raster::Crs::Epsg(4258))); + assert_eq!(info_crs, "EPSG:4258"); + let Some(Geometry::Point(c)) = ds.features.first().and_then(|f| f.geometry.clone()) else { panic!("a point") }; + assert!((c.x - 5.9).abs() < 1e-3 && (c.y - 52.0).abs() < 1e-3, "{c:?}"); +} + +#[test] +fn geojson_crs_not_understood_is_said_and_coordinates_kept() { + let long = "x".repeat(100_000); + let declared = [ + (r#"{"type":"link","properties":{"href":"http://example.com/crs","type":"proj4"}}"#.to_string(), "of type \"link\""), + (named("urn:ogc:def:crs:OGC:1.3:CRS83"), "CRS83"), + (named("EPSG:1"), "EPSG:1"), + (named("EPSG:99999999999"), "EPSG:99999999999"), + (named("EPSG:abc"), "EPSG:abc"), + (named("Amersfoort / RD New"), "Amersfoort"), + (named("VN-2000 / UTM zone 48N EPSG:28992"), "VN-2000"), + (named("OGC:CRS84:EPSG"), "OGC:CRS84:EPSG"), + (named("x:EPSG:28992"), "x:EPSG:28992"), + (named("urn:ogc:def:crs:EPSG::28992:extra"), "extra"), + (named("urn:x:EPSG:6.6:28992"), "urn:x"), + (named("http://example.com/def/crs/EPSG/0/28992"), "example.com"), + (named("EPSG:28992:28992"), "EPSG:28992:28992"), + (named("+proj=utm +zone=31 +datum=WGS84 EPSG"), "+proj=utm"), + (named("PROJCS[\\\"RD\\\",EPSG]"), "PROJCS"), + (named(&format!("EPSG:28992{long}")), "…"), + (r#"{"type":"name","properties":{"name":28992}}"#.to_string(), "of type \"name\""), + (r#"{"type":"name","properties":{}}"#.to_string(), "of type \"name\""), + (r#"{"type":"name"}"#.to_string(), "of type \"name\""), + (r#"{"type":5}"#.to_string(), "of an unknown shape"), + (r#"{}"#.to_string(), "of an unknown shape"), + (r#""EPSG:28992""#.to_string(), "of an unknown shape"), + (r#"28992"#.to_string(), "of an unknown shape"), + (r#"true"#.to_string(), "of an unknown shape"), + (r#"["EPSG:28992"]"#.to_string(), "of an unknown shape"), + (r#"[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]"#.to_string(), "of an unknown shape"), + (format!(r#"{{"type":"{long}"}}"#), "…"), + ]; + for (crs, shown) in &declared { + let (ds, messages, info_crs) = geojson_paths("odd.geojson", &rd_geojson(Some(crs))); + // Not taken as WGS 84 without saying so: one message naming the declaration. + assert_eq!(messages.len(), 1, "{crs:.60}: {messages:?}"); + assert!(messages[0].starts_with("GeoJSON crs ") && messages[0].contains(shown), "{crs:.60}: {}", messages[0]); + assert!(messages[0].contains("longitude/latitude"), "{}", messages[0]); + assert!(messages[0].len() < 400, "a declaration is clipped in a message: {}", messages[0].len()); + // Read as they are, nothing converted or lost. + assert_eq!(ds.features.len(), 2, "{crs:.60}"); + assert_eq!(ds.extent(), Extent::new(183548.0, 438459.0, 196442.0, 454549.0), "{crs:.60}"); + assert_eq!(ds.crs, None); + assert_eq!(info_crs, "EPSG:4326"); + } +} + +#[test] +fn geojson_crs_is_read_only_from_the_feature_collection() { + // Not part of the legacy text: on a feature, a geometry or a lone Feature. Ignored, quietly. + let crs = named("EPSG:28992"); + let docs = [ + format!( + r#"{{"type":"FeatureCollection","features":[{{"type":"Feature","crs":{crs},"properties":{{}},"geometry":{{"type":"Point","coordinates":[188000,445000]}}}}]}}"# + ), + format!( + r#"{{"type":"FeatureCollection","features":[{{"type":"Feature","properties":{{}},"geometry":{{"type":"Point","coordinates":[188000,445000],"crs":{crs}}}}}]}}"# + ), + format!(r#"{{"type":"Feature","crs":{crs},"properties":{{}},"geometry":{{"type":"Point","coordinates":[188000,445000]}}}}"#), + format!(r#"{{"type":"Point","coordinates":[188000,445000],"crs":{crs}}}"#), + ]; + for text in &docs { + let (ds, report) = crate::io::read_geojson(text, "x").unwrap(); + assert!(report.messages.is_empty(), "{text}"); + assert_eq!(ds.crs, None, "{text}"); + assert_eq!(ds.extent(), Extent::new(188000.0, 445000.0, 188000.0, 445000.0), "{text}"); + } + // A FeatureCollection whose features aren't an array is read whole, its crs still honoured. + let (ds, report) = crate::io::read_geojson(&format!(r#"{{"type":"FeatureCollection","crs":{crs},"features":7}}"#), "x").unwrap(); + assert!(ds.features.is_empty() && report.messages.is_empty()); +} + +#[test] +fn geojson_features_that_cant_be_converted_are_skipped_and_counted() { + // A coordinate the declared system can't convert (UTM 1e9 m east), or converts to beyond + // longitude/latitude (Web Mercator 1e9 m east is longitude 8983°): the feature is dropped. + for (crs, good, bad) in [("EPSG:32617", "[500000,4649776]", "[1e9,0]"), ("EPSG:3857", "[1000000,6000000]", "[1e9,0]")] { + let text = format!( + r#"{{"type":"FeatureCollection","crs":{},"features":[ + {{"type":"Feature","properties":{{"n":1}},"geometry":{{"type":"Point","coordinates":{good}}}}}, + {{"type":"Feature","properties":{{"n":2}},"geometry":{{"type":"Point","coordinates":{bad}}}}}, + {{"type":"Feature","properties":{{"n":3}},"geometry":{{"type":"Point","coordinates":{good}}}}}]}}"#, + named(crs) + ); + let (ds, messages, info_crs) = geojson_paths("far.geojson", &text); + assert_eq!(ds.features.len(), 2, "{crs}"); + assert_eq!(ds.features.iter().map(|f| f.id).collect::>(), [1, 3], "{crs}: ids are the features' places in the file"); + assert_eq!(messages.len(), 1, "{crs}: {messages:?}"); + assert!(messages[0].contains(&format!("1 features couldn't be converted from {crs}")), "{}", messages[0]); + assert_eq!(info_crs, crs); + assert!(ds.crs_incomplete, "{crs}: a projection can't be defined again over lost features"); + let Some(Geometry::Point(c)) = ds.features.first().and_then(|f| f.geometry.clone()) else { panic!("a point") }; + assert!(c.x.abs() < 180.0 && c.y.abs() < 90.0, "{crs}: {c:?}"); + if crs == "EPSG:32617" { + assert!((c.x + 81.0).abs() < 1e-6 && (c.y - 42.0).abs() < 1e-3, "{c:?}"); + } + } +} + +/// A point in Web Mercator that converts, one that doesn't (longitude 8983°), and a polygon. +const MERC_OK: &str = r#"{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[1000000,6000000]}}"#; +const MERC_BAD: &str = r#"{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[1e9,0]}}"#; +const MERC_POLY: &str = r#"{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[1000000,6000000],[1100000,6000000],[1100000,6100000],[1000000,6000000]]]}}"#; + +fn merc_collection(features: &[&str]) -> String { + format!(r#"{{"type":"FeatureCollection","crs":{},"features":[{}]}}"#, named("EPSG:3857"), features.join(",")) +} + +#[test] +fn geojson_dropped_features_do_not_set_the_geometry_type() { + // Two points that can't be placed and one polygon that can: a polygon layer, as read whole. + let (ds, _, _) = geojson_paths("kinds.geojson", &merc_collection(&[MERC_BAD, MERC_POLY, MERC_BAD])); + assert_eq!(ds.geometry_type, GeometryType::Polygon); + assert_eq!(ds.features.len(), 1); + // Nothing that can be placed: no geometry type at all, both ways. + let (ds, messages, _) = geojson_paths("none.geojson", &merc_collection(&[MERC_BAD, MERC_BAD])); + assert_eq!(ds.geometry_type, GeometryType::None); + assert!(ds.features.is_empty() && ds.crs_incomplete); + assert_eq!(messages.len(), 1, "{messages:?}"); +} + +#[test] +fn geojson_sampling_skips_dropped_features() { + // Unplaceable features at every other place must not use up the sample: a read asking for + // two of the three placeable ones gets two, and says it matched three. + let text = merc_collection(&[MERC_BAD, MERC_OK, MERC_BAD, MERC_OK, MERC_BAD, MERC_OK]); + let p = dir().join("geojson-crs").join("alternating.geojson"); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, &text).unwrap(); + for in_memory in [crate::io::MAX_BYTES, 0] { + let src = geojson::GeoJsonSource::open_with(&p, "x", in_memory).unwrap(); + assert_eq!(src.info().count, Some(3)); + let all = src.read(&Request { extent: None, limit: 100 }).unwrap(); + assert_eq!((all.features.len(), all.matched, all.skipped, all.complete), (3, 3, 0, true)); + let some = src.read(&Request { extent: None, limit: 2 }).unwrap(); + assert_eq!((some.features.len(), some.matched, some.skipped, some.complete), (2, 3, 0, false)); + assert!(some.features.iter().all(|f| f.geometry.is_some())); + let ids: Vec = all.features.iter().map(|f| f.id).collect(); + assert_eq!(ids, [2, 4, 6]); + } +} + +#[test] +fn geojson_unreadable_feature_alone_does_not_make_the_layer_incomplete() { + // A declared system, one feature with geometry that can't be read, and all the others + // placed: nothing was lost to the system, so Define Projection stays available, both ways. + let bad = r#"{"type":"Feature","properties":{},"geometry":{"type":"Nope"}}"#; + let (ds, messages, _) = geojson_paths("unreadable.geojson", &merc_collection(&[MERC_OK, bad])); + assert_eq!(ds.features.len(), 1); + assert!(!ds.crs_incomplete); + assert_eq!(messages.len(), 1, "{messages:?}"); + assert!(messages[0].contains("unreadable geometry"), "{}", messages[0]); +} + #[test] fn damaged_geojson_is_an_error_not_a_panic() { let d = dir().join("geojson-junk"); diff --git a/crates/engine/tests/geojson_crs.rs b/crates/engine/tests/geojson_crs.rs new file mode 100644 index 000000000..bbbe1738e --- /dev/null +++ b/crates/engine/tests/geojson_crs.rs @@ -0,0 +1,166 @@ +//! A GeoJSON FeatureCollection's legacy `crs` member (issue #930) is honoured when it is added, +//! whole or read in place, and what is written is still WGS 84 with no `crs`. +use rustgis_engine::{ + Project, Session, + data::{io, raster::Crs}, + geom::Extent, +}; +use serde_json::{Value, json}; +use std::path::PathBuf; + +/// Arnhem's municipal outline's box and a point, in RD New (EPSG:28992). +fn rd(crs: &Value) -> String { + json!({ + "type": "FeatureCollection", + "name": "aoi_gemeente_arnhem", + "crs": crs, + "features": [ + {"type": "Feature", "properties": {"naam": "Arnhem"}, "geometry": {"type": "Polygon", "coordinates": + [[[183548, 438459], [196442, 438459], [196442, 454549], [183548, 454549], [183548, 438459]]]}}, + {"type": "Feature", "properties": {"naam": "punt"}, "geometry": {"type": "Point", "coordinates": [188000, 445000]}}, + ] + }) + .to_string() +} + +fn declared(name: &str) -> Value { + json!({"type": "name", "properties": {"name": name}}) +} + +fn file(name: &str, text: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("rustgis-geojson-crs-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + let p = d.join(name); + std::fs::write(&p, text).unwrap(); + p +} + +fn add(s: &mut Session, p: &PathBuf, stream: bool) -> Result { + let r = s.run("layer.addData", &json!({"path": p, "stream": stream}))?; + r["layer"].as_u64().map(|id| id as u32).ok_or_else(|| format!("no layer in {r}")) +} + +fn in_arnhem(e: &Extent) -> bool { + (5.8..6.0).contains(&e.min_x) && (5.8..6.0).contains(&e.max_x) && (51.9..52.1).contains(&e.min_y) && (51.9..52.1).contains(&e.max_y) +} + +#[test] +fn layer_lands_in_arnhem_by_the_declared_crs_both_ways() { + let p = file("arnhem.geojson", &rd(&declared("urn:ogc:def:crs:EPSG::28992"))); + for stream in [false, true] { + let mut s = Session::new(Project::default()); + let id = add(&mut s, &p, stream).unwrap(); + let layer = s.map().layer(id).unwrap(); + assert_eq!(layer.is_streamed(), stream); + assert!(in_arnhem(&layer.extent()), "stream {stream}: {:?}", layer.extent()); + if !stream { + assert_eq!(layer.dataset.crs, Some(Crs::Epsg(28992))); + } + } +} + +#[test] +fn export_is_wgs84_without_crs_and_returns_to_the_native_numbers() { + let p = file("arnhem-export.geojson", &rd(&declared("EPSG:28992"))); + let mut s = Session::new(Project::default()); + let id = add(&mut s, &p, false).unwrap(); + let out = p.with_file_name("out.geojson"); + s.run("share.exportLayer", &json!({"layer": id, "path": out})).unwrap(); + let text = std::fs::read_to_string(&out).unwrap(); + assert!(!text.contains("\"crs\""), "{text}"); + let (back, _) = io::read_geojson(&text, "x").unwrap(); + assert!(in_arnhem(&back.extent()), "{:?}", back.extent()); + // In its own system again, the layer is where the file put it. + let native = io::project(&s.map().layer(id).unwrap().dataset, Crs::Epsg(28992)).unwrap(); + let e = native.extent(); + assert!((e.min_x - 183_548.0).abs() < 0.01 && (e.max_x - 196_442.0).abs() < 0.01, "{e:?}"); + assert!((e.min_y - 438_459.0).abs() < 0.01 && (e.max_y - 454_549.0).abs() < 0.01, "{e:?}"); +} + +#[test] +fn define_projection_still_works_on_a_layer_with_a_declared_crs() { + let mut s = Session::new(Project::default()); + // Declared one way; the user knows better: the native numbers are reinterpreted. + let p = file("arnhem-redefine.geojson", &rd(&declared("EPSG:32631"))); + let id = add(&mut s, &p, false).unwrap(); + assert!(!in_arnhem(&s.map().layer(id).unwrap().extent())); + s.run("layer.defineProjection", &json!({"layer": id, "crs": "EPSG:28992"})).unwrap(); + let layer = s.map().layer(id).unwrap(); + assert!(in_arnhem(&layer.extent()), "{:?}", layer.extent()); + assert_eq!(layer.dataset.crs, Some(Crs::Epsg(28992))); + // And a file that says nothing, then Define Projection, as before the fix. + let p = file("arnhem-silent.geojson", &rd(&Value::Null)); + let id = add(&mut s, &p, false).unwrap(); + assert!(!in_arnhem(&s.map().layer(id).unwrap().extent())); + s.run("layer.defineProjection", &json!({"layer": id, "crs": "EPSG:28992"})).unwrap(); + assert!(in_arnhem(&s.map().layer(id).unwrap().extent())); +} + +#[test] +fn a_crs_of_any_shape_never_panics() { + let long = "9".repeat(200_000); + let mut nested = json!(1); + for _ in 0..100 { + nested = json!([nested]); + } + let crs = [ + Value::Null, + json!(1), + json!(1e308), + json!(true), + json!("EPSG:28992"), + json!(long), + json!([]), + json!({}), + nested.clone(), + json!({"type": "name"}), + json!({"type": "name", "properties": null}), + json!({"type": "name", "properties": []}), + json!({"type": "name", "properties": {"name": null}}), + json!({"type": "name", "properties": {"name": [1]}}), + json!({"type": "name", "properties": {"name": nested}}), + json!({"type": "name", "properties": {"name": long}}), + json!({"type": "name", "properties": {"name": format!("EPSG:{long}")}}), + json!({"type": "name", "properties": {"name": "EPSG:-1"}}), + json!({"type": "name", "properties": {"name": "EPSG:65535"}}), + json!({"type": "name", "properties": {"name": "\u{0}EPSG:\u{202e}28992\u{1f}"}}), + json!({"type": "name", "properties": {"name": format!("ÉPSG:é{}", "€".repeat(60))}}), + json!({"type": "link", "properties": {"href": "http://127.0.0.1:9/crs", "type": "ogcwkt"}}), + json!({"type": ["name"], "properties": {"name": "EPSG:28992"}}), + ]; + for (n, crs) in crs.iter().enumerate() { + let p = file(&format!("junk{n}.geojson"), &rd(crs)); + for stream in [false, true] { + let mut s = Session::new(Project::default()); + if let Ok(id) = add(&mut s, &p, stream) { + // Added, drawn and described. + let _ = s.map().layer(id).map(|l| l.extent()); + let _ = s.run("info.project", &Value::Null); + } + } + } + // The same on features and geometries (not read), and a crs after the features. + for text in [ + r#"{"type":"FeatureCollection","features":[{"type":"Feature","crs":7,"properties":{},"geometry":{"type":"Point","coordinates":[1,2],"crs":[]}}]}"#, + r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[1,2]}}],"crs":{"type":"name","properties":{"name":"EPSG:32631"}}}"#, + r#"{"type":"Feature","crs":{"type":"name","properties":{"name":"EPSG:28992"}},"properties":{},"geometry":null}"#, + ] { + let p = file("odd.geojson", text); + for stream in [false, true] { + let mut s = Session::new(Project::default()); + let _ = add(&mut s, &p, stream); + } + } +} + +#[test] +fn a_crs_after_the_features_is_honoured() { + let text = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[500000,4649776]}}],"crs":{"type":"name","properties":{"name":"urn:ogc:def:crs:EPSG:6.6:32617"}}}"#; + let p = file("late.geojson", text); + for stream in [false, true] { + let mut s = Session::new(Project::default()); + let id = add(&mut s, &p, stream).unwrap(); + let e = s.map().layer(id).unwrap().extent(); + assert!((e.min_x + 81.0).abs() < 0.01 && (e.min_y - 42.0).abs() < 0.01, "stream {stream}: {e:?}"); + } +} diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 64b8ce184..896d83b52 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -15,7 +15,7 @@ not every EPSG code; azimuthal equidistant systems such as Equi7 need the deskto | Format | Extensions or source | How to add it | Notes | | --- | --- | --- | --- | -| GeoJSON | `.geojson`, `.json` | Home › Add › Add Data, or drop the file | Features with typed attributes; a `GeometryCollection` keeps its first readable member. Desktop FeatureCollections larger than 32 MiB use an in-place reader; layers with more than 100,000 features stream a view at a time. Other GeoJSON files load whole, up to 256 MiB. | +| GeoJSON | `.geojson`, `.json` | Home › Add › Add Data, or drop the file | Features with typed attributes; a `GeometryCollection` keeps its first readable member. A FeatureCollection's legacy `crs` member (`{"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::28992"}}`) is honoured for EPSG codes (`EPSG:28992`, `urn:ogc:def:crs:EPSG::28992`, `urn:ogc:def:crs:EPSG:6.6:28992`) and for `CRS84`: a projected or other non-WGS 84 system converts the coordinates to longitude/latitude and the layer keeps it as its coordinate system (Layer properties, export in it, Define Projection); `CRS84` and `EPSG:4326` are longitude/latitude already, so the coordinates stay as they are and the layer has no other coordinate system. Coordinates are x/y (easting/northing, or longitude/latitude) whatever the EPSG axis order. A `crs` that isn't understood (a `link`, WKT or PROJ text, any other name, an unknown code) is named in a notice and the coordinates are read as longitude/latitude. A feature whose coordinates can't be converted, or convert to beyond ±180.5° longitude or ±90.5° latitude (the declared system doesn't fit the numbers), is left out and counted in a notice, and Define Projection then asks for the file to be added again. A `crs` on a feature, a geometry or a lone Feature is ignored. GeoJSON is always written in WGS 84 without a `crs`. Desktop FeatureCollections larger than 32 MiB use an in-place reader (which honours `crs` too, reading the file once more to check the features convert); layers with more than 100,000 features stream a view at a time. Other GeoJSON files load whole, up to 256 MiB. | | Delimited text | `.csv`, `.tsv`, `.txt` | Add Data, or Home › Add › Add Data › Delimited Text… to choose the columns from the file's header | Comma, semicolon, tab or pipe separated (found from the header, or `"delimiter"`); `info.fileColumns` lists the columns and the ones found. Points from x/y columns (named like longitude/latitude or easting/northing, or chosen with `"x"` and `"y"`), or any geometry from a WKT/EWKT column (named like `wkt` or `geometry`, or `"wkt"`). Coordinates in another system are converted from `"crs"` (`EPSG:2263`, or WKT) or an EWKT `SRID=`; without either, coordinates outside longitude/latitude are pointed out. A file without coordinates opens as a standalone table. | | Shapefile | `.shp` with `.dbf` (and `.shx`), or zipped (`.zip`) | Add Data or drop the file; in the web app, pick or drop the `.shp` and `.dbf` together, with the `.shx` and `.prj` if there are (they open as one layer), or the zip | A zip may hold several Shapefiles, in folders, and other vector files (GeoJSON, CSV, GeoParquet, FlatGeobuf); each becomes a layer, at most 32 for the whole archive (Shapefiles first, so other files are left out when Shapefiles fill it). `.txt` files and macOS `__MACOSX` entries are passed over, and a member that can't be read is skipped and named in a notice. Converted to longitude/latitude from the coordinate system in its `.prj` (EPSG, Esri or OGC WKT, or WKT2 as PROJ and QGIS write it; Web Mercator, UTM, State Plane and other common projections); without a `.prj` it must be in longitude/latitude. On the desktop, one with more than 100,000 shapes (counted from its `.shx`) is read in place, a view at a time. | | KML / KMZ | `.kml`, `.kmz` | Add Data, drop the file, or upload it in the web app | A layer for each kind of geometry (points, lines, polygons), with each placemark's name, description, folder path and extended data. Styles are kept: the layer is drawn by unique values of its `style` field, with each KML style's line and fill colours, widths and icon colours (up to 64 styles). Multi-geometries and `gx:Track`s are read. Ground overlays (pictures on a `LatLonBox`, turned by its `rotation`) are raster layers kept in the project: the picture inside a KMZ, or a file beside a KML (not a web address, nor outside its folder); a `gx:LatLonQuad` isn't drawn. |