From 78fbe307b4aefddd7add822103192ee98758fd41 Mon Sep 17 00:00:00 2001 From: jeafreezy Date: Fri, 2 Oct 2026 08:05:53 +0200 Subject: [PATCH] chore: finalized geoparquet integration: --- .github/workflows/CI.yml | 7 + CHANGELOG.md | 17 + Cargo.lock | 1 + README.md | 15 +- ROADMAP.MD | 31 +- crates/cli/src/main.rs | 262 +++++++++-- crates/engine/src/engine.rs | 126 +++-- crates/engine/src/lib.rs | 4 + crates/engine/tests/geoparquet.rs | 4 +- crates/geoparquet/Cargo.toml | 4 + crates/geoparquet/README.md | 162 ++++++- crates/geoparquet/src/coverage.rs | 108 +++++ crates/geoparquet/src/ingest.rs | 142 +++++- crates/geoparquet/src/lib.rs | 234 +++++++--- crates/geoparquet/src/maintenance.rs | 190 ++++++++ crates/geoparquet/src/manifest.rs | 3 + crates/geoparquet/src/pruning.rs | 250 ++++++++++ crates/geoparquet/src/sorting.rs | 144 ++++++ crates/geoparquet/tests/policies.rs | 279 ++++++++++- crates/geoparquet/tests/search.rs | 2 +- crates/python/Cargo.toml | 4 +- crates/python/README.md | 37 ++ crates/python/pyproject.toml | 2 +- crates/python/python/superstac/__init__.py | 2 + crates/python/python/superstac/_superstac.pyi | 27 +- crates/python/src/async_client.rs | 123 +++-- crates/python/src/geoparquet.rs | 162 +++++++ crates/python/src/lib.rs | 2 + crates/python/src/shared.rs | 44 +- crates/python/src/sync_client.rs | 83 +++- crates/python/tests/test_wheel.py | 51 ++ crates/search/src/aggregator.rs | 7 +- crates/search/src/backend.rs | 11 + crates/search/src/response.rs | 3 + crates/search/src/translator.rs | 36 +- docs/app/page.tsx | 4 +- docs/content/docs/cli/overview.md | 4 +- docs/content/docs/guides/benchmarks.md | 58 +++ docs/content/docs/guides/geoparquet.md | 296 ++++++++++++ docs/content/docs/guides/meta.json | 4 +- docs/content/docs/index.mdx | 1 + docs/content/docs/meta.json | 3 +- .../docs/python/geoparquet-notebook.mdx | 39 ++ docs/content/docs/python/meta.json | 3 +- docs/content/docs/reference/search.md | 2 +- docs/content/docs/reference/status.md | 5 +- .../content/docs/reference/troubleshooting.md | 24 +- docs/content/docs/releases.mdx | 58 +++ docs/content/docs/rust/overview.md | 10 +- docs/content/docs/start/installation.md | 2 +- docs/lib/layout.shared.tsx | 1 + docs/next-env.d.ts | 4 +- docs/public/benchmarks/geoparquet-v0.3.json | 16 + docs/public/benchmarks/geoparquet-v0.3.png | Bin 0 -> 65119 bytes docs/public/benchmarks/geoparquet-v0.3.svg | 195 ++++++++ .../notebooks/superstac-geoparquet.ipynb | 442 ++++++++++++++++++ scripts/plot-geoparquet-benchmark.py | 30 ++ scripts/publish-crates.py | 2 +- scripts/test-geoparquet-notebook.py | 66 +++ 59 files changed, 3535 insertions(+), 313 deletions(-) create mode 100644 crates/geoparquet/src/coverage.rs create mode 100644 crates/geoparquet/src/maintenance.rs create mode 100644 crates/geoparquet/src/pruning.rs create mode 100644 crates/geoparquet/src/sorting.rs create mode 100644 crates/python/src/geoparquet.rs create mode 100644 docs/content/docs/guides/benchmarks.md create mode 100644 docs/content/docs/guides/geoparquet.md create mode 100644 docs/content/docs/python/geoparquet-notebook.mdx create mode 100644 docs/content/docs/releases.mdx create mode 100644 docs/public/benchmarks/geoparquet-v0.3.json create mode 100644 docs/public/benchmarks/geoparquet-v0.3.png create mode 100644 docs/public/benchmarks/geoparquet-v0.3.svg create mode 100644 docs/public/notebooks/superstac-geoparquet.ipynb create mode 100644 scripts/plot-geoparquet-benchmark.py create mode 100644 scripts/test-geoparquet-notebook.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a008980..38f557d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,6 +23,8 @@ jobs: toolchain: '1.88.0' - uses: Swatinem/rust-cache@v2 - run: cargo test --workspace --exclude superstac-python --locked + - run: cargo test -p superstac-cli -p superstac-engine --features geoparquet --locked + - run: cargo check -p superstac-python --no-default-features --locked wheels: name: Wheel (${{ matrix.target }}) @@ -61,6 +63,11 @@ jobs: python -m pip install --no-deps --force-reinstall dist/*.whl - name: Test the installed wheel run: python -m unittest discover -s crates/python/tests -v + - name: Execute the GeoParquet notebook against local fixtures + if: matrix.target == 'x86_64-unknown-linux-gnu' + run: | + python -m pip install pyarrow nbformat nbclient ipykernel + python scripts/test-geoparquet-notebook.py - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.target }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 299b58c..3ba5860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ releases. ## [Unreleased] +### Added — v0.3.0 +- Optional Rust/CLI GeoParquet backend and GeoParquet-enabled Python wheels. +- Scoped, named, resumable STAC metadata ingestion with progress and storage caps. +- Snapshot and automatic API fallback modes, freshness checks, and gap-aware scope unions. +- Per-catalog sorting, local collection summaries, row-group pruning, and a reproducible benchmark. +- Explicit acquisition-window incremental overlays, atomic compaction, and cleanup with reader protection. +- Sync/async Python ingestion, backend configuration, maintenance, and type hints. +- Automatically versioned Parquet provenance and search diagnostics. +- A GeoParquet Jupyter/Colab notebook, release highlights page, and documented local benchmark. + +### Changed — v0.3.0 +- Dataset manifests use version 2; legacy version-1 inventories remain readable. +- Release publishing includes the GeoParquet crate before the engine; CI checks both feature modes. + +### Fixed — v0.3.0 +- Forward sort fields to live STAC searches instead of silently discarding them. + ## [0.2.0] - 2026-09-27 ### Added diff --git a/Cargo.lock b/Cargo.lock index bade785..e2dd50e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3519,6 +3519,7 @@ dependencies = [ "superstac-config", "superstac-core", "superstac-engine", + "superstac-geoparquet", "superstac-search", "tokio", ] diff --git a/README.md b/README.md index 6e199dc..19015cc 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,12 @@ normalized to canonical names — regardless of which catalog they came from. > **Status: alpha.** APIs and YAML schema are not yet stable. Pre-1.0; expect > breaking changes. +**[What’s new in v0.3](https://spatialnode.com/superstac/docs/releases)** · [Try GeoParquet in Colab](https://colab.research.google.com/drive/1JIx8j0Vi7jypKe5Q6klgCzYQMFHMaKGc?usp=sharing) · [Local search benchmark](https://spatialnode.com/superstac/docs/guides/benchmarks) + ## Features - **Federated search:** Query multiple STAC catalogs concurrently with spatial, temporal, collection, and item filters. +- **Reusable satellite-data inventories:** Save metadata for your area of interest and explore it repeatedly without querying every provider again. Fall back to live catalogs when saved coverage is missing or stale. - **Collection discovery:** Find available collections and the catalogs that serve them. - **Consistent names:** Map provider-specific collection IDs and asset keys to canonical names through configurable aliases. - **Deduplicated results:** Merge items across catalogs by item ID, with source provenance available in Rust and CLI results. @@ -69,6 +72,7 @@ cargo add superstac-search cargo add superstac-engine cargo add superstac-cli cargo add superstac-config +cargo add superstac-geoparquet ``` ### From source @@ -106,7 +110,9 @@ superstac --json search -c landsat-c2-l2 -l 10 | jq '.metadata' superstac collections microsoft sentinel-2-l2a ``` -Run `superstac --help` for the full surface. +Run `superstac --help` for the full surface. For saved metadata inventories, build +with `--features geoparquet` and follow the [GeoParquet guide](crates/geoparquet/README.md). +Published Python wheels include this backend. ## Configuration @@ -178,11 +184,18 @@ async fn main() -> Result<(), Box> { | [`superstac-core`](https://crates.io/crates/superstac-core) | domain models, errors, storage trait | | [`superstac-config`](https://crates.io/crates/superstac-config) | YAML config loading | | [`superstac-search`](https://crates.io/crates/superstac-search) | federated search logic | +| [`superstac-geoparquet`](https://crates.io/crates/superstac-geoparquet) | Reusable satellite-data inventories for repeated searches | | [`superstac-engine`](https://crates.io/crates/superstac-engine) | runtime (health, introspection, search orchestration) | | [`superstac-cli`](https://crates.io/crates/superstac-cli) | the `superstac` binary | ## Logs and debugging +Include `superstac --version` or Python’s `superstac.__version__` in bug reports. +Search diagnostics also include `metadata.superstac_version`. Provider requests +identify the library as `superstac/` through the User-Agent header. +New ingested and compacted Parquet files record the writer version in their footer. + + Logs flow through `tracing`. The default level comes from `settings.log_level` in your config; override at runtime: diff --git a/ROADMAP.MD b/ROADMAP.MD index 3d74679..d4e174c 100644 --- a/ROADMAP.MD +++ b/ROADMAP.MD @@ -3,29 +3,36 @@ Our priority is reliable federated STAC search. These are areas of focus, subject to feedback and capacity, with no fixed delivery dates. -Last Updated: September 28th, 2026 +Last updated: October 2, 2026 + +## Implemented for v0.3 + +- [x] [Reusable satellite-data inventories with GeoParquet](https://github.com/Spatialnode/superstac/issues/31): scoped ingestion with progress and resume, local search and API fallback, Python integration, scope unions, sorting, incremental refresh, compaction and cleanup. + + ## Short term -- Fix search and iterator edge cases, backed by focused tests. -- Improve configuration errors, logging, and documentation. -- Support private catalogs through custom headers and environment-based credentials. +- [ ] [Support private catalogs](https://github.com/Spatialnode/superstac/issues/32) through per-catalog headers and environment-based credentials, consistently across search, discovery, health checks and ingestion. +- [ ] [Add AI/agent integration](https://github.com/Spatialnode/superstac/issues/22): define a supported tool interface for catalog discovery and bounded searches, with a working agent example. Existing Python APIs can be called by agents; a dedicated integration is still pending. +- [ ] [Explore WASM bindings](https://github.com/Spatialnode/superstac/issues/21): validate a browser-compatible API search client first, including CORS and runtime constraints. Native filesystem and GeoParquet support should not be assumed to work in browsers. ## Near term -- Add catalog preferences and improve health-based fallback. -- Introduce SQLite persistence. -- Improve pagination and result handling across Python, Rust, and the CLI. +- [ ] Add explicit preferred-catalog ordering and failover between providers. Improve recovery of catalogs unavailable at startup. Health checks, retries, concurrent search and snapshot-to-API fallback already exist. +- [ ] Implement SQLite persistence for catalog registration, provider configuration and settings. +- [ ] Expose resumable federated pagination and bounded/lazy result iteration across Python, Rust and the CLI. Provider pagination is already followed internally, but callers currently receive materialized, capped results rather than a federated cursor or lazy stream. ## Long term -Explore broader authentication, additional storage backends, and ecosystem -integrations as real use cases emerge. +Explore OAuth/token refresh and asset signing, additional configuration storage +backends, and ecosystem integrations as real use cases emerge. These extend the +basic credential support above rather than duplicate it. ## v1.0 release -- Stable public APIs and configuration format. -- Well-tested core search, with predictable results and failure handling. -- Clear documentation, supported-platform coverage, and migration guidance. +- [ ] Stable public APIs and configuration format, with a compatibility policy. +- [ ] Release confidence in core search, with predictable results and failure handling. Existing tests are a foundation; scale, interoperability and failure-recovery coverage remain ongoing work. +- [ ] Maintain documentation, supported-platform coverage and migration guidance for every stable release. Documentation and packaging already exist; this is a continuing release requirement. The v1.0 release depends on core stability; longer-term ideas are optional. diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index cb0111c..0c70c57 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -51,7 +51,11 @@ struct Cli { } #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -enum SearchMode { Live, Snapshot, Auto } +enum SearchMode { + Live, + Snapshot, + Auto, +} #[derive(Subcommand)] enum Command { @@ -63,6 +67,10 @@ enum Command { /// Inspect available collections. Collections(CollectionsArgs), + /// Reclaim retired Parquet files (dry run unless --apply). + Cleanup(MaintenanceArgs), + /// Rewrite snapshots into larger files and deduplicate records. + Compact(CompactArgs), } #[derive(Args)] @@ -87,6 +95,9 @@ struct SearchArgs { /// Specific item IDs to fetch (repeatable). #[arg(long = "id")] ids: Vec, + /// Repeatable STAC sort field, e.g. --sortby=-datetime. + #[arg(long)] + sortby: Vec, } #[derive(Args)] @@ -111,6 +122,9 @@ struct IngestArgs { /// Scope name; refreshes only this name. Omit to derive it from coverage. #[arg(long)] name: Option, + /// Overlay records acquired since this timestamp; requires an existing --name. + #[arg(long, requires = "name")] + incremental_since: Option, /// Retained Parquet budget including old/partial generations; 0 disables the cap. #[arg(long, default_value_t = 1024)] max_dataset_mib: u64, @@ -130,6 +144,23 @@ struct IngestArgs { timeout_seconds: u64, } +#[derive(Args)] +struct MaintenanceArgs { + #[arg(long, default_value = "./data")] + output: PathBuf, + #[arg(long)] + apply: bool, +} +#[derive(Args)] +struct CompactArgs { + #[arg(long, default_value = "./data")] + output: PathBuf, + #[arg(long, default_value_t = 10000)] + items_per_file: usize, + #[arg(long, default_value_t = 1024)] + max_dataset_mib: u64, +} + #[derive(Args)] struct CollectionsArgs { /// Catalog ID. If omitted, lists all collections across catalogs. @@ -159,20 +190,33 @@ async fn main() -> ExitCode { } if let Command::Ingest(args) = &cli.command { - if cli.dataset.is_some() || !cli.geoparquet.is_empty() || cli.mode.is_some() || cli.max_snapshot_age_seconds.is_some() { + if cli.dataset.is_some() + || !cli.geoparquet.is_empty() + || cli.mode.is_some() + || cli.max_snapshot_age_seconds.is_some() + { eprintln!("error: ingest reads APIs; use --output to choose its dataset destination"); return ExitCode::FAILURE; } return run_ingest(db.as_ref(), args, cli.json, cli.quiet).await; } + if matches!(&cli.command, Command::Cleanup(_) | Command::Compact(_)) { + return run_maintenance(&cli.command, cli.json); + } + let mode = cli.mode.unwrap_or_else(|| { - if cli.dataset.is_some() || !cli.geoparquet.is_empty() { SearchMode::Snapshot } else { SearchMode::Live } + if cli.dataset.is_some() || !cli.geoparquet.is_empty() { + SearchMode::Snapshot + } else { + SearchMode::Live + } }); if (mode == SearchMode::Auto && (cli.dataset.is_none() || !cli.geoparquet.is_empty())) || (mode == SearchMode::Snapshot && cli.dataset.is_none() && cli.geoparquet.is_empty()) || (mode == SearchMode::Live && (cli.dataset.is_some() || !cli.geoparquet.is_empty())) - || (mode != SearchMode::Auto && cli.max_snapshot_age_seconds.is_some()) { + || (mode != SearchMode::Auto && cli.max_snapshot_age_seconds.is_some()) + { eprintln!("error: auto requires --dataset; snapshot requires --dataset or --geoparquet; live takes neither; max-snapshot-age-seconds applies only to auto"); return ExitCode::FAILURE; } @@ -186,9 +230,11 @@ async fn main() -> ExitCode { } #[cfg(feature = "geoparquet")] match if mode == SearchMode::Auto { - Ok(SuperSTACEngine::automatic(db, root.clone(), std::time::Duration::from_secs( - cli.max_snapshot_age_seconds.unwrap_or(86400) - ))) + Ok(SuperSTACEngine::automatic( + db, + root.clone(), + std::time::Duration::from_secs(cli.max_snapshot_age_seconds.unwrap_or(86400)), + )) } else { SuperSTACEngine::from_dataset(db, root) } { @@ -203,7 +249,9 @@ async fn main() -> ExitCode { } else { #[cfg(not(feature = "geoparquet"))] { - eprintln!("error: rebuild superstac-cli with --features geoparquet to search snapshots"); + eprintln!( + "error: rebuild superstac-cli with --features geoparquet to search snapshots" + ); return ExitCode::FAILURE; } #[cfg(feature = "geoparquet")] @@ -233,7 +281,9 @@ async fn main() -> ExitCode { let exit = match cli.command { Command::Search(args) => run_search(&engine, args, cli.json).await, Command::Collections(args) => run_collections(&engine, args, cli.json).await, - Command::Ingest(_) => unreachable!("ingestion is handled before engine startup"), + Command::Cleanup(_) | Command::Compact(_) | Command::Ingest(_) => { + unreachable!("ingestion is handled before engine startup") + } }; engine.shutdown().await; @@ -250,50 +300,137 @@ async fn run_ingest( use superstac_engine::{ingest_catalog, IngestOptions, IngestScope}; let mut catalogs = Vec::new(); for id in &args.catalogs { - if catalogs.iter().any(|c: &superstac_core::models::catalog::Catalog| &c.id == id) { + if catalogs + .iter() + .any(|c: &superstac_core::models::catalog::Catalog| &c.id == id) + { eprintln!("error: duplicate catalog '{id}'"); return ExitCode::FAILURE; } match db.get_catalog(id) { Ok(catalog) => catalogs.push(catalog.clone()), - Err(e) => { eprintln!("error: {e}"); return ExitCode::FAILURE; } + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::FAILURE; + } } } let mut completed = Vec::new(); for catalog in catalogs { - let mut options = IngestOptions::new(&args.output, IngestScope { - collections: args.collections.clone(), bbox: args.bbox, datetime: args.datetime.clone(), - }); + let mut options = IngestOptions::new( + &args.output, + IngestScope { + collections: args.collections.clone(), + bbox: args.bbox, + datetime: args.datetime.clone(), + }, + ); options.page_size = args.page_size; options.items_per_file = args.items_per_file; options.request_timeout = std::time::Duration::from_secs(args.timeout_seconds); options.resume = args.resume; options.name = args.name.clone(); - options.max_dataset_bytes = if args.max_dataset_mib == 0 { None } else { + options.incremental_since = args.incremental_since.clone(); + options.max_dataset_bytes = if args.max_dataset_mib == 0 { + None + } else { match args.max_dataset_mib.checked_mul(1024 * 1024) { Some(bytes) => Some(bytes), - None => { eprintln!("error: storage budget is too large"); return ExitCode::FAILURE; } + None => { + eprintln!("error: storage budget is too large"); + return ExitCode::FAILURE; + } } }; - if !args.no_progress && !quiet { options.progress = Some(ingest_progress()); } - if !quiet { eprintln!("ingesting '{}' into {}", catalog.id, args.output.display()); } + if !args.no_progress && !quiet { + options.progress = Some(ingest_progress()); + } + if !quiet { + eprintln!("ingesting '{}' into {}", catalog.id, args.output.display()); + } match ingest_catalog(&catalog, options).await { Ok(snapshot) => { - if !quiet { eprintln!("completed '{}' scope '{}': {} items, {} pages, {} files", catalog.id, - snapshot.name, snapshot.items, snapshot.pages, snapshot.files.len()); } + if !quiet { + eprintln!( + "completed '{}' scope '{}': {} items, {} pages, {} files", + catalog.id, + snapshot.name, + snapshot.items, + snapshot.pages, + snapshot.files.len() + ); + } completed.push(snapshot); } Err(e) => { eprintln!("error: {e}"); - if json { print_json(&completed); } + if json { + print_json(&completed); + } return ExitCode::FAILURE; } } } - if json { print_json(&completed); } + if json { + print_json(&completed); + } ExitCode::SUCCESS } +fn run_maintenance(command: &Command, json: bool) -> ExitCode { + #[cfg(feature = "geoparquet")] + { + let result = match command { + Command::Cleanup(args) => superstac_engine::cleanup_dataset(&args.output, args.apply) + .and_then(|report| { + serde_json::to_value(report).map_err(|e| { + superstac_core::errors::SuperSTACError::SearchFailed(e.to_string()) + }) + }), + Command::Compact(args) => { + let budget = if args.max_dataset_mib == 0 { + None + } else { + match args.max_dataset_mib.checked_mul(1048576) { + Some(n) => Some(n), + None => { + eprintln!("error: budget too large"); + return ExitCode::FAILURE; + } + } + }; + superstac_engine::compact_dataset(&args.output, args.items_per_file, budget) + .and_then(|manifest| { + serde_json::to_value(manifest).map_err(|e| { + superstac_core::errors::SuperSTACError::SearchFailed(e.to_string()) + }) + }) + } + _ => unreachable!(), + }; + match result { + Ok(value) => { + if json { + print_json(&value); + } else { + println!("{}", serde_json::to_string_pretty(&value).unwrap()); + } + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = (command, json); + eprintln!("error: rebuild with --features geoparquet"); + ExitCode::FAILURE + } +} + #[cfg(not(feature = "geoparquet"))] async fn run_ingest( _db: &dyn superstac_core::storages::factory::StorageBackend, @@ -307,26 +444,45 @@ async fn run_ingest( #[cfg(feature = "geoparquet")] fn ingest_progress() -> superstac_engine::ProgressCallback { - use std::{io::{IsTerminal, Write}, sync::{Arc, Mutex}, time::{Duration, Instant}}; + use std::{ + io::{IsTerminal, Write}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; use superstac_engine::IngestPhase; let terminal = std::io::stderr().is_terminal(); let last = Mutex::new(None::); Arc::new(move |event| { let mut last = last.lock().unwrap_or_else(|e| e.into_inner()); let final_event = matches!(event.phase, IngestPhase::Completed | IngestPhase::Failed); - let interval = if terminal { Duration::from_millis(250) } else { Duration::from_secs(2) }; - if !final_event && event.phase != IngestPhase::Retrying && last.is_some_and(|t| t.elapsed() < interval) { + let interval = if terminal { + Duration::from_millis(250) + } else { + Duration::from_secs(2) + }; + if !final_event + && event.phase != IngestPhase::Retrying + && last.is_some_and(|t| t.elapsed() < interval) + { return; } *last = Some(Instant::now()); - let estimate = event.total_items.filter(|total| *total > 0 && *total >= event.items_received) + let estimate = event + .total_items + .filter(|total| *total > 0 && *total >= event.items_received) .map(|total| { let percent = event.items_received as f64 * 100.0 / total as f64; let eta = if event.items_per_second > 0.0 { - format!(", ETA ~{:.0}s", (total - event.items_received) as f64 / event.items_per_second) - } else { String::new() }; + format!( + ", ETA ~{:.0}s", + (total - event.items_received) as f64 / event.items_per_second + ) + } else { + String::new() + }; format!(" / ~{total} ({percent:.1}%{eta})") - }).unwrap_or_default(); + }) + .unwrap_or_default(); let line = format!("{} {:?}: {} received{}, {} saved | {} pages, {} files, {:.1} MiB saved | {:.0}s, {:.0} items/s{}", event.catalog_id, event.phase, event.items_received, estimate, event.items_saved, event.pages, event.files, event.bytes_saved as f64 / 1048576.0, @@ -335,8 +491,12 @@ fn ingest_progress() -> superstac_engine::ProgressCallback { let mut stderr = std::io::stderr().lock(); if terminal { let _ = write!(stderr, "\r\x1b[K{line}"); - if final_event { let _ = writeln!(stderr); } - } else { let _ = writeln!(stderr, "{line}"); } + if final_event { + let _ = writeln!(stderr); + } + } else { + let _ = writeln!(stderr, "{line}"); + } let _ = stderr.flush(); }) } @@ -358,7 +518,10 @@ fn parse_bbox(s: &str) -> Result { match parts.as_slice() { [w, s, e, n] => Ok(Bbox::new(*w, *s, *e, *n)), - _ => Err(format!("expected 4 comma-separated floats, got {}", parts.len())), + _ => Err(format!( + "expected 4 comma-separated floats, got {}", + parts.len() + )), } } @@ -380,8 +543,8 @@ fn init_tracing(settings: &Settings, verbose: bool, quiet: bool) { } }; - let filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(default_level)); + let filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level)); tracing_subscriber::fmt() .with_writer(std::io::stderr) @@ -390,11 +553,7 @@ fn init_tracing(settings: &Settings, verbose: bool, quiet: bool) { .init(); } -async fn run_collections( - engine: &SuperSTACEngine, - args: CollectionsArgs, - json: bool, -) -> ExitCode { +async fn run_collections(engine: &SuperSTACEngine, args: CollectionsArgs, json: bool) -> ExitCode { match (args.catalog, args.collection) { (Some(catalog_id), Some(collection_id)) => { describe_collection(engine, &catalog_id, &collection_id, json).await @@ -466,11 +625,7 @@ async fn describe_collection( if json { print_json(&c); } else { - println!( - "{} — {}", - c.id, - c.title.as_deref().unwrap_or("(no title)") - ); + println!("{} — {}", c.id, c.title.as_deref().unwrap_or("(no title)")); println!("catalog: {}", catalog_id); println!(); match serde_json::to_string_pretty(&c) { @@ -500,12 +655,16 @@ async fn describe_collection( async fn run_search(engine: &SuperSTACEngine, args: SearchArgs, json: bool) -> ExitCode { let query = SearchQuery { collections: args.collections, - ids: if args.ids.is_empty() { None } else { Some(args.ids) }, + ids: if args.ids.is_empty() { + None + } else { + Some(args.ids) + }, intersects: None, bbox: args.bbox, datetime: args.datetime, limit: Some(args.limit), - sortby: None, + sortby: (!args.sortby.is_empty()).then_some(args.sortby), }; let response = match engine.search(query).await { @@ -516,11 +675,12 @@ async fn run_search(engine: &SuperSTACEngine, args: SearchArgs, json: bool) -> E } }; - let exit = if response.metadata.catalogs_queried > 0 && response.metadata.catalogs_succeeded == 0 { - ExitCode::FAILURE - } else { - ExitCode::SUCCESS - }; + let exit = + if response.metadata.catalogs_queried > 0 && response.metadata.catalogs_succeeded == 0 { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + }; if json { print_json(&response); return exit; diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index 0e9f021..8ae245b 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -13,12 +13,12 @@ use crate::{ health::HealthManager, types::SharedStorage, }; +use std::time::Duration; use superstac_core::{ errors::SuperSTACError, models::catalog::{Catalog, CatalogFilters}, storages::factory::StorageBackend, }; -use std::time::Duration; use superstac_search::{ executor::SearchExecutor, @@ -42,6 +42,8 @@ pub struct SuperSTACEngine { executor: SearchExecutor, snapshot_catalog_ids: Option>, auto_search: bool, + #[cfg(feature = "geoparquet")] + local_backend: Option>, pub started: AtomicBool, } @@ -75,6 +77,8 @@ impl SuperSTACEngine { executor: SearchExecutor::new(client), snapshot_catalog_ids: None, auto_search: false, + #[cfg(feature = "geoparquet")] + local_backend: None, started: AtomicBool::new(false), } } @@ -101,7 +105,9 @@ impl SuperSTACEngine { let ids = paths.keys().cloned().collect(); let backend = superstac_geoparquet::GeoParquetBackend::new(paths)?; let mut engine = Self::from_shared(storage); - engine.executor = SearchExecutor::with_backend(Box::new(backend)); + let backend = Arc::new(backend); + engine.executor = SearchExecutor::with_backend(Box::new(backend.clone())); + engine.local_backend = Some(backend); engine.health_manager = None; engine.snapshot_catalog_ids = Some(ids); Ok(engine) @@ -113,14 +119,24 @@ impl SuperSTACEngine { pub fn from_dataset( storage: Box, root: impl AsRef, + ) -> Result { + Self::from_shared_dataset(Arc::new(Mutex::new(storage)), root) + } + + #[cfg(feature = "geoparquet")] + pub fn from_shared_dataset( + storage: SharedStorage, + root: impl AsRef, ) -> Result { let backend = superstac_geoparquet::GeoParquetBackend::from_dataset(root)?; for id in backend.catalog_ids() { - storage.get_catalog(id)?; + storage.lock().get_catalog(id)?; } let ids = backend.catalog_ids().cloned().collect(); - let mut engine = Self::new(storage); - engine.executor = SearchExecutor::with_backend(Box::new(backend)); + let mut engine = Self::from_shared(storage); + let backend = Arc::new(backend); + engine.executor = SearchExecutor::with_backend(Box::new(backend.clone())); + engine.local_backend = Some(backend); engine.health_manager = None; engine.snapshot_catalog_ids = Some(ids); Ok(engine) @@ -134,21 +150,53 @@ impl SuperSTACEngine { root: impl Into, max_snapshot_age: Duration, ) -> Self { - let mut engine = Self::new(storage); - let backend = superstac_geoparquet::auto::AutoBackend::new(root.into(), max_snapshot_age, engine.client.clone()); + Self::from_shared_automatic(Arc::new(Mutex::new(storage)), root, max_snapshot_age) + } + + #[cfg(feature = "geoparquet")] + pub fn from_shared_automatic( + storage: SharedStorage, + root: impl Into, + max_snapshot_age: Duration, + ) -> Self { + let mut engine = Self::from_shared(storage); + let backend = superstac_geoparquet::auto::AutoBackend::new( + root.into(), + max_snapshot_age, + engine.client.clone(), + ); engine.executor = SearchExecutor::with_backend(Box::new(backend)); engine.health_manager = None; engine.auto_search = true; engine } - fn require_remote_discovery(&self) -> Result<(), SuperSTACError> { - if self.snapshot_catalog_ids.is_some() || self.auto_search { - return Err(SuperSTACError::SearchFailed( - "collection discovery is not yet supported in snapshot or auto mode".into(), - )); + async fn discovery_catalogs(&self) -> Result, SuperSTACError> { + self.ensure_started().await?; + let mut catalogs = self.storage.lock().list_catalogs(None)?; + #[cfg(feature = "geoparquet")] + if let Some(backend) = &self.local_backend { + catalogs.retain(|c| { + self.snapshot_catalog_ids + .as_ref() + .is_some_and(|ids| ids.contains(&c.id)) + }); + for catalog in &mut catalogs { + let backend = backend.clone(); + let copy = catalog.clone(); + let collections = tokio::task::spawn_blocking(move || backend.collections(©)) + .await + .map_err(|e| SuperSTACError::SearchFailed(e.to_string()))??; + catalog.supported_collections = Some(collections.into_keys().collect()); + } } - Ok(()) + if self.auto_search { + for catalog in &mut catalogs { + catalog.supported_collections = + Some(capabilities::fetch_supported_collections(&self.client, catalog).await?); + } + } + Ok(catalogs) } /// Run health checks against all catalogs, then introspect `/collections` @@ -198,7 +246,7 @@ impl SuperSTACEngine { self.started.store(false, Ordering::Relaxed); } - /// Run health checks against every catalog, update storage with results. Then update the `supported_collections` sets for any newly-healthy catalogs. + /// Run health checks against every catalog, update storage with results. Then update the `supported_collections` sets for any newly-healthy catalogs. async fn introspect_capabilities(&self) -> Result<(), SuperSTACError> { let healthy_catalogs = { let storage = self.storage.lock(); @@ -217,9 +265,7 @@ impl SuperSTACEngine { "introspected /collections" ); let mut storage = self.storage.lock(); - if let Err(e) = - storage.update_supported_collections(&catalog.id, Some(set)) - { + if let Err(e) = storage.update_supported_collections(&catalog.id, Some(set)) { tracing::warn!( catalog = %catalog.id, error = %e, @@ -253,7 +299,6 @@ impl SuperSTACEngine { /// Federated search. Applies source selection (filter catalogs that /// can't possibly serve the requested collections), then fans out. pub async fn search(&self, query: SearchQuery) -> Result { - self.ensure_started().await?; let (candidate_catalogs, options) = { @@ -271,7 +316,9 @@ impl SuperSTACEngine { catalogs } else if self.auto_search { let mut catalogs = storage.list_catalogs(None)?; - for catalog in &mut catalogs { catalog.supported_collections = None; } + for catalog in &mut catalogs { + catalog.supported_collections = None; + } catalogs } else if let Some(true) = settings.search_healthy_catalogs_only { storage.list_catalogs(Some(CatalogFilters { @@ -320,14 +367,13 @@ impl SuperSTACEngine { .into_iter() .filter(|c| c.supports_any_of(&query.collections)) .collect(); - // Perform the federated search across the selected catalogs. let mut response = self .executor .federated_search(catalogs, query, options) .await?; - + response.metadata.unsupported_collections = unsupported; Ok(response) @@ -336,12 +382,7 @@ impl SuperSTACEngine { /// Aggregated view: every collection ID known across healthy catalogs, /// with the catalogs that serve each. pub async fn list_collections(&self) -> Result, SuperSTACError> { - self.require_remote_discovery()?; - self.ensure_started().await?; - let catalogs = { - let storage = self.storage.lock(); - storage.list_catalogs(None)? - }; + let catalogs = self.discovery_catalogs().await?; Ok(discovery::aggregate_collections(&catalogs)) } @@ -351,12 +392,7 @@ impl SuperSTACEngine { &self, collection_id: &str, ) -> Result, SuperSTACError> { - self.require_remote_discovery()?; - self.ensure_started().await?; - let catalogs = { - let storage = self.storage.lock(); - storage.list_catalogs(None)? - }; + let catalogs = self.discovery_catalogs().await?; Ok(discovery::catalogs_supporting(&catalogs, collection_id)) } @@ -365,12 +401,7 @@ impl SuperSTACEngine { pub async fn collections_by_catalog( &self, ) -> Result>, SuperSTACError> { - self.require_remote_discovery()?; - self.ensure_started().await?; - let catalogs = { - let storage = self.storage.lock(); - storage.list_catalogs(None)? - }; + let catalogs = self.discovery_catalogs().await?; Ok(discovery::collections_by_catalog(&catalogs)) } @@ -383,7 +414,6 @@ impl SuperSTACEngine { catalog_id: &str, collection_id: &str, ) -> Result, SuperSTACError> { - self.require_remote_discovery()?; self.ensure_started().await?; let catalog = { @@ -391,11 +421,21 @@ impl SuperSTACEngine { storage.get_catalog(catalog_id)?.clone() }; + #[cfg(feature = "geoparquet")] + if let Some(backend) = &self.local_backend { + let backend = backend.clone(); + let id = collection_id.to_owned(); + return tokio::task::spawn_blocking(move || { + backend.collections(&catalog).map(|mut c| c.remove(&id)) + }) + .await + .map_err(|e| SuperSTACError::SearchFailed(e.to_string()))?; + } + let local_id = catalog.resolve_collection(collection_id).to_string(); - let stac_client = - stac_io::api::Client::with_client(self.client.clone(), &catalog.url) - .map_err(|e| SuperSTACError::SearchFailed(format!("stac client init: {}", e)))?; + let stac_client = stac_io::api::Client::with_client(self.client.clone(), &catalog.url) + .map_err(|e| SuperSTACError::SearchFailed(format!("stac client init: {}", e)))?; stac_client .collection(&local_id) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d3f6aae..5b23a18 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -17,6 +17,10 @@ pub use types::SharedStorage; #[cfg(feature = "geoparquet")] pub use superstac_geoparquet::{ ingest::{ingest_catalog, IngestOptions}, + maintenance::{cleanup as cleanup_dataset, compact as compact_dataset, CleanupReport}, manifest::{DatasetManifest, IngestScope, Snapshot}, progress::{IngestPhase, IngestProgress, ProgressCallback}, }; + +/// Version of the SuperSTAC engine in this build. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/engine/tests/geoparquet.rs b/crates/engine/tests/geoparquet.rs index 6bc46d6..c8ad778 100644 --- a/crates/engine/tests/geoparquet.rs +++ b/crates/engine/tests/geoparquet.rs @@ -77,11 +77,11 @@ async fn snapshot_search_ignores_remote_health_and_inventory() { ); assert!(response.metadata.unsupported_collections.is_empty()); assert_eq!(response.items[0].item.id, "scene"); - assert!(engine.list_collections().await.is_err()); + assert_eq!(engine.list_collections().await.unwrap()[0].id, "actual"); assert!(engine .describe_collection("snapshot", "actual") .await - .is_err()); + .unwrap().is_some()); engine.shutdown().await; } diff --git a/crates/geoparquet/Cargo.toml b/crates/geoparquet/Cargo.toml index f9907c6..217fca9 100644 --- a/crates/geoparquet/Cargo.toml +++ b/crates/geoparquet/Cargo.toml @@ -1,5 +1,9 @@ [package] name = "superstac-geoparquet" +description = "Scoped STAC GeoParquet ingestion, search, and dataset maintenance for SuperSTAC" +readme = "README.md" +keywords = ["stac", "geoparquet", "geospatial"] +categories = ["science::geo"] version.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/crates/geoparquet/README.md b/crates/geoparquet/README.md index 5aec7a8..de958c8 100644 --- a/crates/geoparquet/README.md +++ b/crates/geoparquet/README.md @@ -1,7 +1,13 @@ # SuperSTAC GeoParquet -Ingest reusable STAC metadata scopes, then search local snapshots or route to -live APIs when local coverage is missing or stale. Imagery is not downloaded. +**Build your satellite-data inventory. Search it again and again.** + +Save metadata for your area of interest, then explore different dates and scenes +without repeating every provider request. When saved coverage is missing or stale, +auto mode queries the live catalogs. Imagery stays with the provider. + +For example, retain Sentinel-2 metadata for a study area and revisit it throughout +your analysis. Refresh that inventory when you need newer records. ```sh cargo build -p superstac-cli --features geoparquet @@ -79,18 +85,32 @@ superstac --mode snapshot --dataset ./data --json search \ --bbox=-4,40,-3.5,40.5 --limit 20 ``` -The newest single scope covering the entire query is selected for each catalog. -Queries are allowed to differ from ingestion within the recorded coverage. -Scope unions are not inferred: a query spanning two separately ingested months -needs a single covering scope or live fallback. Coverage failures appear in +The newest single scope covering the query is preferred. Otherwise, complete +scopes can jointly cover the requested collections, bounding box and time range. +Coverage is checked as a union, not just an enclosing bounding box: gaps still +require live fallback. Intersects queries use their enclosing bbox conservatively. +Overlapping records are deduplicated by source-local collection and item ID; +newest scopes and incremental delta records take precedence. Queries can differ +from ingestion within this recorded coverage. Coverage failures appear in per-catalog response metadata. Snapshot mode has no freshness cutoff, performs no provider monitoring, and never falls back to HTTP. An empty complete snapshot -correctly returns no items. Collection discovery and sorting are not supported. +correctly returns no items. Collection discovery describes the **saved inventory**, including locally derived +STAC Collection extents; it does not claim to reproduce the provider's full +collection metadata. Sorting is supported per catalog with repeated +`--sortby=-datetime` / `--sortby=eo:cloud_cover` (Python: `sortby=["-datetime"]`). +Null values sort last. Sorted queries scan all candidate rows and retain only the +requested top results; the final federated response is not globally sorted. Snapshot engines retain their loaded manifest; reopen one to see updates. File metadata prunes irrelevant collections, dates and bounding boxes. Remaining -files are scanned in batches of 1,024 rows. There is no row-group predicate -pushdown or spatial index yet. +files are scanned in batches of 1,024 rows, pruning row groups using collection, +ID, instant datetime, and available GeoParquet bbox statistics. Missing or nullable +statistics are handled conservatively. New files use 1,024-row groups and order +records by acquisition time within each file. Overlays and multi-scope unions +scan without pruning so an older version of a moved item cannot reappear. +Compaction restores pruning for incremental scopes by resolving their overlays. +Decoding is bounded, but deduplication keeps item keys in memory (proportional to +scanned unique items). Local collection summaries also scan the saved files. ### Automatic selection @@ -100,7 +120,7 @@ superstac --mode auto --dataset ./data --max-snapshot-age-seconds 86400 \ --bbox=-4,40,-3.5,40.5 --limit 20 ``` -For each configured catalog, auto uses one complete, covering scope whose source +For each configured catalog, auto uses complete covering scopes whose source URL still matches and whose age is within the configured maximum (default one day). Age is measured from ingestion **start**, conservatively accounting for long downloads. A value of zero effectively forces live requests. Freshness is @@ -116,7 +136,8 @@ or `source=snapshot` on stderr when logging is enabled. **Auto is read-only:** it does not persist API results, launch ingestion, create a dataset, or mark limited search results as complete coverage. Use `ingest` to save a new scope explicitly. A missing dataset is a cache miss, not an error. -Collection discovery and sorting are not supported in auto mode. +Collection discovery in auto mode queries the live APIs on demand; it does not +start background monitoring. Sorting is forwarded to live APIs on fallback. ### Live and standalone files @@ -140,8 +161,7 @@ checkpoint. Raise the budget and resume to continue. No automatic eviction or deletion occurs. Staleness only affects auto routing; it does not delete archives. Retired generations remain for active readers, so -refreshes require headroom for both old and new files. Cleanup, compaction and -incremental synchronization are future work. A dataset-wide OS lock prevents +refreshes require headroom for both old and new files. Explicit cleanup, compaction and incremental refresh are available below. A dataset-wide OS lock prevents concurrent writers and releases on process exit. ```text @@ -160,6 +180,116 @@ With the engine's `geoparquet` feature enabled, use `ingest_catalog` and `IngestOptions::progress` accepts an `Arc`. Callbacks receive `IngestPhase` and counters, must return promptly, and should -not panic. They can be adapted to application UIs or future Python callbacks. -The Python build feature exists, but Python client methods for these operations -are not exposed yet. Remote object storage is not supported. +not panic. `cleanup_dataset` and `compact_dataset` are also re-exported by +`superstac-engine`. Remote object storage is not supported. + +## Incremental refresh + +```sh +superstac ingest --catalog earth-search --name madrid-feb \ + --collection sentinel-2-l2a --datetime 2025-02-01/2025-03-05 \ + --bbox=-4.5,39.5,-3,41 --output ./data \ + --incremental-since 2025-02-25T00:00:00Z +``` + +The named scope must already exist. Keep its source, collections, spatial bounds +and start time unchanged; its end may advance. `--incremental-since` must overlap +the old end (no gaps). The API fetches only the specified acquisition-time window; +returned records override earlier copies with the same collection/item ID. +Resume uses the same options and verifies that the base scope has not changed. + +This is an acquisition-window overlay, not a provider change feed. It cannot +detect deletions or revisions/late arrivals outside that window. Use an overlap +appropriate to the provider, and periodic full refreshes (omit +`--incremental-since`) when those changes matter. Full refresh replaces that +scope's inventory. Incremental refresh deliberately preserves its original +freshness timestamp: untouched history has not been verified again. + +## Compaction and cleanup + +```sh +superstac compact --output ./data --items-per-file 10000 --max-dataset-mib 2048 +superstac cleanup --output ./data # preview paths and reclaimable bytes +superstac cleanup --output ./data --apply # delete unreferenced Parquet files +``` + +Compaction streams each scope, deduplicates its records, and publishes replacement +files atomically. It retains coverage and freshness timestamps. It requires space +for both old and new generations; a failure leaves the old manifest readable. +Cleanup protects manifest files and durable checkpoint files. Applied cleanup +refuses active snapshot readers or writers; drop/recreate long-lived snapshot +clients before reclaiming files. Preview is safe while readers are active. +Empty directories and checkpoint metadata are retained. + +## Python + +Published wheels include GeoParquet. Source builds use the `geoparquet` Cargo +feature, enabled in `pyproject.toml`; minimal Rust builds can omit it. Check +`superstac.geoparquet_available` at runtime. + +```python +from superstac import Client + +client = Client.from_yaml("superstac.yml") +client.ingest( + "earth-search", "./data", name="madrid-feb", + collections=["sentinel-2-l2a"], datetime="2025-02-01/2025-03-01", + bbox=[-4.5, 39.5, -3, 41], + progress=lambda event: print(event["phase"], event["items_saved"]), +) +local = Client.from_yaml("superstac.yml", mode="snapshot", dataset="./data") +results = local.search( + collections=["sentinel-2-l2a"], datetime="2025-02-10/2025-02-15", + bbox=[-4, 40, -3.5, 40.5], sortby=["-datetime"], limit=20, +) +``` + +`Client`, `AsyncClient`, `open()` and `from_yaml()` accept `mode`, `dataset`, and +`max_snapshot_age_seconds`. Async clients expose awaitable `ingest`, +`compact_dataset`, and `cleanup_dataset`. Constructors and `from_yaml` remain +synchronous. Callbacks receive dictionaries, run on the worker thread, and should +return promptly; callback exceptions are reported as unraisable Python exceptions +without aborting ingestion. `shutdown()` stops monitoring; release the client to +release its snapshot reader lock. Existing snapshot clients retain their loaded +manifest; reopen after ingestion or compaction to see updates. + +Ingestion keyword options include `name`, `collections`, `datetime`, `bbox`, +`resume`, `incremental_since`, `page_size`, `items_per_file`, `max_dataset_mib`, +`timeout_seconds` and explicit `all=True`. Unknown options raise an error. + +## Performance checks + +Run the reproducible synthetic benchmark without downloading provider data: + +```sh +cargo test -p superstac-geoparquet --release benchmark_scoped_search -- --ignored --nocapture + +SUPERSTAC_BENCH_ITEMS=1000000 \ + cargo test -p superstac-geoparquet --release benchmark_scoped_search -- --ignored --nocapture +``` + +It compares the same bounded scanner with and without row-group pruning, verifies +identical results, and reports median latency over five runs. The fixture contains +time-ordered point items; it is a best-case selective time query, not a promise +about provider latency, arbitrary geometries, or gigabyte production archives. + + +## Writer provenance and support reports + +Provider requests use `User-Agent: superstac/`. Report CLI versions with +`superstac --version`, Python versions with `superstac.__version__`, or Rust engine +versions with `superstac_engine::VERSION`. Search diagnostics include +`metadata.superstac_version`. + +Newly ingested and compacted Parquet files preserve standard `geo` and +`stac-geoparquet` metadata and add `created_by=superstac/` plus a JSON +`superstac` key containing `version`, `crate`, and `written_at`. Older files are +unchanged until rewritten. To inspect one with PyArrow: + +```python +import json +import pyarrow.parquet as pq +footer = pq.read_metadata("path/to/part.parquet") +print(footer.created_by) +print(json.loads(footer.metadata[b"superstac"])) +``` diff --git a/crates/geoparquet/src/coverage.rs b/crates/geoparquet/src/coverage.rs new file mode 100644 index 0000000..fdb728a --- /dev/null +++ b/crates/geoparquet/src/coverage.rs @@ -0,0 +1,108 @@ +//! Conservative union coverage in collection, space, and acquisition time. +use crate::{error, manifest::Snapshot}; +use stac::{api::Search, Bbox}; +use superstac_core::errors::SuperSTACError; + +#[derive(Clone, Copy)] +struct Region { + x: [f64; 2], + y: [f64; 2], + t: [i128; 2], +} +fn region(bbox: Option, datetime: Option<&str>) -> Result { + let (start, end) = datetime + .map(stac::datetime::parse) + .transpose() + .map_err(error)? + .unwrap_or_default(); + let nanos = |t: chrono::DateTime| { + i128::from(t.timestamp()) * 1_000_000_000 + i128::from(t.timestamp_subsec_nanos()) + }; + Ok(Region { + x: bbox.map_or([f64::NEG_INFINITY, f64::INFINITY], |b| [b.xmin(), b.xmax()]), + y: bbox.map_or([f64::NEG_INFINITY, f64::INFINITY], |b| [b.ymin(), b.ymax()]), + t: [start.map_or(i128::MIN, nanos), end.map_or(i128::MAX, nanos)], + }) +} +impl Region { + fn subtract(mut self, other: Self, out: &mut Vec) { + let intersection = Self { + x: [self.x[0].max(other.x[0]), self.x[1].min(other.x[1])], + y: [self.y[0].max(other.y[0]), self.y[1].min(other.y[1])], + t: [self.t[0].max(other.t[0]), self.t[1].min(other.t[1])], + }; + if intersection.x[0] > intersection.x[1] + || intersection.y[0] > intersection.y[1] + || intersection.t[0] > intersection.t[1] + { + out.push(self); + return; + } + macro_rules! split { + ($axis:ident) => { + if self.$axis[0] < intersection.$axis[0] { + let mut part = self; + part.$axis[1] = intersection.$axis[0]; + out.push(part); + self.$axis[0] = intersection.$axis[0]; + } + if self.$axis[1] > intersection.$axis[1] { + let mut part = self; + part.$axis[0] = intersection.$axis[1]; + out.push(part); + self.$axis[1] = intersection.$axis[1]; + } + }; + } + split!(x); + split!(y); + split!(t); + } +} + +pub(crate) fn covers(scopes: &[&Snapshot], search: &Search) -> Result { + let bbox = if let Some(geometry) = &search.intersects { + let mut item = stac::Item::new("bounds"); + item.set_geometry(geometry.clone()).map_err(error)?; + item.bbox + } else { + search.items.bbox + }; + // Do not infer a union for wrapping or 3D boxes. + if bbox.is_some_and(|b| !b.is_valid() || !matches!(b, Bbox::TwoDimensional(_))) { + return Ok(false); + } + let requested = region(bbox, search.items.datetime.as_deref())?; + let collections: Vec> = if search.collections.is_empty() { + vec![None] + } else { + search.collections.iter().map(Some).collect() + }; + for collection in collections { + let mut missing = vec![requested]; + for scope in scopes { + if !scope.scope.collections.is_empty() + && collection.is_none_or(|c| !scope.scope.collections.contains(c)) + { + continue; + } + let covered = region(scope.scope.bbox, scope.scope.datetime.as_deref())?; + let mut next = Vec::new(); + for part in missing { + part.subtract(covered, &mut next); + } + missing = next; + if missing.is_empty() { + break; + } + // Bound pathological fragmentation; falling back is safe. + if missing.len() > 100_000 { + return Ok(false); + } + } + if !missing.is_empty() { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/geoparquet/src/ingest.rs b/crates/geoparquet/src/ingest.rs index 693e79f..4677d82 100644 --- a/crates/geoparquet/src/ingest.rs +++ b/crates/geoparquet/src/ingest.rs @@ -21,6 +21,7 @@ use std::{ collections::{BTreeMap, HashSet}, fs::{self, File, OpenOptions}, path::{Path, PathBuf}, + sync::Arc, time::Duration, }; use superstac_core::{errors::SuperSTACError, models::catalog::Catalog}; @@ -40,6 +41,8 @@ pub struct IngestOptions { /// Includes current, old, and incomplete generations under runs/. No eviction. pub max_dataset_bytes: Option, pub progress: Option, + /// Re-fetch acquisition times since this RFC3339 timestamp and overlay an existing named scope. + pub incremental_since: Option, } impl IngestOptions { @@ -54,6 +57,7 @@ impl IngestOptions { name: None, max_dataset_bytes: Some(1024 * 1024 * 1024), progress: None, + incremental_since: None, } } } @@ -76,6 +80,8 @@ struct Checkpoint { last_error: Option, #[serde(default)] visited_requests: HashSet, + #[serde(default)] + base_fingerprint: Option, } /// Ingest or refresh one named scope, retaining other scopes for the catalog. @@ -146,6 +152,30 @@ async fn ingest_inner( .valid() .map_err(error)?; scope.datetime = search.items.datetime.clone(); + let mut request_search = search.clone(); + if let Some(since) = &options.incremental_since { + if options.name.is_none() { + return Err(error("incremental refresh requires an explicit scope name")); + } + let since = chrono::DateTime::parse_from_rfc3339(since) + .map_err(error)? + .with_timezone(&Utc); + let (start, end) = scope + .datetime + .as_deref() + .map(stac::datetime::parse) + .transpose() + .map_err(error)? + .unwrap_or_default(); + if start.is_some_and(|start| since < start) || end.is_some_and(|end| since > end) { + return Err(error("incremental since must lie within requested scope")); + } + request_search.items.datetime = Some(format!( + "{}/{}", + since.to_rfc3339(), + end.map(|e| e.to_rfc3339()).unwrap_or_else(|| "..".into()) + )); + } let initial = PageRequest { url: UrlBuilder::new(&catalog.url) .map_err(error)? @@ -153,7 +183,7 @@ async fn ingest_inner( .to_string(), method: "POST".into(), body: Some( - serde_json::to_value(search) + serde_json::to_value(request_search) .map_err(error)? .as_object() .ok_or_else(|| error("invalid search request"))? @@ -172,6 +202,9 @@ async fn ingest_inner( .map_err(error)?; lock.try_lock_exclusive() .map_err(|e| error(format!("dataset already being written or cannot lock: {e}")))?; + let lock = Arc::new(lock); + // Initialize the shared-reader lock so copied read-only datasets can be opened. + drop(crate::maintenance::lock(&root, ".readers.lock", false)?); fs::create_dir_all(root.join("checkpoints")).map_err(error)?; fs::create_dir_all(root.join("runs")).map_err(error)?; let manifest = if root.join("manifest.json").exists() { @@ -196,6 +229,47 @@ async fn ingest_inner( }) }) .unwrap_or(default_name); + let base = if options.incremental_since.is_some() { + let base = existing + .and_then(|scopes| scopes.get(&name)) + .ok_or_else(|| error("incremental refresh requires an existing named scope"))?; + let (old_start, old_end) = base + .scope + .datetime + .as_deref() + .map(stac::datetime::parse) + .transpose() + .map_err(error)? + .unwrap_or_default(); + let (new_start, new_end) = scope + .datetime + .as_deref() + .map(stac::datetime::parse) + .transpose() + .map_err(error)? + .unwrap_or_default(); + let since = + chrono::DateTime::parse_from_rfc3339(options.incremental_since.as_ref().unwrap()) + .map_err(error)? + .with_timezone(&Utc); + if base.source_url != catalog.url + || base.scope.collections != scope.collections + || base.scope.bbox != scope.bbox + || old_start != new_start + || old_end.is_some_and(|old| since > old) + || new_end.is_some_and(|new| old_end.is_none_or(|old| new < old)) + { + return Err(error("incremental refresh must preserve source, collections, bbox and start; overlap the old end and never shrink coverage")); + } + Some(base.clone()) + } else { + None + }; + let base_fingerprint = base + .as_ref() + .map(|base| serde_json::to_vec(base).map(|bytes| format!("{:x}", Sha256::digest(bytes)))) + .transpose() + .map_err(error)?; let key = format!( "{:x}", Sha256::digest(format!("{}\0{name}", catalog.id).as_bytes()) @@ -220,6 +294,7 @@ async fn ingest_inner( || saved.snapshot.source_url != catalog.url || saved.snapshot.scope != scope || saved.initial != initial + || saved.base_fingerprint != base_fingerprint { return Err(error("checkpoint differs from this request; resume with the original scope and page size")); } @@ -273,6 +348,7 @@ async fn ingest_inner( started_at: Utc::now(), completed_at: None, complete: false, + overlays: options.incremental_since.is_some(), pages: 0, items: 0, files: Vec::new(), @@ -282,6 +358,7 @@ async fn ingest_inner( run: run.strip_prefix(&root).map_err(error)?.to_owned(), last_error: None, visited_requests: HashSet::new(), + base_fingerprint, } }; progress.resume(&checkpoint.snapshot); @@ -297,6 +374,7 @@ async fn ingest_inner( &checkpoint_path, &mut checkpoint, &options, + &lock, progress, ) .await; @@ -310,6 +388,13 @@ async fn ingest_inner( } return Err(e); } + if let Some(base) = base { + // Delta files precede older files so duplicate IDs resolve to the new item. + checkpoint.snapshot.files.extend(base.files); + checkpoint.snapshot.items += base.items; + // A delta does not revalidate old records: do not claim whole-scope freshness. + checkpoint.snapshot.started_at = base.started_at; + } checkpoint.snapshot.complete = true; checkpoint.snapshot.completed_at = Some(Utc::now()); let mut manifest = if root.join("manifest.json").exists() { @@ -335,6 +420,7 @@ async fn download( checkpoint_path: &Path, state: &mut Checkpoint, options: &IngestOptions, + writer_lock: &Arc, progress: &mut Progress, ) -> Result<(), SuperSTACError> { let mut buffer = Vec::new(); @@ -423,7 +509,9 @@ async fn download( let run = root.join(&state.run); let dataset_root = root.to_owned(); progress.emit(IngestPhase::Writing, None); + let writer_lease = writer_lock.clone(); let files = tokio::task::spawn_blocking(move || { + let _writer_lease = writer_lease; write_files(&dataset_root, &run, rows, remaining_bytes) }) .await @@ -455,6 +543,9 @@ async fn fetch_page( for attempt in 0..3 { let mut builder = client.request(method.clone(), &request.url); for (key, value) in &request.headers { + if key.eq_ignore_ascii_case("user-agent") { + continue; + } builder = builder.header(key, value); } if let Some(body) = &request.body { @@ -586,7 +677,7 @@ fn next_request( }) } -fn write_files( +pub(crate) fn write_files( root: &Path, run: &Path, rows: Vec, @@ -600,7 +691,8 @@ fn write_files( .push(item); } let mut files = Vec::new(); - for (collection, items) in groups { + for (collection, mut items) in groups { + items.sort_by_key(|item| (item.properties.datetime, item.id.clone())); let directory = run.join(format!( "collection={}", collection @@ -646,17 +738,41 @@ fn write_files( } } // Infer across the whole bounded file, preserving source-local properties. - stac::geoparquet::WriterBuilder::new(budget::BudgetWriter::new( - file.as_file(), - &mut remaining, - )) - .options(stac::geoarrow::Options { - drop_invalid_attributes: false, - }) - .build(items) - .map_err(error)? - .finish() + let (state, batch) = stac::geoparquet::WriterState::new( + stac::geoarrow::Options { + drop_invalid_attributes: false, + }, + items, + ) + .map_err(error)?; + let mut metadata = state.into_metadata().map_err(error)?; + metadata.push(parquet::format::KeyValue::new( + "superstac".to_owned(), + Some( + serde_json::json!({ + "version": env!("CARGO_PKG_VERSION"), + "crate": "superstac-geoparquet", + "written_at": Utc::now().to_rfc3339(), + }) + .to_string(), + ), + )); + let defaults = stac::geoparquet::WriterOptions::default(); + let mut properties = parquet::file::properties::WriterProperties::builder() + .set_max_row_group_size(1024) + .set_created_by(concat!("superstac/", env!("CARGO_PKG_VERSION")).to_owned()) + .set_key_value_metadata(Some(metadata)); + if let Some(compression) = defaults.compression { + properties = properties.set_compression(compression); + } + let mut writer = parquet::arrow::ArrowWriter::try_new( + budget::BudgetWriter::new(file.as_file(), &mut remaining), + batch.schema(), + Some(properties.build()), + ) .map_err(error)?; + writer.write(&batch).map_err(error)?; + writer.close().map_err(error)?; file.as_file().sync_all().map_err(error)?; let bytes = file.as_file().metadata().map_err(error)?.len(); let (_, path) = file.keep().map_err(error)?; diff --git a/crates/geoparquet/src/lib.rs b/crates/geoparquet/src/lib.rs index ba06ccf..45fdab5 100644 --- a/crates/geoparquet/src/lib.rs +++ b/crates/geoparquet/src/lib.rs @@ -1,13 +1,17 @@ //! Searches local STAC GeoParquet files or managed datasets with coverage manifests. //! //! Files retain source-local collection and asset names. Searches scan bounded -//! batches on Tokio's blocking pool; there is no spatial index or predicate -//! pushdown yet. Each query reopens the file, so publish replacements atomically. +//! batches on Tokio's blocking pool with conservative row-group pruning. Managed +//! snapshots support coverage unions, ordered overlays, and explicit maintenance. pub mod auto; mod budget; +mod coverage; pub mod ingest; +pub mod maintenance; pub mod manifest; pub mod progress; +mod pruning; +mod sorting; use std::{ collections::HashMap, @@ -39,6 +43,7 @@ pub struct GeoParquetBackend { snapshots: std::collections::BTreeMap>, root: Option, + _reader_lock: Option>, } impl GeoParquetBackend { @@ -61,11 +66,13 @@ impl GeoParquetBackend { .collect(), snapshots: Default::default(), root: None, + _reader_lock: None, }) } /// Opens one committed dataset generation; readers retain it during refreshes. pub fn from_dataset(root: impl AsRef) -> Result { let root = root.as_ref(); + let reader_lock = maintenance::lock(root, ".readers.lock", false)?; let manifest = manifest::DatasetManifest::read(root)?; if manifest.catalogs.is_empty() { return Err(error("dataset has no completed snapshots")); @@ -73,7 +80,8 @@ impl GeoParquetBackend { Ok(Self { paths: HashMap::new(), snapshots: manifest.catalogs, - root: Some(root.to_owned()), + root: Some(root.canonicalize().map_err(error)?), + _reader_lock: Some(Arc::new(reader_lock)), }) } @@ -81,41 +89,93 @@ impl GeoParquetBackend { self.paths.keys().chain(self.snapshots.keys()) } - fn select_snapshot( + /// Collection summaries describe the saved inventory, not the provider's full archive. + pub fn collections( &self, catalog: &Catalog, + ) -> Result, SuperSTACError> { + let mut collections = std::collections::BTreeMap::new(); + let mut paths = Vec::new(); + if let Some(root) = &self.root { + for snapshot in self + .snapshots + .get(&catalog.id) + .into_iter() + .flat_map(|s| s.values()) + .filter(|s| s.source_url == catalog.url) + { + for id in &snapshot.scope.collections { + collections.entry(catalog.canonical_collection(id).to_owned()).or_insert_with(|| stac::Collection::new(catalog.canonical_collection(id), "Saved SuperSTAC inventory; extent describes locally stored metadata only")); + } + for file in &snapshot.files { + paths.push(manifest::resolve_file(root, &file.path)?); + } + } + } else { + paths.extend(self.paths.get(&catalog.id).into_iter().flatten().cloned()); + } + paths.sort(); + paths.dedup(); + let mut initialized = std::collections::HashSet::new(); + for path in paths { + visit_items(&path, None, &AtomicBool::new(false), |item| { + if let Some(id) = &item.collection { + let id = catalog.canonical_collection(id).to_owned(); + if initialized.insert(id.clone()) { + collections.insert(id.clone(), stac::Collection::new_from_item(&id, "Saved SuperSTAC inventory; extent describes locally stored metadata only", &item)); + } else if let Some(collection) = collections.get_mut(&id) { + collection.add_item(&item); + collection.links.clear(); + } + } + Ok(true) + })?; + } + Ok(collections) + } + + fn select_snapshots<'a>( + &'a self, + catalog: &Catalog, search: &stac::api::Search, max_age: Option, - ) -> Option<&manifest::Snapshot> { - self.snapshots - .get(&catalog.id)? - .values() - .filter(|snapshot| { - snapshot.source_url == catalog.url - && snapshot.ensure_coverage(search).is_ok() + ) -> Result, SuperSTACError> { + let mut scopes: Vec<_> = self + .snapshots + .get(&catalog.id) + .into_iter() + .flat_map(|s| s.values()) + .filter(|s| { + s.source_url == catalog.url && max_age.is_none_or(|max| { chrono::Utc::now() - .signed_duration_since(snapshot.started_at) + .signed_duration_since(s.started_at) .to_std() .ok() .is_some_and(|age| age <= max) }) }) - .max_by_key(|snapshot| (snapshot.started_at, &snapshot.name)) + .collect(); + scopes.sort_by(|a, b| (b.started_at, &b.name).cmp(&(a.started_at, &a.name))); + if let Some(scope) = scopes.iter().find(|s| s.ensure_coverage(search).is_ok()) { + return Ok(vec![*scope]); + } + if coverage::covers(&scopes, search)? { + Ok(scopes) + } else { + Ok(vec![]) + } } - /// A single complete, fresh scope must cover the whole query. Scope unions - /// are deliberately not inferred from overlapping bounding boxes/intervals. pub fn can_answer( &self, catalog: &Catalog, query: SearchQuery, max_age: std::time::Duration, ) -> Result { - let search = prepare_query(catalog, query)?; - Ok(self - .select_snapshot(catalog, &search, Some(max_age)) - .is_some()) + Ok(!self + .select_snapshots(catalog, &prepare_query(catalog, query)?, Some(max_age))? + .is_empty()) } pub(crate) async fn search_with_age( @@ -127,14 +187,21 @@ impl GeoParquetBackend { ) -> Result, SuperSTACError> { let cap = query.limit.unwrap_or(10).min(options.max_items_per_catalog); let search = prepare_query(catalog, query)?; + let mut overlays = false; let paths = if let Some(root) = &self.root { - let snapshot = self.select_snapshot(catalog, &search, max_age).ok_or_else(|| error(format!( - "query exceeds recorded coverage, source differs, or snapshot is stale for catalog '{}'; narrow the query or ingest another scope", catalog.id - )))?; + let snapshots = self.select_snapshots(catalog, &search, max_age)?; + if snapshots.is_empty() { + return Err(error(format!("query exceeds recorded coverage, source differs, or snapshot is stale for catalog '{}'", catalog.id))); + } + overlays = snapshots.len() > 1 || snapshots.iter().any(|s| s.overlays); let mut paths = Vec::new(); - for file in &snapshot.files { - if file.may_match(&search)? { - paths.push(manifest::resolve_file(root, &file.path)?); + for snapshot in snapshots { + for file in &snapshot.files { + if (overlays || file.may_match(&search)?) + && !paths.contains(&root.join(&file.path)) + { + paths.push(manifest::resolve_file(root, &file.path)?); + } } } paths @@ -148,23 +215,44 @@ impl GeoParquetBackend { return Ok(Vec::new()); } let catalog = catalog.clone(); + let reader_lease = self._reader_lock.clone(); let cancelled = Arc::new(AtomicBool::new(false)); let _cancel_on_drop = CancelOnDrop(cancelled.clone()); tokio::task::spawn_blocking(move || { + let _reader_lease = reader_lease; let mut results = Vec::new(); + let mut seen = std::collections::HashSet::new(); + let sorted = !search.items.sortby.is_empty(); for path in paths { - let remaining = cap - results.len(); - if remaining == 0 || cancelled.load(Ordering::Relaxed) { + visit_items( + &path, + if overlays { None } else { Some(&search) }, + &cancelled, + |mut item| { + if !seen.insert((item.collection.clone(), item.id.clone())) { + return Ok(true); + } + if search.matches(&item).map_err(error)? { + if options.unify_response { + unifier::unify_item(&mut item, &catalog); + } + let result = SearchItem { + catalog_id: catalog.id.clone(), + seen_in: vec![catalog.id.clone()], + item, + }; + if sorted { + sorting::insert(&mut results, result, &search.items.sortby, cap)?; + } else { + results.push(result); + } + } + Ok(sorted || results.len() < cap) + }, + )?; + if !sorted && results.len() >= cap { break; } - results.extend(scan( - path, - catalog.clone(), - search.clone(), - options, - remaining, - cancelled.clone(), - )?); } Ok::<_, SuperSTACError>(results) }) @@ -188,14 +276,12 @@ fn prepare_query( catalog: &Catalog, mut query: SearchQuery, ) -> Result { - if query - .sortby - .as_ref() - .is_some_and(|fields| !fields.is_empty()) - { - return Err(error( - "sortby is not supported by the GeoParquet backend yet", - )); + if query.sortby.as_ref().is_some_and(|fields| { + fields + .iter() + .any(|f| f.trim_start_matches(['+', '-']).is_empty()) + }) { + return Err(error("sortby fields must be nonempty")); } query.collections = query .collections @@ -209,16 +295,24 @@ fn prepare_query( .map_err(error) } -fn scan( - path: PathBuf, - catalog: Catalog, - search: stac::api::Search, - options: BackendSearchOptions, - cap: usize, - cancelled: Arc, -) -> Result, SuperSTACError> { - let file = File::open(&path).map_err(|e| error(format!("{}: {e}", path.display())))?; +/// Stream rows with bounded decoding; the visitor returns false to stop. +pub(crate) fn visit_items( + path: &std::path::Path, + search: Option<&stac::api::Search>, + cancelled: &AtomicBool, + mut visitor: impl FnMut(stac::Item) -> Result, +) -> Result<(), SuperSTACError> { + let file = File::open(path).map_err(|e| error(format!("{}: {e}", path.display())))?; let builder = ParquetRecordBatchReaderBuilder::try_new(file).map_err(error)?; + let groups: Vec<_> = builder + .metadata() + .row_groups() + .iter() + .enumerate() + .filter(|(_, group)| search.is_none_or(|search| pruning::may_match(group, search))) + .map(|(i, _)| i) + .collect(); + tracing::debug!(path = %path.display(), row_groups_total = builder.metadata().num_row_groups(), row_groups_read = groups.len(), "GeoParquet scan"); let metadata = builder .geoparquet_metadata() .transpose() @@ -227,11 +321,16 @@ fn scan( let schema = builder .geoarrow_schema(&metadata, true, Default::default()) .map_err(error)?; - let reader = builder.with_batch_size(1024).build().map_err(error)?; - let mut reader = GeoParquetRecordBatchReader::try_new(reader, schema).map_err(error)?; - let mut results = Vec::new(); - while !cancelled.load(Ordering::Relaxed) { - let Some(batch) = reader.next() else { break }; + let reader = builder + .with_row_groups(groups) + .with_batch_size(1024) + .build() + .map_err(error)?; + let reader = GeoParquetRecordBatchReader::try_new(reader, schema).map_err(error)?; + for batch in reader { + if cancelled.load(Ordering::Relaxed) { + return Err(error("snapshot search cancelled")); + } let batch = batch.map_err(error)?; let schema = batch.schema(); let items = stac::geoarrow::from_record_batch_reader(RecordBatchIterator::new( @@ -240,26 +339,13 @@ fn scan( )) .map_err(error)? .items; - for mut item in items { - if cancelled.load(Ordering::Relaxed) { - return Err(error("snapshot search cancelled")); - } - if search.matches(&item).map_err(error)? { - if options.unify_response { - unifier::unify_item(&mut item, &catalog); - } - results.push(SearchItem { - catalog_id: catalog.id.clone(), - seen_in: vec![catalog.id.clone()], - item, - }); - if results.len() == cap { - return Ok(results); - } + for item in items { + if !visitor(item)? { + return Ok(()); } } } - Ok(results) + Ok(()) } struct CancelOnDrop(Arc); diff --git a/crates/geoparquet/src/maintenance.rs b/crates/geoparquet/src/maintenance.rs new file mode 100644 index 0000000..0081157 --- /dev/null +++ b/crates/geoparquet/src/maintenance.rs @@ -0,0 +1,190 @@ +//! Explicit maintenance. Publication is atomic; cleanup refuses active readers. +use crate::{ + budget, error, + ingest::write_files, + manifest::{atomic_json, resolve_file, DatasetManifest}, + visit_items, +}; +use fs2::FileExt; +use serde::Serialize; +use std::{ + collections::HashSet, + fs::{self, File, OpenOptions}, + path::{Path, PathBuf}, + sync::atomic::AtomicBool, +}; +use superstac_core::errors::SuperSTACError; + +pub(crate) fn lock(root: &Path, name: &str, exclusive: bool) -> Result { + let file = if !exclusive && root.join(name).exists() { + File::open(root.join(name)).map_err(error)? + } else { + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(root.join(name)) + .map_err(error)? + }; + if exclusive { + file.try_lock_exclusive() + } else { + FileExt::try_lock_shared(&file) + } + .map_err(|e| { + error(format!( + "dataset is in use; retry maintenance after readers/writers finish: {e}" + )) + })?; + Ok(file) +} + +#[derive(Debug, Serialize)] +pub struct CleanupReport { + pub files: Vec, + pub bytes: u64, + pub applied: bool, +} + +/// Remove only unreferenced Parquet files. Checkpoint files remain protected. +/// Dry run by default; an exclusive reader lock prevents deleting live generations. +pub fn cleanup(root: impl AsRef, apply: bool) -> Result { + let root = root.as_ref().canonicalize().map_err(error)?; + let _writer = lock(&root, ".writer.lock", true)?; + let _readers = lock(&root, ".readers.lock", apply)?; + let manifest = if root.join("manifest.json").exists() { + DatasetManifest::read(&root)? + } else { + DatasetManifest::default() + }; + let mut referenced = HashSet::new(); + for snapshot in manifest.catalogs.values().flat_map(|s| s.values()) { + for file in &snapshot.files { + referenced.insert(resolve_file(&root, &file.path)?); + } + } + let checkpoints = root.join("checkpoints"); + if checkpoints.exists() { + for entry in fs::read_dir(checkpoints).map_err(error)? { + let path = entry.map_err(error)?.path(); + if path.extension().is_some_and(|e| e == "json") { + let value: serde_json::Value = + serde_json::from_reader(File::open(path).map_err(error)?).map_err(error)?; + let files = value + .pointer("/snapshot/files") + .and_then(|v| v.as_array()) + .ok_or_else(|| error("invalid checkpoint; cleanup aborted"))?; + for file in files { + let path = file + .get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| error("invalid checkpoint file path"))?; + referenced.insert(resolve_file(&root, Path::new(path))?); + } + } + } + } + fn collect(path: &Path, out: &mut Vec) -> Result<(), SuperSTACError> { + if !path.exists() { + return Ok(()); + } + for entry in fs::read_dir(path).map_err(error)? { + let entry = entry.map_err(error)?; + let kind = entry.file_type().map_err(error)?; + if kind.is_symlink() { + return Err(error("cleanup refuses symlinks")); + } + if kind.is_dir() { + collect(&entry.path(), out)?; + } else if entry.path().extension().is_some_and(|e| e == "parquet") { + out.push(entry.path()); + } + } + Ok(()) + } + let mut files = Vec::new(); + collect(&root.join("runs"), &mut files)?; + files.retain(|f| !referenced.contains(f)); + files.sort(); + let bytes = files.iter().try_fold(0u64, |total, path| { + Ok::<_, SuperSTACError>(total + fs::metadata(path).map_err(error)?.len()) + })?; + if apply { + for file in &files { + fs::remove_file(file).map_err(error)?; + } + } + Ok(CleanupReport { + files: files + .into_iter() + .map(|p| p.strip_prefix(&root).unwrap().to_owned()) + .collect(), + bytes, + applied: apply, + }) +} + +/// Rewrite each scope into larger bounded files and remove duplicate item IDs. +/// Newest occurrence wins. Old files are retained until explicit cleanup. +pub fn compact( + root: impl AsRef, + items_per_file: usize, + max_dataset_bytes: Option, +) -> Result { + if items_per_file == 0 || items_per_file > 1_000_000 { + return Err(error("items_per_file must be 1..=1000000")); + } + let root = root.as_ref().canonicalize().map_err(error)?; + let _writer = lock(&root, ".writer.lock", true)?; + let mut manifest = DatasetManifest::read(&root)?; + fs::create_dir_all(root.join("runs")).map_err(error)?; + let mut remaining = max_dataset_bytes + .unwrap_or(u64::MAX) + .checked_sub(budget::used_bytes(&root.join("runs"))?) + .ok_or_else(|| error("existing files exceed storage budget"))?; + let run = tempfile::Builder::new() + .prefix("compact-") + .tempdir_in(root.join("runs")) + .map_err(error)? + .keep(); + #[cfg(unix)] + File::open(root.join("runs")) + .and_then(|f| f.sync_all()) + .map_err(error)?; + for snapshot in manifest.catalogs.values_mut().flat_map(|s| s.values_mut()) { + let mut rows = Vec::new(); + let mut files = Vec::new(); + let mut seen = HashSet::new(); + for file in &snapshot.files { + visit_items( + &resolve_file(&root, &file.path)?, + None, + &AtomicBool::new(false), + |item| { + if seen.insert((item.collection.clone(), item.id.clone())) { + rows.push(item); + } + if rows.len() >= items_per_file { + let written = + write_files(&root, &run, std::mem::take(&mut rows), remaining)?; + remaining = + remaining.saturating_sub(written.iter().map(|f| f.bytes).sum::()); + files.extend(written); + } + Ok(true) + }, + )?; + } + if !rows.is_empty() { + let written = write_files(&root, &run, rows, remaining)?; + remaining = remaining.saturating_sub(written.iter().map(|f| f.bytes).sum::()); + files.extend(written); + } + snapshot.overlays = false; + snapshot.items = files.iter().map(|f| f.items).sum(); + snapshot.files = files; + } + atomic_json(&root.join("manifest.json"), &manifest)?; + Ok(manifest) +} diff --git a/crates/geoparquet/src/manifest.rs b/crates/geoparquet/src/manifest.rs index bf377db..55df075 100644 --- a/crates/geoparquet/src/manifest.rs +++ b/crates/geoparquet/src/manifest.rs @@ -43,6 +43,9 @@ pub struct Snapshot { pub completed_at: Option>, /// True means pagination exhausted for the scope, not a provider-wide archive. pub complete: bool, + /// Delta overlays require newest-first deduplication before filtering. + #[serde(default)] + pub overlays: bool, pub pages: u64, pub items: u64, pub files: Vec, diff --git a/crates/geoparquet/src/pruning.rs b/crates/geoparquet/src/pruning.rs new file mode 100644 index 0000000..757142d --- /dev/null +++ b/crates/geoparquet/src/pruning.rs @@ -0,0 +1,250 @@ +//! Statistics are only used when exclusion is certain; absent/null stats scan normally. +use parquet::{ + basic::{LogicalType, TimeUnit}, + file::{metadata::RowGroupMetaData, statistics::Statistics}, +}; +use stac::api::Search; + +pub(crate) fn may_match(group: &RowGroupMetaData, search: &Search) -> bool { + for column in group.columns() { + let field = column.column_path().string(); + let Some(stats) = column.statistics() else { + continue; + }; + if field == "id" || field == "collection" { + let values = if field == "id" { + &search.ids + } else { + &search.collections + }; + if values.is_empty() { + continue; + } + if let Statistics::ByteArray(stats) = stats { + if let (Some(min), Some(max)) = (stats.min_opt(), stats.max_opt()) { + if !values + .iter() + .any(|v| v.as_bytes() >= min.data() && v.as_bytes() <= max.data()) + { + return false; + } + } + } + } + // A nullable datetime may represent an interval item. Never exclude those + // using the non-null rows' instant statistics. + if field == "datetime" && stats.null_count_opt() == Some(0) { + if let ( + Some(datetime), + Statistics::Int64(stats), + Some(LogicalType::Timestamp { unit, .. }), + ) = ( + &search.items.datetime, + stats, + column.column_descr().logical_type(), + ) { + if let (Ok((start, end)), Some(min), Some(max)) = ( + stac::datetime::parse(datetime), + stats.min_opt(), + stats.max_opt(), + ) { + let scale = match unit { + TimeUnit::MILLIS(_) => 1_000_000i128, + TimeUnit::MICROS(_) => 1_000, + TimeUnit::NANOS(_) => 1, + }; + let ns = |t: chrono::DateTime| { + i128::from(t.timestamp()) * 1_000_000_000 + + i128::from(t.timestamp_subsec_nanos()) + }; + if start.is_some_and(|t| ns(t) > i128::from(*max) * scale) + || end.is_some_and(|t| ns(t) < i128::from(*min) * scale) + { + return false; + } + } + } + } + // GeoParquet covering bbox struct. Null or missing bounds are not pruned. + if stats.null_count_opt() == Some(0) { + if let (Some(bbox), Statistics::Double(stats)) = (search.items.bbox, stats) { + if bbox.is_valid() { + if let (Some(min), Some(max)) = (stats.min_opt(), stats.max_opt()) { + if min.is_finite() + && max.is_finite() + && match field.as_str() { + "bbox.xmin" => *min > bbox.xmax(), + "bbox.xmax" => *max < bbox.xmin(), + "bbox.ymin" => *min > bbox.ymax(), + "bbox.ymax" => *max < bbox.ymin(), + _ => false, + } + { + return false; + } + } + } + } + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{visit_items, GeoParquetBackend}; + use std::{collections::HashMap, fs::File, sync::atomic::AtomicBool, time::Instant}; + use superstac_core::models::catalog::Catalog; + use superstac_search::{ + backend::{BackendSearchOptions, SearchBackend}, + query::SearchQuery, + translator::to_stac_search, + }; + + fn fixture(path: &std::path::Path, count: usize) { + let start = chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc); + let items = (0..count) + .map(|i| { + let mut item = stac::Item::new(format!("scene-{i:08}")); + item.collection = Some("scenes".into()); + item.properties.datetime = Some(start + chrono::Duration::minutes(i as i64)); + item.set_geometry(Some( + serde_json::from_value(serde_json::json!({"type":"Point","coordinates":[1,1]})) + .unwrap(), + )) + .unwrap(); + item + }) + .collect(); + stac::geoparquet::WriterBuilder::new(File::create(path).unwrap()) + .writer_options(stac::geoparquet::WriterOptions::new().with_max_row_group_size(1024)) + .build(items) + .unwrap() + .finish() + .unwrap(); + } + fn query(count: usize) -> SearchQuery { + let start = chrono::DateTime::parse_from_rfc3339("2020-01-01T00:00:00Z").unwrap() + + chrono::Duration::minutes(count as i64 - 20); + SearchQuery { + collections: vec!["scenes".into()], + ids: None, + bbox: None, + intersects: None, + datetime: Some(format!("{}/..", start.to_rfc3339())), + limit: Some(20), + sortby: None, + } + } + #[tokio::test] + async fn statistics_prune_time_groups_without_changing_results() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("items.parquet"); + fixture(&path, 4096); + let query = query(4096); + let search = to_stac_search(query.clone()); + let builder = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new( + File::open(&path).unwrap(), + ) + .unwrap(); + let groups = builder.metadata().row_groups(); + assert_eq!(groups.len(), 4); + assert_eq!(groups.iter().filter(|g| may_match(g, &search)).count(), 1); + let mut baseline = Vec::new(); + visit_items(&path, None, &AtomicBool::new(false), |item| { + if search.matches(&item).unwrap() { + baseline.push(item.id); + } + Ok(true) + }) + .unwrap(); + let backend = GeoParquetBackend::new(HashMap::from([("test".into(), path)])).unwrap(); + let catalog = Catalog::new( + "test", + None::, + "http://127.0.0.1:9", + None::, + None, + ) + .unwrap(); + let result = backend + .search( + &catalog, + query, + BackendSearchOptions { + max_items_per_catalog: 20, + unify_response: false, + }, + ) + .await + .unwrap(); + assert_eq!( + result.iter().map(|r| r.item.id.clone()).collect::>(), + baseline + ); + } + + /// Reproducible local benchmark; no provider access and no persistent data. + #[tokio::test] + #[ignore = "performance benchmark; run explicitly in release mode"] + async fn benchmark_scoped_search() { + let count = std::env::var("SUPERSTAC_BENCH_ITEMS") + .ok() + .map(|s| s.parse().unwrap()) + .unwrap_or(100_000usize); + assert!(count >= 4096); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("items.parquet"); + fixture(&path, count); + let bytes = std::fs::metadata(&path).unwrap().len(); + let query = query(count); + let search = to_stac_search(query.clone()); + let backend = + GeoParquetBackend::new(HashMap::from([("test".into(), path.clone())])).unwrap(); + let catalog = Catalog::new( + "test", + None::, + "http://127.0.0.1:9", + None::, + None, + ) + .unwrap(); + let mut baseline_ms = Vec::new(); + let mut pruned_ms = Vec::new(); + for _ in 0..5 { + let start = Instant::now(); + let mut ids = Vec::new(); + visit_items(&path, None, &AtomicBool::new(false), |item| { + if search.matches(&item).unwrap() { + ids.push(item.id); + } + Ok(ids.len() < 20) + }) + .unwrap(); + baseline_ms.push(start.elapsed().as_secs_f64() * 1000.); + let start = Instant::now(); + let found = backend + .search( + &catalog, + query.clone(), + BackendSearchOptions { + max_items_per_catalog: 20, + unify_response: false, + }, + ) + .await + .unwrap(); + pruned_ms.push(start.elapsed().as_secs_f64() * 1000.); + assert_eq!( + found.iter().map(|r| r.item.id.clone()).collect::>(), + ids + ); + } + baseline_ms.sort_by(f64::total_cmp); + pruned_ms.sort_by(f64::total_cmp); + println!("items={count} parquet_bytes={bytes} repeats=5 baseline_median_ms={:.3} pruned_median_ms={:.3}",baseline_ms[2],pruned_ms[2]); + } +} diff --git a/crates/geoparquet/src/sorting.rs b/crates/geoparquet/src/sorting.rs new file mode 100644 index 0000000..d14e9eb --- /dev/null +++ b/crates/geoparquet/src/sorting.rs @@ -0,0 +1,144 @@ +use crate::error; +use serde_json::Value; +use std::cmp::Ordering; +use superstac_core::errors::SuperSTACError; +use superstac_search::response::SearchItem; +fn field<'a>(value: &'a Value, key: &str) -> &'a Value { + fn lookup<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + v.get(key) + .or_else(|| key.split('.').try_fold(v, |v, part| v.get(part))) + } + if let Some(key) = key.strip_prefix("properties.") { + return value + .get("properties") + .and_then(|p| lookup(p, key)) + .unwrap_or(&Value::Null); + } + lookup(value, key) + .or_else(|| value.get("properties").and_then(|p| lookup(p, key))) + .unwrap_or(&Value::Null) +} +fn compare(a: &Value, b: &Value) -> Ordering { + match (a, b) { + (Value::Null, Value::Null) => Ordering::Equal, + (Value::Null, _) => Ordering::Greater, + (_, Value::Null) => Ordering::Less, + (Value::Number(a), Value::Number(b)) => { + if let (Some(a), Some(b)) = (a.as_i64(), b.as_i64()) { + return a.cmp(&b); + } + if let (Some(a), Some(b)) = (a.as_u64(), b.as_u64()) { + return a.cmp(&b); + } + a.as_f64() + .partial_cmp(&b.as_f64()) + .unwrap_or(Ordering::Equal) + } + (Value::String(a), Value::String(b)) => a.cmp(b), + (Value::Bool(a), Value::Bool(b)) => a.cmp(b), + _ => a.to_string().cmp(&b.to_string()), + } +} +pub(crate) fn insert( + results: &mut Vec, + item: SearchItem, + sort: &[stac::api::Sortby], + cap: usize, +) -> Result<(), SuperSTACError> { + let value = serde_json::to_value(&item.item).map_err(error)?; + let mut position = results.len(); + for (i, other) in results.iter().enumerate() { + let other = serde_json::to_value(&other.item).map_err(error)?; + let mut order = Ordering::Equal; + for spec in sort { + let a = field(&value, &spec.field); + let b = field(&other, &spec.field); + order = if matches!( + spec.field.trim_start_matches("properties."), + "datetime" | "start_datetime" | "end_datetime" | "created" | "updated" + ) { + match ( + a.as_str() + .and_then(|v| chrono::DateTime::parse_from_rfc3339(v).ok()), + b.as_str() + .and_then(|v| chrono::DateTime::parse_from_rfc3339(v).ok()), + ) { + (Some(a), Some(b)) => a.cmp(&b), + _ => compare(a, b), + } + } else { + compare(a, b) + }; + // Nulls sort last in either direction. + if spec.direction == stac::api::Direction::Descending && !a.is_null() && !b.is_null() { + order = order.reverse(); + } + if order != Ordering::Equal { + break; + } + } + if order == Ordering::Equal { + order = compare(&value["collection"], &other["collection"]) + .then_with(|| compare(&value["id"], &other["id"])); + } + if order == Ordering::Less { + position = i; + break; + } + } + if position < cap { + results.insert(position, item); + results.truncate(cap); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn fractional_datetimes_numbers_and_nulls_sort_before_limit() { + let mut results = Vec::new(); + for (id, date) in [ + ("fraction", "2025-01-01T00:00:00.1Z"), + ("instant", "2025-01-01T00:00:00Z"), + ] { + let mut item = stac::Item::new(id); + item.properties.datetime = Some( + chrono::DateTime::parse_from_rfc3339(date) + .unwrap() + .with_timezone(&chrono::Utc), + ); + insert( + &mut results, + SearchItem { + item, + catalog_id: "test".into(), + seen_in: vec![], + }, + &[stac::api::Sortby::asc("datetime")], + 1, + ) + .unwrap(); + } + assert_eq!(results[0].item.id, "instant"); + assert_eq!( + compare( + &serde_json::json!(9007199254740992u64), + &serde_json::json!(9007199254740993u64) + ), + Ordering::Less + ); + assert_eq!( + compare(&Value::Null, &serde_json::json!(5)), + Ordering::Greater + ); + assert_eq!( + field( + &serde_json::json!({"id":"root","properties":{"id":"property"}}), + "properties.id" + ), + "property" + ); + } +} diff --git a/crates/geoparquet/tests/policies.rs b/crates/geoparquet/tests/policies.rs index f488548..43cb2a9 100644 --- a/crates/geoparquet/tests/policies.rs +++ b/crates/geoparquet/tests/policies.rs @@ -60,6 +60,10 @@ fn server(responses: Vec) -> (String, std::thread::JoinHandle<()>) { if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") { let headers = String::from_utf8_lossy(&request[..end]); assert!(headers.starts_with("POST /search ")); + assert!(headers.to_ascii_lowercase().contains(&format!( + "user-agent: superstac/{}", + env!("CARGO_PKG_VERSION") + ))); let length: usize = headers .lines() .find_map(|line| { @@ -133,6 +137,31 @@ async fn retains_scopes_refreshes_one_and_reports_durable_progress() { let manifest = DatasetManifest::read(dir.path()).unwrap(); assert_eq!(manifest.version, 2); assert_eq!(manifest.catalogs["test"].len(), 2); + let path = dir + .path() + .join(&manifest.catalogs["test"]["scope-a"].files[0].path); + let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new( + File::open(path).unwrap(), + ) + .unwrap(); + let footer = reader.metadata().file_metadata(); + assert_eq!( + footer.created_by(), + Some(concat!("superstac/", env!("CARGO_PKG_VERSION"))) + ); + let kv = footer.key_value_metadata().unwrap(); + assert!(kv.iter().any(|v| v.key == "geo")); + assert!(kv.iter().any(|v| v.key == "stac-geoparquet")); + let producer: Value = serde_json::from_str( + kv.iter() + .find(|v| v.key == "superstac") + .unwrap() + .value + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(producer["version"], env!("CARGO_PKG_VERSION")); let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); assert_eq!( backend @@ -154,10 +183,14 @@ async fn retains_scopes_refreshes_one_and_reports_durable_progress() { ); let mut union = query("a"); union.collections.push("b".into()); - assert!(backend - .search(&source, union, search_options()) - .await - .is_err()); + assert_eq!( + backend + .search(&source, union, search_options()) + .await + .unwrap() + .len(), + 2 + ); let events = events.lock().unwrap(); assert!(events .iter() @@ -181,6 +214,7 @@ fn local_snapshot(root: &std::path::Path, source: &Catalog) -> Snapshot { started_at: Utc::now(), completed_at: Some(Utc::now()), complete: true, + overlays: false, pages: 1, items: 1, files: vec![DataFile { @@ -222,7 +256,7 @@ async fn auto_uses_fresh_scope_and_falls_back_for_stale_missing_or_broken_data() let backend = AutoBackend::new( dir.path().into(), Duration::from_secs(86400), - reqwest::Client::builder().no_proxy().build().unwrap(), + reqwest::Client::builder().no_proxy().user_agent(concat!("superstac/", env!("CARGO_PKG_VERSION"))).build().unwrap(), ); assert_eq!( backend @@ -236,7 +270,7 @@ async fn auto_uses_fresh_scope_and_falls_back_for_stale_missing_or_broken_data() let stale = AutoBackend::new( dir.path().into(), Duration::ZERO, - reqwest::Client::builder().no_proxy().build().unwrap(), + reqwest::Client::builder().no_proxy().user_agent(concat!("superstac/", env!("CARGO_PKG_VERSION"))).build().unwrap(), ); assert_eq!( stale @@ -334,3 +368,236 @@ async fn budget_failure_preserves_published_data_and_can_resume() { "new" ); } + +#[tokio::test] +async fn incremental_overlay_compaction_cleanup_and_reader_protection() { + let mut moved = item("same", "a"); + moved + .set_geometry(Some( + serde_json::from_value(json!({"type":"Point","coordinates":[20,20]})).unwrap(), + )) + .unwrap(); + let delta = json!({"type":"FeatureCollection","features":[moved,item("new","a")],"links":[]}); + let first = json!({"type":"FeatureCollection","features":[item("same","a"),item("old","a")],"links":[]}); + let (url, thread) = server(vec![first, delta]); + let source = catalog(&url); + let dir = tempfile::tempdir().unwrap(); + let original = ingest_catalog(&source, ingest_options(dir.path(), "one", "a")) + .await + .unwrap(); + let mut options = ingest_options(dir.path(), "one", "a"); + options.incremental_since = Some("2025-01-01T00:00:00Z".into()); + let refreshed = ingest_catalog(&source, options).await.unwrap(); + thread.join().unwrap(); + assert_eq!(original.started_at, refreshed.started_at); + assert!(refreshed.overlays); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + let mut q = query("a"); + q.bbox = Some(stac::Bbox::TwoDimensional([0., 0., 2., 2.])); + let result = backend.search(&source, q, search_options()).await.unwrap(); + assert_eq!(result.len(), 2); + assert!( + !result.iter().any(|i| i.item.id == "same"), + "old moved item must not reappear" + ); + let mut q = query("a"); + q.sortby = Some(vec!["-id".into()]); + q.limit = Some(2); + let result = backend + .search(&source, q.clone(), search_options()) + .await + .unwrap(); + assert_eq!( + result + .iter() + .map(|i| i.item.id.as_str()) + .collect::>(), + vec!["same", "old"] + ); + let manifest = + superstac_geoparquet::maintenance::compact(dir.path(), 100, Some(1048576)).unwrap(); + assert_eq!(manifest.catalogs["test"]["one"].items, 3); + assert!(!manifest.catalogs["test"]["one"].overlays); + assert!(superstac_geoparquet::maintenance::cleanup(dir.path(), true).is_err()); + // Old readers remain usable until dropped. + assert_eq!( + backend + .search(&source, q.clone(), search_options()) + .await + .unwrap() + .len(), + 2 + ); + drop(backend); + let dry = superstac_geoparquet::maintenance::cleanup(dir.path(), false).unwrap(); + assert!(dry.bytes > 0); + assert!(!dry.applied); + let applied = superstac_geoparquet::maintenance::cleanup(dir.path(), true).unwrap(); + assert_eq!(applied.files, dry.files); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + assert_eq!( + backend + .search(&source, q, search_options()) + .await + .unwrap() + .len(), + 2 + ); + let collections = backend.collections(&source).unwrap(); + assert!(collections.contains_key("a")); +} + +#[tokio::test] +async fn temporal_union_requires_gap_free_coverage() { + let dir = tempfile::tempdir().unwrap(); + let source = catalog("http://127.0.0.1:9"); + let mut left = local_snapshot(dir.path(), &source); + left.name = "left".into(); + left.scope.datetime = Some("2025-01-01T00:00:00Z/2025-01-15T00:00:00Z".into()); + let mut right = left.clone(); + right.name = "right".into(); + right.scope.datetime = Some("2025-01-15T00:00:00Z/2025-02-01T00:00:00Z".into()); + let mut manifest = DatasetManifest { + version: 2, + catalogs: BTreeMap::from([( + "test".into(), + BTreeMap::from([("left".into(), left), ("right".into(), right)]), + )]), + }; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut q = query("a"); + q.datetime = Some("2025-01-01T00:00:00Z/2025-02-01T00:00:00Z".into()); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + assert!(backend + .can_answer(&source, q.clone(), Duration::from_secs(86400)) + .unwrap()); + assert_eq!( + backend + .search(&source, q.clone(), search_options()) + .await + .unwrap() + .len(), + 1 + ); + manifest + .catalogs + .get_mut("test") + .unwrap() + .get_mut("right") + .unwrap() + .scope + .datetime = Some("2025-01-16T00:00:00Z/2025-02-01T00:00:00Z".into()); + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + assert!(!backend + .can_answer(&source, q, Duration::from_secs(86400)) + .unwrap()); +} + +#[tokio::test] +async fn union_rejects_spatial_holes_and_unknown_collections() { + let dir = tempfile::tempdir().unwrap(); + let source = catalog("http://127.0.0.1:9"); + let mut a = local_snapshot(dir.path(), &source); + a.name = "a".into(); + a.scope.bbox = Some(stac::Bbox::TwoDimensional([0., 0., 1., 1.])); + let mut b = a.clone(); + b.name = "b".into(); + b.scope.bbox = Some(stac::Bbox::TwoDimensional([1., 1., 2., 2.])); + let mut manifest = DatasetManifest { + version: 2, + catalogs: BTreeMap::from([( + "test".into(), + BTreeMap::from([("a".into(), a), ("b".into(), b)]), + )]), + }; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut q = query("a"); + q.bbox = Some(stac::Bbox::TwoDimensional([0., 0., 2., 2.])); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + assert!(!backend + .can_answer(&source, q.clone(), Duration::from_secs(60)) + .unwrap()); + manifest + .catalogs + .get_mut("test") + .unwrap() + .get_mut("a") + .unwrap() + .scope + .bbox = Some(stac::Bbox::TwoDimensional([0., 0., 1., 2.])); + manifest + .catalogs + .get_mut("test") + .unwrap() + .get_mut("b") + .unwrap() + .scope + .bbox = Some(stac::Bbox::TwoDimensional([1., 0., 2., 2.])); + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let backend = GeoParquetBackend::from_dataset(dir.path()).unwrap(); + assert!(backend + .can_answer(&source, q.clone(), Duration::from_secs(60)) + .unwrap()); + q.collections.clear(); + assert!(!backend + .can_answer(&source, q, Duration::from_secs(60)) + .unwrap()); +} + +#[tokio::test] +async fn maintenance_protects_checkpoints_and_failed_compaction_preserves_manifest() { + let (url, thread) = server(vec![page("old", "a")]); + let source = catalog(&url); + let dir = tempfile::tempdir().unwrap(); + let snapshot = ingest_catalog(&source, ingest_options(dir.path(), "one", "a")) + .await + .unwrap(); + thread.join().unwrap(); + let before = std::fs::read(dir.path().join("manifest.json")).unwrap(); + let bytes = snapshot.files.iter().map(|f| f.bytes).sum::(); + assert!(superstac_geoparquet::maintenance::compact(dir.path(), 10, Some(bytes + 1)).is_err()); + assert_eq!( + before, + std::fs::read(dir.path().join("manifest.json")).unwrap() + ); + let run = dir.path().join("runs/checkpoint"); + std::fs::create_dir(&run).unwrap(); + let protected = run.join("protected.parquet"); + std::fs::copy(dir.path().join(&snapshot.files[0].path), &protected).unwrap(); + let mut partial = snapshot.clone(); + partial.files[0].path = "runs/checkpoint/protected.parquet".into(); + std::fs::write( + dir.path().join("checkpoints/interrupted.json"), + json!({"snapshot":partial}).to_string(), + ) + .unwrap(); + let orphan = run.join("orphan.parquet"); + std::fs::copy(&protected, &orphan).unwrap(); + let report = superstac_geoparquet::maintenance::cleanup(dir.path(), true).unwrap(); + assert!(report + .files + .contains(&"runs/checkpoint/orphan.parquet".into())); + assert!(protected.exists()); + assert!(!orphan.exists()); + assert_eq!( + before, + std::fs::read(dir.path().join("manifest.json")).unwrap() + ); +} diff --git a/crates/geoparquet/tests/search.rs b/crates/geoparquet/tests/search.rs index 91be601..151a420 100644 --- a/crates/geoparquet/tests/search.rs +++ b/crates/geoparquet/tests/search.rs @@ -152,7 +152,7 @@ async fn rejects_invalid_queries_and_corrupt_files() { q.datetime = Some("invalid".into()); assert!(backend.search(&catalog, q, options()).await.is_err()); let mut q = query(); - q.sortby = Some(vec!["datetime".into()]); + q.sortby = Some(vec!["-".into()]); assert!(backend .search(&catalog, q, options()) .await diff --git a/crates/python/Cargo.toml b/crates/python/Cargo.toml index 5bd5d2f..5a5c8b3 100644 --- a/crates/python/Cargo.toml +++ b/crates/python/Cargo.toml @@ -31,6 +31,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["rt-multi-thread"] } +superstac-geoparquet = { path = "../geoparquet", version = "0.3.0", optional = true } + [features] default = [] -geoparquet = ["superstac-engine/geoparquet"] +geoparquet = ["superstac-engine/geoparquet", "dep:superstac-geoparquet"] diff --git a/crates/python/README.md b/crates/python/README.md index 75e460a..51629d7 100644 --- a/crates/python/README.md +++ b/crates/python/README.md @@ -137,3 +137,40 @@ client = AsyncClient.from_yaml("superstac.yml") # async ## License MIT. + +## GeoParquet inventories (v0.3) + +Wheels include GeoParquet support (`superstac.geoparquet_available`). Both clients +accept `mode="snapshot"` or `mode="auto"`, `dataset="./data"`, and +`max_snapshot_age_seconds=86400`, including through `from_yaml()` and `open()`. +The default is `mode="live"` without a dataset. + +```python +from superstac import Client + +client = Client.from_yaml("superstac.yml") +client.ingest("earth-search", "./data", name="madrid", + collections=["sentinel-2-l2a"], + bbox=[-3.8, 40.3, -3.6, 40.5], + datetime="2025-02-01/2025-02-08", + progress=lambda event: print(event["phase"], event["items_saved"])) +local = Client.from_yaml("superstac.yml", mode="snapshot", dataset="./data") +result = local.search(collections=["sentinel-2-l2a"], + bbox=[-3.8, 40.3, -3.6, 40.5], + datetime="2025-02-01/2025-02-08", sortby=["-datetime"]) +``` + +`AsyncClient.ingest`, `compact_dataset`, and `cleanup_dataset` are awaitable. +Use `incremental_since="2025-02-05T00:00:00Z"` with the same named scope to overlay +an acquisition-time window. This does not detect deletions or changes outside the +window; periodic full refreshes remain necessary. Auto mode falls back to live +APIs when complete fresh local coverage is unavailable, and does not persist live +results. Previews and imagery remain remote. + +Compaction publishes new files; cleanup defaults to a preview: +`client.cleanup_dataset("./data")`. Pass `apply=True` to remove retired files +once snapshot clients have been released. `shutdown()` stops monitoring but does +not release the snapshot reader lock while the client remains alive. + +See the [GeoParquet guide](https://spatialnode.com/superstac/docs/guides/geoparquet) +for scope coverage, progress, storage budgets, and maintenance semantics. diff --git a/crates/python/pyproject.toml b/crates/python/pyproject.toml index 9f3cb3d..c219316 100644 --- a/crates/python/pyproject.toml +++ b/crates/python/pyproject.toml @@ -35,5 +35,5 @@ Changelog = "https://github.com/spatialnode/superstac/blob/main/CHANGELOG.md" # importable as `superstac._superstac` and re-exported by __init__.py. python-source = "python" module-name = "superstac._superstac" -features = ["pyo3/extension-module"] +features = ["pyo3/extension-module", "geoparquet"] diff --git a/crates/python/python/superstac/__init__.py b/crates/python/python/superstac/__init__.py index be549b7..ebfa125 100644 --- a/crates/python/python/superstac/__init__.py +++ b/crates/python/python/superstac/__init__.py @@ -5,6 +5,7 @@ Client, Search, __version__, + geoparquet_available, ) __all__ = [ @@ -12,4 +13,5 @@ "AsyncClient", "Search", "__version__", + "geoparquet_available", ] \ No newline at end of file diff --git a/crates/python/python/superstac/_superstac.pyi b/crates/python/python/superstac/_superstac.pyi index 9d17a92..0616223 100644 --- a/crates/python/python/superstac/_superstac.pyi +++ b/crates/python/python/superstac/_superstac.pyi @@ -1,6 +1,7 @@ -from typing import Any, Iterator +from typing import Any, Iterator, Callable __version__: str +geoparquet_available: bool class Search: @@ -22,6 +23,9 @@ class Client: providers: list[dict[str, Any]] | None = None, settings: dict[str, Any] | None = None, storage: str = "memory", + mode: str = "live", + dataset: str | None = None, + max_snapshot_age_seconds: int = 86400, ) -> None: ... @classmethod def open( @@ -30,9 +34,16 @@ class Client: *, id: str | None = None, storage: str = "memory", + mode: str = "live", + dataset: str | None = None, + max_snapshot_age_seconds: int = 86400, ) -> "Client": ... @classmethod - def from_yaml(cls, yaml_path: str, *, storage: str = "memory") -> "Client": ... + def from_yaml(cls, yaml_path: str, *, storage: str = "memory", mode: str = "live", dataset: str | None = None, max_snapshot_age_seconds: int = 86400) -> "Client": ... + + def ingest(self, catalog_id: str, output: str, *, progress: Callable[[dict[str, Any]], None] | None = None, collections: list[str] = ..., bbox: list[float] | None = None, datetime: str | None = None, name: str | None = None, resume: bool = False, incremental_since: str | None = None, page_size: int = 500, items_per_file: int = 10000, max_dataset_mib: int = 1024, timeout_seconds: int = 60, all: bool = False) -> dict[str, Any]: ... + def cleanup_dataset(self, dataset: str, *, apply: bool = False) -> dict[str, Any]: ... + def compact_dataset(self, dataset: str, *, items_per_file: int = 10000, max_dataset_mib: int = 1024) -> dict[str, Any]: ... # engine lifecycle def start(self) -> None: ... @@ -84,6 +95,9 @@ class AsyncClient: providers: list[dict[str, Any]] | None = None, settings: dict[str, Any] | None = None, storage: str = "memory", + mode: str = "live", + dataset: str | None = None, + max_snapshot_age_seconds: int = 86400, ) -> None: ... @classmethod async def open( @@ -92,12 +106,19 @@ class AsyncClient: *, id: str | None = None, storage: str = "memory", + mode: str = "live", + dataset: str | None = None, + max_snapshot_age_seconds: int = 86400, ) -> "AsyncClient": ... @classmethod def from_yaml( - cls, yaml_path: str, *, storage: str = "memory" + cls, yaml_path: str, *, storage: str = "memory", mode: str = "live", dataset: str | None = None, max_snapshot_age_seconds: int = 86400 ) -> "AsyncClient": ... + async def ingest(self, catalog_id: str, output: str, *, progress: Callable[[dict[str, Any]], None] | None = None, collections: list[str] = ..., bbox: list[float] | None = None, datetime: str | None = None, name: str | None = None, resume: bool = False, incremental_since: str | None = None, page_size: int = 500, items_per_file: int = 10000, max_dataset_mib: int = 1024, timeout_seconds: int = 60, all: bool = False) -> dict[str, Any]: ... + async def cleanup_dataset(self, dataset: str, *, apply: bool = False) -> dict[str, Any]: ... + async def compact_dataset(self, dataset: str, *, items_per_file: int = 10000, max_dataset_mib: int = 1024) -> dict[str, Any]: ... + # engine lifecycle async def start(self) -> None: ... async def shutdown(self) -> None: ... diff --git a/crates/python/src/async_client.rs b/crates/python/src/async_client.rs index bc345c1..5788698 100644 --- a/crates/python/src/async_client.rs +++ b/crates/python/src/async_client.rs @@ -1,20 +1,20 @@ -use std::sync::Arc; use std::sync::atomic::Ordering; +use std::sync::Arc; use pyo3::exceptions::{PyKeyError, PyRuntimeError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyType}; - use superstac_config::init_from_yaml; +use crate::runtime::into_py; use crate::search::PySearch; use crate::shared::{ - Inner, add_catalog_impl, add_catalogs_impl, add_provider_impl, add_providers_impl, build_inner, search_query_from_kwargs, update_catalog_impl, update_provider_impl, update_settings_impl + add_catalog_impl, add_catalogs_impl, add_provider_impl, add_providers_impl, build_inner, + search_query_from_kwargs, update_catalog_impl, update_provider_impl, update_settings_impl, + Inner, }; use crate::utils::{err_to_py, parse_storage_kind, pythonize_obj}; -use crate::runtime::into_py; - #[pyclass(name = "AsyncClient", module = "superstac._superstac")] pub struct PyAsyncClient { @@ -32,7 +32,10 @@ impl PyAsyncClient { catalogs = None, providers = None, settings = None, - storage = "memory" + storage = "memory", + mode = "live", + dataset = None, + max_snapshot_age_seconds = 86400 ))] fn new( config: Option>, @@ -40,32 +43,33 @@ impl PyAsyncClient { providers: Option>, settings: Option>, storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, ) -> PyResult { Ok(Self { - inner: build_inner( - config, - catalogs, - providers, - settings, - storage, - )?, + inner: build_inner(config, catalogs, providers, settings, storage)? + .with_search_backend(mode, dataset, max_snapshot_age_seconds)?, }) } - /// pystac-client compat: connect to a single STAC catalog at `url`. /// Async: returns a coroutine that resolves to a fully-started client. #[classmethod] - #[pyo3(signature = (url, *, id = None, storage = "memory"))] + #[pyo3(signature = (url, *, id = None, storage = "memory", mode = "live", dataset = None, max_snapshot_age_seconds = 86400))] pub fn open<'py>( _cls: &Bound<'_, PyType>, py: Python<'py>, url: &str, id: Option<&str>, storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, ) -> PyResult> { let inner = Inner::from_storage_kind(storage)?; inner.register_open_catalog(url, id)?; + let inner = inner.with_search_backend(mode, dataset, max_snapshot_age_seconds)?; let engine = Arc::clone(&inner.engine); into_py(py, async move { engine.start().await.map_err(err_to_py)?; @@ -74,16 +78,26 @@ impl PyAsyncClient { } #[classmethod] - #[pyo3(signature = (yaml_path, *, storage = "memory"))] - pub fn from_yaml(_cls: &Bound<'_, PyType>, yaml_path: &str, storage: &str) -> PyResult { + #[pyo3(signature = (yaml_path, *, storage = "memory", mode = "live", dataset = None, max_snapshot_age_seconds = 86400))] + pub fn from_yaml( + _cls: &Bound<'_, PyType>, + yaml_path: &str, + storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, + ) -> PyResult { let backend_kind = parse_storage_kind(storage)?; let backend = init_from_yaml(backend_kind, yaml_path).map_err(err_to_py)?; Ok(Self { - inner: Inner::from_backend(backend), + inner: Inner::from_backend(backend).with_search_backend( + mode, + dataset, + max_snapshot_age_seconds, + )?, }) } - fn start<'py>(&self, py: Python<'py>) -> PyResult> { let engine = Arc::clone(&self.inner.engine); into_py(py, async move { engine.start().await.map_err(err_to_py) }) @@ -97,7 +111,6 @@ impl PyAsyncClient { }) } - #[pyo3(signature = (catalog, *, provider = None))] fn add_catalog( &self, @@ -203,7 +216,15 @@ impl PyAsyncClient { let engine = Arc::clone(&self.inner.engine); into_py(py, async move { let resp = engine.search(query).await.map_err(err_to_py)?; - Python::attach(|py| Py::new(py, PySearch { response: resp, cached_items: None })) + Python::attach(|py| { + Py::new( + py, + PySearch { + response: resp, + cached_items: None, + }, + ) + }) }) } @@ -309,19 +330,63 @@ impl PyAsyncClient { }) } + #[pyo3(signature = (catalog_id, output, *, progress = None, **kwargs))] + fn ingest<'py>( + &self, + py: Python<'py>, + catalog_id: &str, + output: &str, + progress: Option>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let job = crate::geoparquet::prepare(&self.inner, catalog_id, output, progress, kwargs)?; + into_py(py, async move { + let result = crate::geoparquet::run(job).await?; + Python::attach(|py| pythonize_obj(py, &result)) + }) + } + + #[pyo3(signature = (dataset, *, apply = false))] + fn cleanup_dataset<'py>( + &self, + py: Python<'py>, + dataset: String, + apply: bool, + ) -> PyResult> { + into_py(py, async move { + let result = + tokio::task::spawn_blocking(move || crate::geoparquet::cleanup(&dataset, apply)) + .await + .map_err(|e| PyRuntimeError::new_err(e.to_string()))??; + Python::attach(|py| pythonize_obj(py, &result)) + }) + } + + #[pyo3(signature = (dataset, *, items_per_file = 10000, max_dataset_mib = 1024))] + fn compact_dataset<'py>( + &self, + py: Python<'py>, + dataset: String, + items_per_file: usize, + max_dataset_mib: u64, + ) -> PyResult> { + into_py(py, async move { + let result = tokio::task::spawn_blocking(move || { + crate::geoparquet::compact(&dataset, items_per_file, max_dataset_mib) + }) + .await + .map_err(|e| PyRuntimeError::new_err(e.to_string()))??; + Python::attach(|py| pythonize_obj(py, &result)) + }) + } + fn __repr__(&self) -> String { let storage = self.inner.storage.lock(); format!( "SuperSTACAsyncClient(catalogs={}, started={}, providers={})", - storage - .list_catalogs(None) - .map(|c| c.len()) - .unwrap_or(0), + storage.list_catalogs(None).map(|c| c.len()).unwrap_or(0), self.inner.engine.started.load(Ordering::Relaxed), - storage - .list_providers(None) - .map(|p| p.len()) - .unwrap_or(0) + storage.list_providers(None).map(|p| p.len()).unwrap_or(0) ) } } diff --git a/crates/python/src/geoparquet.rs b/crates/python/src/geoparquet.rs new file mode 100644 index 0000000..8be2866 --- /dev/null +++ b/crates/python/src/geoparquet.rs @@ -0,0 +1,162 @@ +//! Python adapters keep blocking IO and HTTP outside the GIL. +use crate::shared::Inner; +#[cfg(feature = "geoparquet")] +use crate::utils::{depythonize_into, err_to_py, pythonize_obj}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; +use serde_json::Value; + +#[cfg(feature = "geoparquet")] +pub struct Job( + superstac_core::models::catalog::Catalog, + superstac_geoparquet::ingest::IngestOptions, +); +#[cfg(not(feature = "geoparquet"))] +pub struct Job; +#[cfg(not(feature = "geoparquet"))] +fn unavailable() -> PyErr { + PyValueError::new_err("rebuild superstac with the geoparquet feature") +} + +pub fn prepare( + inner: &Inner, + catalog_id: &str, + output: &str, + progress: Option>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult { + #[cfg(feature = "geoparquet")] + { + #[derive(serde::Deserialize, Default)] + #[serde(default, deny_unknown_fields)] + struct Options { + collections: Vec, + bbox: Option, + datetime: Option, + name: Option, + resume: bool, + incremental_since: Option, + page_size: Option, + items_per_file: Option, + max_dataset_mib: Option, + timeout_seconds: Option, + all: bool, + } + let input: Options = match kwargs { + Some(kw) => depythonize_into(kw.as_any(), "ingest options")?, + None => Options::default(), + }; + let filtered = + !input.collections.is_empty() || input.bbox.is_some() || input.datetime.is_some(); + if input.all == filtered { + return Err(PyValueError::new_err( + "provide collections, bbox or datetime, or explicitly all=True", + )); + } + let scope = superstac_geoparquet::manifest::IngestScope { + collections: input.collections, + datetime: input.datetime, + bbox: input + .bbox + .map(serde_json::from_value) + .transpose() + .map_err(|e| PyValueError::new_err(e.to_string()))?, + }; + let mut options = superstac_geoparquet::ingest::IngestOptions::new(output, scope); + options.name = input.name; + options.resume = input.resume; + options.incremental_since = input.incremental_since; + if let Some(n) = input.page_size { + options.page_size = n; + } + if let Some(n) = input.items_per_file { + options.items_per_file = n; + } + if let Some(n) = input.timeout_seconds { + options.request_timeout = std::time::Duration::from_secs(n); + } + options.max_dataset_bytes = budget(input.max_dataset_mib.unwrap_or(1024))?; + if let Some(callback) = progress { + Python::attach(|py| { + if callback.bind(py).is_callable() { + Ok(()) + } else { + Err(PyValueError::new_err("progress must be callable")) + } + })?; + options.progress = Some(std::sync::Arc::new(move |event| { + Python::attach(|py| { + let result = + pythonize_obj(py, event).and_then(|event| callback.call1(py, (event,))); + if let Err(error) = result { + error.write_unraisable(py, Some(callback.bind(py))); + } + }) + })); + } + let catalog = inner + .storage + .lock() + .get_catalog(catalog_id) + .map_err(err_to_py)? + .clone(); + Ok(Job(catalog, options)) + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = (inner, catalog_id, output, progress, kwargs); + Err(unavailable()) + } +} +#[cfg(feature = "geoparquet")] +fn budget(mib: u64) -> PyResult> { + if mib == 0 { + Ok(None) + } else { + mib.checked_mul(1048576) + .map(Some) + .ok_or_else(|| PyValueError::new_err("budget too large")) + } +} +pub async fn run(job: Job) -> PyResult { + #[cfg(feature = "geoparquet")] + { + let result = superstac_geoparquet::ingest::ingest_catalog(&job.0, job.1) + .await + .map_err(err_to_py)?; + serde_json::to_value(result).map_err(|e| PyValueError::new_err(e.to_string())) + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = job; + Err(unavailable()) + } +} +pub fn cleanup(dataset: &str, apply: bool) -> PyResult { + #[cfg(feature = "geoparquet")] + { + serde_json::to_value( + superstac_geoparquet::maintenance::cleanup(dataset, apply).map_err(err_to_py)?, + ) + .map_err(|e| PyValueError::new_err(e.to_string())) + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = (dataset, apply); + Err(unavailable()) + } +} +pub fn compact(dataset: &str, items: usize, mib: u64) -> PyResult { + #[cfg(feature = "geoparquet")] + { + serde_json::to_value( + superstac_geoparquet::maintenance::compact(dataset, items, budget(mib)?) + .map_err(err_to_py)?, + ) + .map_err(|e| PyValueError::new_err(e.to_string())) + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = (dataset, items, mib); + Err(unavailable()) + } +} diff --git a/crates/python/src/lib.rs b/crates/python/src/lib.rs index fd12ec7..bbb2b61 100644 --- a/crates/python/src/lib.rs +++ b/crates/python/src/lib.rs @@ -15,6 +15,7 @@ use pyo3::prelude::*; use crate::{{async_client::PyAsyncClient, sync_client::PyClient}, search::PySearch}; mod search; +mod geoparquet; mod shared; mod utils; mod runtime; @@ -31,6 +32,7 @@ fn _superstac(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add("__version__", VERSION)?; + m.add("geoparquet_available", cfg!(feature = "geoparquet"))?; Ok(()) } diff --git a/crates/python/src/shared.rs b/crates/python/src/shared.rs index 93a2322..fa0abb1 100644 --- a/crates/python/src/shared.rs +++ b/crates/python/src/shared.rs @@ -5,6 +5,9 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; +use crate::utils::{ + depythonize_into, derive_id_from_url, err_to_py, parse_storage_kind, pythonize_obj, +}; use superstac_config::config::{CatalogConfig, CatalogProviderConfig}; use superstac_core::{ models::{ @@ -16,7 +19,6 @@ use superstac_core::{ }; use superstac_engine::{SharedStorage, SuperSTACEngine}; use superstac_search::query::SearchQuery; -use crate::utils::{err_to_py, parse_storage_kind, pythonize_obj, depythonize_into, derive_id_from_url}; #[derive(Clone)] pub struct Inner { @@ -38,6 +40,44 @@ impl Inner { Ok(Self::from_backend(parse_storage_kind(kind)?.init())) } + pub fn with_search_backend( + mut self, + mode: &str, + dataset: Option<&str>, + max_age: u64, + ) -> PyResult { + if mode == "live" && dataset.is_none() { + return Ok(self); + } + if !matches!(mode, "snapshot" | "auto") || dataset.is_none() { + return Err(PyValueError::new_err( + "mode must be live (without dataset), snapshot or auto (with dataset)", + )); + } + #[cfg(feature = "geoparquet")] + { + let root = dataset.unwrap(); + self.engine = Arc::new(if mode == "snapshot" { + SuperSTACEngine::from_shared_dataset(self.storage.clone(), root) + .map_err(err_to_py)? + } else { + SuperSTACEngine::from_shared_automatic( + self.storage.clone(), + root, + std::time::Duration::from_secs(max_age), + ) + }); + Ok(self) + } + #[cfg(not(feature = "geoparquet"))] + { + let _ = (&mut self, max_age); + Err(PyValueError::new_err( + "rebuild superstac with the geoparquet feature", + )) + } + } + pub fn register_open_catalog(&self, url: &str, id: Option<&str>) -> PyResult<()> { let catalog_id = match id { Some(s) => s.to_string(), @@ -194,7 +234,6 @@ pub fn search_query_from_kwargs(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult< depythonize_into(kw.as_any(), "search query") } - #[allow(clippy::too_many_arguments)] pub fn build_inner( config: Option>, @@ -236,4 +275,3 @@ pub fn build_inner( Ok(inner) } - diff --git a/crates/python/src/sync_client.rs b/crates/python/src/sync_client.rs index 30d29d3..d624aae 100644 --- a/crates/python/src/sync_client.rs +++ b/crates/python/src/sync_client.rs @@ -11,7 +11,9 @@ use superstac_core::errors::SuperSTACError; use crate::runtime::block_on; use crate::search::PySearch; use crate::shared::{ - Inner, add_catalog_impl, add_catalogs_impl, add_provider_impl, add_providers_impl, build_inner, search_query_from_kwargs, update_catalog_impl, update_provider_impl, update_settings_impl + add_catalog_impl, add_catalogs_impl, add_provider_impl, add_providers_impl, build_inner, + search_query_from_kwargs, update_catalog_impl, update_provider_impl, update_settings_impl, + Inner, }; use crate::utils::{err_to_py, parse_storage_kind, pythonize_obj}; @@ -31,7 +33,10 @@ impl PyClient { catalogs = None, providers = None, settings = None, - storage = "memory" + storage = "memory", + mode = "live", + dataset = None, + max_snapshot_age_seconds = 86400 ))] fn new( config: Option>, @@ -39,9 +44,13 @@ impl PyClient { providers: Option>, settings: Option>, storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, ) -> PyResult { Ok(Self { - inner: build_inner(config, catalogs, providers, settings, storage)?, + inner: build_inner(config, catalogs, providers, settings, storage)? + .with_search_backend(mode, dataset, max_snapshot_age_seconds)?, }) } @@ -49,17 +58,21 @@ impl PyClient { /// catalog id defaults to the first dot-separated label of the hostname. /// Calls `start()` before returning so the result is ready to `search()`. #[classmethod] - #[pyo3(signature = (url, *, id = None, storage = "memory"))] + #[pyo3(signature = (url, *, id = None, storage = "memory", mode = "live", dataset = None, max_snapshot_age_seconds = 86400))] fn open( _cls: &Bound<'_, PyType>, py: Python<'_>, url: &str, id: Option<&str>, storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, ) -> PyResult { let inner = Inner::from_storage_kind(storage)?; inner.register_open_catalog(url, id)?; + let inner = inner.with_search_backend(mode, dataset, max_snapshot_age_seconds)?; let engine = Arc::clone(&inner.engine); @@ -70,12 +83,23 @@ impl PyClient { /// Load catalogs/providers/settings from a `superstac.yml` file. #[classmethod] - #[pyo3(signature = (yaml_path, *, storage = "memory"))] - fn from_yaml(_cls: &Bound<'_, PyType>, yaml_path: &str, storage: &str) -> PyResult { + #[pyo3(signature = (yaml_path, *, storage = "memory", mode = "live", dataset = None, max_snapshot_age_seconds = 86400))] + fn from_yaml( + _cls: &Bound<'_, PyType>, + yaml_path: &str, + storage: &str, + mode: &str, + dataset: Option<&str>, + max_snapshot_age_seconds: u64, + ) -> PyResult { let backend_kind = parse_storage_kind(storage)?; let backend = init_from_yaml(backend_kind, yaml_path).map_err(err_to_py)?; Ok(Self { - inner: Inner::from_backend(backend), + inner: Inner::from_backend(backend).with_search_backend( + mode, + dataset, + max_snapshot_age_seconds, + )?, }) } @@ -320,19 +344,48 @@ impl PyClient { } } + /// Download a complete scoped metadata inventory. Progress callbacks receive dicts. + #[pyo3(signature = (catalog_id, output, *, progress = None, **kwargs))] + fn ingest( + &self, + py: Python<'_>, + catalog_id: &str, + output: &str, + progress: Option>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let job = crate::geoparquet::prepare(&self.inner, catalog_id, output, progress, kwargs)?; + let result = block_on(py, crate::geoparquet::run(job))?; + pythonize_obj(py, &result) + } + + #[pyo3(signature = (dataset, *, apply = false))] + fn cleanup_dataset(&self, py: Python<'_>, dataset: String, apply: bool) -> PyResult> { + let result = py.detach(move || crate::geoparquet::cleanup(&dataset, apply))?; + pythonize_obj(py, &result) + } + + #[pyo3(signature = (dataset, *, items_per_file = 10000, max_dataset_mib = 1024))] + fn compact_dataset( + &self, + py: Python<'_>, + dataset: String, + items_per_file: usize, + max_dataset_mib: u64, + ) -> PyResult> { + let result = py.detach(move || { + crate::geoparquet::compact(&dataset, items_per_file, max_dataset_mib) + })?; + pythonize_obj(py, &result) + } + fn __repr__(&self) -> String { let storage = self.inner.storage.lock(); format!( "SuperSTACSyncClient(catalogs={}, started={}, providers={})", - storage - .list_catalogs(None) - .map(|c| c.len()) - .unwrap_or(0), + storage.list_catalogs(None).map(|c| c.len()).unwrap_or(0), self.inner.engine.started.load(Ordering::Relaxed), - storage - .list_providers(None) - .map(|p| p.len()) - .unwrap_or(0) + storage.list_providers(None).map(|p| p.len()).unwrap_or(0) ) } } diff --git a/crates/python/tests/test_wheel.py b/crates/python/tests/test_wheel.py index f92bc2b..be1a013 100644 --- a/crates/python/tests/test_wheel.py +++ b/crates/python/tests/test_wheel.py @@ -16,6 +16,7 @@ def log_message(self, *args): pass def do_HEAD(self): + assert self.headers.get("User-Agent") == f"superstac/{superstac.__version__}" self.send_response(200) self.end_headers() @@ -24,6 +25,7 @@ def do_POST(self): self.do_GET() def do_GET(self): + assert self.headers.get("User-Agent") == f"superstac/{superstac.__version__}" path = self.path.split("?", 1)[0] status = 200 if path.endswith("/collections"): @@ -88,6 +90,7 @@ def check_results(self, result): self.assertEqual(result.to_geojson(), result.item_collection_as_dict()) self.assertEqual(result.to_geojson()["type"], "FeatureCollection") self.assertEqual(result.metadata["catalogs_failed"], 0) + self.assertEqual(result.metadata["superstac_version"], superstac.__version__) self.assertEqual(result.metadata["catalogs_queried"], 2) def test_installed_metadata(self): @@ -138,5 +141,53 @@ def test_configuration_and_invalid_storage(self): client.shutdown() + +@unittest.skipUnless(getattr(superstac, "geoparquet_available", False), "GeoParquet feature disabled") +class GeoParquetWheelTests(unittest.TestCase): + setUpClass = WheelTests.__dict__["setUpClass"] + tearDownClass = WheelTests.__dict__["tearDownClass"] + catalogs = WheelTests.catalogs + client = WheelTests.client + def test_snapshot_ingest_progress_sort_discovery_and_maintenance(self): + import tempfile + import gc + with tempfile.TemporaryDirectory() as root: + client = self.client(ids=("first",)) + events = [] + snapshot = client.ingest("first", root, collections=["test-scenes"], name="one", progress=events.append) + self.assertTrue(snapshot["complete"]) + self.assertEqual(events[-1]["phase"], "completed") + local = Client(catalogs=self.catalogs("first"), mode="snapshot", dataset=root) + self.assertEqual([i["id"] for i in local.search(collections=["test-scenes"], sortby=["-id"], limit=1).items()], ["shared"]) + self.assertEqual(local.list_collections()[0]["id"], "test-scenes") + self.assertEqual(local.get_collection("test-scenes")["id"], "test-scenes") + auto = Client(catalogs=self.catalogs("first"), mode="auto", dataset=root) + self.assertEqual(len(auto.search(collections=["test-scenes"])), 2) + client.compact_dataset(root) + with self.assertRaises(Exception): + client.cleanup_dataset(root, apply=True) + del local + gc.collect() + self.assertGreater(client.cleanup_dataset(root)["bytes"], 0) + self.assertTrue(client.cleanup_dataset(root, apply=True)["applied"]) + client.shutdown() + auto.shutdown() + + def test_async_ingest_and_snapshot(self): + import tempfile + async def run(root): + client = self.client(AsyncClient, ids=("first",)) + events = [] + saved = await client.ingest("first", root, collections=["test-scenes"], name="one", progress=events.append) + self.assertTrue(saved["complete"]) + local = AsyncClient(catalogs=self.catalogs("first"), mode="snapshot", dataset=root) + self.assertEqual(len(await local.search(collections=["test-scenes"])), 2) + self.assertEqual((await local.list_collections())[0]["id"], "test-scenes") + await local.shutdown() + await client.shutdown() + with tempfile.TemporaryDirectory() as root: + asyncio.run(run(root)) + + if __name__ == "__main__": unittest.main() diff --git a/crates/search/src/aggregator.rs b/crates/search/src/aggregator.rs index 9bef054..74ea891 100644 --- a/crates/search/src/aggregator.rs +++ b/crates/search/src/aggregator.rs @@ -17,11 +17,7 @@ impl SearchAggregator { let flat: Vec = results.into_iter().flatten().collect(); let pre_dedup_count = flat.len(); - let items = if deduplicate { - Self::dedup(flat) - } else { - flat - }; + let items = if deduplicate { Self::dedup(flat) } else { flat }; let total_items = items.len(); let duplicates_removed = pre_dedup_count - total_items; @@ -30,6 +26,7 @@ impl SearchAggregator { SearchResponse { items, metadata: SearchMetadata { + superstac_version: env!("CARGO_PKG_VERSION").to_owned(), catalogs_queried, catalogs_succeeded: catalogs_queried - catalogs_failed, catalogs_failed, diff --git a/crates/search/src/backend.rs b/crates/search/src/backend.rs index d05bb7e..e8e4fb9 100644 --- a/crates/search/src/backend.rs +++ b/crates/search/src/backend.rs @@ -29,3 +29,14 @@ pub trait SearchBackend: Send + Sync { options: BackendSearchOptions, ) -> BoxFuture<'a, Result, SuperSTACError>>; } + +impl SearchBackend for std::sync::Arc { + fn search<'a>( + &'a self, + catalog: &'a Catalog, + query: SearchQuery, + options: BackendSearchOptions, + ) -> BoxFuture<'a, Result, SuperSTACError>> { + (**self).search(catalog, query, options) + } +} diff --git a/crates/search/src/response.rs b/crates/search/src/response.rs index b4fa8aa..9958eef 100644 --- a/crates/search/src/response.rs +++ b/crates/search/src/response.rs @@ -21,6 +21,9 @@ pub struct SearchItem { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SearchMetadata { + /// Producer version for support reports. Older serialized responses omit it. + #[serde(default)] + pub superstac_version: String, pub catalogs_queried: usize, pub catalogs_succeeded: usize, pub catalogs_failed: usize, diff --git a/crates/search/src/translator.rs b/crates/search/src/translator.rs index 28c6e6d..d4b1c52 100644 --- a/crates/search/src/translator.rs +++ b/crates/search/src/translator.rs @@ -1,27 +1,21 @@ -use stac::api::Search; use crate::query::SearchQuery; +use stac::api::Search; /// Convert a SuperSTAC `SearchQuery` into the typed `stac::api::Search` the /// stac-io client expects. Field-by-field copy with no canonicalization — /// alias rewriting happens upstream in the STAC API backend. -pub fn to_stac_search( - query: SearchQuery, -) -> Search { +pub fn to_stac_search(query: SearchQuery) -> Search { let mut search = Search::new(); if !query.collections.is_empty() { - search = search.collections( - query.collections, - ); + search = search.collections(query.collections); } if let Some(ids) = query.ids { search = search.ids(ids); } - if let Some(intersects) = - query.intersects - { + if let Some(intersects) = query.intersects { search = search.intersects(intersects); } @@ -29,13 +23,23 @@ pub fn to_stac_search( search = search.limit(limit as u64); } - if let Some(datetime) = - query.datetime - { + if let Some(datetime) = query.datetime { search = search.datetime(datetime); } if let Some(bbox) = query.bbox { - search = search.bbox(bbox); -} + search = search.bbox(bbox); + } + if let Some(fields) = query.sortby { + search.items.sortby = fields + .into_iter() + .map(|s| { + if let Some(field) = s.strip_prefix('-') { + stac::api::Sortby::desc(field) + } else { + stac::api::Sortby::asc(s.strip_prefix('+').unwrap_or(&s)) + } + }) + .collect(); + } search -} \ No newline at end of file +} diff --git a/docs/app/page.tsx b/docs/app/page.tsx index 57e202d..eecc740 100644 --- a/docs/app/page.tsx +++ b/docs/app/page.tsx @@ -46,8 +46,8 @@ export default function Home() {
- - v0.2 · Alpha + + v0.3 · What’s new

Many catalogs.
One search.

Search across STAC catalogs through one interface, with Python, Rust, or the command line.

diff --git a/docs/content/docs/cli/overview.md b/docs/content/docs/cli/overview.md index 8b32226..ab8d4f6 100644 --- a/docs/content/docs/cli/overview.md +++ b/docs/content/docs/cli/overview.md @@ -31,7 +31,7 @@ superstac search -c sentinel-2-l2a --id scene-id | `-l`, `--limit N` | Maximum items per catalog; default `10`. | | `--id ID` | Item ID; repeatable. | -For negative coordinates, use `--bbox=-10,35,5,45` to avoid flag parsing ambiguity. `intersects` and `sortby` are not CLI options. +For negative coordinates, use `--bbox=-10,35,5,45` to avoid flag parsing ambiguity. `intersects` is not a CLI option. Sorting accepts repeated `--sortby=-datetime` or `--sortby=eo:cloud_cover` flags. ## Collection discovery @@ -60,3 +60,5 @@ The JSON search response contains `items` and `metadata`. Each item is wrapped w > The current tracing subscriber uses its default writer, which can put logs on standard output. For reliable JSON piping, set `logging_enabled: false` in your configuration before using `jq` or another parser. `--quiet` changes the level but does not disable all logs. `RUST_LOG` overrides the CLI-selected logging level when logging is enabled. Per-catalog failures may appear inside a successfully serialized response; check the metadata rather than relying only on the exit code. + +See [GeoParquet inventories](/docs/guides/geoparquet) for ingestion, backend selection, and dataset maintenance. diff --git a/docs/content/docs/guides/benchmarks.md b/docs/content/docs/guides/benchmarks.md new file mode 100644 index 0000000..df61430 --- /dev/null +++ b/docs/content/docs/guides/benchmarks.md @@ -0,0 +1,58 @@ +--- +title: Local search benchmark +description: Reproduce a selective GeoParquet metadata search and see where row-group pruning helps. +--- + +## Less metadata to decode + +For a selective time query over **100,000 synthetic STAC items**, local search took +**4.2 ms with row-group pruning**, compared with **411.3 ms without it** in this run. +Both paths used the same bounded decoder and returned identical records. + +![Median search latency: 411.3 milliseconds without row-group pruning and 4.2 milliseconds with pruning, over 100,000 synthetic items.](/superstac/benchmarks/geoparquet-v0.3.svg) + +| Measurement | Value | +| --- | --- | +| Fixture | 100,000 time-ordered point items, one collection | +| File size | 858,390 bytes; minimal, highly compressible metadata | +| Layout | 1,024 rows per group | +| Query | The last 20 records by acquisition-time filter; no sorting | +| Statistic | Median of five sequential runs; OS cache may be warm | +| Environment | Apple M3 Pro, 36 GiB RAM, macOS 15.5, Rust 1.88.0 | +| Build | SuperSTAC v0.3 development code, optimized release profile | +| Recorded | October 2, 2026 | + +The baseline decodes row groups sequentially until the result limit. The optimized +path consults Parquet statistics and skips groups that cannot match. The benchmark +asserts that the returned IDs are identical on every repetition. + +## What this means for your workflow + +Time-ordered metadata can make repeated, selective time searches much cheaper. +This is useful when exploring an already saved inventory. It does **not** show +that SuperSTAC is faster than a provider API: no remote API was measured, and +initial ingestion time is excluded. + +This fixture is a favorable pruning case, not a production Sentinel archive. +Broad queries, complex geometry, large asset dictionaries, sorting, scope unions, +and incremental overlays can cost more. Unions and overlays disable pruning until +safe to filter without resurrecting old item versions; compacting an incremental +scope resolves its overlays. Deduplication retains item keys in memory. + +## Reproduce it + +```sh +cargo test -p superstac-geoparquet --release benchmark_scoped_search -- --ignored --nocapture +``` + +The fixture is generated locally and removed afterward. There are no provider +requests. Output includes the item count, file bytes, and both median latencies. +Increase `SUPERSTAC_BENCH_ITEMS` only when you have enough memory: fixture generation +currently materializes the synthetic items, unlike the bounded ingestion buffers. + +[Download measurements as JSON](/superstac/benchmarks/geoparquet-v0.3.json) · +[Download the figure](/superstac/benchmarks/geoparquet-v0.3.png) · +[Plotting script](https://github.com/spatialnode/superstac/blob/main/scripts/plot-geoparquet-benchmark.py) + +Start with the [GeoParquet notebook](/docs/python/geoparquet-notebook) or read the +[dataset guide](/docs/guides/geoparquet) before measuring your own inventory. diff --git a/docs/content/docs/guides/geoparquet.md b/docs/content/docs/guides/geoparquet.md new file mode 100644 index 0000000..df9bf12 --- /dev/null +++ b/docs/content/docs/guides/geoparquet.md @@ -0,0 +1,296 @@ +--- +title: GeoParquet inventories +description: Save the metadata for your area of interest and explore it repeatedly without querying every provider again. +--- + +**Build your satellite-data inventory. Search it again and again.** + +Save metadata for your area of interest, then explore different dates and scenes +without repeating every provider request. When saved coverage is missing or stale, +auto mode queries the live catalogs. Imagery stays with the provider. + +For example, retain Sentinel-2 metadata for a study area and revisit it throughout +your analysis. Refresh that inventory when you need newer records. + +```sh +cargo build -p superstac-cli --features geoparquet +``` + +Catalog IDs must exist in `superstac.yml`. Collection arguments use configured +canonical names; stored items retain the provider's original names. + +## Ingestion and progress + +```sh +superstac ingest --catalog earth-search --name madrid-feb \ + --collection sentinel-2-l2a --datetime 2025-02-01/2025-03-01 \ + --bbox=-4.5,39.5,-3,41 --output ./data --max-dataset-mib 1024 +``` + +Repeat `--catalog` to ingest multiple catalogs sequentially. Supply a collection, +spatial or temporal filter, or explicitly use `--all` to ingest an entire catalog. +Ingestion runs only when requested, never automatically at startup or on search. + +Progress goes to stderr; `--json` stdout remains machine-readable. Terminals get +an updating line, redirected output gets throttled log lines. `--no-progress` +disables progress; `--quiet` also suppresses summaries. Events distinguish items +received from items saved in a durable checkpoint, and report pages, file count, +compressed bytes saved, elapsed time and throughput. A provider's optional +`numberMatched` (or `context.matched`) supplies an approximate percentage and ETA. +Without a total, only counters are displayed. Estimates can change during paging. + +`--page-size 500` controls the API page size, not the total result count. All next +links are followed, including on empty pages. Initial requests use POST `/search`; +GET/POST continuation links, relative URLs, and STAC body/header merging are +supported. Connection errors and HTTP 429/5xx receive up to three attempts. +`--timeout-seconds 60` bounds each request; long Retry-After pauses stop with a +resumable checkpoint. + +`--items-per-file 10000` is a row target, flushed at page boundaries and split by +collection. A 64 MiB response-byte threshold also flushes buffers; individual +API responses are capped at 32 MiB. Parsed objects and encoding require extra +memory. Files infer schemas across a bounded group; unsupported attributes or +schema combinations fail instead of being silently discarded. Asset/link hrefs +are made absolute before writing. + +## Retained scopes and refresh + +`--name madrid-feb` identifies a scope within a catalog. A successful ingest using +the same name replaces only that scope; other names and catalogs are retained. +Changing the bounds under the same name intentionally replaces its old coverage. + +Without a name, SuperSTAC reuses a matching existing scope, or derives a stable +name from normalized coverage. Different regions/months therefore coexist. + +Repeat an interrupted command with `--resume`, keeping name, scope, catalog and +page size the same. Resume starts at the last checkpoint; uncommitted buffered +pages may be fetched again. Expired provider continuation tokens require a new +run without `--resume`. If a matching snapshot is already complete and has no +checkpoint, resume returns it without downloading again (it does not refresh it). + +The version-2 manifest groups snapshots as `catalogs[catalog_id][scope_name]`. +Version-1 manifests remain readable as a scope named `legacy`. The next successful +ingestion publishes version 2; interrupted unnamed version-1 jobs can be resumed. +Older SuperSTAC versions cannot read a manifest after that upgrade. + +Snapshots record source URL, scope, start/completion times, counts and file +bounds. Completion means pagination was exhausted for the requested scope. It +does not make a changing provider transactionally consistent or guarantee that +an older snapshot is current. + +## Search modes + +### Snapshot only + +```sh +superstac --mode snapshot --dataset ./data --json search \ + --collection sentinel-2-l2a --datetime 2025-02-10/2025-02-15 \ + --bbox=-4,40,-3.5,40.5 --limit 20 +``` + +The newest single scope covering the query is preferred. Otherwise, complete +scopes can jointly cover the requested collections, bounding box and time range. +Coverage is checked as a union, not just an enclosing bounding box: gaps still +require live fallback. Intersects queries use their enclosing bbox conservatively. +Overlapping records are deduplicated by source-local collection and item ID; +newest scopes and incremental delta records take precedence. Queries can differ +from ingestion within this recorded coverage. Coverage failures appear in +per-catalog response metadata. Snapshot mode has no freshness cutoff, performs +no provider monitoring, and never falls back to HTTP. An empty complete snapshot +correctly returns no items. Collection discovery describes the **saved inventory**, including locally derived +STAC Collection extents; it does not claim to reproduce the provider's full +collection metadata. Sorting is supported per catalog with repeated +`--sortby=-datetime` / `--sortby=eo:cloud_cover` (Python: `sortby=["-datetime"]`). +Null values sort last. Sorted queries scan all candidate rows and retain only the +requested top results; the final federated response is not globally sorted. + +Snapshot engines retain their loaded manifest; reopen one to see updates. +File metadata prunes irrelevant collections, dates and bounding boxes. Remaining +files are scanned in batches of 1,024 rows, pruning row groups using collection, +ID, instant datetime, and available GeoParquet bbox statistics. Missing or nullable +statistics are handled conservatively. New files use 1,024-row groups and order +records by acquisition time within each file. Overlays and multi-scope unions +scan without pruning so an older version of a moved item cannot reappear. +Compaction restores pruning for incremental scopes by resolving their overlays. +Decoding is bounded, but deduplication keeps item keys in memory (proportional to +scanned unique items). Local collection summaries also scan the saved files. + +### Automatic selection + +```sh +superstac --mode auto --dataset ./data --max-snapshot-age-seconds 86400 \ + search --collection sentinel-2-l2a --datetime 2025-02-10/2025-02-15 \ + --bbox=-4,40,-3.5,40.5 --limit 20 +``` + +For each configured catalog, auto uses complete covering scopes whose source +URL still matches and whose age is within the configured maximum (default one +day). Age is measured from ingestion **start**, conservatively accounting for +long downloads. A value of zero effectively forces live requests. Freshness is +rechecked per query and the latest manifest is read; old provider health and +collection inventories do not suppress searches. + +Missing/stale coverage or missing/corrupt local data routes the whole catalog +query to its API. Local partial results are not merged with live results. A +fresh snapshot with zero matches does not trigger a live request. Catalog +timeouts still apply to the overall attempt. Tracing logs report `source=live` +or `source=snapshot` on stderr when logging is enabled. + +**Auto is read-only:** it does not persist API results, launch ingestion, create +a dataset, or mark limited search results as complete coverage. Use `ingest` to +save a new scope explicitly. A missing dataset is a cache miss, not an error. +Collection discovery in auto mode queries the live APIs on demand; it does not +start background monitoring. Sorting is forwarded to live APIs on fallback. + +### Live and standalone files + +```sh +superstac --mode live search --collection sentinel-2-l2a --limit 10 +superstac --geoparquet earth-search=./items.parquet search --limit 20 +``` + +Without `--mode`, existing behavior is preserved: a dataset/file argument selects +snapshot mode, otherwise live mode. Standalone files have no managed coverage or +freshness guarantees. Repeat `--geoparquet CATALOG=PATH` for multiple catalogs. + +## Storage policy + +The ingestion default is a **1,024 MiB budget** for files retained under `runs/`, +including active, old, orphaned and checkpointed generations. Increase it with +`--max-dataset-mib`; `0` explicitly disables the cap. Metadata/checkpoints and +filesystem overhead are not included. The writer stops before exceeding its +remaining byte budget, retains the last published manifest, and leaves a +checkpoint. Raise the budget and resume to continue. + +No automatic eviction or deletion occurs. Staleness only affects auto routing; +it does not delete archives. Retired generations remain for active readers, so +refreshes require headroom for both old and new files. Explicit cleanup, compaction and incremental refresh are available below. A dataset-wide OS lock prevents +concurrent writers and releases on process exit. + +```text +data/ + manifest.json + checkpoints/ # may contain provider continuation tokens/headers + runs/snapshot-.../ + collection=.../part-....parquet +``` + +## Rust + +With the engine's `geoparquet` feature enabled, use `ingest_catalog` and +`IngestOptions`, `SuperSTACEngine::from_dataset`, or +`SuperSTACEngine::automatic(storage, path, max_age)`. + +`IngestOptions::progress` accepts an `Arc`. +Callbacks receive `IngestPhase` and counters, must return promptly, and should +not panic. `cleanup_dataset` and `compact_dataset` are also re-exported by +`superstac-engine`. Remote object storage is not supported. + +## Incremental refresh + +```sh +superstac ingest --catalog earth-search --name madrid-feb \ + --collection sentinel-2-l2a --datetime 2025-02-01/2025-03-05 \ + --bbox=-4.5,39.5,-3,41 --output ./data \ + --incremental-since 2025-02-25T00:00:00Z +``` + +The named scope must already exist. Keep its source, collections, spatial bounds +and start time unchanged; its end may advance. `--incremental-since` must overlap +the old end (no gaps). The API fetches only the specified acquisition-time window; +returned records override earlier copies with the same collection/item ID. +Resume uses the same options and verifies that the base scope has not changed. + +This is an acquisition-window overlay, not a provider change feed. It cannot +detect deletions or revisions/late arrivals outside that window. Use an overlap +appropriate to the provider, and periodic full refreshes (omit +`--incremental-since`) when those changes matter. Full refresh replaces that +scope's inventory. Incremental refresh deliberately preserves its original +freshness timestamp: untouched history has not been verified again. + +## Compaction and cleanup + +```sh +superstac compact --output ./data --items-per-file 10000 --max-dataset-mib 2048 +superstac cleanup --output ./data # preview paths and reclaimable bytes +superstac cleanup --output ./data --apply # delete unreferenced Parquet files +``` + +Compaction streams each scope, deduplicates its records, and publishes replacement +files atomically. It retains coverage and freshness timestamps. It requires space +for both old and new generations; a failure leaves the old manifest readable. +Cleanup protects manifest files and durable checkpoint files. Applied cleanup +refuses active snapshot readers or writers; drop/recreate long-lived snapshot +clients before reclaiming files. Preview is safe while readers are active. +Empty directories and checkpoint metadata are retained. + +## Python + +Published wheels include GeoParquet. Source builds use the `geoparquet` Cargo +feature, enabled in `pyproject.toml`; minimal Rust builds can omit it. Check +`superstac.geoparquet_available` at runtime. + +```python +from superstac import Client + +client = Client.from_yaml("superstac.yml") +client.ingest( + "earth-search", "./data", name="madrid-feb", + collections=["sentinel-2-l2a"], datetime="2025-02-01/2025-03-01", + bbox=[-4.5, 39.5, -3, 41], + progress=lambda event: print(event["phase"], event["items_saved"]), +) +local = Client.from_yaml("superstac.yml", mode="snapshot", dataset="./data") +results = local.search( + collections=["sentinel-2-l2a"], datetime="2025-02-10/2025-02-15", + bbox=[-4, 40, -3.5, 40.5], sortby=["-datetime"], limit=20, +) +``` + +`Client`, `AsyncClient`, `open()` and `from_yaml()` accept `mode`, `dataset`, and +`max_snapshot_age_seconds`. Async clients expose awaitable `ingest`, +`compact_dataset`, and `cleanup_dataset`. Constructors and `from_yaml` remain +synchronous. Callbacks receive dictionaries, run on the worker thread, and should +return promptly; callback exceptions are reported as unraisable Python exceptions +without aborting ingestion. `shutdown()` stops monitoring; release the client to +release its snapshot reader lock. Existing snapshot clients retain their loaded +manifest; reopen after ingestion or compaction to see updates. + +Ingestion keyword options include `name`, `collections`, `datetime`, `bbox`, +`resume`, `incremental_since`, `page_size`, `items_per_file`, `max_dataset_mib`, +`timeout_seconds` and explicit `all=True`. Unknown options raise an error. + +## Performance checks + +Run the reproducible synthetic benchmark without downloading provider data: + +```sh +cargo test -p superstac-geoparquet --release benchmark_scoped_search -- --ignored --nocapture +SUPERSTAC_BENCH_ITEMS=1000000 cargo test -p superstac-geoparquet --release benchmark_scoped_search -- --ignored --nocapture +``` + +It compares the same bounded scanner with and without row-group pruning, verifies +identical results, and reports median latency over five runs. The fixture contains +time-ordered point items; it is a best-case selective time query, not a promise +about provider latency, arbitrary geometries, or gigabyte production archives. + + +## Writer provenance and support reports + +Provider requests use `User-Agent: superstac/`. Report CLI versions with +`superstac --version`, Python versions with `superstac.__version__`, or Rust engine +versions with `superstac_engine::VERSION`. Search diagnostics include +`metadata.superstac_version`. + +Newly ingested and compacted Parquet files preserve standard `geo` and +`stac-geoparquet` metadata and add `created_by=superstac/` plus a JSON +`superstac` key containing `version`, `crate`, and `written_at`. Older files are +unchanged until rewritten. To inspect one with PyArrow: + +```python +import json +import pyarrow.parquet as pq +footer = pq.read_metadata("path/to/part.parquet") +print(footer.created_by) +print(json.loads(footer.metadata[b"superstac"])) +``` diff --git a/docs/content/docs/guides/meta.json b/docs/content/docs/guides/meta.json index 1ae8dd9..f955d5f 100644 --- a/docs/content/docs/guides/meta.json +++ b/docs/content/docs/guides/meta.json @@ -3,7 +3,9 @@ "pages": [ "catalogs", "aliases", - "resilience" + "resilience", + "geoparquet", + "benchmarks" ], "defaultOpen": true } diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index a37dc2f..5efbc4c 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -14,6 +14,7 @@ The same Rust engine powers a Python package, a Rust library, and a command-line For the background and motivation, read [Introducing SuperSTAC: Many catalogs. One search.](https://www.spatialnode.net/articles/introducing-superstac-many-catalogs-one-search2a5e11) + diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 261966c..eb3c755 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -8,6 +8,7 @@ "python", "rust", "cli", - "reference" + "reference", + "releases" ] } diff --git a/docs/content/docs/python/geoparquet-notebook.mdx b/docs/content/docs/python/geoparquet-notebook.mdx new file mode 100644 index 0000000..08b1f7b --- /dev/null +++ b/docs/content/docs/python/geoparquet-notebook.mdx @@ -0,0 +1,39 @@ +--- +title: GeoParquet notebook +description: Build a reusable inventory for your study area and explore it in Jupyter or Colab. +--- + +Build a satellite-data inventory for a small area around Madrid, then reuse it +to explore dates and scenes without querying every provider again. + +Download the GeoParquet notebook, +then open it in Jupyter or upload it through **File → Upload notebook** in +[Google Colab](https://colab.research.google.com/). + +[Open directly in Google Colab](https://colab.research.google.com/drive/1JIx8j0Vi7jypKe5Q6klgCzYQMFHMaKGc?usp=sharing). +The notebook also works by download/upload. It requires SuperSTAC v0.3.x; before +the release is published, install a locally built wheel and skip its installation cell. + +## What you will try + +1. Ingest a small Sentinel-2 metadata scope around Madrid with progress. +2. Search and sort locally, discover saved collections, and optionally view an online thumbnail. +3. Inspect the Parquet writer/version metadata with PyArrow. +4. Save a second adjacent scope and search their union. +5. Query outside saved coverage through automatic API fallback. +6. Extend a named scope with an incremental acquisition-window overlay. +7. Compact files and preview/apply cleanup in the temporary demo directory. +8. Search with `AsyncClient` and download the inventory as a ZIP. + +The example downloads metadata, not imagery. It needs internet for installation, +API requests, and previews. No GPU or API key is needed for the default public +catalog. Results may be empty or change with provider availability; the notebook +checks per-catalog failures rather than treating every empty response as success. + +Files in Colab are temporary. Download the ZIP before ending the runtime. +Incremental refresh is not a provider change feed: full refreshes remain necessary +to reconcile deletions or changes outside the chosen acquisition window. + +Continue with the [GeoParquet guide](/docs/guides/geoparquet), +[benchmark methodology](/docs/guides/benchmarks), or +[version reporting](/docs/reference/troubleshooting#identify-your-version). diff --git a/docs/content/docs/python/meta.json b/docs/content/docs/python/meta.json index a44e413..04a97d3 100644 --- a/docs/content/docs/python/meta.json +++ b/docs/content/docs/python/meta.json @@ -4,7 +4,8 @@ "overview", "notebook", "async", - "api" + "api", + "geoparquet-notebook" ], "defaultOpen": true } diff --git a/docs/content/docs/reference/search.md b/docs/content/docs/reference/search.md index 4f5d583..0159b83 100644 --- a/docs/content/docs/reference/search.md +++ b/docs/content/docs/reference/search.md @@ -13,7 +13,7 @@ Python clients take keyword arguments. Rust uses `SearchQuery`. Both are transla | `intersects` | GeoJSON geometry dictionary, optional | Spatial geometry, not a Feature or FeatureCollection. | | `datetime` | String, optional | Datetime instant or interval passed to upstream catalogs. | | `limit` | Nonnegative integer, optional | Maximum collected items per catalog, default `10`; also capped by `max_items_per_catalog`. Prefer a positive value. | -| `sortby` | List of strings, optional | Present in the query model but **not forwarded by the current translator**. Do not rely on it. | +| `sortby` | List of strings, optional | Per-catalog ordering, e.g. `["-datetime"]`. Forwarded to live APIs and supported locally; no global federated ordering. | ## Spatial and temporal filtering diff --git a/docs/content/docs/reference/status.md b/docs/content/docs/reference/status.md index bfa1c93..ab8750b 100644 --- a/docs/content/docs/reference/status.md +++ b/docs/content/docs/reference/status.md @@ -3,11 +3,12 @@ title: Status and limitations description: What the current alpha supports and what is still being built. --- -SuperSTAC is a pre-1.0, MIT-licensed project. This documentation targets version 0.2.0. APIs and YAML shapes may change. +SuperSTAC is a pre-1.0, MIT-licensed project. This documentation targets version 0.3.0. APIs and YAML shapes may change. ## Available - Python sync and async clients backed by Rust. +- GeoParquet inventories, coverage-aware local search, incremental overlays, compaction and cleanup. - A Tokio-based Rust engine and command-line tool. - In-memory catalog and provider registration. - Collection discovery and source selection. @@ -27,7 +28,7 @@ The package searches item metadata; it does not download, mosaic, render, or rep ## Current alpha behavior to account for -YAML and Python dictionary configuration have different required fields. Python `matched()` counts returned records. Python item dictionaries omit per-item provenance wrappers. `sortby` is currently ignored by translation. Health monitors and global monitoring settings have limitations documented in [resilience](/docs/guides/resilience/). +YAML and Python dictionary configuration have different required fields. Python `matched()` counts returned records. Python item dictionaries omit per-item provenance wrappers. `sortby` orders results per catalog; the merged response is not globally sorted. Health monitors and global monitoring settings have limitations documented in [resilience](/docs/guides/resilience/). These are implementation constraints, not promises about future releases. Track development in the [repository](https://github.com/spatialnode/superstac) and [changelog](https://github.com/spatialnode/superstac/blob/main/CHANGELOG.md). diff --git a/docs/content/docs/reference/troubleshooting.md b/docs/content/docs/reference/troubleshooting.md index 1d49431..9e4f52d 100644 --- a/docs/content/docs/reference/troubleshooting.md +++ b/docs/content/docs/reference/troubleshooting.md @@ -28,9 +28,9 @@ The implemented method is `search.to_geojson()` (or `search.item_collection_as_d The limit is per catalog and bounded by `max_items_per_catalog`. Deduplication compares item IDs; it can collapse equal IDs even across different collections. Inspect `duplicates_removed` and try `deduplicate_items=False` if your sources reuse IDs. -## Sorting or cloud-cover filters have no effect +## Sorting and cloud-cover filters -The current engine does not forward `sortby`, and does not implement a cloud-cover/CQL2 filter interface. Unsupported Python keyword arguments may be ignored. Use only [documented search fields](/docs/reference/search/). +`sortby` is supported per catalog in v0.3; it does not globally sort the merged response. The engine does not implement a cloud-cover/CQL2 filter interface. Unsupported Python keyword arguments may be ignored. Use only [documented search fields](/docs/reference/search/). ## A catalog never recovers @@ -38,7 +38,7 @@ Catalogs unhealthy at startup currently do not get a background monitor. Shut do ## JSON piping fails -Set `logging_enabled: false` in YAML before piping CLI JSON. The current tracing writer can mix log output into stdout, and `--quiet` does not disable warnings. +Use `--json` for machine-readable output. CLI logs and ingestion progress go to stderr; avoid merging stderr into stdout when piping JSON. `--quiet` does not disable warnings. ## Python cannot load the extension @@ -47,3 +47,21 @@ Use Python 3.9+ and ensure the package is installed in the interpreter you are r ## Report a reproducible issue Include the package version or commit, Python/Rust version, operating system, a minimal configuration without credentials, the query, and failure metadata. Open an issue on [GitHub](https://github.com/spatialnode/superstac/issues). + + +## Identify your version + +```sh +superstac --version +python -c "import superstac; print(superstac.__version__)" +``` + +Rust applications can report `superstac_engine::VERSION`. Search response metadata +includes `superstac_version`, including searches with per-catalog failures. +Provider requests use `User-Agent: superstac/`. This identifies the +library, not an individual user, and does not send telemetry to SuperSTAC. + +Newly ingested or compacted Parquet files record `created_by` and a JSON +`superstac` metadata entry with their writer version and write time. Existing +files are not changed until rewritten. Include that version when reporting a +file-reading problem; it may differ from your currently installed version. diff --git a/docs/content/docs/releases.mdx b/docs/content/docs/releases.mdx new file mode 100644 index 0000000..e78c895 --- /dev/null +++ b/docs/content/docs/releases.mdx @@ -0,0 +1,58 @@ +--- +title: What’s new +description: Release highlights, migration notes, and examples for SuperSTAC. +--- + +import { Cards, Card } from 'fumadocs-ui/components/card'; + +## v0.3 — Reusable satellite metadata inventories + +**Build your satellite-data inventory. Search it again and again.** + +Save metadata for your area of interest, explore different dates and scenes, and +reuse that inventory throughout your analysis without repeating every provider +request. Auto mode queries live catalogs when saved coverage is missing or stale. +Previews and imagery stay with the provider. + +- **GeoParquet in Python, Rust and the CLI:** scoped ingestion, named inventories, + progress callbacks, resumable downloads and storage budgets. +- **Local or live search:** snapshot mode searches saved coverage; auto mode uses + fresh complete local coverage and falls back to provider APIs when needed. +- **Maintain an inventory:** acquisition-window incremental refresh, atomic + compaction and explicit cleanup of retired files. +- **Explore saved metadata:** gap-aware scope unions, per-catalog sorting, + collection summaries, and row-group pruning for selective searches. +- **Easier support reports:** version information in search diagnostics and newly + written Parquet footers; provider requests identify the SuperSTAC version. + + + + + + + +### Upgrade notes + +Python wheels include GeoParquet. Rust and CLI consumers opt in with the +`geoparquet` feature; the new crate is named +[`superstac-geoparquet`](https://crates.io/crates/superstac-geoparquet). +Version-1 manifests remain readable and upgrade to version 2 on publication. +Older builds cannot read the upgraded manifests. + +Incremental refresh overlays an acquisition-time window; it does not detect +provider deletions or changes outside that window. Full refresh remains available. +Auto searches do not automatically persist API results. Sorting is per catalog, +not global across a federated response. + +## v0.2 — Python clients and documentation + +Released September 27, 2026. Introduced Rust-backed synchronous and asynchronous +Python clients, native dictionary results, configuration and collection discovery, +with multi-platform wheel packaging and the documentation website. + +The Python package replaced the legacy v0.1 alpha implementation; existing users +should follow the current client API. + +See the [complete changelog](https://github.com/spatialnode/superstac/blob/main/CHANGELOG.md) +and [published GitHub releases](https://github.com/spatialnode/superstac/releases) +for the release history and availability. diff --git a/docs/content/docs/rust/overview.md b/docs/content/docs/rust/overview.md index ece2e12..63b4634 100644 --- a/docs/content/docs/rust/overview.md +++ b/docs/content/docs/rust/overview.md @@ -9,14 +9,14 @@ description: Embed SuperSTAC in an asynchronous Rust application with explicit c ```toml title="Cargo.toml" [dependencies] -superstac-core = "0.2" -superstac-config = "0.2" -superstac-search = "0.2" -superstac-engine = "0.2" +superstac-core = "0.3" +superstac-config = "0.3" +superstac-search = "0.3" +superstac-engine = "0.3" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` -Use Rust 1.88 or newer as declared by the workspace. These examples target SuperSTAC 0.2. +Use Rust 1.88 or newer as declared by the workspace. These examples target SuperSTAC 0.3. ## A complete search diff --git a/docs/content/docs/start/installation.md b/docs/content/docs/start/installation.md index b6f2a64..3dd05f3 100644 --- a/docs/content/docs/start/installation.md +++ b/docs/content/docs/start/installation.md @@ -41,7 +41,7 @@ cargo add superstac-core superstac-config superstac-search superstac-engine cargo add tokio --features macros,rt-multi-thread ``` -These docs target SuperSTAC 0.2. See the [Rust guide](/docs/rust/overview/). +These docs target SuperSTAC 0.3. See the [Rust guide](/docs/rust/overview/). ## Command-line tool diff --git a/docs/lib/layout.shared.tsx b/docs/lib/layout.shared.tsx index 2174269..92ceb75 100644 --- a/docs/lib/layout.shared.tsx +++ b/docs/lib/layout.shared.tsx @@ -7,6 +7,7 @@ export function baseOptions(): BaseLayoutProps { githubUrl: 'https://github.com/spatialnode/superstac', links: [ { text: 'Documentation', url: '/docs' }, + { text: 'What’s new', url: '/docs/releases' }, { text: 'Python', url: '/docs/python/overview' }, { text: 'Rust', url: '/docs/rust/overview' }, ], diff --git a/docs/next-env.d.ts b/docs/next-env.d.ts index a419cbe..ce4e94a 100644 --- a/docs/next-env.d.ts +++ b/docs/next-env.d.ts @@ -1,7 +1,7 @@ /// /// -import "./.next/dev/types/routes.d.ts"; -import "./.next/dev/types/root-params.d.ts"; +import "./.next/types/routes.d.ts"; +import "./.next/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/docs/public/benchmarks/geoparquet-v0.3.json b/docs/public/benchmarks/geoparquet-v0.3.json new file mode 100644 index 0000000..3540ecb --- /dev/null +++ b/docs/public/benchmarks/geoparquet-v0.3.json @@ -0,0 +1,16 @@ +{ + "version": "0.3.0", + "date": "2026-10-02", + "hardware": "Apple M3 Pro, 36 GiB RAM", + "os": "macOS 15.5", + "rust": "1.88.0", + "profile": "release", + "fixture": "100000 time-ordered synthetic point STAC items, one collection, 1024-row groups", + "parquet_bytes": 858390, + "query": "20 latest acquisition-time records, no sort, one catalog", + "repetitions": 5, + "baseline_median_ms": 411.332, + "pruned_median_ms": 4.195, + "identical_results_asserted": true, + "limitations": "Local bounded scanner comparison; warm OS cache possible; no remote API baseline; minimal metadata, not production imagery or gigabyte archives." +} diff --git a/docs/public/benchmarks/geoparquet-v0.3.png b/docs/public/benchmarks/geoparquet-v0.3.png new file mode 100644 index 0000000000000000000000000000000000000000..8c7498cb98e3e04b81e153cf08ed4da7b7b9bd3c GIT binary patch literal 65119 zcmeFZcT|(<*ENhfjw2QvrKkw#C^B>b0i}*Qigcy7fP#ROfb^PR8AL%qL3$IUNJ%0! zkPu>_2%$(xLQMpu1`;4ZfRN<9GSBaM-tV7pt@qFGTkB;l7gmTgv<$5;MJut}2-%m|RMMX(f;q1fU;737P%F59H`5Q|9fu72b6Y1cW2poKL+dfD@ zKpe&Yv&Z@Ln==9e9D?EB*Q~;GmnQaSip+2itQ1VdrwINgYV>l?pMRVjp2)K!C;4>X zoUtNUEY{I?zwc+!H#H)@Rr|jx2r2KIk9?`4YVyhe>YR4+dPl%onLJnwe%q_oWfo zb~t+(-E}wz+AuQ3mWoMQNQ1g+5Bslh{|Yh9ymXENW95Z=4r{Ltn(NzyN54EO;hLyU z$}KvwL%}lIU;JLsYU>gsq1JafsP19EPWs}otf+JNq%6E}F>v$Zt$Z*X&)#^x>`{8V@CZlu7)F{zl{-^KkK;fY^_ z1s)H8kT30v{<+dGPIZ zQL4^pu9@7b!pG>Z~h$wSE3?C0NO;fskUHH+@JTTCZG-qSY2i> z)p`~%O>@)}%krZG<2Cw)^mmVGGwR%ScZ1rrJfLFES9Ufb(r_M!6+X4v;kvuE>Mkj@ zy--$7&c}9qP1dnt4Cd#P>IH}dNVO!M_i7)Dl;Ne(P zlqkl_F+ZF-_zrQzKe@8Ba=vu)4}tHGm$3|Ze~o+P{6LAZO|TxZJ$$uItn|vAs|&5X zwqX@Fxx-Z7+Ma2QxZf*#k1T06Z_FQ`7(#ZIr@+|=)#)SAi{*`+hB4Pl1R6>H&)J?A z_%w8Ze>^^Ahs8DOzO3kOj@J%clD4mK>$svik))-CO{u;~NLSBh{B!WcRHCMHa-Tk1 zr#(?hthPtX@v;x;GDfl1*Ji0jZ-}V6OXYD~npt$Lifii&Up9AjPO85sBH>|w%}s@- zpB5_MfYVO#*bIDfjz2~P%j2>cVtKQbsFvdyPSJN`Cvj1zGsa;wn>F753GAIsK_I5a(l^e0Ft|8W z9t18*;dDHsg{?%1;o(l|dI#_B5{fI|=cq$P^R&Z~+C*KVYswtQ*fvlVrqo+%~aQrn&yj3?$^ec|l7^3(x8zBrktazL>m#;<4o&(0&rQNQ7`ukGk#UpVNL0 zaR15#PO%)Dj*b~>aEallBk^>M;Mcj_px(x<0fLd)kc?@LM&ceZr(=}}i-2uq-1?ND z42>2zIyi{YC|(XH^k{Z$txi{r&skU)(o)!j9Oo0T?*cr#_h*pJW8 zYFt;;dM~-v`B0T??C!T^hLG%intfBFOvtlgO*=bPQda|i7Te5G;i)jmu*D&-c>MGb zDbPG&thR+ZC*H-#+}SaMjE#L%V(~CLy@!UL@Da7YN8_W=@PVa$n@_Uo`}|wfgI%@6 zthu4PJhKlQoCPwKn;A}?V3|1KT<|Yzd$oM}w@V%8dLb8`14m{~#O!X9<;u|O0~2yO zVGSc5m1OrnVNr_RxRC5C&yP+3r+S{_qMBz=Fx6{Wi zBFD?jH?^!AntQUY0u(i7%`%*M&N@97GL+eO z(XW0(LnNK&62-9eWSd(P)VOg47bQctem?gPzj-0OsE_&})l?p4HWn+Uw38b~<$a`->Rv^Ag+Rmy zLYKbZ?=)gXGT^Ah`PJqhkhmqD`VUy%E>7ya~>z z{auDj62N)A(sV8-OgS=SPM0+fSqQSg(}eW+6^@3)IxEzDzkXzG1Nk!8nwITTB{>Gg zhBUA2GDm1uSyuYZ?r5zc zDC9xj*i%EPh?u3uuc0d8D?f~=J}JuhUDe8&I&Z38wNU|Ob=5%1TFVczd4oj4HYl0zJy|nTNG!c-#W|HCMF)^Q zj~j>wQ19}<_AzR}b7l!v8I8-ha724{{=I7fj%PiiBR<7S!|a+EQ5kIHxHiAm7bWAl zJ3iH<)10a1pgylV1YPOVi-E*-7~El7!ZL$eUKT7)n&G;LQ6rvUWcU@J&TsIdM{YoS zk^V(Y$nqK9Rr+u^cYCg8b~Kd+AUd9C>sZpxsm1RsR#q>{!D^(ASyq6X$NA0PZG%`N zwLKWWp*eikL3E)A@xA`KN*EPp8){MC%*8H~&H_X}sU7ND)!E!S1<=QY`98q+2Hpl;`87sRI6(1)R`6zqKq@31Q49b8>?Q(&;HJuy|h(nFuKLi&=b(hqOt0)|+lb$T8E+;9s_EjHu3y{I8OU*TUMU^Q# zhSY$LJU-u4P}|*}%TG{oYyUe(11hI8m!9J&5(=Qp*h^6%XG~9>j{+yhzH^ z^5k67XLb*+ZjjyCer9I6Ge771-y2k+xxa?g#0aNe|2~u)QajsIw6NFI?MbO=?R_sS zeC6{26~OF7O4R_bY+M>Jp89cvE3q+|?kGoB>pX58{K7rMQE3)H{Yia4CqPX*>?O(; zozxx3Y#W#_dHsf7-Z6zCB|CSb^?U2e{cb`y70Ur~AA zF*H_JRu%$%I95HZS73Ipj5S_|M?p7k!~lcF^W-@tNJrjs#~& z0Rn$R1RpxE##s8{@$qQhI=Q!2aD${S&+ z@H+mVwnsP}Qc-pD$?d^`j7HwjQU#)7X*33X1+d?)-9L@=RsC=?vsczXekHuhG5fpp zM9R!^GgHa_JL$VKBfD{qj;*cy+Lhy$bajd}=BFHFp|0r9gM=mlx`+C*)o}(ET!p3V zxFQ3*_!^&9!=CpT94LPOhw?7 zp^Rq-{JxY~8{Cp5wNTMNFz_RUYo8`^&?nGeY6=z`0_`q4aC|Kx%?!o)o}kq3|d z-&kx|I1dSJ6^RbGQZ%xeOW=| zBeL?ps%o3(03?iJA2`)i7kqpt2}@4`&6B?pz4utz5+=T*J)vDu1^)AR zx5epNm=XFugu$!lXx(ZRoZR3V@9*!fsXY8n>i17npK8zX zd$~PVfd3M%=@bL$p97b@><$;W5Y(p|<^*p`TWy_Aw%$?+T2{9;-#oYppo50QH4G?k zXpRR5^-JwdP;3d`v~!1Lku&Pfz;-qh=kj3<-7A2Wmwz$r2>zpHR|K*U-;x(LQh%+5 zq8>1qs3|M~Z+&B6-FRJBkzq}C63f8s~fS9*Bl%ksi!Lf}*vP<%gDu^lDlj%qSg^Q3T^u*?vC%R5zE-NZO zass^=g)Jp-Z)(#kJ|+$Xvgo)#93Jo6IC(1oWXao8BwSRooe!dZ`pE&g(1zm zz>q$uJKf+U4cy77#4wnHfE;6XeG3NuO=)NhNaBLN?jxEa+A5rW+6QSr`H%OFaGcK{ij3MmJCe?0Rh9f-lysBRo={(k)saXuoX{_w&!Rct};Jh3>-rsYf-dFS;r zp;v%hAo&T=QvI{oO>0lq&W@sT-bV%0xn_#`n{@PWXUP!c)6+vrb_+~7T8=h^<8a58 z-LX)XmmN}nNqXaiMF4T;ys^FONeZZ15fy4_0djZaFaNG2$^{2dI5Tv)1@+8|Tk zbrah+>ivnVPzY=TZms7`Whm=LBJTumji))OXW4!a@swDdv><5LisFA z<1xwVSt!H*eXt7yB|fqDgL-ts`U9|vkX_yTBzsRqt=&F6H`k*DHMHqzA=qxAd26Oe zo6&4dy0J76vT%C8+>s4!qZ<(9VsoFl{C}ZPAo@ORiC4c3P{>7%#M~8lSUe!q--90h z!w)k`mk? zsykh6JtjhF2bLDvLXE9(hPQmiIZXMz2La72b{C9Idq16po^|IM%r|nl7OkdaKMneezg~x_X0; z^;mHW)ny>9B5gy~#oG3-Bdd)*z6@vt+wevWXozpd#V|u=XgfWcB8QI4C=yI7mCF7b z8Yl@CV!A|_l+pe+pMeSwhwT;%u>!+3vu`rYZf*ax!A$>_3aJ9omiL#3X0itu`;IJw zDjwZnPS>bP{_hwBGNw1gn~t;UNE(5XA?%FHV$_g~yoAt08m_G}?p)Ned=s2E%{r}L z#&39tRwnK@UBoLuXOnP9In!#r_6G=yZfS7yYrB1MI)SQF-PEIFxjFvFH(T3))9(q=2k$0&Bs)Rc>*1Fctiw z@y|3;1&6lw1RmQejSXu>q)-$%_`B7L@#Q{0PokA@4S8)&Ymrmr+OzVfc=cf^9 zN_-`<@efM)d*9Dw14;N5XV^ws)ul)X-SJ2WRwJF2v7<*f%E_mWl|Gl$jq;Fr-WA2{ z(Ugbviu6A5#q{z_A-(XJ3MJNC z?+*mx$(W8IvM2lJ2LN^r{|$BzjQ1aj+Hfczo!0cfyPM!y>){?(mTH5ub-wzm-iB4I z)D@ogWNVf3$l`4m`e0DS;#`9&ALM>I7PP>oUv@~LF3|zy{yoFlX0`a{cFU8oIxowB z2>GV#wOSB66C@;T3MntLsD^yP(7 ze#bIe4EjfZXHla3`xi<$h3^_z$?I-ana=VEtMz|Ap{a1td^v9(aD+vH9>o_~Pw6!N>~M!H^N*55WEdviDl1!(2Vl?>ykEkQ3myEg+p zW8)~cObG3eXsE3D0ozI83|X4D1Kd&*b*Y0jZ+$sLbS~I&cn$)oVV-G0ZIsDh!578m z3>qc`^o1K z4=FdFO{b4uRchKQGBWqR0Hm0z`IR-ug`u_<;%l@Iq#1u%7;vdH4P88%i01Rld0IA7 zK#xBBUVf=8PBQX#|H4vBhvbIV@&(A%+<5H_{6s)h+v^f!hNnZ`X59y+|2b%Jzs9@$ zlQO%Et*x#1$z6xp92V6L^eOhs2@#7swRPwG`gSh!J7`qjX6N0E|8zrrhflzHwQ#vt zd%w?j>-mg4PXan!0@7GnB2O!%H~UeUlxq2FicCTDasHGe6mO@(E@U)|EsxgAtLeZ_ zxxh!<<27>3`s@KgbSp1OkDpa^c;GT%xo4>GMqE1$g|qR{vvmoA_@X(R?)lVO_S%0j zpLFEMU2aDK&HWUY!Q(31%=hckB;RGmj@2h@kR0{LE-n~vcSyyYN!D9?w0RTj*_XdM zen?paXVCzq#rKR4ikLb2R=bn@w}Etf)oa8vM=^8^%IX}wIJ1nWwgj}zWLRG{8x_;r z4z`M|w4>30VzuZOw~kR5MEHFxs} zQ4p+>1VIrWy?Ntm-cK%Q6Q?G^sEG%p8uj6d1Fk=I#QowC4HQf-3-KHSp-pj0N(n); zr^-44IwVusqeE5@$*{qxC`ZVd80(0ilW|jE#D%!CZ+UmmzIf+9t2swS%yXCPm`WmM zHc1lN8tIN63q{o5Ee^d?Cp*cHX@pHgcs8>rSdn%^#_!5UPCL~lnL^C8MGn#@eA)!R zhEd)68dr)E4~``SMXCSxXo^} z-!fQJglDe&UW1^bD@~nQTmrDJO6*+HLgBq@6zzTnstI7o=Y*I@Enwt&7}TvR~L- z9B(qNFXc}Q?VgGf+GvJ8*zGB9C zv?*;=4)mVXR;_cx6D7Prrnkj8>QJ)3yH@C;aI)U!-wI$J*O?8M6CFmvM$(nlt!)Xf zcgAC;I-gW`s}gQOukOA$|GtH+T{twYHGYyF_F{g($@pV=dr@Dhv<}gjK z@!<_e%t0!2moZ!g=!>yhJySlBrq#j%hyJ6k?wsSRt4B`n8u9P0`%R%krn5o%;U2jv zHufRJM&6SH(<#Qv1TLy;U;@+j?i{K$%QH{HSfT%1*iO0E)(*}8HD{i;m1_Pzj1fPf z3ldZDYl^X=NFRR7!s$_eTYrf$d3>Q%iMX>~-aHJn7vB?}1wnq@KrX*qu*0y~Hf&pW zyUfX~ME=W#`ts#lU)wq-Ommr3?~sM>d^KUxKq`T$+<1xRS14CsA5$y^5z>Dutc>lZ~#;gf9{ZnzE24`^6rp+9*P z9^-X;ajj7v8VUE$Io&W4Iu_39LA%xVXrO9*KCGWYgSkt`*jGx=Q&hS7^3Y8)l|u?2 zKUvx5d%;ErvoTtwGAs~eF&9ftbd^UH?iZ^Xj$-Eg2k$LP`q`9-=# zk6)z!okECxhyq3Wjy!jVh6-m)gw;mMmf!J_3*Dlj@DDgky;tX)Mh-p)P}QzVP!aU5y%v6MISTFjs7M|>L8~#|R z;x`_)%4WDHobSEbaVDV4tBgFVVYpw3$1a$zx&CGbTDRI>y|c)14I~m9klk;6+M~*_}$*;<8s&XbQFMyN48xNoXl#exJPtnj=1rh``v@_S_u%*SCEX<5S?3XaH+UV9n70)&>s)QAj7&Gt|!Mu&cS zC(03aCZ?n$Tb1!sIH)%W%==#OB|M)oAuwXz`k6Yp+Cz2$#)_V|u&~?bNPvlH-IVF` z_8H~d49-NEj1(=EQ`y2-ic@_ya5{-zqiI(+-lk3yl<`$0y@EwfZkS)&$so3!0 z(k{u-@8z{;&Y&8+j*&VIILsqlj7wH7BQRtm%c_Oare-em3K$x6*(Uih+;vWiWfiVh z%l@}F#Vp?X)jJe0XA_r5cd8liBF_YB4lQSuDN*LccNm$YTwhcV)Q(R*Ks(}H@YjCe zMDe~i5kl%R`<@=k(q&=1PYS$yGdLp#-mn_v&%b9foaNMffj)7pR6xMwz%O#(ojaz= z%`4HO6~cACBdWbxSU-%zK7n`lfByj9`%C>u^jfDX4}|U)Z;|&01n!M$GKZYOI|7+D z3PYF>KET$np<@k+rWzB$-pa=X1UCK#V>JHP<1?7uz5l-Y1|a1BzW6^pgm<^#+yxoO z!=+EAhpGjSm1(~T}M zdB=RJu-4H!yMuzskjTk}>V0>x;Icyq8qgBVgz1rC# zj=O*8@v;VJ%1P_-{QiSHHS-zOCUgE)JDQr5uuA2<-me9W9L9Cd4?o*^cecH!F3NYz z<{+hldQ}V%JyJ#|_@E~saQP8Wdts^QzEM78^U9yjF>VhL3OAr;l^9AC@4a2v`{hdC zdbO#+yCchQI(}{U@^n5aCUiyJW?`(-IU{x9#6$~WPCoY;~Ub;Dmna!@O zUGT*(Yt%PH3#3lJ8Z;+RlDV*NpEb^c54;04*X2Yz1#zGacIwg0p0H|5-oB}4LUCix zLotWafM%%e*Rs&Q#QohsLSzEV`Vc%0E>h7HudS2X!k(uWh6kb1ElenrI6 zo6loo#w5aQPK{o^HOSuKn=C)APkldc_5#(ia}i5zju@ODjVM34)tx|u!zChFpDde zgb%!AGyC*Z=r~^gY18R!`Cdf*OgRW-+m4d-7g3Dn(s?(8n70Ufc|xByOG_c3l$pbC z|2bwL85P*DY>b_aXS(7-Y?9-oBuyB=T;ftLz(<7ewFZUecTgbfN@^$pW890iLh6|zR}eFq-*^LfB?j$D&sjru_bb;Wnbgj zUhI!})vuIjcHS`S@q5>AugTM|ERz##oulMKDa((yHpc>_r;AByUd_Q#lO^W@w%a^y z9*(T7oHuWyu%)s>Y;7pO51BdfJVsSDAHG>~g;!MR`e{U_UBeOTM5Em%RuJn7kA)Y{ z8Y<(quJ;ExM=rF1c{3&n=S0X2i4t&`X)HfM$UpkTKJl8E@p4Y7<^*lAVIu;ZY2S zN@mMki-FOXw*xpu5vQc0o-p5X3FT##6(oVtKlaO&K3%r&CYo#{-_|A1dTH;Or``gbi{aCW8&twB z(z)g26L5}twz_}5ggnjv(G4vNYqdbK}p=vzZSZgZgKuV&g3>uRpeBRhZ=|4DJV6qtXFA&91Z3@OK$VXsS=Zs zj0x}7?$Zg_2tl{Um|v;wz7or0QG&{@tpC8dJwN67f|viuX8exQis8gdXw&v{X`WE@ zPHf7GLKc9r>6Ale3F_>P9?Zth4B2HG^vsFQ0r2xpZST<)l7D1IO6vWSrF7S1x+xY*06iz5jWu#8vZH z&s4q1NIf7G{q9T6nTYlNG$i3Mvp4a5vR2J?LRPA|mWrAFboa0)l_LkE^DXIY2HJl- zlF(y+%#t!0CviGzsmWF&PuNUY`w4gb)06&M&$h?!>V!RnNkL!!IarV3wl%By+G$K! z$;%*Y1^f60B~@DECWe+C(VM6_4!}OZ4lsk_r*r5Zw;wXI8P-bp2dJ<~&) zw(^D9*EULfn@8slEizm}y+nO4$VSVFesGHHl(tvQO4OuWC~Y;A$$zcr?+vpam6a(F zHaHVu#g}!`6#TbC3@zS)B>>>oNj`VuSI=`~{vb-#ELl=luDOL1HgVM%zEDy`xD2$x zwpE_?xlM{EwJ2GC_xm$@aui?23Y>6`TCVpU8R%rYf?l@ir0Rp)uYe~-G-(;w5?ANX zC%-*4w^)gMHO&ih;=4RVoGY_GAR^N8&9o%L7 zWA7|Qb3(eJeBMzIlXvhIl?g$=?go9Hla)CmlAzr1BBSs$$?3g+E_|)JUb6IpqT}Uc zX5ZE7;!nHISjMrrqm2sfF}xdDtC5J0(Pv`D3#EBdu+}{R(>-BnSIJuIc`-_v<`#Kr z1HgHgpK?h$D=6t|(Zm;M%~+TG#;X|e&gk9BCq{w?EC{K=(o8Ty`dxo_9 zYnx-XQ*?i9KN=Z=eGW#hNxN@x;OmO*fjGT_w=>{O^Z7`O;9j$GuESOU{U8UCd?aMzQ zWJl#OGvM+LBc;6CJvefMr(5ob4o-{aaj`eaCD^uqs25v5mF354`{5ORvg&}D()Gt+ z-!0%9C;SuSMKxr9w=~zapKOf~%oV(Er26Q`CqY8ymw4gQ3+3wFKef1c-pfHVU1=;e zbTYP9^e|s4)XvH?Yc1=j@RqfpfWq}+pU|*!wuX-=Z=11Ny)|XrDpPP+xm@wGSFZVc zMwfgW*~L6Eb^PJ`#M+PE`Au`3qy@hDQMqO7kE43?pSBh&QCU&w%? zzJEAupR+j{lw5UlPzk%)m#)A6O*!GK@NbFYno13nz=ZH=L}4{BVwl!%7rzeXgpfPG zR0iMt+Asz!z{!*e?P^k^b&)NL6~n>PiA7-J#k^mYuZh&t$SyLiEe9mih~u|4D39`n zLusfPA?AuExH6YQ1wW zQ}g9TW{>!EdyEV4z~z|F)eJ=B6ni!0-?qrOkK6G(hO=Efm7qAPC+boKQ)sYb+XcS7 zX`KIc=aFNp|?&*CUnnCrW0F6)lnY$>uU1$D8WijOROVqKqf zfhiyQg2~mDch9Ek1_`1^M|qF;x%KIW_F;}k0ZZTX?w+}{p_Fn(%Nj26O~YO9>VO$* za_doTx}9n?b{XC0uvj*DEbuW!KQ)l`xRJosraIL_{jC z$~vTZyItI#W!xE$i)?!)q?@vg-ygM9?dcRTBN8xw?tpDnwpXMV9zWfSHO7=13yvv` zAb3jlnJJrjJ}B(D28u_H>fjY%a0KL=)VsGlc(T;g&x*l_!Db2Xrsf1l}D7ChQMs1|o)_k1W z7ITjGBPdv^>fp+XS>K@O5jjV$a8-WzK&K^CFKW3dO0=|?b1BwxAyfYL1c?KLIJZQ2 zrTZRX5MJLO<%ZCuS65feYuN~gh8H{u7Ng4fwpWa3xqY6uqgcB!1X3teTO*L9cdOZy4v ztK3x-)|=|A0$7j-&1z*^%*FaceM;$I8B4HwOlhoL4#vBRzQrjIJrN)L6#Fh3?Gh_0 zQ!ZkBUuK@@Q?m4rr&iW3$kVqHJjPs^p9g=MvT;u0wl=*kDE5-HVZCbTa6s7-_fVgH z!G8IYk`t+b_}hoZEC2wKEz5U4H(^49^|xg_P?Shv_Ue|Igt-qstxMDx4Nq0QZ*=XP z@5xI)np5c2NYHwNAAEN3L7g}td0F;FIBOzos_+0oatmy#StZQmg*|tlz>6HR+8lO6 zJ;ZrR&W*pNrQD;p*s~1@;Y6?U>hlUw}O_q{*kd zuQ7h|Lu)71B7?^xIwX#`I(uoAi`-ebRI+lqn0}CERACsnbIKD~5e0z_uGS9pq_ym1 z{mo&k9P^3{i{D~|wpoSXL!Sb>U&rG;b7ZeAe$O;CUUuFfW1KfdK)Gh+`?MDsBkX2F z$+|+;;gmgbXKdHD4k@SUij8HS30wcHGWG292z2QM*=^ou-*mh&Fn1X%-n~C=tb|qa zIcVXSt$`R$HLaG6pUua*_RJ6Ht_j!vc&a%4_C#{Ly+`@Zmu(mLwz_20RMxOJ8#k59 zyqJe71te>s)m`pPo({Xa_~}~PVwrGpYBnK^;ar%20#5B#nXhr?r(@CAm zshRiPbRx`pmcN}6$m=&Ckh=bys3;Eh0h@Voo*R+_;&%mC1gAyzzJ2PMFL~uA!ZHPz zCUn?2v$fy;NH8JkSyI4Q>R3;XCTe|kK>fz%xDU#x!}#3YLit+ZOC^_aN+rvAd-J=B z&(?gm_*sp2;elKYK3p0=bBuJ67yU=1YA9-eeS6X9k#|p7@Ha3-n~=+E$AGGga3I2C ziom@P*fXszMKVs=$Xj>ZV*{tc=U)2_nOGQ29-&f;NS%9m^a^|$bFpSP!+=L4%g9-i5k zCgu`-u3%+V-f8y^NK03TrI?wa_vZnGV_W)x62!+Z#hgFu z&+?rUMM@>v+U25V?z7oG$ob)F%dC*hQKr#gmVe83QHvtjsOb{YiqJKc>Ul)iVmj4R)ib564P#xQe^0iB^mY5cdKz6Mf!z zP<4O1ct?V=P5~~wga$(Ou{1F)e7hv0CQmc(tb(O}iKOF);z~-T)-`*~QNu#x%BKNf zD+UFkHOG%5^f_pU_c=dDYq2R;y$Q)p>DF1^RZ=;7dWNuq?BhGh5;u2F`} zGTJM0ns;TMOW^+4nDu27_}`b3k)Jr!SvyC zSOnS+Zx`6vR>aGXvzp)g>en+Ls%jSX_uLk|W%179x7$W!bJ^P=RsiwRt0X_YKKIOR zCXM~}6}1#PX6L)w=H_t8xcuxz#*t~3KG#r}aNr6NO1G}cw19n+_j5^IzN58AN%TPp zNk;aLA;Q$c_#Z3dvXB14d((t4LeAml zY{ICBPVNzKtV6hjUvpEmd-oiW5=T$iYlNSCwE6O1)mkD3G`Wdv$qyKDmhju+9r+R{ zuI*L;w@HRLQddDEpjwi!4A>Pcv2~50C;<%I;!nB=&zuu(9M#veStRP(I8;ZGPD>Mg z?e9I=u4USMjf***;T&)yc1}?>v5Fcj{-M|}-lBhLXu`IThPLY%;(Dk2pWK-Y;#l&nq}+8a>ynRo4)s-A4y3^Gx5@CF^$mO8Kx~aA;q~?VTNMrt&0zoA{^1>r0FgnyzPQ9>loZ8BhrZK9yGV*gqsR4(uz~Z zV>b}p0KxJ(5dKy?*zdT>dM`}3o&`&UNo1EOgaZNiS6$k^qY0!YS?JcvSOxLmLd4mK5U9 zJ)2rJkV2$AU1#=c`*(6yr#lQFj=(BV^GAQDwJqe7TSt3txLGo=%S#B6>`7%x5;-qlEzqkrie9^r2h`D^r<9gN| zy%@9l@a2x41<_xu3dX8yP$@nc0dPUIwH9D<8V0Nmae0+zEL*aW!x7(5jvq%`FD2G^Tz1d14- zCNoRb{PjdSj+O5_T$G?F3LP`!noU%@b)>L?+5H}nj*xdV26|y@vSsa}IsVys!_-Hd z4gOM25`QV{6?4=F)gC(Lmx}Oopq?=+<=KAn(4t(=?+IAkGPMoaOy09 z^HQrFtbJu*oq((b#wzeXk8)V&0>Dbl+qz>$>;=AVT^W+%$W@ zb6q8@wQU7NwQ0k6(c9rrH}%4;=sO!XXZ9;*YbG8cyU1gS8J6bYNd_}bGtZ4H)NvX4 zu(M%cHMF7E4{VJ%*Ozu)KK`Nj*g4|6+ls=1vAGFEbJ+=;g?ASJc@ZhAWyM(3gL)#n z?fM}_;R3MFxx_?)dqhwoU~b9f4(TtyQUe}*Z?qx#S8>10JicAljqkO5Ig{adu~K%> z`?HM-&_7cIF2u@;IY%3=FVNVyiNEgPE~A1CrJvM|-0W`X2j>QiJxQ;C zH}hiT#$sCR@grWGLl!>8;AswO8l#UmzFhv=YHsw@!G2JLm%LeIXPEk-+bQZKC$(NJ zcCJxb8s?cX!c_3)H_MM7gVDgOXpFD{LeeBpHqCnvBxc&Qy7QK}$GPtwIesboVa5v8 ze~V_9Y}~%z>t?QV3KwO%SW&5w{hc$1@9>1oOoWj0>`ylEc}i5W0*X zzzssH($HY#tL;~D-;?~MY*90XmWOP+7Iv|yt3PhrgLHhlyS6up>+LhAlayoO_b7KX!V6^gH6Wi&v8<+<*{!ZP8G|4vdm1$4lLQbxUV3h zp-c2UU)s%u^)`(hWC~v>M!w)b*ko&x&8SUXCOc~`2y&h`a`(&XM0kCFJq%^()|@?C zzjj+)-+~O0HSq$^IkA5o$bbnwmNj>CiwcnXZl{7i-$5Z9BRnCpuJ!?V&TarCFrlQCdHP1*k|e{ZNks5q74C3XA0 z(Wvtn6n~>xXDiJvpv^hWYz_=|5^<~%%|pc&0&g9DDH$7QcAwCs1RF2K{5A8$$z3rN ziK9QyL>Z=5Ax*Mv1g}^eN*NlR?0jNwQ0vjdKl!NtDuoNfkt>A;Q&gVbW$jy*y6-&; zA}$`fXqgc$`?nUz#M*T2|Ha;WhBXFTZAA8F23(EU}@m zR|_rL4uy{XRctq^w@ZU&714VC1L$esx_NPtpi&yg;hnsYtp;=2p9+@>0)}wmDjaBD z(Ji?Z7`4UouR-P0PEhLiA#eGx9afIIt5`&WT%gKQdv`1nts1p_e9v;Fbbu0|YWI@b z!q0pcVTA@S^45TMN-)y`q=~MjETG~&Vzo^su(bF|dG6O!9*mU_bkm*&&`f1q z{+=sdvRNiZs-J#|w(DL@m@S-73zi{mFBMPZ3d+tY`3!Vc><|6+4q$GP;7iHjrNb)o z92x>HiuAp#0Z+)KTz2|$5&1@G$2}Kcb>`eJo;2It-oGnhN%_aMzn1<0?UmTl)I`$` zQImE-E}=kT)w-N^YoQc^)OcB0Nhg5rR}HB{d~MLuwD`L*DA<^@PiL#ywJ(C6aIoWe zXfQa$=PP4e18=Tondz##)q1SxKlMKo#wSI+HJjj;s%2N&w`ePsnHg(JsIfRGcZ8@h zwl%8d(fJjP3)&YFt)(RQ)YdAM8uUDdqxx7NecQF*(K23m7vC#M!U*hvCQq_u^4+8K z@di*aPrJ;b*#?8Vno{Kw*qEl()OBT6W-H_VH~ry$ZB!9?Tl!1O@?Ohd?Z{ zDFH$pdGCwgrhV4`cL*>O`!)aX2hb6F#cBV;%A^qnhs0q*R~6Mefx#^mKu%gnEAkO^EmzN zg`y{lDISdk$UT7vCcgN-ZCOE273P?-F;7&+6-3qc(o~`irpyuA@O`-keZtt4F-ah0 zt#Sui_@dZ*e*O2314u~FQwrh@P$np;+FdnvV)pz!;aImdfCm%Q1%UbRNC4^>huYuv zB<9I^)3DIl_N3t^=+nMB8`S-9lmzYr6VL))1FcLm&DBoHPKmBpmwZQ}jK$>j@+_K6K~?z(4%!i+`Pjf87cHgQwupmwzbW z@0Q9PBWcMdC$27OJDUcmG`~~a0aDw=on#!^>veVRS@ON)~_i6fBHAn#% z`4%#Nqnt*vQ$35!M(B(nfw_8^&-BHBm=I}`dTGQ)IIA$P6ri_MkpanZoQ{VI_FfOI zbdYC|X@o;(@r(fQ$vlw$A~bax82V=KnY>{ZOpG++0UW@#VN?(1>KPDJXT#oXYy1dEb7$h3lzFW{jt% z7zI>Ln4tm)1CLJqtzd4IC37*K?;ehX%s8{1fOGJ&F{Smx3d$WKq)B)9JbaK0SXhGF zzwH9bS0_<62q{{u^~I{CO8re8;|i^>cqd?rDt-_yz4P;n$e%x^QyYA>oDaP==Q{zR z@^xX;E!qM1)Tu@UYaL(>S!i*3joVLvmR{gx_Qp9wud7pd>IK>|J7$2@(d;4|>@R-2 zA~{ny=1nsu8bgh`5X+Z^PS!$E&-l2agDPN`cGxy(F*e1Scd1nFUUH zP`hKD1fo(PfCf8iAyU$jFc&8R_(p-iD_I^DmH5cbr7F1uJmg9L4J7HsBw*rRyVDQE zXu6k1GNKH=Z_a!O<_+snlD=(`Vu#JGxfk01=Oza47pJGXPAz=F9DV#8?CZESqDa?# zzo6RWj0mY7YAth0Y%i5NM$zX~uXg|WO*RaOy^8HQr`wp2&B7Qixu%fpvf!~c@d9a9 zX9?(UT0W!Rjt7nb&y)ITE$V71v%?~U=#9x#5^M932s0o;4^!XIj}IL%4wMP**y)Kd zg(T4(vk^a|*TGSy&W{v6-6R&;=Hr&Hb=eV=1Pz42nRw@4(B--p^D|o3LhR0_FE6DaKJZj>4QXI*AHcFnxgWtK+z0Zf|npjk!zB%|5a;q$#;!_0H!fW zdOD1?%ic~Ckdr2FIuqSi`PuY&3kp4vzn#mNvlbq}2t{gL6KF(Oo_nwPTf_)zvNCAKzc3&$G8qdmNTaA9ib z$^C;5?8^xqz{Rj$cV4=&)z>iiFr&d$Mtx5^bOF>xdGxgF%~41)tifY-cKyJ)lb)IV ztTKB_qLAG+_)4*|knQT1$7`T$N6$*x0e;!i!BQaGf^Mx$9lIrU8I(IoK*enb9=Fe|?d9p=#Oj zt=INmT$fp`hiUV<;0EsEXuAwhWsx@T;aC|DU`6_=X9_xu2~>jqM68xqgzaBPS&wfc zb56b($E)1DK3QXTAoeM?|2Yfz0`d2apPae2>D0A`gxjm^7@>8`!W!bV(%MV`S;IrN za32gonP$63cgB%ts}73f<^6d^OP!m`{f5Qs{M9R@a{F7uh8-BWwuu+HHs@Kym%kw- z_M7VXGofv|Ev3~}xAM+APq1+|$!2!@Bvk{KnNyN)NO2o^J>kJxP4$L~pbcU+n5(Tk zJv_PvJi_BZ;JTZZk190*mBM{n1FOVwKzHkM>^HCpjkH_;ogPldJO}iUO)M*A`_h=T zY0rbg7p8#L7iPe<*@j)j>N?OvFd5l0TR(GI!M81>-Slud7G;d{1Fzjxk=dNv#amjH z@^iMHGi2Zwb)1ROGz&+UOwtTu4V^>#inlyT=(0S$F_u=oxOTo8dv($EPjy9W-7dj{ z_DSesyxW)+o=8P{?{o0h&j0-V)_2PUR4D`1Nd?yfm_1)sQ^zuN=v33Nv+CYr1g)Cw zAsZ7Q^A1cMjq79~&DWh@^wK2*_^l(TWi@0a+0u>1=?ZBWST+0xl1(SWD)P>-a8d^t z+h1jGa9k0zRy&w0HwBbMjpnc}03zcWX7FQxl*CS` z(*B_BloU-A)HllG?SZ^I5qOP}opa)5D>^BiNnrXKtMP2q#YkLLmiD$hHMNH-dTC{^7(N$Q$HO8t7^5|nMOoQ!}^n~6jf7h(Tx&n{kH*R7&~i) zq=EZ>?XJ;p+BHJJ)5+ywOiMp{xSQDqMg|YZp=&|Ub)T{k`t1_p!Q*8*QtoK36B`Za z!cj*eYf<@;Tert;GGhQ!kF~RrujDPq70Mc(-?c5CNa2*x$qr!^F35N;_q8=)8gLi5 zRX3$Ls6G!S%1+C~twA;u#Ll320j|JSm@^ku1nDSU{|OW@b;^uFS$S9he?IY<55%8? zFIQBxiHuWM5zXWhdprw5?aY<81{%&;C17&94XFi02=$jITTlgIK)H!)n*t^qF72J{ zljlMJ_9(4wD*Z(`?}m=#!D?OMw%DSE)~xtSxZ%DNy3!6EEDk zTN*F6GvaJgHoEi4xs&NaaoYoVKM(Kq=2gfB!nM>`0O5h!VtHV5s-SS1UW`{2*#$(0 z#=A^*S5|O_LcbvT4J1=`T(HWtd_}6dTnBh89OnVRjxtwnp(UD zUWAwjQ;5|c_%6`P8zirdYo5W+CAp>2)TyB2!ToFYcUEc4dtLTQ_PB>?Yt(cNK#c=n zYf;-2;n?1_YHS-^JQ8e0%c0HH968UJybZ%Z44p3y+0tW$%d2MOtIA>=*IEt z2TM`o)5Al=SIQ;NNfK9qVXAnoG06eXQ#Qr|VMkbS>p`tkvq|qR<-+qAyBogE!a5J)AH3^XHRxl0ZRj~LF}l> zM*CzSI}6`KG6XdNB9x|O>8;)<`dPy(8aa<2uI6TQhD7Py03vO|fJ11r!GI(XJl;_t z^Z+ruD=B4dpUJ^|SOiVI9WHSl&*JP-n!N)wfc48&ONo8UvOS(lTf$Re;wxc5`KbG# zg9~#eaBD!HomO=gfJsEwbJC|0=2ANwYu2)K(*VRoyET3@aT|cDa?W!+d&zUVF_gsH zAvHI0MPN6E2xEOqISxs4&EJoGrAplaA{_q{4}s*;u59c9RDTo!&7H}HxRVXVow(1( z`Tb&LU^kaGhV6^h``7>Wed11>?J%gK<*mL4bfd9;p(lxD4iFt&6(&Ux83K5$qWf&2 zK0^dcvmW=EzJaNAE=-WeTnZm{#AIXF+iNh2+>>{diqMmNTiDi}GL=)egUoAn-l;in z={MgRz;SX32YNKWy-THV=9RFp?WLjUs~Y+8h3cBxxli1T>=+W&@@q?2y${%pbib(w z(+K|99+~?#KM1e+?KnW5+P&Nv#AILl5Lh9#GT+~UCUn!T9NnBdr~<6chz}LPhJKA5 zSwJqwso1=8Sr<%sO9YIe=D@9K0MsgqiXddTJto9nM)iO3mRopkyWGIo8Pw0nUHRKY zJ2+l~U`^t}l;2|USCG*E{o<;i8E+!LG0H9Y3-%rPDuyVy@-XnRCj7! zxIAe8&uZw~Z2?rUZtw|CzeDD&XlzI09MLf27kyv@^sB3sIZ*7K|))1zD+(>0d$%<)t4Cm7y zsyf61){JJG^IgSl@cR<<=W3~Rrhp0_}U3k}^Uk7RHgI7*=`|#NefJyj|(TBGnv01>`m27R9Y0Wo6LOtwCG0GjZ!VnQL%h%%|W+( zVN+VL#fIYXS$UjmD|eM`J^&wc(^XDK22UN(@+Na`$QmnHcg13#dOEL$;lIpEd`M@;lQP?Ju+k~i$5n8F<=FM(w7lg;$=>@3gX9$)Rm#3{&Hn^H7(b$vrAp@)4T+`s zJXW=Hsc?gY&*?X6QgdM{>p<6|42%GkPXV1dVpMv&oEi=E!hZq5K-&WwO3beH(#(rd zq>0#;T%FhQnPP4oQIbZPly6OcpOXgz^#X^(MBI^rqf|0dWT}m|)pnrw?118cftIDQ zTz7_S!=FMgRHRu4Zd_9i)gg8N5Akwwth5IZR^ZEL_lHe<6L@F&XE~I5FrvvInG0~f| zF?k{Hjvtc@odLkgG!Ph3QNQ_r@L5CYQ z6~nEX$NREtCeerTW}p@!+aA?DgtcRW;3uO#zyZtUGE@39h|2=~k!I_4s)wghvwAP4 zX4|^?K|UB;&9g{Nb?0S3=|sa*-dG{>NmB2TTe{-~F&Kv;kT#H3S$QyOr)Ms;Kc7x? zpgEK0sYt7D#AcO1E3>jF=gC9EV-h5L69}@Hi%G1paGvb`YLNF3$gP&^0M=mcU0hV! zS)`AONSw7Ss0UzF&87Uv+^|3mb**?8^rSm7wPuI%Lfg$K6IcSa;a!d|!uPm^e!FD5 zQ~Z9I`@1a;aj?{`G|rc+_2KtRZj~M^l<6QvHhQN_{+PPee>#5>pY$~Klek08E$lef zr0gSzRj|$DW(Ozuk-qh&pjxDw&2)$O$~`Ms(UR*hXlk|q!RNgaEf1h^EABr`IOb01 z=M?04DV?ce6PjR=nA?q)cXLoXb>^<7SVGE&kd}j0w^I?(1t(P#j?Wt2SoF-fU=ST^ zjRa=MliI;?B_meMwD0A<9&ZIRkHsSLCkA31qnKa6K}W`xI#k3y0R~akn2D!1qxTBc zR-?;U1albylS+C*gWqrIz99^&pL=;?ochX<)R(mv8JAoCt-a4y$D(JA8ShU!cYr@N<{Jv*aij48nLuCD? z8pVohcTTc6>G_FqW2!e_a*tp^3eZaJj)q(% zlGPW^OxuKv%KckPU;SDjPkNu{n~*#j`$$nan5E#?YntQPOL26IC%8(bDj@6T=LsM$ z3+b~RfA;cjCoYUvsAbU&k| zwNnOyzT8EhS+{nh!#-8S>J#SZ)%u*imPmW^dCMR?E_&;jvPNe)veB>Fb znVD^2JBzy14)Gy{2@=gK(0b;V*aKo9Cs4@VjwOE6VtAf845;vzkUnebBVNOj^r=0d z@Q_=S4?Wy4tIAVqvA|`_Wsp)y;ylIyoC(nd8{hs4%svSi0Dye)azUiJCJMJ)do58d z_-^TyJ)og<2D91mL}{{%iZM~h?ymlE10Cy`s3XZPFvP;P4e)JhfBLKw!flwxt3YF* z@+I;^w9itF@);I^95Z*1T(+)}V7}BU79L$}o555`NS$+K&o!`3e68E<9qk2znbrX2 z*9@J`sp?5$NYxZGlI6+v2K8LdIbl~~yfx3j>IADL+AWWURfYv8EgJy1wfVC^niwDwg z=J3?Osl}jh1@!EAc^JQnWc(BKXSzc%*gqelD%m-SbqyKz-GCIhWzC~$&9?cIHcICt z{RSA68o&}DVsxd?rX|Q`;GajVawR$A`)V#bGRLKw?mifsbhpaQ6i^Jd>f4ezocdMV z^lsy>eF8lxSHY1#QdD5%QKJE>NPv7NQXs_zNf#)q9wTd`XnJsKjZ6@Ufuj1S_GNJF z0;fLacBy-ADA(%ZFfjRr1DYm3t!fx`#RudJbaD)t_xOZ~7u>0a1T(=(HjtDK=&DVc z{`}xyu5vU7H`GhCdbg{*-@LLEp*}M9o+P~BzRY4_U6^Sb3X`0jk0kp5yG+5h0Mk$C77=s-6$Fp# zo=rcrQOTE;MQ-k+3oW`dIH6b!j)E!yv=$JPbMc?B50 zuRb03kOsn}q+QpMU8r1KY>#6A#gqAO^|!j*X6Vx`!_85neJtSmd@r%6Jgp{I#_a)a z7cOu^{z`w*na}W;<|MG|+-K?sNI1?atCk*s5Rn5UfIagyB$WlJTbA*4(7tMEYJm&W z^x-otpNO>hY47RRFDysjv*o%U0K^F^6wgAULShvSv<`iW28}Z0q$bMkgBn2g8%+Z% zQpPnAz#~z1HL1Sfn#o(0Rj(wz0_8V33XR`BC(R8{QH?Xn<&|HESKpSiu0mQ$fZpyI-OK^{ho zmjpux((>y%KzT4grXN>sVOP|qHh-1+z&q!=eq|m)gKtuZ8b}R;TuTWjs$#_y3mOGN*!G<0ezs7RGLLvaD9G#RuIy7S32G$uL~<}URg$6 z_ydKkxl%WKltN?Mlb+)<#h(3b_PF^g(%igO)SLGQE56K`>l~@$2GOp$u`jFM)EKpD zeYMkWBchp-0s zY2#JC6!)qyPc6e&P_I7MUf706mV@M8^ByQPr{)Bq-*`=Cszcm9+@_$0mqFX5(3(fJ z$3k%mRMR>>r1PeTKd-`~i>+Wfc~4~r%m9wkAlYfbO0}A@C5oy1tD06#dF$_TECnhi+^oX|r8u04uhRPrgZ=3AQ{tl*1lm1;dx1WRPFbB82ftiEdn_o0{imYF7t9EY?l zM`2TddDzN=$ylJ_sQJ%g|j*~4UpDR1#i~O z78e8+0^P=7$_!8X{P&YJ^=(mBk3BK6Kl%W*fN8W+(fVp!zi6qFtf0m+{Lc^W5z^j@ z*vpL$UGaaOFpGx@yX4M5{*;&coQR#JGBzq7s7_c1d)PdNY`c>3)KuE!ZLTShTnFCj2#8T)fP_lvmnCujau)Cx>~lpH#`m zRqYB%JeOPfl16c$3}q6`Q^vhftp+#U%y~UXjz#OlSUIXP=eCxVob47;7`AsZz?F=X zfuTfuz&dlMQ|;(%Hmgraf@K05i15>#@IYu9aQL6O8tJE_EFqe!TXpBSIETwiV`V$P zmtesOc_tk$->oM)w_QZ*YUVS&)*Q~O+-QfNf4P|WBt#pKdcoF0*K&h{uck9&2S?{0 zTaewI?S^p&ErFut9Tv~suAG8cJc|l?tyUt}B$VxoH{Fq*3hUj9W4)~Zz{k&ba$}=B)zrUdobwc+gF*%v1ic2jO5^e-Y&PyS9Y*Y&!WtpoozR0)=JiF06CG;IWmfpelxu zD;QC(Qb2tk_Wmk^G@5C2m$G6tp}~oPAuCo(HEpJ_kM=4~f^ePhqyn2XbyIcQ zt)OCV``V+UGa`QeosKkTkyCV8XQhSH#&aqUKH#8;hCw^eF33bG%Ey^#9;PW zen3(HeR1gX(=Y$f#r*4w|E1_4ILtjdi!z6CbCc^ERpsV+e5X1m+L&57mOK<BPSTNo-@LB3}Un(7#L<^6HAbs{knY_OrKG~RPgVs;C0;64-BM4r?qcGK% z#=XcefdsH>2h)>nZ9^>d7<(em37i-QErsgMNm{UZa;S%Dw1(bmG63{_rmILDH9uDK z!nV5kE|fg10r!N4+P{vFw|`XE0E@f9wz!i8gz~(x1%U}rxMx4fxG~vjb8wXsZfn* zEc*SxI&*WYS8k_!yM@w?4eSh^cMy)^#Oxvytoufg#?2pQLv<{w^5KK`xkCvb52oztORNc+eZ9k27%f$S+>@R5jl-(%IV9b!dpywPyha z+>-ez5Zk0(TKk1z1mIvCn|_?MVE1X2Xa5{V>hziVRu`UAMtEfUZLHD;|4><4e{%z4 z1cXmP02;x2PHK8w`k z2PL0-t;@zt@qm0uh~5R7^mZCE?uLjhbt2Gtaew2s02F$B>3}|~yvzJ6zMl7u%kk-9 zAs~PWHBF^^Cq-^^uE4#fIurRF$2{gNc3N#fJ4DZkU|u-3&(zxrE}r}D*OR8jio&`5 z<)F_0)_&Jxk*BQwG1!6uT@2s;@H(^yyB#hj=4MbTm@))HdcUZ^sWzDM*#%S)3HVh% zrMNAzwkHXOe|(4oQbcnLVB$$HD~CEO?*F zLe1{{cJ2gDrI<;`l?$rZ=;;g_oM8Va(z4HeDB@Qv<(vQmq3TVNpAO{SF=;TUDT>tr zMhG!+DL|RDWeAd5{Z1ZqOnTCZ!c_2LNA0W9dgPFGHu<$W)qAr{gH$Mi!S>t4^FjEd z{gK-JJgzASj4E{-38yqb;LH@Ilh>+xKcL2~QH@0KI6?WC+sq|m?^}SOj!9F`TU%D7 zLY_*25{p4n_M2^{E%2;#NKuz-=f<>bmS78BV!y5&je!JMfcdFLC|?@pyYv?=jduWM z+27`K%W?Wh=~bVsUiI#S*o*-j{%5nnme+9?0H2_V0|c4Vs#1K$PR*8J`Bc#sYn zJv#2v(?02+WY6ntVp0F;Vi#C?5(8}aABzxEc1*%VXTO1hV|Vi}XxRaY;T{mThwXzw z3}o%|_i~j-umC?f1q2cyfEXx|>ao4|*RlybvIHn~m9KX7Nt?)22>+W^u0mi0gb6Xf zQK)oXj@ECl1qgOa&D_|yN_x2;Zx&4Mc@P8HO(7|h7`m1VLkH|cfl$a~Du{vwP{}6= z_;9iHRgs-!=OWTvGTeX&w$(5(a7#exle@q-jUq-qfEgEo-kT{X>viR87hmCba$&HK zb_3Cwe{;u1r8n}6p;Sihjoqi`r7q~&)@r+|u){EJfI&-_VrZv$@>($v8nJo081gRa z?1wz;41x@>Hf)iMITCx*mkWP*E?Vpe551Wx-|HW4Ie4x(x|rG7vRV6CC+jR?t|nj5 zN*}z3&6oAwp3Y~$AEn&yQoA>sL@YC3*fNTcVGOlI!f)!7ym`IuzArKMrfzGh!V=le z>*@|d8nl2Ld%;!=24#!~iOoz$72ycMQ!;USIuo1&F1L2;qm)6UCiBC)1vRtQ;L*yJ zjk*g6GR-gwc)V}EbQ<$G+Vb&+ER85%6)}iNYf&k-)2pgf&;!P!bpQ^nKh2)1CZNFK z0v81gCOHaj&7UbKY-iUr9Q<;mQhx0D$B~znBt(a(r`3mYbkVtQU{GJ|A#O})Y=ry}!lKl}^Zu~9l@Bk&c;wT^Gb(1j+d zyo>x9O|PMKb9~X4KNCQaFJ1($Y^g&vb_ed{g1lPCGA4P z7W$lb+{OqCVmS_A&uY0{w)B%0f6b;q-6a_a$!;y?-2=?O7exgx7p%p8#j9-(YU%*_ znLLfW&#s$%%=&Wq6tWW!IAaC`*$L3<6s7pEEjFX?n}yP#%s=i{GZMO7e&|H zG{1TKDloFrla_UL!N5z*%}xzz<*O2~vGW+5(X?QHiZTTv8OTgoOk%PxaqOSWj|XmA zF{q75r0C0pVZhw=YDC{nY2PlPay~DA%zh!wdCQwz1QBZZQ#I6?&rr}bN!}}hfa0;0$&381Hd&+chJOJBUc16PdRf!l@hp;SyHjlJ=73ME zGO?z{?cLiHF-Xpg-JjddC9J7~D?m`R&V_-wIObz8dy1D_SE7$!=g(fmOW>E}N-?lL zjs@mPbx*IpelY9F;io`|O}cKG`Q$jrmrkFn)lFZ?k77|~l=K2zQSb-fv_X=(zmPeI zL8lqDL8cd9fMMrBB^EL#=z_8CFE)=`zO-POVDme zK)frd;GH5e8^#Ylg-UQMd3&@fW9v6iJzoUOkmONA5P3L>!&8Ak#Tu2?si^t$E-i1# zs~F863{te?9+q0r?sw2pL+nc~bPb~`>#GQjOhuk-ar)jeewAz0Llsi^-@oBh*{(b@qSx zI~V96{iMlicTfF2Fpz0BUSCT(;HpCEWGN`!f31sqP44%b^hEcHKM%PM)?IIqh@D^% zAjao9VqBGPgO+q%Vdcs%H{q^c{Tah*D2ntmmpc@)DfhsNAO3PPm ziB$m1ExNF>1OmbQSRR%hQ{lpbi2mOXRVh0Uf zsbXp^sUC>9WBTsPm8xdVI;i1bU&}!?7}xUF2hxp2DT@kLNgMzCD6%r*%*m1sU4oR( z9*z_xePFMAW&RT>bq}?^$cO8l&(r?m8Z9-p9@b-+C!%CyBX+;0d}@1m>K%EzeYwoK zj}!EBzCuOlyfu$~-Q^Nmyw)Dqnzaq|9NWwG>%dcpSYVQFT&`NV2!*vukPc|IHQkYb zJ78P#V_#c8W}OCU<$?6|7)OA2{}n&tw9)=|_Yv~2<4T;qql^X-VNH3yrOSACJQYqx zO~-qJq4z(3=pSB+iP0mhAb2@O_l;gIe$Gba;NzuNq}ikuwU_u>0G=hEecByBAJ244e4ms>`k>2Wa`eg7zCWtPkmh98J(uFbG%!#Vb?TO|9@D*3=mY;|NGH3~+4TC(j!N1Qay+h<`~)?;nbua3&&us841`->ZI`1`5f zr%z9sEG}OcL+nSJbw<=#9sNaRiA&Lqx));BYVVwyDmJrO+tHzF$P;j{DCr>_+aLP; z=Ml-PpB0*8#LIf{8H!07qy(eJri7l3PE4_7r*&E*@n9YL*qX3~-coOz-J(WMXDz#O z@~=UzE2o*nU2Wr5{1P5oM!e}t+cOt^7SXWhJeS`b$}H~sa#XBTVa4*8`FR!*qZ;Mr z$js^BF+;HfRzbH^i7~|XRu}ZPhaVR|dOMbe46;l3W<=Gk+->vRk4~bd`;^1v%q}0~JKMx4Y_9#`-hhcJMZ$rByCXUZg z48iv-ZJLZR7f~}8WEc{P%_1r`W~-{dZl>`6!tbUm5$Wz}7F6Fk!$~TA`{PJJr^msr`>XxDaj-=V?OLw{g2uu)4!K1VvKw}izvNd4Hq48f(e!(;Uwm0R3HObw zlf~ge^OU87%Ij%OCwysi1RV03={6PC1QL#bO?ipXAMzTWa?@I|4xBFYe(8r>Bu;fy zV%UejkCO0}y)l4IM@tVB=!@4QUg?}lH#TmI@5%~YTw@V+|8;}RqPV=+kD4okj{Zu9 zW=$g#uS0yG)uuSVx8}E3UoX@!ts0yXcU%6JI^u+MRsSlzJ-pzQ8Dplb7bsns`U8-+2?1dTw0%DKMp+ByG}}Ol3WNq z@;lU;l>zW+@1FW5;Lxu|4U|unsNmMF9GNnsxslsdQ#FI^3}q5}Il!ATLm`v;aja4< zFRuw(=XCmZpFohdcNU#K-HI}P8qkyxczI@jYcb15`kN0CyuzO~aNr%pd6UGR$=%r+vTFDcSO{AHqxv)%Ieo_OAJ@Pa5r0PtgigWixKcKLhox{gK8u4 z-g;(hk!7d7O@*(3RgV@m_TFogJ@~<~o*BJr_t%`7d0!OY-c(E!h-9+?lfpYdoLYCsS z)(>AEB%bnv)-IZk z6wp?Y4B$7RKYZ(H#rlkSW1DvW;incSC<2c>eY)gvFMJ9=9H*7ryuxqRxjef5=f_bp zA|Ncqtm&7r(m?Cq;pJF;U}pNbd<-uSN&^G)G(5A3(#*zg>HUPetkX67lEZbrJOU^} zFTb9RQSb{vXWou5HX#Re?77VI)FN+rI0e#+W0NJ}{rl7#bUZq079EMVIaD&d!ty{92C8NB>tI1 z_PZ{L&(EiV#i%paC69Y<1=oK_#f$l#M`d?X80m`KNZWRM`zWagR+GobWAB`c4amZo zhE!I;$CE*r*r%JIX`1?R?Wm;ZnGZa1n8fhNlXt}Z57w%HfziQfc9URqd#E*4(2?9-R@mC4%n!;78B8&DX~?GZILTs=qIa$f zwLDe>Umrl$m5hj6JYn(;I^#}lL*m}K5PM~(9$n%7(~=_YHVsfL-7R;^>_=&~apt3* zTU$P@U>xutbn*&3O@Hf_6YU2SKAbhG%56n%9rQclVb?G5=m^+L-r_S>8#$WfB-v3P zLNCv$kw3leeQ(Q`g=V0@FzVy!OLAD^Alr=_-=|<3&}^&zQit0R+nbIz+Vq%diJeVY z7TBzQyY_){JDx+;f;r>Q*}oIiBT=+GBVS7lO4FrLZt_{+7MYfOsoHVk=W$W60yyCT z6t3)7A6lG&o=^+GJqNR#!ONe|0yEs#F0oL=A}yRrXT#6YL`C2yMNk4$;S z(rWG#XU&@<`xX;)z_8c#zETsT-$&))ue)>q{{4~IDO%{h_^+>u?^&DA{=qj!1Jo#buKkk=+~{2--D%E z_(L#tfLT*7McT*U^2Nrb!D35$N$5WO!4mN9rTb4$%l72g4!%CjX4mfXQz$#-g%d?# zblzk6nwmR^MGc+sE{mP8#r`P5L+aYF=cO*6 z0)raPa8~J9twIyoBD3alt6N`WXc^^#XB9^)S1cu^;~Z+q9V?5^o<+>low&w}L4h7Y6X)?igt-3S1bRQWRxns_~&bjd_{9<3|wcfc{uj5}r zA{@#5RctRm->BA%xHob~Hd3wAcbY@%R#%jb`1r|B^RNbJAqbX&H#1&FJmEF0<+d5f zj{pT_`B-l4>tk-~8G&LSy4~AQhi*%TBPE!}P>-KszG^$q%0=9gse=oCx|I=;u6KSK zT*wD+|M>||H3Y|f1~EUat#N$jBft5TR>WOHW>(NTrRUO1Z~wjGX#>obe{b{GAG@;b zbO~e`zJ&V}hrNYGhZ2R@8ek`Esztzkm&cX7DIQ<&oz7GW!ejN%`#Va>zc)qoD&RW> zcfBX^;Vavw|NRZa%epb&b=!_1bS{ut9NS8gM(p>BnX=19-nqoClx*Zvg&<+2%7ih4 zDX@n1sfJ*7P$}KAzWQ~PuXe;>e$jV-RqMPZY}yf*BQU_Btc#uCN$$&<#jd(XcLMcBv1R4xk)ua_?c81z=vCZ; z9&<88tVP;OYmzR^iHF&}On8v_ESzO@#?gIgNRZ5ecXD4E_%>}zy@C%}!ixJ)stukx zR*P~8yG}Kpt-1Th(cMeyI->~dw+=07qO|2)9qQ093qAv7Bi+#q5?(|l?F;$7DIZ!f z*2)rYT^lIdN1wn`-oo{37WUjIENqWsN%?5!Kg)5$LU)5_7YW%#XI z4*z!DuAp^rXzHokYu{X&O?pTnSM503UsozzZBD%%Fx)y;VwGNEKlr3OL++;RCAQBl zRw}X|+z+c{I>{tR*(J8(Z@(I^8kmb@Q#~OYdne+Ka-vpumJ$cjdp&}3dqwHSAy~5` z2|l9LLpmU>+=V7Fl>!A0*_c}icS2b2f!U`YGgxdMQJ^o@I`K5%x)lCBnS}1-cPPlj zw@E*=lpE}3+bT6dTxJj9bLhu#Rk<#^RCFv?XQd($wDeLcI<+Ar%C{vCu zjz1xIZ*Q2f-U_s-#`2r$#hq9f*g09_faOk@{<*ht>Gh5GpPU&U zI~o>`O9z}|iFUOco-ci1zMipPHiZwCXC7I{}G2QO?OzgLuL{`=R> zYtcgDQtQ(#Ux%IlC?8dbBazF-wL1OohB76NS+1qY^pl9=xXzV|DAQuUvwJZ@vI6@edgrhq5xUdv+sQ+N&BYVX8T#|J z<(WFt^EKObN`1-9@4tgPU}0xz2=>+{;mYB?t^n31L!WcSlykRl^543Yt|FXoh$*XZ zwrHI%oVxQoL$BsH!H+7q2TPAM6kPzJnQ*c~*#%GJXdL6l}|^>Nxv z+d(G-G}n=eW7|ukTv6Fd8?D}JTiXlWdy-crtuuI!4%6QD zUGml3m}`#$i}@4qF%bQ!V=0<%P%7sj8+9kfbE`dR-+r)2`{>igCex`3&?CJgIYvbi zhs%^HS01Pxn6F}RO&H%3+CGgiq@8x(5_$>D0j=?*mi7U43n727bEMA}CYN|>RU(__z zsc`Z;;alk7C}rw@q+pby^xPe;XH=sjV8dB#K{h2bTi8r^e!!bng;L?kW4!tmS$Mf; z{WwFzz1c-f{_t>ortd(Jc`HLrD3iDbB|-9qO16?6;`*{?ks1fWL94=~ViSz_7g`M} zoHDma?Nkb?m}bdR*Mxf10z=2ESC1L^@J*!u=l-r1)+JB?feNfWCvh^Bc7U@QSv#DY z&aGdq)ku*X+mz%N7#@hSfj@I+J%w33b1}jISAO;H!HU={q}c6mT6HgObSfOpxY8VM zMss8dxpkn|p{qBm(Le7kQ z^%{-}>mmC-YBaq4e4(hY_S8M&(K7qtn>9G%?ZG0;oxeBTHDUXZj_do&Bvz2E<~q?$ zZ)Me!ZP^es)4hpqyf*UW5n;~o1Yv(t(6Gi^700Vnq&8H#a${{0{}7SNG~n=ZBg;Xy z%cQ}DX~NMxHKl4>E6G3nI{(PE=*<8Dt8TfnWoi`I11>r`l9mw&X15jT=yqMK6^CTJ zBs}&&;1Av?oJ#D#}}*96^FeczlaI(KzD+n!5; zJ&Y;&=_#U#g${WimeR3Hv$Vb8%34brgYUz3mi3D~HgM+#9F~(s`<-4ei;F2$e=JpH zSGN9j`G@>5{)?wlM4kQBFgh}rqZSVtsvXHgFe{R)4TnM6N87>5QBVDN3G+l~$$!8m zfNucj7yVS{YY#ra0-9#euk5^E zt?amP@#?MZ?8DIF$8)Lrb#ZKlT#THrV7$JOP22qI{VjA>`$%3e&U1_ND*pi=xmK<( zugJam0f%~CiQxSEKa=Ol`!RKoA|m&N{qz7=V5em2;EXf&^>zjtPi{}=E9!$5S=w`LYTP=}IqOltR%T&1G3Ccn635h~MJER+=I|OM&Q0a~Vq`O;??(Ptj?uMbeK^Ph-iJ@bNVc@y=-uLrc>mP5P zm(TTvwHOx6T%6ZF_dbr#-sk?33L&R{@$;+9rju_-SkISgwg!|!qe!m&ZhCUW z+8Xe;vRW?a{R5HaeOK~~rH8YG>TZ-QVcks zz~nZcU~7Q>X<%To%@=qLLRsV(H3Y(+@O;naO|`E|I=6X3%LSqQn$K-{=U`1VB%DNs z%W{fS*m*v88Io!+=c_8Sjs8ECAbvizr198G$wf|NqyyQRp0xhWk&|FF-PBt+l{5uE z%h9bTCsq!v7B47@-pcewRF<#L}{@ce3D^~+!Ufqu_tKds&=$IM$N{Mal@BDL4FZnx!%XLWh;?(WTZhQHbk z-}r9Lr>fLz009`6I8OQ7e0z({oqMF&q}{$WkoA3k)9q+)eCg%E8v=thH~XLD_(hWq zURY~uoqV?7a_&z18f4aZB;FL#2}q1`(c~nD_Ds{}bV*0CXr^x1pc-nN%4=;(p8;&e z7o<_P$CJRLPi}4O=(BJKwwIW-MpN`L9w>1f&jD(ePJ2T`&#%{FIPtXqQ^Bi!pRLsQ zrr?W>ue8*8Z5o^*RT%ZT#-v}L16fRXzPtuSKrN+b@%M+QFqtWw`6^utgcyh)Fkq!S ztIlLqem%BByf0MFLUu+|70L+58?>9J_f{LHS?_YYl1IBo^B1K#N#)dk${S}l1f7%S zyyu$u-4Q!*k;terV-_mnQ)75v*D;ofLuRA&f+Wz@-fS#MAy_tMdo2BP=UrR65KM#= zlZg4tC)_^FDbS%dw|TW)6VMQ^b>e+HVqTSt1|OViZz+neF7pjA;6Q_&T;IYlkHmQs ziUsG7g44N7zMB)`SLz=LyCj9Z4!l_Z@v7!Qbcv(+_=r2&Oi%0OxT7(O`}NT>>uQ^8 z^5eyG!{U4~+Op;K0w>S;jEjwc4 zzboXvl@5E50)q*^&(#p745ERCl36v^Hr7Xzn5|WkY!^<~A7F#9t0GO;bvzuQ9v6=P zs$)UzTF@Mrl!-M&o8?0!VInY>$%9wqg%su3rvLyk(T0;Qq;PVB2O)h@QV{^egrbY8 z7%Sz5)*<<7K;GWom<|NUzZK@=HgW@|gkkT_Nolv#oS@TNavX3oK|Es49FEZi*&NBP z8>oF}qPwu$lzF{1ywPQ!TrPruvr;OC+k8a+_9X$|+s)Xg!XtUE|3S?4;3r@D_ZgwJ z9!pP6c)z@8e&_R`TySXdAOK@wHJ+S_rnMM}Qst0|;YIZoG{sPe+^wH)4Pw`H+IBAS zzZ-aHWcS>PJR(~14XYJt=6Kukqt=iq8&plFJ)o;Z0cc4d+C#P=9@%GjbZ`9MpOP;l z!TZG}q0zwSRrEj~cbslf(q#7zCDCER3e%m)YH>-SvXiI&2^cnCphSf5%fm&Uc_-%q zHatxX20{O0z3N4_OK&62K`$D1`Fb0R?T*Smt8VY(-E0Q=Z;g&tpVJLF^>&YNqYi~Uplspx zUHXzb>ipGi2z|+8nF^UcTAfanW8MF{f}cxP((@>T&kMQS9sZi)=cML3iZ>Q`f%9R( z;JG963M(A<@4X**iStdy8huROcTM1&W8UERcAxDuguD? z0r*#Pfwjn~!5)XlZiJD4ol2%6#4-(FF1e8b&e#4%0^70bvjr z8e;YwXkkWf_5Pgz^BkLYuKRTO`tE;Sy_S)Q`Cpw7N4-}zdb<~U4U%zs^YZFo&RvUW z%^>JI#v@!@D(Fq1o4XNRQTK0^6T7bU&$5M3q!%y06@HbI8YdHYV4cKW`-FHyr(Ub(0wL$t?YNDbiBSauTle3)dq94OJN z31ZW&QO9e-i9hpmU5>t{S-vhqISZal>po;j`Xu6d{3BOBN z`Afw{;vKNVt1qQ`i$Fo^9^0U~f79#??|hCHvjTQqcIo`^)7sW-3Ame}v`=q=)U5y= zW2XQ(C9#qR=NKj0&SneL!bw{l?zN3hZd@$W`GDO1vcQOr6N$n5`q0tKkPfFkByY1j z80`|C(eO>^)wcqiBf#gRFHlJ3$mX@5es3Q@dXc^EEvHeT$E@3EGaXtl8@EUVrx$J1 zJYH!r-%p_TVk@}yfRX33o)H=VXOCH|V#g#f1CA|WuBc%QxsXd+e%3+_-mf0MH~%7i z!v>~~L|p!?Njd);n@@Kp;?$ec);V>{Z1l?2e4%JWA?~BqshT+NnJ#JAa~7a&+Y}Vk z8$&MQrrd)IBBfC?8mpFU5ot&Bh3zv6w>}K|A$V~}aLh+j^iw!>iy<0?Us7qk-A)=& zn|aK}h&N9>5I`eD5-?;973yuyTPOfB|9b|=+>nrWk@Wr#NC|*PgYL^7_InWRZa~cQ zyl*3lHHo_4H)_VPY6KQN1eKN9#^Ntx{O`V&>BC_x+ z|J^|#^OxqO@+WvZj3SRK%)%VnsglDxyzdl6i4P32N!gV@5>AmJ9WiD%Z2Z!bSOqf% zydmDWC{rSO|4{5EE7^5$aJyph$8-}FEzMGVCyl^Pvymi)PGKbvnmv1z*j^=Iuio)L zxsEgGXB!@*m5Y&=F^4}0GXLI9Be|6Afs-ULX>)7cat2SkP(mhwf#Q3&ky5XV6q8@i zSCv|-DTS8-8`J^Y2mM%H*(Nxt~VEF)?G5WotV)3B2O_shmFY z>*)>>Ww@uZX{&cq37dt@=B3s|)6f)>T&=db^9fp{ls~B-N53|d0rmO{n$Ghpn-Y^L zJVf?*3h0^=@|>9=UL&zjurY4Ie&<}_hY+}eKj~(Fm{ZKY&3&6I4-YyGbsm`$YIWXy zeY@97c_&s`0U(B$g$;Pjn&2S6)jEr&a((OlHZ#m!7YN5m$ne#~+*znLx~2SU=Fde* zVWk>JOFE^PfZ5k2jM@LtyfB}__H`H(9_-ieX@fF_l^l%A+4;!7 z;6>rad-|3}ACu<%Y`RQSHTxEX4KcePmj^XiE{<0B=uWq;80XDntLE3-suN2lll}zEUiS(Y%yEE{`*l+ezdHJq?>;XW-Y(Jsj zX|oSwvBtFR^0u}KFp4FG-Mkq8)H(jWEcvWnp=h2Ca3bVlPBLyMOY~2+^++;vQM8sd zq>^^S7I69!-udcSK^mi@NO*LtcTbkvG@CFEMz0YY-uqMFiiY|xN5ZZ?V8b3$?`RSM zg9HYZQmJ8(N~ZiWzx^nn^IduZcnI}q3P(H@{5lc%#&FLIOgb7MM-GYMMLR6aMZNz$ zcy?wwXa+0SR&P;%q4LHBy0*ISq?pQVN^$hizbn@8_Nv8s_Y#Ccgk?Gy4bcWIXaY7x zQmn+?r^&ZA0*5D0N`27n1uwc@q1V?Ey=iXNF)E)1?DeA+?amw#?|ESp(faW{UyNN- zK+4+ba9@vCz+)Uam@B)ydOID){gKM|Ch$w?r?n-eEufN3Y`;AZS-dk&iFG9B_bv;M zB7R6c4(|Q&dieXH1-hgSJ6Zn~C}UBOc_~3G z=9V?jXIO0np8q{KP8bARf1J{MxN}Z}KyUqGvCT&RjW4A>rya}$&j~K9^8l40kHtj7 zb7a4y(@;trVs;VF8JP~Dpr!lI#UFXF?!UWrejkMO90v1QLzIAI=2GY#>+tm0PvFG% zf1Nt$meJ&VvP(_Yg>_%Rhlr3ho8Vdrtb6TqA|~q2M_6X>4zK;%`-ht&0rug`QBDcy zc!1hr(}_;Pt2lmJ9ql&1LYKp-kMM*0Gq}B%a~~7!Ha2+a$Ihr`oT%sdK@Fzm7yDen zSE}fxv_sJeX3auDIj9J3a|6 zAlB}58z=1$YWsKfP#eo@y{1P~>$Ed6Dp3crt|i-PjXND2H^^)}+P;UU zn~tEv4T!J4RMa#tE5dG>bDy+912W6+_xFoD3JJU=?y;(gvYo=-J_v#JqyxThLznxr zX8q?o!Uq@sDl%~eKI$o?h^7OAjgDd}M+L1zomoRtLTyO=A4}dyTq${KasZGGW&;SI z6rgiwR97YQU~-@}kNk%=b>EG*q*0#LGnbM2eIRA!&3+r8bI9bL$gi(c^0ln7-mfj1 zfq~13cvK;o+sq0l>NS5(ul6U*Qx6+=w#mY;Ef(aec~ zNB0M`K34{rIx;b&8PJ5&mp7$8hX$TIV-k&}0zlUAnwa8%X_MM(WVyMl$4JL4I^~bdo5yb?$5)R}^t75gSfdocXN)ln0Fq?;K!xrD& zMmp6L<(EXvrGZyIw--=lT2rwp#x4URqr7tcPN@fS{x2310A}=&Qq(74udt81MIxoa z@=SdZu!_cVu44HeJH2*FMg7n8a>B^q;^;8O<6Zb#dRu@beYza5oj z@jcJ~CG+fZ?0JTSZvIluopHv8qwg}2>ZDeEk^&MgJFkxOq&E1V#swieogvuAi`;Ix z_3QCkTiIyWZ6t}mR#l%u!rlqjEIaQc^su?hF=~qf!ZUW*;<8_$*WspO(jP^?Kl3Xw zfnG^rRHK3(NRV*X^s<*f0oan$>8TotfLqiArKs;gy{Ol{d;+}+OMgUa93i_dU~^Um z&&{oSNHd_(li@Ka|Af5o7W@a0Pn^~oRWrFzl&cX3hVsf6;HAnZT>#9`?r*nJuPqxx z%mG-aDUn2+iuWF8MIPa}=fD)6)+`0WrKj~MR`3)nZgrF7RC9zlBjlfK5#Z_uP92jP zcoh5StwJ4;4IBz@_g~s6A0zgkoeeBpQHR|LigY2_nb&D+r2`-~aLiLTC54&PV2_H)p0t3QuIzHejNw*P1D4cesf)Q2;+R zZB4W3GX6g%ElotgM2;A%P@M(V^q$XNb!U%l_S3V&bojq{h>XnkA=U2QG(CX-(5My2 z&(J9pIm82d>tL>YN_lr8+tWwFZ>iZP`rp6*^d%nCMp51>7oC!EIW^RGBfiWm}dk+ZG-#G5lDI+`|@ zoeWWLm~sN63JuV6d^*g{X#i%K%#}#RA5M^uA_i-Nw3?l-!7DA^nw_fgq9uJ{l!%5! zix|BYFX56T(&8qwfTbY|042hIs6Ff99G z*o(B_Lo~zl=Z>`ZLVJ}^0sB#aq6|iGeEahoh@LhIY1B4rxnhe%z}BAtkWC6QXE$ef zEnbMwz9<~vpkO7oD){a!nSlNN?yQ*tZU|f)eH<+0zLDUklFZ#A6FG9$1}r0?4`Yvt z!0Pka-k<%&ZGT3p4F)ACN$*f%_N!jfmlWN-rgI=F>E-SL6a=xt^6?fkLwLUl$zxC@9&Rdc&Iu&RrtRnAv zd*kg1yKi~WyN-^Ssc|hdx)$;|OtQIg|6#M*U})G2gpbSXccy)DKAyD-iFbT-ny|5a z9aV36q-0M0FBxEM>CJ&?+Nx5VEznSLMVs;?;TwS%eAX;Y9p2KS;jYs^{-^Rne&^*r zS5_9xdelua_-L^?aaDI%@oecc!<_5>ELA*uyM7O*<=c~q`e5js*vhq(B@9bkr0Yfk zu5j-WC~jy0tl6*tX;k3BSC74c-?w#h+l`Q`6dV1q4Z1m_Qz-a;zPY0$TL%W6^{ zP|hM)D^rw_ajT1PI~KD?3q z4OaA?Gkv&>ZZ@9hmnAmqu99B9@Un89rhcd`6(^S@o=`KL+Us}EqgL}-F zdWAYObDY2rhR;#zY0Zz-)1SBZ!gf#lBZ>U^eMjr~7%Bv~6%V-L=Q z`~LHTHeL3b-M!+;jCwyNYUaBLZSgVzo8KL_L=<5h#++pE1j)>z+w;|UYLFeeJ>VY{cC>^XW&SWXuz6rh~BY*#ly zzj7IV9FF5ihOs-ME)$GK;)rDo^8J%;;{d@?U3=_TTgsNPmM7N$t*C)Ysa)&b>RRKm zji&u5#JEMR(0IAuSu67%(6XH#Hq8NBq!gc&y8q;OrGwG^L@W*fYEjDyW4CE!qY7V9 zdSQCDg6FRyNWbD1oCF0Ak|FGE;Nrxw{cijs12q$AR)i8R`58Yo+Dr*POZPdN*xj9?$`)gCfzq!*=oImPd=v}-rkvvuxJ-=y_1ZAb0Te)nqg+WeD_$?bUO z!|rcpBQn+_MU_q%Z5@K1M}1@UhbvdqBw$RxUXieB#R$VEFLyxh#L@^~D$lBpXtc*1 zT5j$hXW0{Wu4o?FrdNePiyi*67XzET@q?Mw;qcS^gS-HE5rS8u8{OwjW<8Y zw7{x2qZ=s*ykwZa3W@RX7HtE#JhoN-g+UQFsy`%jja<$zSW=i7G4n^se4y_LrL z4)g}p#ijE43OXyzVbIF*IA%zh^afWYW&<(ajA}YCUD@%Y{3NrPGMO#GP=~Jq>R9Kh zn)+kX7s_u3*+M$$9pEFvsLWdAK2R>X_Z*WTW7q9a0C8%Oux)%d?X=8aQE&5lF)RKa z?AH-m2C*OcefGG=hkX(&yIE)(DR4#H)Z}q1oGxe@_@`9k+d^=aYC>qt=Nfei`Hll>$NciQU?!KYBckALJ zIUSd~!!Cd8I%0OG${WOS!zk`p@EEk6#`k7vhwKfy)D-^X8`|GL)GE-2jOv+@plvXU z6spdhgys7V27Znm)K!tK6=W{e*OZvs>kI`BR=lqLL2S;^P{?n4=xX*}H zB95A^*Wcc5wsp^3k5ZoZV+Zey%mIZifdXndvIxxoWxex8xt$!Gqj^L-Ls|y-b=SU#Lb%K+P#6s;> z=6zjnetYs*_1@E4^LiS$8`#j66M?#nI=E=ygH;k-6zqUe>Uw)wX2=^JO!xWTBb=U# zY4N+HRW8_n&+(No4-=Q9-AZf4PV^gco7r|SZ~14w1#WB#F=pR8gz@S}LB@CR7tra8 zOsNEB={?dKTF3C>?zH}3Ti@$r1eLDx3D*yM6h8tl zsxSXen?kQAKy6+~ki?+Bgcvk2b!s@2gU35z{}$il^jZ(7kG^=`g)hBk$ufV zy3Oyn%L1p}V#8ygCYFTrh7v+a*C>uK6ro_l&3Pc_Q6a3-s#@P#=`B~Xp|u(0*=2_a zKb3hg|C5zP;Exs38ojfAR>V1y!ctIppOw7){LkSay5`ucg?8VnnC2!6UinIm8bBg? zb1D4{KKs}JnQlb?X<}xvE=9XN))LwJv9XSki`2J!hL784<(0}zvm%#At52824LT|H z@3mS5iW2RLEZ3i#;h!ybu5l81J%qpAE8pggE*n&&cG!wsYbP=%j}W8+Ay5nNQ!J&} z{E;^zl%8u_(Gt=hpv{f`%73*EU?vjO{C`ldU`sp}Nws2oKmL|V7SC;G3_0JO30N=5 zGeSigi%hA&)NmWhJUWO$!=@bRk{Gb6CWkH-I$LAKbc2KCc6Q%xZI3jN0XO;VZIDhxvN$pgSx$c54gXJEr5fT|%sGMnQFDJ8=XVM#Z7VQ7lH)D76K^19PfWM3sk}YE(4Y?E zN68G-gr?)6=a6A=lrw_diFtJ2C=XWnHgWPjgL}q)Ob{|1LYXMofdZbg>wUAXu0r-q zEh6r)@5*eyG}@S7c}%27*k7}AP?3H-24}hVMb7&m7gUdS~VsZ2>cE2zi zzOOY?v`3WxFnR>y{G~7UaDYk3xmJ}$^MG+9l@Tf?2!*8X{*=TVevIsNe(9N8o-1K$c-Bg?%3T`HG)hR@FmPt;gm zaE&4jN7C>IilT_%2nKeb#fA>M_5XIhbdYq|iHsH2PDF_l(Bxmuk#)lAz#U6s*Om{a zY#hdy6r%mDRuq-e$b+PcZcsq9MAno>&G7>R%5tNm+UuxBr z+snLN7h#vC`}_t)_RY({nJRBLEDe(v-JF+FrG{m-;@6u5Waa+Lt+i-o4u zN8bZXc46am^#HQ}_M{UsLW=(ZX*-(5yy%!L#Yx*asPUI;w%g$x&`zbH#kjm^BFJzm za*t*1^2uNXk3Qlbq?SY0*FWoV;fJh|=gITQDt7rXsSW`aAn*iiQNb^>`Bg-}p~fO& zW`xpI{+6(d!1jg0o|(VRJm)7+)J~LNTYLoquW+$_{YmV^ znjEoU1Y`Vx2g#!-nX94DXO+HbXJ-5&b3|;rvd}j?ycT2nrRGR#;%`oZ8|lZ?i{y(T zB3Wo@XWzdEvaWGp(>J!dSQZDH*q}}PtfmS{UH z;}bi{!8Exy39+sMbS2@qK|xpz+pCX~PWaGPlZvh9*bkxVE}v>G{V2+feHy!VliXe_ zCIGAYgKXB50US>lbPb42U;c3zRM2Ls!B7Jkqm25o6bPd%AM-AjNtvoSi@4p+$rtNR zs^?h+NVkz>14wx|$fMZ?C64#RjIbxs(1{B+n2?UvPF-#-bt2NAqqmw(X1vqNcF;vf zrKWZ{+$K3<#wA5BEl^IWLRQRA7E=m+11F2Z*+d7Yb-2-bgbaXD{p~bSOrIl^#}qBz zY6oJE!4MurgJb?`g2G5@a@fal-Jv1Il~)Jl|fOg1{La~H4BWGWM;LgdXN~Bgnaqt z{k29mDS^tU=vBF{V`FI7nZ5A}B8HgML79*yl1GxKNHhh};w@c%OdF1>b#iOQjn%4Hl5mxw zNm*D7mi~(B`xmo{#yf41*>x>2_E2pv8bd;vhJgWuJI9-xxOe*B#3~)iMWBvA!{&yd ztmmiY2Z`om6Br#x-Q*u0^}Fa=tDCX!i%bJ2hz`G)zdU5HGBG-iYJ{V0|E{`efSgv( z7&l$S^I;>Nw2j~#Za{mFr+DH;@k`#ye`iQKDZEisQ_Y0@w4bH;Y!Zc}erPhynyYy@H-PbqO>%0@Gan-8-ZwJ*9%%Cy;~f5u^j zjoN+)sP!4XOx>@j(G2F)=oMB)q`O#+<>JvcNB{k^yZ*p(9QD!NBKz;k2c<+)Ui^V^ zl1nWv;wp-6ZXQ9+cu7o^_w*gX=BC?RF_=k8J?GKXB6oewEAIR1)=0gxSBl9qKDi`_ zPjT{B__+3wDOC_j(K{IaNUD+(XtMmdw7sI5C+72^Y;U&i=9lI{Xn$~q`hVNfVSjR( z){a}6MeLLf-feoo@DnryVeg!AhaYJNRiVN(Et|t|r4o}DmmtY05(U&GNooJ=d}QxT z1~>hhONH8Y&bMkI&-h~y=S30U(~3ae(fmr^NOlP-y<><^`0|C;#X`1vm?~g<(|GLJ zq3LqO4Y~b4`wT3RX7Q0lA4sX9*aL}aq0H-yM`1yd_6&?-A7T)(ZSUVeEzZV zGZS>2Z1|S!&a3d56?O>+d2E z8CuV+y$s8{w614j26%ngl%npMC7UlQAhyjw(+Mp7eGqQ+ls|NRw#+Pvgj`9&7ha>B z+o8p8vsNUQ?7%az#c)+-t;*)lwakf4rL%_D)woj7Yu@MT;}_!QDbMW-pXFj7G;e?W z>k)<{*6J#WJkmxIy^Y1x2{hopKlM_Ryh!baSZBDCaNIR zu2Bu$k?AjGW(IDI1@9ej3uMAsI$=lqS7XGgeV7Ml0n>S^xngywd^C9WO{_tUpzgFj z{=m6rNA~EDD*dKEV*QJ#!o z6hAS1_TOm2S27n0fS?Al%#6v0Qxwn(f{9#R4{GyZzzMMIev0XmDTReLau;5-_M(dH!+m__BeNH$uNNgWl0v zwYhD%sc_|ryQ`sIAC+K`8g?3ZPpl8J`D50rf>0=wW4w@B4}^9&Xv2dRHPb3gmhmS2 zC;$h9M&t>vD}ZE7>s|FLDUr}xjPJ*5Nr~CqOY1KM=b#-&Y;NXmP62_pf&a)4c9?!B z;=-mp&+6OyC1uy=LC@Mx?$Q=NWw`;6o4H;7roOdf&%B4+&!Znu3YwK@+hGsCL7A~R zfxf;J2&ZJvK>hHVq9zK|^6y)B20KOaXn+lpj{4lZRRXDNnpXD}jt$ z?8_*p$MeMNpTn7am)E1g3BwOT)~rFjal&7nR0K0tKY3wlRIG2F3ZA+6-yYO}1#>zA zyFhCzi}mI7l}${k|H@`mnN-o$#L`-JIR{#twim2A9@5`c_HO8da8;&78o&Kqr{#)O z|I%nOSUa}H!-_yZrmL=weYb}kEIn1)8$M88s(o1t|3AoW#+ia8GOGH3Cs#H zwO34$LFeKiuFMzo7e?DR=^w9ye^oWKw{=vcIW-3GfK zoEoce8SCk|>{eS?AL>n_&D|7>FRjed6l$lNuTEyAk%!TRkHsAN7~!Luu~+?PhwOBZ z(LWQlrQPi-JJO7e$vcJWKnbTB?_MSRBxVmro@sJ(>ObYro0?j0}aa0qBDfxjkO-c-=(Ct8wq3 zrcs`RuxqM&&6Lqba$?X~8odb@>*O@N?H%jZe73ircSj=^&;%usVjW!p+)GOZ877>| zmJy?*+z(VmXl}Vurw`)HdrLc*{g}Gq)?;ZMbbVLIMMK4t`UiEp0GSanL}Pe_1;kux zd-we!zH7{X<0X#{CN306kxeHnv+-Mznl@9<1aTP#l<~cV$%;Irv^$K;V4njI{8+Bd zd8=hVq0=f)8xcnLPV;nYcM`fc0RE=ikyyD@EoH}@(kdwdz1(~hDQ~kQG$uuH5p3O= z)#jj$oC?;e(y*VK{z(A3_W1*6EHMNQI2#4(3q^hGBqpi2hhPqSxv;0_evaUe-DEm@ zVv520x|OE2^0&867LfQwGJ%sMPo;S+vWU#&HHTuW;EY=5YbllYK0HQgK+dla>}WAz zn83_KG$O_ro&bnWLoj``Csk;s5E852B>TelZ{!1)Ks2Nc9*`yI`%(7(E?SQM7wLQj z3(r5cIn?gKk2 ^pWxr+Y0qm*a<*Z)ZusWvE*8ozC?RYeTD(3s-C=l+nm)$NW9KB z7XRZz;ml{@kuQ`W2k5parR^cDx4FJN@)-icr9Trfy-D_B>^}X<%72a zNu7%{^5AVKyND~ppj-!Bp3Hm_q(@E z;Q~%zV1i|W?tz&dN%oxWhKruhv6!Hv|@rAj3w_0G$&cKjCIg>Lu6hU!# zjb7bkeOnwO(*!|e6AA~T6`2&@*U8&Ap8p#uzQ(}Z`&I2Y%7In736iYa@#14usJEOp zgMg*Ic!Hm~&gRV+J_Pi!fN=7C3_`d+=&yBt3sLvdU$1vLaIJWI4N-;8?URp+0p+iasBwqyJZ3H7@m3uD-JiSH!kD-qDXz}YEHf0e{EyKrLdoPJ5?eSd{JJo>#l(&+`4n!XX2@1|E7RZtXLd zck)veWcpME(NuK)xSE}gXoFFuTo7@Lj_WqJ$am1=wrF?lLMK}-1JGUnzI#bfh!2zjmy72-7Rrp$sIZg< zprnWqL)$q|wmv_+#nd-AHw!P44-{4B^8XkM!K4*FYbR#6Cj3MSbEeV;dnLv!wF8yI z1gJ~?XfxzOdo-wJblKIh;pk0L|2oAuER2Mw^?{S~r+me_u0qJRBcZRZGd_0#DV zxAxmANXY5F9ze@fl{z-a$Fkm4BrzK>$spkumZSN>YhvE7wK{jMwD}ua7#eAy%g&po zJj)R=_=Ip*eKJX3?u?c^|2Wp({qD!~j=V9>?Gng! zPXtW&krWCzFL8K+;=;AZ1fWAJEUJgNX5YW)HZB)I%8`xsvrr`9q^=_>PG(fi8F~mF zd_N|83i1#6R}%wEZyK!QY0g>kKb1rhm3R1?)eBRx6d-#}yDW!*RZ!-7RLR`ZB)xhs zNdzIjOaMIi&R)}r?5ij$&aR`u=2lcP1!|P!rlXZ#7^Ogi&gg~3B)#@bpBH(aM6v)B ziq$^Y>?ILO82xR3tOMxS9=(m!ynKfqbbW*Xj~Uj=uJgow>T8KSkLD{)ititVKoN%) zaXt$T(FSb-_Unba+={82g1xRy^4%*sF)tN}bcs>Uw;)tQ1?JF8oySw0iASQhF=%p8 z15hms3!Xv0hhsdI7!%OIEvDx7@GPSj#XvcBl6W1bO8BISNAz$a!`bk8t;H}CP~er+ zamVBcRt$asS{g19L<&VkW&ELn*l_`*0+(pMlzK{B7v~QMG+bihJRT#~RL+tUcd(X( zdyMblAOPF#u@PjYE_9~xg<1V8#NTu*f)6HhOp=TPyp(*VehgeMfC1sSA0W5J*Efb?`VkkAAc$$XcGX86%6r=gzQ-! z*V1G1r2pR&wJtMEhB<-E43B9t@Lc$b_3CG3+rd2e!Z~RR>^1*gz|TB|g`vJGfb#Q2 zi`DiAyt^c`k+_uQ2mp6^y7u_`pXO4=WAOHW_-w!{FKas8%E#&?g3|%g;|uyVRM**D z^#rJe4N?4mv#^;n|GmklHws)WT~EK~|6K9^|7U~wouBV0wYb770VH1rmm!Q3Xfobz zj6s>Yyc-zT#bwoC0|xR;`*-@EegYh6_8QavgrEGK{@H>{{Ea!{zAOTEYbijQus>;~ zUK9tc_>noN0G;gR9x&fdy|Zc;U=X)^idgS_m;NyOHtM;@1bSan4OJ|=1H+gE6X?{C za_!frUPkVsV~a{4PJlS(Wx%MEp#{vCctpHDNxVK@PX&n7D?*&r4tGPl^_H^rU-Bt3 z0F<$D`S3(q465d!!1e2YPt#FT$IL=3ML7(N0st;{QqEd+!%+{oA8OnGb3bFV#l5A9 z#|>xzQe^7y&$j?bw>=WC9YHk0TBL+Ole<4x=R7Gza=L6e3_Sflkg~R`YhAa)M5*3~ zl=rvJ!;oCpcu0JqEwUg{XiNrMX5(Rs#bmCozl84?r_=V0)jp!vK0(}; zt^`r_n>u;Cg=>ZrvS+BKKquZ+1N-)^^Vk>WujP4{htu5_%Mi_;O`#s-N!3UA!|sOD zzm*UBkY7KxC*u2$2FKIdzWif1U{^VS!~rRHe3<3X5Ob{+lmh{Fh8Lg9Cf9yEd4%R} znXh-77W*qDae)EXM=OJ`oMh8E;H){+wM;P+bme>VlBqn_`l(`Q|E~}6JRdMnMq-2I zM(gsX|HDJSV{8p0;MlHack>kROF{t5NMh3oSk5sZD`dPc1RCHH1oQh|XVehTe@k*O z;Gv<}t@{j=fffkQouzROJeE+8K*tImb*|BkyJ%gfi+KtaYU6YXzpH#Y<*sz5r*&V2 z`-E|Y`JiAVcKd&vGE@8)Wd#6lp#Xo=XjHg~O)1U+D3Uve_z_8)1L5pe>hzEqCKxaV zu*wkbbDo3GoW{7YZqgr-AFC8Es*qM;ZMF=qnsNTL{r%z^n3xt{$u_eTy?p5_1|;`R zcVjP}IM}Ch1u&#G&THmal3(-BqR?aSau~4ts{+FUI1CdNcnIZFUytA4zhvU2fjX)H z?{e+GEqh4L?NI2qLNYKocSJMS4ddMbexM>s!cp+>#GP_pY+%`R84;&`@-AnN^$Esh z9@n=SQ(l`*84Nm|Kfm8nIzM(xx455xA6CDzd-P_vUGH@F{?J(#qqZY&qTHHjHM!)p zBks30d_Y-#UPTWu&OkctGYc`nhIeV@6kf@&`)rAWt`2iMfkAc+gR=NdK_DVd!?`t* zC+z+SYre7vi0`>)4Fb^pq2w}w?cvW#_LLSyTTyU&#k8&s=jOKPaWDVB#BHw77(bP4 z?v^L`)XR54Ay&5+*blaMMu~)3$dHgWQJ2LBS^@`22EWs;kz+5U3fQmB*BW4Kq8*qb zx-(6w7jFHaF{p=G*(V^N6MOKhme`N|n(cr5Tx}`F`qh-p{I_1~f9rf|A~{B>FbK%0u1_SUpg%>zpSj0i`}_Z^HsrXQS_M0+ zR$;a32`KS|!Y&dKj++O~1C6~6nB*OPhV~~S?kB^&KVn7iUdaKg0Nndu_>I+TbNk#` zteo?w440DeV|H>fv~lR4t@rl9RVJx2*FlA5nU><3S6%47#DrKU#m~ny?BnZ}LkX8J zpmD?eSTvL@(my2y4f{Z2Q(`e^T%yGUbP@=yNe9V_N^OMp8K|sp1T#B~LQs={xj)7T z-{MzRfE#b@}s}RnRuMV+oFxuBi?=*`>RmJi^0H9)`FdvINfW1_7K3c3frFS?K^=2T@W%376 zgHGxOu-$H3?=BI&t-i0x#f^_9pT9GaNhL8w^K8h888c{ppZdAMgn56hBxK*W`lR*$ zs_m@5qKw+Tj}ih(8gzpoA)ShVq=b|~2#mx4QqnLmbSexXA&rEfAR$sxBJI#1H8AAR z-OLbYNPoHvZ}ujc60uwcl4n)qs>+J?1_bjiK0DY6t%R0bnQ zS$wfr3uAV`Z+Rzv@dpIbF#EOdN2X_;5732$n2$h4rv3`Ipom0VitHe`x4d%my8@Xe4lSQaJqYP zEn{+3++gKtvz;I??Y#WJ z3SxPQNlt&A`MezOnw8k6>L|>d&*pYecf4PSU-2-i(F3xiIsBUx1#pbNidwySEJ~R&qm0Tss_$HSVhU-5V z$%z^y0FyOdH0ro&<9o`n6&X21V}N9Di)1pkE43KrND94s6pqMD08s&V+FvPB@J1p* zN_hpKxURPr_LB@PUN^r^e1n2XTp6FtKr(^V=kz#o#3dYT%?OQ|gZkm1Q1GFkZ>5hm zG=tSN8?oQMMd{`+P*Wx5upQQcdh>hVdqRV#!W(owigWMd&l#!Q5Cmn6b4J<0sM2!$WzTZ4-U=&w)r@4mV@T<0&~sv=*U#KaDW<9X zpQNyVK|tCXA3IT^+*`26Fj?J{AGTOoJsl~i<>uU4Q{d6 zU8^BtxnOz5+=a)F|KxW4xhyWGGf*)&|EOb2{nKe}tG-mhBOKAvux!I-2jaE{ScmYcHKGIHgmu@_ZXT%Vk^PKhoVbTw(iYb`|i=P z9*@~mt?uISvge>P;D}APlW-g=y^Xx%xpu%tJc$dR7Mr0M^XSWzO$PS)pQUN??~j^N z`Wq>veM+6LQF{giNCsg)zJ9k6U}j z;Mc{1bUe4}os2-}_Wtqqku^EEv91RdvE=~D?30aD)+5867WMrMIcB^y0YI9}+%+EX z-M@XXb9`t;D}qViak8GHp!Ze(!t2a*!^+d5(yF6{l=qfHBPLi`=x0b8#D_gD2DZ=S6xZc4GFw2AKylq-4yo<43O`55$z@>%qmptbpoRkwM`-_0GN1;7%P-0@I8GZ z(x&ScYLKr`D<*(P%*jB1wO)4s>q)>t*{=j4K;buQSS-{~Wj99*rau2;c=HDF45SXy z9w_&kQ%;5dGS3EPUFF8}VvPvgv5Rblbys2T4wbx$g+&F`1n7u}lfhsW(GF~#?`LR( zBnfvH-`1AUo?JgLHiAC5alF_;%xh|&S^L8+ZY&XAwIs&IWIR;; zAU5lKV@$^7G=Ye1K%Z!oJj%YkbCt!P>B|+H^yE#>XMjX?Gzy50WN&4~fMu8w61PO_ z9<$J%uFNN1_RfN3lrJje2s-^yf&DY6w&>;iKGD*n&s`IqW9+yEEt!!z-AjJNoSfJq z9SVVO#h?>&LU)&)-)~5RIOg}Q8QqhEzMb8rKDefarL+tSRqMHTrvVw7{nNhgs03^Z zkHFWTPUMC4(T!7!;?tcWni0LnY9s8Qf;Oy@bed1Xs7|FQ%1cUS;6zT@SJ%`+gwj_n3>Zghu zTVrUlk*zRV6z13LJR39&?DFFLj|(EiE=!oi9Kiz-%(=(0m>1-&?b<;!w77Po`303+x_uH zuTynu!U)Ondc22zWSH8jFH>$*M1qJwl&Vm*48K{GBO)Ie&1@3P6TSL;)PI?#)L?6; z>hX3i*rKpqrQul%2~Zje6L<;iy zhqCRw_6}e#c=a`#*tcyda$Tt8f0r9H{@|jdO?9v@HSSn+;%R1zMfMgq^WX4S=s&&8!BTCh;4yP0p>0u0O&uo@x~C-TBs+jan*U`6f>u>GMJYD%+$+<8zJBGkhSx06ZA;HDyC>u zoMburP(XmIH%iZnmh0l=8k0M2&Q}6zTS`jVD+)7xa_f2ccmQg7 zmqF54r2+ryAnj}nGYe0yWnT??2HusSga>;+dq&y!cP|(*Vo5p!Z`191Y}P;_4ZB<) zHB{snx^;w~5&!(`INtcuCs8VR{I=Iw&8Ky`pt**o3$2^4bvSrdXy~s|Wc3Hg9E!)g z)GVSl84}{&pysqd&C0znhQc4dSeD3cNUbGB8|m52tH7T3I)fLxH`!M7jjO@;;G}D4 zzSXPgi%CO}c#1iIEjrs@L!mVr$Ixo@9&5voLG;e*Q~m3>?7Ym1RuK26I=S;bg zUrI?1e_l$de^{URYMMzC)ugf><>7jD`t{As7~(Q~*eolFB}M*``W=DnY48ViVtw{~ zd7e-1;+}hS8F5zYq;`JnnnTDctm)Rj^&`teDT?$+usR@VwiKaT;+v1>z5Bc7>6zO|()1LsC$W?=8c8P#psm5yFH3RDs+RDt{E4wQ?&c0R?eNk5z-vQa~r%EUh6;jyF0 z{T+AOTpnyeEGp|uh$(q{hIXBaFfHwwtlN?yfl^pybzkkQ>R8R^8`ZklT=s3XyN04; zI|Op}HZ#8j}fwJX!KAkaf$3^joaVv>u$D8S*~8u#J;G# zHeBquPxOPgvg1Ep{mba?5fPx{_~o@f-L|_(Onr`&c|{KMrWF;4)%=AMfWFI)lcGj9 zLVQg-&oZ2_)xQPfD46k$2t2I;$b$M$*o!=@PfXmj`m5beR+8!HdEikG}74 zo0W1?-&HFtMwj(~4Gm=X_)CS62O)C*uY`oxV865&xQ`raG@zmp-jnHo{2h{9sH->~ zA;TZz@Bc8o05!Aop6M3a2#$GLMMcL1 zf0qKywA_Hg3X6ltU;;$$9+hZ4YvTtCSyP=ARvgvgvT-n4f_4W%cbE7MDhVfojjWls z?T1PEM5~Z+_}5dUG>q0*^n`s&O=xkUYOU5jwj@iv9WZFZ6SK%eqVr7;0CI~PkB8O~1xsH*#RXq*7lBy#N*bUgzg%#%@5h#8W&H$Qsph_LC1iO1b^1e`97R~Xgem8u%YM~&lkyrFlm?d1dz7T1PLid;xMw)%?}r= zf377SdI12=5TsGg>qqAOZ0D_rChN}GT<*$sN|iEtE13El!4QK-Q|+(2Q0`5&m;l>>p|Ue9y!nZFsmg4jypZMPWJV##FH7;x3=9ch_`3ge+=9_AFw3-BVA>u zJ8iVvu??!o-)(A_&hlO*t_mam;?a;0`Yh8}de_fenKeKSJ+e*oh_2Xn|2eikchSmW z{S@S4r(%206A2$L2_G8uI)jFIvxpSGvc7i|%4vAihm~pCn&=t)Tb=8MI@Q*l<45!HM>+n`7Ze&f3ZLLoBjsA-*_L<-dQ z$y4NSBI@ut40MxOa)T~6#fWFhX2BL6eFj@Mxs$9ox3sQZ;c>#U(jJU%q+!CKvL*_@ z%54-hE<(iDs#jf0r)&~%yqnbyj9`5l1uS7h91YWr)#s3r)^V1HvP7rNG#$^CyvRk> zXw4UIugh7Yk%Uc>>W$+`qa;0z3PD6%K4npk8L8oFpS^r~gtA!(puZ^M^1oIR!cwM7d+I{FO&0eXH^nU7udVJg31#GE1g zn$zfQu8;3-b$*+^+(0D-mxodU?{oZW#T`70X*PI7|~8K}m_q^r(v4LbV*n*lN5rR9k* z3mW*8ues_hA5h{#u-mOGhQS?NqpkvEN?z2nTIvy~0y@p>tUVe(h*h$#-7eE@*C(>`75m1i#Vb^N`twwRgal(-dk;wak)u1Wxp>)L`&>Xe| z7zJ=}RQ4^w1nnH<7~X%IG5bZdVM7a4YO_o3=_wLqn~bh5_V7`BhqT^%Xp0ny?lsd9^d-u9(9TJw0@ zWj&T92#iPt*F;2cc63n4@}F%GE_o2S&e zT868TUp%Um^1xOe<3*SDKRv($0kh308mv+b2 z6Kx=qesD@pg~s8e93>p4xa7`+U-b0^dAq*xrxu%OU-S5}@p~RlOk$7_8^nvzkZBOa zc7rs!K2XTBt(u}N%?w&R-slv3y(}fSYh~s}*1WTN_hgVW5;L;#z-71FTUKS-zdG?Q z$K~7aI+pO@)FT#%3K|Uc9J+Nj<1%YY1;9lntn>Hs^~3tjm|-gcbYNosde#zvhHsaD z04g-~8jS9@ozI1+Q!P-8^|CIGSXxT#`N05U+3_C%9Ykm~@SBK>XBLT92Ryfi*~hlj zX#{yf_m<+Be8^Yje$^OG7w0N}cnS^+2UXJWmBz=c*PFSb($O_m$>?s{+#&dtXw#;( zzgnOzp5Kgg)?fUHrikQK?4}*SjC_0W9kN2-kRtA~G1+Ka=69VZ@h@nr)5*Mfw=Vnk znG?~ltbg^Cu?SFSVn&$A>Z0LNzLWb9WSX!??8~(k^6t)GYMaW)(Js*YWlX}6_WXPM z*nw_;o+}Re8Ao%a7~mH=3rQ}Hec>id_~$dF^Ss4bEhZnOcbu!McK2ePUG`h)9Bw&X zkIZmnN;AgqXq=GTs&e3;KHo1$2VNu5sf5hf^rZp5GL#>Wk>K)ssGFz5tVPLhQmN}y z6HHF= zw64UWQ4AQSc8?NjcdNTA&&etvlwxL7VtkzS)8SPclcl*>=Zv@3667#awogHvEYK!8 zxY=)`_wAJnfC!PAZB%U=j<%mYZZ6+Mf^CA4qGbe<^Ekpt;J$4O3Nm=-;ss?e|+Nr_M?NMwXBO<9ECZ_k(i!DY4U1#+2m)iC; z_9)@tVl!c*?g$BXVwmwEYkIK`JB7!8cvc9A(-G(MJ_2>YGG`yjLIzpDFQ^^bN0bT)N%NKLRrF3iNrPeVB`u>TAi?ZGz zF96v&sD#yJZWih_hOCeK0}yH^e+M}5xm%2744ipV^B@Q_u|Hx z56B<-elOAaW`3_Lo$Xt<*-w?HS+%UXxhiq{fN{{nYsCeqL*KoWb?}|* z#O|dnzN@hKHMJ43w*Wv-^f%lp;c)n?&^Ff)D1&@g4x#1PFTn$t_jadqEzrCriu>Zk zKh*117H5v4*BPwA{+F%E_u`8wO)SLgo*)-Exkj&8hZ%lWu~^zax}f!&4fbPs*Sqhj z%zHdhQz6&;j_Fb`dyAxJ>0?PQR+)~94>#8`O^PEU-Y1u=!g5nQz)bIUH*NgH=9hGs z?tv=85ccd@k-@~u8UUv)s1|>0{=nenbh5KXVL7l1fJh8m96)SO{!Qreq~|NfEp4tc zn~YSlOk5Ld^X0f}L|tSZLq6HRZ)7)=pM3c3 zxBF<-%EQb3mQu@q&sb9cURsG6!)Q^v!+#?o&Hp8<-6FsgUnWqj)bA`xTr$@{Y!~wH zR9NuCKeLH|mjPEC-9M8Y_%6RcK8VIdn$m2FMO*@I-w}@`fMU)6zd8vI{{e;ny%6}| t5_nk_|KE$|KfK}p_l5t{Z&NRwQv_7e$2ZhB;_$%rMER*w$zw>+{{YFJPxk-- literal 0 HcmV?d00001 diff --git a/docs/public/benchmarks/geoparquet-v0.3.svg b/docs/public/benchmarks/geoparquet-v0.3.svg new file mode 100644 index 0000000..fbdd396 --- /dev/null +++ b/docs/public/benchmarks/geoparquet-v0.3.svg @@ -0,0 +1,195 @@ + + + + + + + + 2026-10-02T07:28:32.021067 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 300 + + + + + + + + + + + + + 400 + + + + + + + + + + + + + 500 + + + + Median search latency · 5 runs · lower is better + + + + + + + Without row-group pruning + + + + + + With row-group pruning + + + + + + + + + + + + + + 411.3 ms + + + 4.2 ms + + + Selective local search over 100,000 metadata records + + + + Synthetic time-ordered points · Apple M3 Pro · Rust release build + Same results verified; this is not a provider API comparison. + + + + + + + + diff --git a/docs/public/notebooks/superstac-geoparquet.ipynb b/docs/public/notebooks/superstac-geoparquet.ipynb new file mode 100644 index 0000000..4dc7e28 --- /dev/null +++ b/docs/public/notebooks/superstac-geoparquet.ipynb @@ -0,0 +1,442 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fc3d9e9d", + "metadata": {}, + "source": [ + "# Build your satellite-data inventory. Search it again and again.\n", + "\n", + "Save Sentinel-2 metadata for a small study area around Madrid, then explore different dates and scenes without querying the provider every time. You will also try live fallback, refresh the inventory, and inspect its writer version. **Imagery is not downloaded.** Previews and live catalog queries require internet.\n", + "\n", + "Run cells from top to bottom in Jupyter or a standard Google Colab Python runtime. The example makes a few small public catalog requests. Results can change or be empty; failures are displayed explicitly. Colab files disappear when its runtime ends\u2014download the archive at the end.\n", + "\n", + "This notebook requires SuperSTAC **0.3.x**. Before the release is published, install your locally built v0.3 wheel and skip the installation cell. [Open this notebook in Google Colab](https://colab.research.google.com/drive/1JIx8j0Vi7jypKe5Q6klgCzYQMFHMaKGc?usp=sharing)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5f44bfe", + "metadata": { + "tags": [ + "installation" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"superstac>=0.3,<0.4\" pyarrow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "445bc7dc", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import gc\n", + "import json\n", + "import os\n", + "from pathlib import Path\n", + "import shutil\n", + "import tempfile\n", + "import time\n", + "\n", + "import pyarrow.parquet as pq\n", + "import superstac\n", + "from superstac import Client, AsyncClient\n", + "from IPython.display import display, FileLink, Image\n", + "\n", + "print(\"SuperSTAC:\", superstac.__version__)\n", + "print(\"GeoParquet available:\", superstac.geoparquet_available)\n", + "assert superstac.geoparquet_available, \"Install a GeoParquet-enabled v0.3 build.\"\n", + "assert hasattr(Client, \"ingest\"), \"Restart the kernel after installing v0.3.\"\n" + ] + }, + { + "cell_type": "markdown", + "id": "d571088f", + "metadata": {}, + "source": [ + "## 1. Choose a small area and date window\n", + "\n", + "A unique temporary directory prevents this demo from overwriting an existing inventory. Keep the default bounds for a quick run. Raising the search limit does not affect ingestion: ingestion follows all pages within its scope." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d075d504", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "CATALOG_ID = \"earth-search\"\n", + "CATALOG_URL = os.environ.get(\"SUPERSTAC_DEMO_CATALOG_URL\", \"https://earth-search.aws.element84.com/v1\")\n", + "COLLECTION = \"sentinel-2-l2a\"\n", + "BBOX = [-3.8, 40.3, -3.6, 40.5]\n", + "WEEK_ONE = \"2025-02-01T00:00:00Z/2025-02-08T00:00:00Z\"\n", + "WEEK_TWO = \"2025-02-08T00:00:00Z/2025-02-15T00:00:00Z\"\n", + "DATASET = Path(tempfile.mkdtemp(prefix=\"superstac-geoparquet-demo-\", dir=Path.cwd()))\n", + "CATALOGS = [{\"id\": CATALOG_ID, \"url\": CATALOG_URL}]\n", + "SETTINGS = {\"logging_enabled\": False, \"enable_background_health_monitor\": False,\n", + " \"search_healthy_catalogs_only\": False, \"max_retry_attempts\": 1,\n", + " \"per_catalog_timeout_seconds\": 30}\n", + "client = Client(catalogs=CATALOGS, settings=SETTINGS)\n", + "print(\"Dataset:\", DATASET)\n" + ] + }, + { + "cell_type": "markdown", + "id": "cbbc293d", + "metadata": {}, + "source": [ + "## 2. Ingest with progress\n", + "\n", + "The callback reports received and durably saved metadata records. A provider may omit a total count, so a percentage is not always available. If interrupted, repeat the call with `resume=True` and the same scope, name, and page size." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9901b98", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "last_progress = 0.0\n", + "\n", + "def progress(event):\n", + " global last_progress\n", + " now = time.monotonic()\n", + " if now - last_progress >= 1 or event[\"phase\"] in (\"completed\", \"failed\"):\n", + " print(f\"{event['phase']}: {event['items_received']} received, \"\n", + " f\"{event['items_saved']} saved, {event['files']} files\")\n", + " last_progress = now\n", + "\n", + "snapshot = client.ingest(\n", + " CATALOG_ID, str(DATASET), name=\"madrid-week-one\",\n", + " collections=[COLLECTION], bbox=BBOX, datetime=WEEK_ONE,\n", + " page_size=100, items_per_file=1000, max_dataset_mib=128,\n", + " progress=progress,\n", + ")\n", + "assert snapshot[\"complete\"]\n", + "print(\"Saved metadata rows:\", snapshot[\"items\"])\n" + ] + }, + { + "cell_type": "markdown", + "id": "5b408de1", + "metadata": {}, + "source": [ + "## 3. Search and discover locally\n", + "\n", + "This search stays within saved coverage and requires no catalog API request. `sortby` applies before the per-catalog limit. Collection descriptions summarize the saved inventory, not the provider's full collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88de8101", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "local = Client(catalogs=CATALOGS, settings=SETTINGS,\n", + " mode=\"snapshot\", dataset=str(DATASET))\n", + "query = dict(collections=[COLLECTION], bbox=BBOX, datetime=WEEK_ONE,\n", + " sortby=[\"-datetime\"], limit=5)\n", + "started = time.perf_counter()\n", + "result = local.search(**query)\n", + "print(f\"Local search: {(time.perf_counter() - started) * 1000:.1f} ms\")\n", + "print(json.dumps(result.metadata, indent=2))\n", + "assert result.metadata[\"catalogs_failed\"] == 0, result.metadata[\"failures\"]\n", + "print(\"Item IDs:\", [item[\"id\"] for item in result.items()])\n", + "print(\"Collections:\", local.list_collections())\n" + ] + }, + { + "cell_type": "markdown", + "id": "6ef8f0a0", + "metadata": {}, + "source": [ + "### Optional online preview\n", + "\n", + "Saved metadata contains asset URLs, not image pixels. Some providers require signing or authentication. If no thumbnail is advertised, this cell simply reports that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4655ad0c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "thumbnail = next((asset[\"href\"] for item in result.items()\n", + " for key, asset in item.get(\"assets\", {}).items()\n", + " if key == \"thumbnail\" or \"thumbnail\" in asset.get(\"roles\", [])), None)\n", + "if thumbnail:\n", + " display(Image(url=thumbnail, width=500))\n", + "else:\n", + " print(\"No thumbnail URL in these results; inspect item assets for available previews.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "1fb0741e", + "metadata": {}, + "source": [ + "## 4. Inspect the file's writer version\n", + "\n", + "The Parquet footer includes both standard GeoParquet metadata and SuperSTAC provenance. The file's writer version can differ from the version currently installed. Empty scopes have no Parquet files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b9b4317", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "manifest = json.loads((DATASET / \"manifest.json\").read_text())\n", + "files = manifest[\"catalogs\"][CATALOG_ID][\"madrid-week-one\"][\"files\"]\n", + "if files:\n", + " footer = pq.read_metadata(DATASET / files[0][\"path\"])\n", + " print(\"created_by:\", footer.created_by)\n", + " producer = json.loads(footer.metadata[b\"superstac\"])\n", + " print(\"SuperSTAC provenance:\", producer)\n", + " assert producer[\"version\"] == superstac.__version__\n", + " assert b\"geo\" in footer.metadata and b\"stac-geoparquet\" in footer.metadata\n", + "else:\n", + " print(\"Complete empty inventory: no Parquet files to inspect.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "6163aa25", + "metadata": {}, + "source": [ + "## 5. Retain a second scope and search their union\n", + "\n", + "Adjacent scopes can jointly cover a larger query; gaps cannot. Reopen snapshot clients to see a newly published manifest. Duplicate collection/item IDs are returned once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e0e3dae", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "client.ingest(CATALOG_ID, str(DATASET), name=\"madrid-week-two\",\n", + " collections=[COLLECTION], bbox=BBOX, datetime=WEEK_TWO,\n", + " max_dataset_mib=128, progress=progress)\n", + "local.shutdown()\n", + "del local\n", + "gc.collect()\n", + "local = Client(catalogs=CATALOGS, settings=SETTINGS,\n", + " mode=\"snapshot\", dataset=str(DATASET))\n", + "union = local.search(collections=[COLLECTION], bbox=BBOX,\n", + " datetime=\"2025-02-01T00:00:00Z/2025-02-15T00:00:00Z\",\n", + " sortby=[\"-datetime\"], limit=10)\n", + "assert union.metadata[\"catalogs_failed\"] == 0, union.metadata[\"failures\"]\n", + "print(\"Named scopes:\", list(json.loads((DATASET / \"manifest.json\").read_text())[\"catalogs\"][CATALOG_ID]))\n", + "print(\"Union returned:\", len(union))\n" + ] + }, + { + "cell_type": "markdown", + "id": "5dffad0a", + "metadata": {}, + "source": [ + "## 6. Automatic API fallback\n", + "\n", + "Auto uses fresh, complete local coverage where available. March is outside these saved scopes, so this query goes to the API. A zero freshness limit also forces live requests. Auto does not save its API results as complete inventories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c98cdbbe", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "auto = Client(catalogs=CATALOGS, settings=SETTINGS, mode=\"auto\",\n", + " dataset=str(DATASET), max_snapshot_age_seconds=86400)\n", + "manifest_before = (DATASET / \"manifest.json\").read_bytes()\n", + "live_result = auto.search(collections=[COLLECTION], bbox=BBOX,\n", + " datetime=\"2025-03-01T00:00:00Z/2025-03-08T00:00:00Z\", limit=3)\n", + "print(json.dumps(live_result.metadata, indent=2))\n", + "assert live_result.metadata[\"catalogs_failed\"] == 0, live_result.metadata[\"failures\"]\n", + "assert manifest_before == (DATASET / \"manifest.json\").read_bytes()\n", + "auto.shutdown()\n" + ] + }, + { + "cell_type": "markdown", + "id": "1e693a07", + "metadata": {}, + "source": [ + "## 7. Incremental acquisition-window refresh\n", + "\n", + "Extend week two by a day while re-fetching an overlapping window. New records override older records with the same collection/item ID. This does **not** detect deletions or late changes outside that window. Omit `incremental_since` for a full refresh. Untouched history retains its original freshness timestamp." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32d7c36e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "updated = client.ingest(\n", + " CATALOG_ID, str(DATASET), name=\"madrid-week-two\",\n", + " collections=[COLLECTION], bbox=BBOX,\n", + " datetime=\"2025-02-08T00:00:00Z/2025-02-16T00:00:00Z\",\n", + " incremental_since=\"2025-02-14T00:00:00Z\",\n", + " max_dataset_mib=128, progress=progress,\n", + ")\n", + "print(\"Overlay present:\", updated[\"overlays\"])\n", + "assert updated[\"overlays\"]\n" + ] + }, + { + "cell_type": "markdown", + "id": "bf5a91d4", + "metadata": {}, + "source": [ + "## 8. Compact and reclaim retired files\n", + "\n", + "Compaction resolves duplicate versions and rewrites bounded files. Old readers remain usable. Applied cleanup refuses active snapshot readers, so release them first; `shutdown()` alone does not release their inventory handle. The calls below only affect the temporary demo dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c048fa46", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "client.compact_dataset(str(DATASET), items_per_file=1000, max_dataset_mib=128)\n", + "local.shutdown()\n", + "del local\n", + "gc.collect()\n", + "preview = client.cleanup_dataset(str(DATASET))\n", + "print(\"Reclaimable bytes:\", preview[\"bytes\"])\n", + "print(\"Retired files:\", preview[\"files\"])\n", + "cleaned = client.cleanup_dataset(str(DATASET), apply=True)\n", + "print(\"Cleanup applied:\", cleaned[\"applied\"])\n" + ] + }, + { + "cell_type": "markdown", + "id": "a7a30163", + "metadata": {}, + "source": [ + "## 9. Async clients use the same backend\n", + "\n", + "Jupyter and Colab support top-level `await`; there is no need for `asyncio.run()` here. Async ingestion and maintenance use the same keyword options as their synchronous equivalents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5d31b72", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "async_client = AsyncClient(catalogs=CATALOGS, settings=SETTINGS,\n", + " mode=\"snapshot\", dataset=str(DATASET))\n", + "async_result = await async_client.search(**query)\n", + "assert async_result.metadata[\"catalogs_failed\"] == 0\n", + "print(\"Async local results:\", len(async_result))\n", + "await async_client.shutdown()\n", + "del async_client\n", + "gc.collect()\n" + ] + }, + { + "cell_type": "markdown", + "id": "0385f7cd", + "metadata": {}, + "source": [ + "## 10. Download the inventory and support information\n", + "\n", + "The archive includes the manifest, metadata files, and a small environment report. It contains no imagery. Download it before ending a Colab runtime. When filing a bug, include the package version, a minimal query, and failure metadata; remove credentials or sensitive data from any attachment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bca187c8", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "client.shutdown()\n", + "support = {\"superstac_version\": superstac.__version__,\n", + " \"query\": query, \"metadata\": result.metadata}\n", + "(DATASET / \"demo-report.json\").write_text(json.dumps(support, indent=2))\n", + "archive = shutil.make_archive(str(DATASET), \"zip\", root_dir=DATASET)\n", + "print(\"Saved:\", archive)\n", + "try:\n", + " from google.colab import files as colab_files\n", + "except ImportError:\n", + " display(FileLink(Path(archive).name))\n", + "else:\n", + " colab_files.download(archive)\n" + ] + }, + { + "cell_type": "markdown", + "id": "b5b81161", + "metadata": {}, + "source": [ + "## Next steps\n", + "\n", + "- [GeoParquet guide](https://spatialnode.com/superstac/docs/guides/geoparquet)\n", + "- [Measured local-scan benchmark](https://spatialnode.com/superstac/docs/guides/benchmarks)\n", + "- [What\u2019s new](https://spatialnode.com/superstac/docs/releases)\n", + "\n", + "This notebook's single timing is a demo, not a benchmark. The documented benchmark compares the same scanner with and without pruning, asserts equal results, and reports repeated measurements." + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/plot-geoparquet-benchmark.py b/scripts/plot-geoparquet-benchmark.py new file mode 100644 index 0000000..baf69fd --- /dev/null +++ b/scripts/plot-geoparquet-benchmark.py @@ -0,0 +1,30 @@ +"""Render the recorded local benchmark. Requires matplotlib; no network access.""" +import json +from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +root = Path(__file__).resolve().parents[1] +folder = root / "docs/public/benchmarks" +data = json.loads((folder / "geoparquet-v0.3.json").read_text()) +plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, "svg.fonttype": "none"}) +fig, ax = plt.subplots(figsize=(9, 3.8), layout="constrained", facecolor="#f8fafc") +ax.set_facecolor("#f8fafc") +values = [data["baseline_median_ms"], data["pruned_median_ms"]] +ax.barh(["Without row-group pruning", "With row-group pruning"], values, + color=["#94a3b8", "#0f766e"], height=0.48) +ax.invert_yaxis() +ax.set_xlim(0, max(values) * 1.26) +for y, value in enumerate(values): + ax.text(value + max(values) * 0.025, y, f"{value:.1f} ms", va="center", weight="bold", color="#0f172a") +ax.set_xlabel("Median search latency · 5 runs · lower is better") +ax.set_title("Selective local search over 100,000 metadata records", loc="left", pad=16, weight="bold") +ax.spines[["top", "right", "left"]].set_visible(False) +ax.tick_params(axis="y", length=0) +ax.grid(axis="x", alpha=0.15) +ax.set_axisbelow(True) +fig.supxlabel("Synthetic time-ordered points · Apple M3 Pro · Rust release build\nSame results verified; this is not a provider API comparison.", fontsize=9, color="#475569") +fig.savefig(folder / "geoparquet-v0.3.svg") +fig.savefig(folder / "geoparquet-v0.3.png", dpi=180) +print("Wrote benchmark SVG and PNG") diff --git a/scripts/publish-crates.py b/scripts/publish-crates.py index d702fcf..67f6c39 100644 --- a/scripts/publish-crates.py +++ b/scripts/publish-crates.py @@ -8,7 +8,7 @@ root = Path(__file__).resolve().parents[1] version = tomllib.loads((root / "Cargo.toml").read_text())["workspace"]["package"]["version"] -for crate in ("core", "config", "search", "engine", "cli"): +for crate in ("core", "config", "search", "geoparquet", "engine", "cli"): name = f"superstac-{crate}" request = urllib.request.Request( f"https://crates.io/api/v1/crates/{name}/{version}", diff --git a/scripts/test-geoparquet-notebook.py b/scripts/test-geoparquet-notebook.py new file mode 100644 index 0000000..6fc5a96 --- /dev/null +++ b/scripts/test-geoparquet-notebook.py @@ -0,0 +1,66 @@ +"""Execute the published notebook against a local STAC fixture, never a provider. + +Requires an installed v0.3 wheel, pyarrow, nbformat, nbclient, and ipykernel. +""" +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import sys +import tempfile +import threading + +import nbformat +from nbclient import NotebookClient +from jupyter_client import KernelManager +import superstac + + +class Catalog(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + assert self.path == "/search" + assert self.headers.get("User-Agent") == f"superstac/{superstac.__version__}" + query = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + start = query.get("datetime", "2025-02-01T00:00:00Z/..").split("/")[0] + instant = datetime.fromisoformat(start.replace("Z", "+00:00")) + timedelta(hours=1) + instant = instant.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + item = {"type": "Feature", "stac_version": "1.0.0", "id": f"scene-{instant}", + "collection": "sentinel-2-l2a", "geometry": {"type": "Point", "coordinates": [-3.7, 40.4]}, + "bbox": [-3.7, 40.4, -3.7, 40.4], "properties": {"datetime": instant}, + "assets": {}, "links": []} + data = json.dumps({"type": "FeatureCollection", "features": [item], "links": [], "numberMatched": 1}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +root = Path(__file__).resolve().parents[1] +notebook = nbformat.read(root / "docs/public/notebooks/superstac-geoparquet.ipynb", as_version=4) +nbformat.validate(notebook) +notebook.cells = [cell for cell in notebook.cells if "installation" not in cell.metadata.get("tags", [])] +server = ThreadingHTTPServer(("127.0.0.1", 0), Catalog) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +try: + with tempfile.TemporaryDirectory(prefix="superstac-notebook-check-") as directory: + kernel = KernelManager(kernel_name="python3") + kernel.kernel_spec.argv = [sys.executable, "-m", "ipykernel_launcher", "-f", "{connection_file}"] + client = NotebookClient(notebook, km=kernel, timeout=120, resources={"metadata": {"path": directory}}) + try: + client.execute(env={**os.environ, "SUPERSTAC_DEMO_CATALOG_URL": f"http://127.0.0.1:{server.server_port}"}) + finally: + if kernel.has_kernel: + kernel.shutdown_kernel(now=True) + archives = list(Path(directory).glob("*.zip")) + assert len(archives) == 1, "Notebook must export an inventory archive" + print(f"Executed {sum(c.cell_type == 'code' for c in notebook.cells)} code cells; verified inventory export and provenance using local HTTP only.") +finally: + server.shutdown() + server.server_close() + thread.join()