From cf18b44743ed40443f96bddfc4d94ea8801bedb3 Mon Sep 17 00:00:00 2001 From: jeafreezy Date: Sat, 3 Oct 2026 08:23:31 +0200 Subject: [PATCH 1/7] feat:wasm progress --- .github/workflows/CI.yml | 25 ++++ .gitignore | 5 +- Cargo.lock | 95 ++++++++++++++-- Cargo.toml | 1 + README.md | 6 + crates/search/Cargo.toml | 17 ++- crates/search/src/backend.rs | 21 +++- crates/search/src/browser.rs | 148 ++++++++++++++++++++++++ crates/search/src/executor.rs | 2 +- crates/search/src/lib.rs | 4 + crates/search/src/runtime.rs | 29 +++++ crates/search/src/stac_api.rs | 45 +++++--- crates/wasm/Cargo.toml | 26 +++++ crates/wasm/README.md | 106 +++++++++++++++++ crates/wasm/examples/index.html | 39 +++++++ crates/wasm/package-lock.json | 75 ++++++++++++ crates/wasm/package.json | 13 +++ crates/wasm/src/lib.rs | 196 ++++++++++++++++++++++++++++++++ crates/wasm/src/types.d.ts | 53 +++++++++ crates/wasm/tests/browser.mjs | 140 +++++++++++++++++++++++ crates/wasm/tests/types.ts | 9 ++ 21 files changed, 1020 insertions(+), 35 deletions(-) create mode 100644 crates/search/src/browser.rs create mode 100644 crates/search/src/runtime.rs create mode 100644 crates/wasm/Cargo.toml create mode 100644 crates/wasm/README.md create mode 100644 crates/wasm/examples/index.html create mode 100644 crates/wasm/package-lock.json create mode 100644 crates/wasm/package.json create mode 100644 crates/wasm/src/lib.rs create mode 100644 crates/wasm/src/types.d.ts create mode 100644 crates/wasm/tests/browser.mjs create mode 100644 crates/wasm/tests/types.ts diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 38f557d..f89a682 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -26,6 +26,31 @@ jobs: - run: cargo test -p superstac-cli -p superstac-engine --features geoparquet --locked - run: cargo check -p superstac-python --no-default-features --locked + wasm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: '1.88.0' + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: crates/wasm/package-lock.json + - run: cargo install wasm-pack --version 0.13.1 --locked + - run: wasm-pack build crates/wasm --target web --release --locked + - run: npm ci + working-directory: crates/wasm + - run: npx playwright install --with-deps chromium + working-directory: crates/wasm + - run: npm test + working-directory: crates/wasm + - run: npx tsc --noEmit --strict --target ES2022 --module ESNext --moduleResolution bundler tests/types.ts + working-directory: crates/wasm + wheels: name: Wheel (${{ matrix.target }}) runs-on: ${{ matrix.runner }} diff --git a/.gitignore b/.gitignore index 8a41d80..160dbd5 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,7 @@ docs/_build/ ideas/ .env .env.local -.pnpm-store/ \ No newline at end of file +.pnpm-store/ +# Browser binding artifacts +/crates/wasm/pkg/ +/crates/wasm/node_modules/ diff --git a/Cargo.lock b/Cargo.lock index e2dd50e..b768127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -648,7 +648,7 @@ checksum = "74b72642b4c3243fb843c3beb0ec9d0406eecfaa7e01d172e7cdfa437605c884" dependencies = [ "geo 0.31.0", "geo-types", - "geojson", + "geojson 0.24.2", "geozero", "jiff", "json_dotpath", @@ -1186,6 +1186,20 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "geojson" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510c094bfc76ea34d02eee00833254945b70491d79a9c0b050abed6eaa799ffb" +dependencies = [ + "geo-types", + "log", + "serde", + "serde_json", + "thiserror 2.0.18", + "tinyvec", +] + [[package]] name = "geoparquet" version = "0.6.2" @@ -1216,7 +1230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5f28f34864745eb2f123c990c6ffd92c1584bd39439b3f27ff2a0f4ea5b309b" dependencies = [ "geo-types", - "geojson", + "geojson 0.24.2", "log", "serde_json", "thiserror 1.0.69", @@ -1250,6 +1264,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "granit-parser" version = "0.0.2" @@ -3138,6 +3164,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -3356,7 +3393,7 @@ dependencies = [ "geo-types", "geoarrow-array", "geoarrow-schema", - "geojson", + "geojson 0.24.2", "geoparquet", "indexmap 2.13.0", "log", @@ -3372,6 +3409,28 @@ dependencies = [ "wkb", ] +[[package]] +name = "stac" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a069e4454aec3e4cf9340d2b2b242764086e23c452f1ff3deff29b30b2bf0b60" +dependencies = [ + "bytes", + "chrono", + "cql2", + "geojson 1.0.0", + "indexmap 2.13.0", + "log", + "mime", + "serde", + "serde_json", + "serde_urlencoded", + "stac-derive", + "thiserror 2.0.18", + "tracing", + "url", +] + [[package]] name = "stac-derive" version = "0.3.0" @@ -3395,7 +3454,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stac", + "stac 0.16.3", "thiserror 2.0.18", "tokio", "tracing", @@ -3434,7 +3493,7 @@ dependencies = [ "clap", "serde", "serde_json", - "stac", + "stac 0.16.3", "superstac-config", "superstac-core", "superstac-engine", @@ -3474,7 +3533,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stac", + "stac 0.16.3", "stac-io", "superstac-core", "superstac-geoparquet", @@ -3498,7 +3557,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "stac", + "stac 0.16.3", "superstac-core", "superstac-search", "tempfile", @@ -3529,16 +3588,35 @@ name = "superstac-search" version = "0.3.0" dependencies = [ "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "gloo-timers", "reqwest", "serde", "serde_json", - "stac", + "stac 0.16.3", + "stac 0.17.0", "stac-io", "superstac-core", "tokio", "tracing", ] +[[package]] +name = "superstac-wasm" +version = "0.3.0" +dependencies = [ + "futures", + "reqwest", + "serde", + "serde-wasm-bindgen", + "serde_json", + "superstac-core", + "superstac-search", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "syn" version = "2.0.106" @@ -3725,6 +3803,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ + "serde", "tinyvec_macros", ] diff --git a/Cargo.toml b/Cargo.toml index b531f6b..6e79e89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/cli", "crates/python", "crates/geoparquet", + "crates/wasm", ] [workspace.package] diff --git a/README.md b/README.md index 19015cc..bf57381 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,12 @@ Run `superstac --help` for the full surface. For saved metadata inventories, bui with `--features geoparquet` and follow the [GeoParquet guide](crates/geoparquet/README.md). Published Python wheels include this backend. +## Browser / WebAssembly + +Build the browser binding to search live STAC catalogs from JavaScript or TypeScript. +See the [WASM guide](crates/wasm/README.md) for build instructions, API usage, +a browser example, and browser-specific limitations. + ## Configuration Only `id` and `url` are required per catalog. Common optional fields: diff --git a/crates/search/Cargo.toml b/crates/search/Cargo.toml index eebceb4..5ce99ad 100644 --- a/crates/search/Cargo.toml +++ b/crates/search/Cargo.toml @@ -17,12 +17,21 @@ authors.workspace = true superstac-core = { path = "../core", version = "0.3.0" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" -tokio = { version = "1", features = ["time"] } futures = "0.3" -stac = "0.16" -stac-io = "0.2" -reqwest = { version = "0.13", default-features = false } +reqwest = { version = "0.13", default-features = false, features = ["json"] } tracing = "0.1" +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { version = "1", features = ["time"] } +stac-io = "0.2" +stac = "0.16" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# 0.17.0 gates filesystem APIs; later releases conflict with native geo dependencies. +stac = { version = "=0.17.0", default-features = false } +getrandom = { version = "0.3", features = ["wasm_js"] } +getrandom02 = { package = "getrandom", version = "0.2", features = ["js"] } +gloo-timers = { version = "0.3", features = ["futures"] } + [dev-dependencies] tokio = { version = "1", features = ["rt", "macros"] } diff --git a/crates/search/src/backend.rs b/crates/search/src/backend.rs index e8e4fb9..97def11 100644 --- a/crates/search/src/backend.rs +++ b/crates/search/src/backend.rs @@ -1,5 +1,18 @@ //! Search contracts shared by remote and file-backed implementations. -use futures::future::BoxFuture; +/// Futures run on the browser event loop on WASM; native backends remain Send. +#[cfg(not(target_arch = "wasm32"))] +pub type SearchFuture<'a, T> = futures::future::BoxFuture<'a, T>; +#[cfg(target_arch = "wasm32")] +pub type SearchFuture<'a, T> = futures::future::LocalBoxFuture<'a, T>; + +#[cfg(not(target_arch = "wasm32"))] +pub trait BackendBounds: Send + Sync {} +#[cfg(not(target_arch = "wasm32"))] +impl BackendBounds for T {} +#[cfg(target_arch = "wasm32")] +pub trait BackendBounds {} +#[cfg(target_arch = "wasm32")] +impl BackendBounds for T {} use superstac_core::{errors::SuperSTACError, models::catalog::Catalog}; use crate::{query::SearchQuery, response::SearchItem}; @@ -21,13 +34,13 @@ pub struct BackendSearchOptions { /// Dataset locations can be held by the backend; `Catalog::url` is only an API /// endpoint for the STAC backend. This interface does not perform health checks /// or ingestion. Boxed futures allow runtime selection via `dyn SearchBackend`. -pub trait SearchBackend: Send + Sync { +pub trait SearchBackend: BackendBounds { fn search<'a>( &'a self, catalog: &'a Catalog, query: SearchQuery, options: BackendSearchOptions, - ) -> BoxFuture<'a, Result, SuperSTACError>>; + ) -> SearchFuture<'a, Result, SuperSTACError>>; } impl SearchBackend for std::sync::Arc { @@ -36,7 +49,7 @@ impl SearchBackend for std::sync::Arc { catalog: &'a Catalog, query: SearchQuery, options: BackendSearchOptions, - ) -> BoxFuture<'a, Result, SuperSTACError>> { + ) -> SearchFuture<'a, Result, SuperSTACError>> { (**self).search(catalog, query, options) } } diff --git a/crates/search/src/browser.rs b/crates/search/src/browser.rs new file mode 100644 index 0000000..3a0e2e1 --- /dev/null +++ b/crates/search/src/browser.rs @@ -0,0 +1,148 @@ +//! Browser STAC transport. Pagination is polled directly without Tokio tasks. +use reqwest::{Client, Method, Url}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use superstac_core::errors::SuperSTACError; + +fn error(e: impl std::fmt::Display) -> SuperSTACError { + SuperSTACError::SearchFailed(e.to_string()) +} + +#[derive(Deserialize)] +struct Link { + rel: String, + href: String, + method: Option, + body: Option, + #[serde(default)] + headers: std::collections::BTreeMap, + #[serde(default)] + merge: bool, +} + +#[derive(Deserialize)] +struct Page { + #[serde(default)] + links: Vec, + #[serde(flatten)] + fields: Map, +} + +/// Read paginated collections, bounding the result to avoid unbounded allocation. +pub async fn collections( + client: &Client, + base: &str, + cap: usize, +) -> Result, SuperSTACError> { + pages( + client, + endpoint(base, "collections")?, + Method::GET, + None, + "collections", + cap, + ) + .await +} + +pub async fn search( + client: &Client, + base: &str, + search: &stac::api::Search, + cap: usize, +) -> Result, SuperSTACError> { + pages( + client, + endpoint(base, "search")?, + Method::POST, + Some(serde_json::to_value(search).map_err(error)?), + "features", + cap, + ) + .await? + .into_iter() + .map(|item| serde_json::from_value(item).map_err(error)) + .collect() +} + +fn endpoint(base: &str, path: &str) -> Result { + Url::parse(&format!("{}/{}", base.trim_end_matches('/'), path)).map_err(error) +} + +async fn pages( + client: &Client, + mut url: Url, + mut method: Method, + mut body: Option, + field: &str, + cap: usize, +) -> Result, SuperSTACError> { + let mut items = Vec::new(); + let mut headers = std::collections::BTreeMap::::new(); + let mut visited = std::collections::HashSet::new(); + while items.len() < cap { + let signature = format!("{method} {url} {body:?} {headers:?}"); + if !visited.insert(signature) || visited.len() > 10_000 { + return Err(error("pagination did not terminate")); + } + let mut request = client.request(method.clone(), url.clone()); + if method == Method::POST { + if let Some(value) = &body { + request = request.json(value); + } + } + for (key, value) in &headers { + request = request.header(key, value); + } + let response = request + .send() + .await + .map_err(error)? + .error_for_status() + .map_err(error)?; + let page_url = response.url().clone(); + let mut page: Page = response.json().await.map_err(error)?; + let values = page + .fields + .remove(field) + .ok_or_else(|| error(format!("missing {field}")))?; + let values = values + .as_array() + .ok_or_else(|| error(format!("{field} must be an array")))?; + items.extend(values.iter().take(cap - items.len()).cloned()); + if items.len() == cap { + break; + } + let Some(next) = page.links.into_iter().find(|link| link.rel == "next") else { + break; + }; + url = page_url.join(&next.href).map_err(error)?; + if !matches!(url.scheme(), "http" | "https") { + return Err(error("pagination URL must use HTTP or HTTPS")); + } + method = Method::from_bytes(next.method.as_deref().unwrap_or("GET").as_bytes()) + .map_err(error)?; + if method != Method::GET && method != Method::POST { + return Err(error("pagination supports only GET and POST")); + } + if next.merge && method == Method::POST { + let mut merged = body.take().unwrap_or_else(|| Value::Object(Map::new())); + let target = merged + .as_object_mut() + .ok_or_else(|| error("pagination body must be an object"))?; + if let Some(extra) = next.body { + target.extend( + extra + .as_object() + .ok_or_else(|| error("pagination body must be an object"))? + .clone(), + ); + } + body = Some(merged); + } else { + body = next.body; + } + headers = next.headers; + } + Ok(items) +} diff --git a/crates/search/src/executor.rs b/crates/search/src/executor.rs index aaf9721..2195fdf 100644 --- a/crates/search/src/executor.rs +++ b/crates/search/src/executor.rs @@ -1,6 +1,6 @@ use futures::StreamExt; use superstac_core::{errors::SuperSTACError, models::catalog::Catalog}; -use tokio::time::{sleep, timeout}; +use crate::runtime::{sleep, timeout}; use crate::{ aggregator::SearchAggregator, diff --git a/crates/search/src/lib.rs b/crates/search/src/lib.rs index 7d91525..0c0b593 100644 --- a/crates/search/src/lib.rs +++ b/crates/search/src/lib.rs @@ -20,3 +20,7 @@ pub mod unifier; pub mod backend; pub mod stac_api; + +pub mod runtime; +#[cfg(target_arch = "wasm32")] +pub mod browser; diff --git a/crates/search/src/runtime.rs b/crates/search/src/runtime.rs new file mode 100644 index 0000000..7c563fa --- /dev/null +++ b/crates/search/src/runtime.rs @@ -0,0 +1,29 @@ +//! Target-specific timers shared by federation and browser discovery. +#[cfg(not(target_arch = "wasm32"))] +pub use tokio::time::{sleep, timeout}; + +#[cfg(target_arch = "wasm32")] +pub async fn sleep(duration: std::time::Duration) { + // Browser timers use signed 32-bit milliseconds. Split larger durations. + let mut remaining = duration.as_millis(); + loop { + let millis = remaining.min(i32::MAX as u128) as u32; + gloo_timers::future::TimeoutFuture::new(millis).await; + remaining -= u128::from(millis); + if remaining == 0 { + break; + } + } +} + +#[cfg(target_arch = "wasm32")] +pub async fn timeout( + duration: std::time::Duration, + future: F, +) -> Result { + use futures::future::{select, Either}; + match select(Box::pin(future), Box::pin(sleep(duration))).await { + Either::Left((output, _)) => Ok(output), + Either::Right(_) => Err("request timed out"), + } +} diff --git a/crates/search/src/stac_api.rs b/crates/search/src/stac_api.rs index 1cf1528..60aa313 100644 --- a/crates/search/src/stac_api.rs +++ b/crates/search/src/stac_api.rs @@ -1,10 +1,11 @@ //! Item retrieval from a STAC API, including pagination. -use futures::{future::BoxFuture, StreamExt, TryStreamExt}; +#[cfg(not(target_arch = "wasm32"))] +use futures::{StreamExt, TryStreamExt}; use stac::Item; use superstac_core::{errors::SuperSTACError, models::catalog::Catalog}; use crate::{ - backend::{BackendSearchOptions, SearchBackend}, + backend::{BackendSearchOptions, SearchBackend, SearchFuture}, query::SearchQuery, response::SearchItem, translator::to_stac_search, @@ -44,22 +45,32 @@ impl StacApiBackend { let search = to_stac_search(query); - let stac_client = stac_io::api::Client::with_client(self.client.clone(), &catalog.url) - .map_err(|e| SuperSTACError::SearchFailed(format!("stac client init: {}", e)))?; + if cap == 0 { + return Ok(Vec::new()); + } + #[cfg(not(target_arch = "wasm32"))] + let raw_items: Vec = { + let stac_client = stac_io::api::Client::with_client(self.client.clone(), &catalog.url) + .map_err(|e| SuperSTACError::SearchFailed(format!("stac client init: {}", e)))?; - let stream = stac_client - .search(search) - .await - .map_err(|e| SuperSTACError::SearchFailed(format!("search request: {}", e)))?; + let stream = stac_client + .search(search) + .await + .map_err(|e| SuperSTACError::SearchFailed(format!("search request: {}", e)))?; - // Stream of `stac::api::Item` (= `serde_json::Map`), - // paginated internally by stac-io. `take(cap)` bounds the total; - // `try_collect` short-circuits on first per-item stream error. - let raw_items: Vec = stream - .take(cap) - .try_collect() - .await - .map_err(|e| SuperSTACError::SearchFailed(format!("stream item: {}", e)))?; + // Stream of `stac::api::Item` (= `serde_json::Map`), + // paginated internally by stac-io. `take(cap)` bounds the total; + // `try_collect` short-circuits on first per-item stream error. + let raw_items: Vec = stream + .take(cap) + .try_collect() + .await + .map_err(|e| SuperSTACError::SearchFailed(format!("stream item: {}", e)))?; + + raw_items + }; + #[cfg(target_arch = "wasm32")] + let raw_items = crate::browser::search(&self.client, &catalog.url, &search, cap).await?; let items: Vec = raw_items .into_iter() @@ -89,7 +100,7 @@ impl SearchBackend for StacApiBackend { catalog: &'a Catalog, query: SearchQuery, options: BackendSearchOptions, - ) -> BoxFuture<'a, Result, SuperSTACError>> { + ) -> SearchFuture<'a, Result, SuperSTACError>> { Box::pin(self.search_catalog(catalog, query, options)) } } diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml new file mode 100644 index 0000000..cdad0eb --- /dev/null +++ b/crates/wasm/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "superstac-wasm" +description = "Browser WebAssembly bindings for SuperSTAC federated search" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[lib] +crate-type = ["cdylib", "rlib"] + +[target.'cfg(target_arch = "wasm32")'.dependencies] +superstac-core = { path = "../core", version = "0.3.0" } +superstac-search = { path = "../search", version = "0.3.0" } +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +serde-wasm-bindgen = "0.6" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.13", default-features = false, features = ["json"] } +futures = "0.3" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/crates/wasm/README.md b/crates/wasm/README.md new file mode 100644 index 0000000..380d695 --- /dev/null +++ b/crates/wasm/README.md @@ -0,0 +1,106 @@ +# SuperSTAC WebAssembly + +Run live federated STAC searches in a browser using the shared Rust search logic. +This package supports catalog configuration, collection discovery, pagination, +collection/asset aliases, deduplication, provenance, concurrency limits, retries, +per-attempt timeouts, and per-catalog failures. + +## Build and try + +From the repository root, using Rust 1.88 or newer: + +```sh +rustup target add wasm32-unknown-unknown +cargo install wasm-pack --version 0.13.1 --locked +wasm-pack build crates/wasm --target web --release --locked +python3 -m http.server 8000 --directory crates/wasm +``` + +Open http://localhost:8000/examples/ to try a live Earth Search query. +`crates/wasm/pkg` contains the generated JavaScript module, WebAssembly binary, +TypeScript declarations, and package metadata. Build artifacts are ignored by Git. +The package has not been published to npm. + +## JavaScript / TypeScript + +```ts +import init, { SuperSTAC } from './pkg/superstac_wasm.js'; + +await init(); +const client = new SuperSTAC({ + catalogs: [{ + id: 'earth-search', + url: 'https://earth-search.aws.element84.com/v1', + // Optional maps: canonical name -> provider-local name. + collection_aliases: {}, + asset_aliases: {}, + }], + settings: { + max_concurrent_catalogs: 8, + per_catalog_timeout_seconds: 30, + max_retry_attempts: 2, // Total attempts, including the first request. + max_items_per_catalog: 1000, + deduplicate_items: true, + unify_response: true, + }, +}); + +const result = await client.search({ + collections: ['sentinel-2-l2a'], + bbox: [6, 49, 7, 50], + limit: 20, +}); +console.log(result.items, result.metadata.failures); +console.log(await client.listCollections()); +client.free(); // Once all outstanding calls have completed. +``` + +`search()` returns a Promise containing `{ items, metadata }`. Each result entry +has `item` (the STAC Feature), `catalog_id`, and `seen_in` provenance. It is not +itself a GeoJSON FeatureCollection. Missing `collections` means no collection +filter. `limit` defaults to 10 and applies **per catalog**, capped by +`max_items_per_catalog`; deduplication then merges the results. + +Invalid configuration throws a JavaScript Error; invalid queries reject the +Promise. Individual catalog failures, including timeouts and CORS failures, +are recorded in `metadata.failures`. Even when every catalog fails, the Promise +resolves with an empty result and failure metadata: always inspect it. + +`listCollections()` returns one entry per catalog with `collections` and a +nullable `error`. Each collection includes its `canonical_id` and original +`collection` metadata. Discovery follows next links, uses the configured +concurrency/timeout, and limits results to 10,000 collections per catalog. +It does not retry or change which catalogs subsequent searches query. + +## Browser scope + +- Endpoints must allow browser requests through CORS. A server-side proxy may + be needed for catalogs that do not. WASM cannot bypass browser restrictions. +- The first search request uses STAC POST `/search`; pagination supports GET + and POST next links, relative URLs, headers, and merged POST bodies. +- Configuration stays in memory. No filesystem YAML loading, GeoParquet, + background health monitoring, or engine capability-based catalog filtering. +- This build targets browsers. Node.js and WASI are not validated targets. +- Browser timers can be delayed in background tabs. Timeouts cover each full + paginated search attempt, and failed attempts discard their partial items. +- Native search uses `stac` 0.16 and `stac-io`; WASM uses `stac` 0.17.0 without + filesystem features and a Fetch transport. The exact WASM version is pinned + because newer STAC releases conflict with the native GeoParquet dependency graph. WASM backend futures are local; + native backend futures retain their Send bounds. + +## Tests + +```sh +cd crates/wasm +npm ci +npm run build +npx playwright install chromium +npm test +npx tsc --noEmit --strict --target ES2022 --module ESNext --moduleResolution bundler tests/types.ts +``` + +Set `CHROME_PATH` to use an existing Chrome executable. The browser suite runs +against a local STAC fixture server and checks pagination, retries, timeouts, +aliases, deduplication, result serialization, validation, and partial failures. +Set `SUPERSTAC_LIVE_TEST=1 npm test` to also query Earth Search; this optional +check needs internet access and the provider's CORS support. diff --git a/crates/wasm/examples/index.html b/crates/wasm/examples/index.html new file mode 100644 index 0000000..4bb3d51 --- /dev/null +++ b/crates/wasm/examples/index.html @@ -0,0 +1,39 @@ + + + + +SuperSTAC browser search + +

SuperSTAC browser search

+

Search Sentinel-2 scenes over western Europe in January 2024 using Earth Search.

+ +
Loading WebAssembly…
+ + diff --git a/crates/wasm/package-lock.json b/crates/wasm/package-lock.json new file mode 100644 index 0000000..dea8b82 --- /dev/null +++ b/crates/wasm/package-lock.json @@ -0,0 +1,75 @@ +{ + "name": "superstac-wasm-tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "superstac-wasm-tests", + "devDependencies": { + "playwright": "1.56.1", + "typescript": "5.9.3" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", + "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", + "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/crates/wasm/package.json b/crates/wasm/package.json new file mode 100644 index 0000000..30af4d1 --- /dev/null +++ b/crates/wasm/package.json @@ -0,0 +1,13 @@ +{ + "name": "superstac-wasm-tests", + "private": true, + "type": "module", + "scripts": { + "build": "wasm-pack build --target web --release --locked", + "test": "node tests/browser.mjs" + }, + "devDependencies": { + "playwright": "1.56.1", + "typescript": "5.9.3" + } +} diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs new file mode 100644 index 0000000..9ba436e --- /dev/null +++ b/crates/wasm/src/lib.rs @@ -0,0 +1,196 @@ +//! JavaScript bindings for browser-hosted live STAC searches. +#![cfg(target_arch = "wasm32")] + +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + time::Duration, +}; +use superstac_core::models::catalog::Catalog; +use superstac_search::{executor::SearchExecutor, options::FederationOptions, query::SearchQuery}; +use wasm_bindgen::prelude::*; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Config { + catalogs: Vec, + #[serde(default)] + settings: Settings, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogConfig { + id: String, + url: String, + #[serde(default)] + collection_aliases: HashMap, + #[serde(default)] + asset_aliases: HashMap>, +} + +#[derive(Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Settings { + deduplicate_items: bool, + unify_response: bool, + max_concurrent_catalogs: usize, + per_catalog_timeout_seconds: u32, + max_retry_attempts: u8, + max_items_per_catalog: usize, +} + +impl Default for Settings { + fn default() -> Self { + Self { + deduplicate_items: true, + unify_response: true, + max_concurrent_catalogs: 8, + per_catalog_timeout_seconds: 30, + max_retry_attempts: 2, + max_items_per_catalog: 1000, + } + } +} + +fn error(value: impl std::fmt::Display) -> JsError { + JsError::new(&value.to_string()) +} + +fn to_js(value: &impl Serialize) -> Result { + value + .serialize(&serde_wasm_bindgen::Serializer::json_compatible()) + .map_err(error) +} + +#[wasm_bindgen] +pub struct SuperSTAC { + catalogs: Vec, + client: reqwest::Client, + options: FederationOptions, +} + +#[wasm_bindgen] +impl SuperSTAC { + #[wasm_bindgen(constructor)] + pub fn new(config: ConfigInput) -> Result { + let config: Config = serde_wasm_bindgen::from_value(config.into()).map_err(error)?; + let settings = config.settings; + if config.catalogs.is_empty() { + return Err(error("at least one catalog is required")); + } + if settings.max_concurrent_catalogs == 0 + || settings.per_catalog_timeout_seconds == 0 + || settings.max_retry_attempts == 0 + || settings.max_items_per_catalog == 0 + { + return Err(error( + "concurrency, timeout, attempts, and item cap must be positive", + )); + } + let mut ids = HashSet::new(); + let mut catalogs = Vec::new(); + for input in config.catalogs { + if !ids.insert(input.id.clone()) { + return Err(error("duplicate catalog id")); + } + let url = reqwest::Url::parse(&input.url).map_err(error)?; + if !matches!(url.scheme(), "http" | "https") + || url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(error("catalog URL must be an HTTP(S) base URL without credentials, query, or fragment")); + } + let mut catalog = + Catalog::new(&input.id, None::, &input.url, None::, None) + .map_err(error)?; + catalog.collection_aliases = input.collection_aliases; + catalog.asset_aliases = input.asset_aliases; + catalogs.push(catalog); + } + let mut options = FederationOptions::default(); + options.deduplicate = settings.deduplicate_items; + options.unify_response = settings.unify_response; + options.max_concurrent = settings.max_concurrent_catalogs; + options.per_catalog_timeout = + Duration::from_secs(settings.per_catalog_timeout_seconds.into()); + options.retry.max_attempts = settings.max_retry_attempts; + options.max_items_per_catalog = settings.max_items_per_catalog; + Ok(Self { + catalogs, + client: reqwest::Client::new(), + options, + }) + } + + /// Search all registered catalogs. Individual catalog failures are in metadata. + pub async fn search(&self, query: QueryInput) -> Result { + let mut value: serde_json::Value = + serde_wasm_bindgen::from_value(query.into()).map_err(error)?; + let object = value + .as_object_mut() + .ok_or_else(|| error("query must be an object"))?; + for key in object.keys() { + if !matches!( + key.as_str(), + "collections" | "ids" | "intersects" | "bbox" | "datetime" | "limit" | "sortby" + ) { + return Err(error(format!("unknown query field: {key}"))); + } + } + object + .entry("collections") + .or_insert_with(|| serde_json::json!([])); + let query: SearchQuery = serde_json::from_value(value).map_err(error)?; + if query.bbox.is_some() && query.intersects.is_some() { + return Err(error("bbox and intersects cannot be combined")); + } + if query.limit == Some(0) { + return Err(error("limit must be positive")); + } + let result = SearchExecutor::new(self.client.clone()) + .federated_search(self.catalogs.clone(), query, self.options) + .await + .map_err(error)?; + Ok(to_js(&result)?.unchecked_into()) + } + + /// Discover collections per catalog, preserving failures and original metadata. + #[wasm_bindgen(js_name = listCollections)] + pub async fn list_collections(&self) -> Result { + use futures::StreamExt; + let results: Vec = futures::stream::iter(self.catalogs.iter().map(|catalog| async move { + let result = superstac_search::runtime::timeout(self.options.per_catalog_timeout, + superstac_search::browser::collections(&self.client, &catalog.url, 10_000)).await; + match result { + Ok(Ok(collections)) => { + let collections: Vec<_> = collections.into_iter().map(|collection| { + let id = collection.get("id").and_then(|id| id.as_str()).unwrap_or(""); + serde_json::json!({ "canonical_id": catalog.canonical_collection(id), "collection": collection }) + }).collect(); + serde_json::json!({ "catalog_id": catalog.id, "collections": collections, "error": null }) + } + Ok(Err(e)) => serde_json::json!({ "catalog_id": catalog.id, "collections": [], "error": e.to_string() }), + Err(e) => serde_json::json!({ "catalog_id": catalog.id, "collections": [], "error": e }), + } + })).buffered(self.options.max_concurrent).collect().await; + Ok(to_js(&results)?.unchecked_into()) + } +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "SuperSTACConfig")] + pub type ConfigInput; + #[wasm_bindgen(typescript_type = "SearchQuery")] + pub type QueryInput; + #[wasm_bindgen(typescript_type = "SearchResponse")] + pub type SearchOutput; + #[wasm_bindgen(typescript_type = "CatalogCollections[]")] + pub type CollectionsOutput; +} + +#[wasm_bindgen(typescript_custom_section)] +const TYPES: &'static str = include_str!("types.d.ts"); diff --git a/crates/wasm/src/types.d.ts b/crates/wasm/src/types.d.ts new file mode 100644 index 0000000..cbfccf0 --- /dev/null +++ b/crates/wasm/src/types.d.ts @@ -0,0 +1,53 @@ +export interface SuperSTACConfig { + catalogs: { + id: string; + url: string; + collection_aliases?: Record; + asset_aliases?: Record>; + }[]; + settings?: { + deduplicate_items?: boolean; + unify_response?: boolean; + max_concurrent_catalogs?: number; + per_catalog_timeout_seconds?: number; + max_retry_attempts?: number; + max_items_per_catalog?: number; + }; +} +export interface SearchQuery { + collections?: string[]; + ids?: string[]; + bbox?: [number, number, number, number] | [number, number, number, number, number, number]; + intersects?: { type: string; coordinates?: unknown; geometries?: unknown[] }; + datetime?: string; + limit?: number; + sortby?: string[]; +} +export interface StacItem { + type: "Feature"; + id: string; + collection?: string; + geometry: Record | null; + properties: Record; + assets: Record; + links: Record[]; + [key: string]: unknown; +} +export interface SearchResponse { + items: { catalog_id: string; seen_in: string[]; item: StacItem }[]; + metadata: { + superstac_version: string; + catalogs_queried: number; + catalogs_succeeded: number; + catalogs_failed: number; + total_items: number; + duplicates_removed: number; + failures: { catalog_id: string; reason: string }[]; + unsupported_collections: string[]; + }; +} +export interface CatalogCollections { + catalog_id: string; + collections: { canonical_id: string; collection: Record }[]; + error: string | null; +} diff --git a/crates/wasm/tests/browser.mjs b/crates/wasm/tests/browser.mjs new file mode 100644 index 0000000..bdf1788 --- /dev/null +++ b/crates/wasm/tests/browser.mjs @@ -0,0 +1,140 @@ +// Exercise the generated JS package and real browser Fetch against a local STAC server. +import { createServer } from 'node:http'; +import { readFile } from 'node:fs/promises'; +import { chromium } from 'playwright'; +import assert from 'node:assert/strict'; + +const calls = []; +let slowClosed = false; +const item = (id) => ({ type: 'Feature', stac_version: '1.0.0', id, + collection: 'local', geometry: null, + properties: { datetime: '2024-01-01T00:00:00Z' }, + assets: { B02: { href: 'https://example.com/blue.tif' } }, links: [] }); +const server = createServer(async (req, res) => { + try { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Headers', 'content-type,x-page'); + res.setHeader('Access-Control-Allow-Methods', 'GET,POST,OPTIONS'); + if (req.method === 'OPTIONS') { res.end(); return; } + const url = new URL(req.url, 'http://localhost'); + if (url.pathname === '/') { + res.setHeader('Content-Type', 'text/html'); res.end('WASM tests'); return; + } + if (url.pathname.startsWith('/pkg/')) { + const name = url.pathname.slice(5); + if (!/^[\w.-]+$/.test(name)) { res.writeHead(404).end(); return; } + res.setHeader('Content-Type', name.endsWith('.wasm') ? 'application/wasm' : 'text/javascript'); + res.end(await readFile(new URL(`../pkg/${name}`, import.meta.url))); return; + } + let raw = ''; + for await (const chunk of req) raw += chunk; + const body = raw ? JSON.parse(raw) : undefined; + calls.push({ path: req.url, method: req.method, body, header: req.headers['x-page'] }); + res.setHeader('Content-Type', 'application/json'); + const send = (features, links = []) => res.end(JSON.stringify({ type: 'FeatureCollection', features, links })); + if (url.pathname.startsWith('/slow/')) { res.on('close', () => { slowClosed = true; }); return; } + if (url.pathname.startsWith('/broken/')) { res.writeHead(503).end('{}'); return; } + if (url.pathname === '/retry/search' && calls.filter(c => c.path === req.url).length === 1) { + res.writeHead(503).end('{}'); return; + } + if (url.pathname === '/a/collections') { + res.end(JSON.stringify({ collections: [{ id: 'local', title: 'Local collection' }], + links: [{ rel: 'next', href: './collections?page=2' }], + ...(url.search ? { collections: [{ id: 'other' }], links: [] } : {}) })); return; + } + if (url.pathname.endsWith('/collections')) { res.end('{"collections":[],"links":[]}'); return; } + if (url.pathname === '/loop/search') { send([], [{ rel: 'next', href: './search', method: 'POST', body }]); return; } + if (url.pathname === '/a/search') { + send([item(url.search ? 'second' : 'shared')], url.search ? [] : [{ rel: 'next', href: './search?page=2' }]); return; + } + if (url.pathname === '/post/search') { + send([item(body?.token ? 'second' : 'shared')], body?.token ? [] : [ + { rel: 'next', href: './search', method: 'POST', body: { token: 'next' }, merge: true, headers: { 'x-page': 'two' } } + ]); return; + } + send([item('shared')]); + } catch (e) { res.writeHead(500).end(JSON.stringify({ error: String(e) })); } +}); +await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); +const origin = `http://127.0.0.1:${server.address().port}`; +// A separate page origin exercises CORS and preflight on the fixture requests. +const pageServer = createServer(server.listeners('request')[0]); +await new Promise(resolve => pageServer.listen(0, '127.0.0.1', resolve)); +const pageOrigin = `http://127.0.0.1:${pageServer.address().port}`; +let browser; +try { + browser = await chromium.launch({ ...(process.env.CHROME_PATH ? { executablePath: process.env.CHROME_PATH } : {}) }); + const page = await browser.newPage(); + await page.goto(pageOrigin); + const result = await page.evaluate(async ({ origin, live }) => { + const { default: init, SuperSTAC } = await import('/pkg/superstac_wasm.js'); + await init(); + const check = (condition, message) => { if (!condition) throw new Error(message); }; + const catalog = id => ({ id, url: `${origin}/${id}`, + collection_aliases: { canonical: 'local' }, asset_aliases: { canonical: { blue: 'B02' } } }); + const make = (ids, settings = {}) => new SuperSTAC({ catalogs: ids.map(catalog), settings }); + const client = make(['a', 'b', 'retry', 'broken']); + const result = await client.search({ collections: ['canonical'], limit: 2 }); + check(result.metadata.total_items === 2, JSON.stringify(result)); + check(result.metadata.catalogs_succeeded === 3 && result.metadata.catalogs_failed === 1, 'partial failures'); + check(result.metadata.duplicates_removed === 2, 'deduplication'); + const shared = result.items.find(i => i.item.id === 'shared'); + check(shared.seen_in.length === 3, 'provenance'); + check(shared.item.collection === 'canonical' && shared.item.assets.blue.href.endsWith('blue.tif'), 'aliases and plain objects'); + check(!('B02' in shared.item.assets), 'asset alias removes original key'); + check(Array.isArray(result.items) && !(shared.item.properties instanceof Map), 'JS serialization'); + const discovered = await client.listCollections(); + check(discovered[0].collections.length === 2 && discovered[0].collections[0].canonical_id === 'canonical', 'paginated discovery'); + check(typeof discovered[3].error === 'string', 'discovery partial failure'); + client.free(); + const capped = make(['a'], { max_items_per_catalog: 1 }); + check((await capped.search({ limit: 20 })).items.length === 1, 'item cap'); capped.free(); + const post = make(['post']); + check((await post.search({ collections: ['canonical'], limit: 2 })).items.length === 2, 'POST pagination'); post.free(); + const slow = make(['slow', 'b'], { per_catalog_timeout_seconds: 1, max_retry_attempts: 1 }); + const timed = await slow.search({}); + check(timed.metadata.catalogs_succeeded === 1 && timed.metadata.failures[0].reason.includes('timeout'), 'browser timeout'); slow.free(); + const loop = make(['loop'], { max_retry_attempts: 1 }); + check((await loop.search({})).metadata.failures[0].reason.includes('pagination'), 'pagination cycle'); loop.free(); + for (const config of [{ catalogs: [] }, { catalogs: [catalog('a'), catalog('a')] }, + { catalogs: [catalog('a')], settings: { max_retry_attempts: 0 } }, + { catalogs: [{ id: 'a', url: 'file:///tmp/data' }] }]) { + let threw = false; + try { new SuperSTAC(config); } catch (e) { threw = e instanceof Error; } + check(threw, 'invalid config throws Error'); + } + const raw = make(['b', 'retry'], { deduplicate_items: false, unify_response: false }); + const unmodified = await raw.search({}); + check(unmodified.items.length === 2 && unmodified.items.every(i => i.item.collection === 'local' && i.item.assets.B02), 'disabled normalization and deduplication'); + raw.free(); + const invalid = make(['a']); + for (const query of [{ collection: ['typo'] }, { limit: 0 }, { bbox: [1, 2] }, { bbox: [0, 0, 1, 1], intersects: { type: 'Point', coordinates: [0, 0] } }]) { + let threw = false; + try { await invalid.search(query); } catch (e) { threw = e instanceof Error; } + check(threw, 'invalid query rejects with Error'); + } + invalid.free(); + if (live) { + const liveClient = new SuperSTAC({ catalogs: [{ id: 'earth-search', url: 'https://earth-search.aws.element84.com/v1' }] }); + const response = await liveClient.search({ collections: ['sentinel-2-l2a'], bbox: [6, 49, 7, 50], + datetime: '2024-01-01T00:00:00Z/2024-01-31T23:59:59Z', limit: 1 }); + check(response.metadata.catalogs_succeeded === 1 && response.items.length === 1, JSON.stringify(response.metadata)); + liveClient.free(); + return { fixtureTests: 'passed', liveItem: response.items[0].item.id }; + } + return { fixtureTests: 'passed' }; + }, { origin, live: process.env.SUPERSTAC_LIVE_TEST === '1' }); + assert.equal(calls.filter(c => c.path === '/retry/search').length, 3); + assert.ok(slowClosed, 'timeout aborts the outstanding Fetch request'); + const postPage = calls.find(c => c.path === '/post/search' && c.body?.token); + assert.deepEqual(postPage.body.collections, ['local']); + assert.equal(postPage.header, 'two'); + assert.equal(calls.filter(c => c.path === '/a/search?page=2').length, 1, 'cap must stop pagination'); + console.log(result); +} finally { + await browser?.close(); + pageServer.closeAllConnections(); + await new Promise(resolve => pageServer.close(resolve)); + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} diff --git a/crates/wasm/tests/types.ts b/crates/wasm/tests/types.ts new file mode 100644 index 0000000..b899dfb --- /dev/null +++ b/crates/wasm/tests/types.ts @@ -0,0 +1,9 @@ +import init, { SuperSTAC, type SearchResponse, type CatalogCollections } from '../pkg/superstac_wasm.js'; +await init(); +const client = new SuperSTAC({ catalogs: [{ id: 'test', url: 'https://example.com' }] }); +const response: SearchResponse = await client.search({ bbox: [0, 0, 1, 1], limit: 2 }); +const collections: CatalogCollections[] = await client.listCollections(); +console.log(response.metadata.total_items, collections[0].catalog_id); +// @ts-expect-error limit must be a number +client.search({ limit: '2' }); +client.free(); From 6a5529839242b57e3d362d0f86c85ab0b94e8276 Mon Sep 17 00:00:00 2001 From: jeafreezy Date: Sat, 3 Oct 2026 12:08:02 +0200 Subject: [PATCH 2/7] chore: wasm wip --- .github/workflows/CI.yml | 28 +---- .github/workflows/wasm.yml | 42 ++++++++ README.md | 4 +- crates/wasm/README.md | 19 +++- crates/wasm/package.json | 6 +- crates/wasm/tsconfig.json | 17 +++ docs/.gitignore | 3 + docs/README.md | 23 +++++ docs/app/layout.tsx | 2 +- docs/components/wasm-playground.tsx | 21 ++++ docs/content/docs/index.mdx | 6 +- docs/content/docs/meta.json | 1 + docs/content/docs/reference/status.md | 3 + docs/content/docs/start/installation.md | 8 +- docs/content/docs/wasm/meta.json | 5 + docs/content/docs/wasm/overview.md | 118 +++++++++++++++++++++ docs/content/docs/wasm/playground.mdx | 23 +++++ docs/package.json | 3 +- docs/public/playground/index.html | 51 +++++++++ docs/public/playground/playground.css | 37 +++++++ docs/public/playground/playground.js | 131 ++++++++++++++++++++++++ docs/scripts/prepare-wasm.mjs | 14 +++ 22 files changed, 529 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/wasm.yml create mode 100644 crates/wasm/tsconfig.json create mode 100644 docs/components/wasm-playground.tsx create mode 100644 docs/content/docs/wasm/meta.json create mode 100644 docs/content/docs/wasm/overview.md create mode 100644 docs/content/docs/wasm/playground.mdx create mode 100644 docs/public/playground/index.html create mode 100644 docs/public/playground/playground.css create mode 100644 docs/public/playground/playground.js create mode 100644 docs/scripts/prepare-wasm.mjs diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f89a682..ddf0b64 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -2,7 +2,7 @@ name: Package checks on: pull_request: - paths: ['crates/**', 'Cargo.toml', 'Cargo.lock', 'scripts/**', '.github/workflows/CI.yml', '.github/workflows/publish.yml'] + paths: ['crates/**', '!crates/wasm/**', 'Cargo.toml', 'Cargo.lock', 'scripts/**', '.github/workflows/CI.yml', '.github/workflows/publish.yml'] workflow_dispatch: workflow_call: @@ -27,29 +27,9 @@ jobs: - run: cargo check -p superstac-python --no-default-features --locked wasm: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: '1.88.0' - targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - cache-dependency-path: crates/wasm/package-lock.json - - run: cargo install wasm-pack --version 0.13.1 --locked - - run: wasm-pack build crates/wasm --target web --release --locked - - run: npm ci - working-directory: crates/wasm - - run: npx playwright install --with-deps chromium - working-directory: crates/wasm - - run: npm test - working-directory: crates/wasm - - run: npx tsc --noEmit --strict --target ES2022 --module ESNext --moduleResolution bundler tests/types.ts - working-directory: crates/wasm + # Pull requests run the standalone WASM workflow, including WASM-only changes. + if: github.event_name != 'pull_request' + uses: ./.github/workflows/wasm.yml wheels: name: Wheel (${{ matrix.target }}) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 0000000..e3fc7b5 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,42 @@ +name: WASM checks + +on: + pull_request: + paths: + - 'crates/wasm/**' + - 'crates/core/**' + - 'crates/search/**' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/wasm.yml' + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + browser: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: '1.88.0' + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: crates/wasm/package-lock.json + - run: cargo install wasm-pack --version 0.13.1 --locked --jobs 2 + - run: npm ci + working-directory: crates/wasm + - run: npm run build + working-directory: crates/wasm + - run: npx playwright install --with-deps chromium + working-directory: crates/wasm + - run: npm run check + working-directory: crates/wasm diff --git a/README.md b/README.md index bf57381..2270896 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,9 @@ Published Python wheels include this backend. Build the browser binding to search live STAC catalogs from JavaScript or TypeScript. See the [WASM guide](crates/wasm/README.md) for build instructions, API usage, -a browser example, and browser-specific limitations. +a browser example, and browser-specific limitations. The docs include a +[live playground](https://spatialnode.com/superstac/docs/wasm/playground) +and a [JavaScript guide](https://spatialnode.com/superstac/docs/wasm/overview). ## Configuration diff --git a/crates/wasm/README.md b/crates/wasm/README.md index 380d695..977c716 100644 --- a/crates/wasm/README.md +++ b/crates/wasm/README.md @@ -12,7 +12,7 @@ From the repository root, using Rust 1.88 or newer: ```sh rustup target add wasm32-unknown-unknown cargo install wasm-pack --version 0.13.1 --locked -wasm-pack build crates/wasm --target web --release --locked +wasm-pack build crates/wasm --target web --release --locked --jobs 1 python3 -m http.server 8000 --directory crates/wasm ``` @@ -21,6 +21,11 @@ Open http://localhost:8000/examples/ to try a live Earth Search query. TypeScript declarations, and package metadata. Build artifacts are ignored by Git. The package has not been published to npm. +The docs also include a [browser guide](../../docs/content/docs/wasm/overview.md) +and [live playground](../../docs/content/docs/wasm/playground.mdx). For local docs +work, `node docs/scripts/prepare-wasm.mjs` from the repository root copies this +build into the docs assets without compiling Rust. + ## JavaScript / TypeScript ```ts @@ -95,10 +100,18 @@ cd crates/wasm npm ci npm run build npx playwright install chromium -npm test -npx tsc --noEmit --strict --target ES2022 --module ESNext --moduleResolution bundler tests/types.ts +npm run check ``` +For routine checks against an already-built package, run just `npm run check`. +It type-checks the public API and runs one headless browser sequentially; it does +not compile Rust or run the native workspace tests. `npm run build` uses one +Cargo worker. The dedicated **WASM checks** GitHub Actions workflow builds from +source and runs the same checks on CI; it can be run manually on its own. + +The TypeScript check includes `ESNext.Disposable` because the generated bindings +expose `Symbol.dispose`. You can still use `client.free()` explicitly. + Set `CHROME_PATH` to use an existing Chrome executable. The browser suite runs against a local STAC fixture server and checks pagination, retries, timeouts, aliases, deduplication, result serialization, validation, and partial failures. diff --git a/crates/wasm/package.json b/crates/wasm/package.json index 30af4d1..26d10b8 100644 --- a/crates/wasm/package.json +++ b/crates/wasm/package.json @@ -3,8 +3,10 @@ "private": true, "type": "module", "scripts": { - "build": "wasm-pack build --target web --release --locked", - "test": "node tests/browser.mjs" + "build": "wasm-pack build --target web --release --locked --jobs 1", + "test": "node tests/browser.mjs", + "check:types": "tsc -p tsconfig.json", + "check": "npm run check:types && npm test" }, "devDependencies": { "playwright": "1.56.1", diff --git a/crates/wasm/tsconfig.json b/crates/wasm/tsconfig.json new file mode 100644 index 0000000..9c3ab41 --- /dev/null +++ b/crates/wasm/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "noEmit": true, + "strict": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": [ + "ES2022", + "DOM", + "ESNext.Disposable" + ] + }, + "include": [ + "tests/types.ts" + ] +} diff --git a/docs/.gitignore b/docs/.gitignore index acc0cec..c1cd0f5 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -7,3 +7,6 @@ out/ public/brand/ !lib/ !lib/** + +# Generated by the WASM build in CI, or prepare-wasm locally. +public/wasm/ diff --git a/docs/README.md b/docs/README.md index d51e720..b5b07d4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -51,3 +51,26 @@ The direct **Open in Colab** links target this file on GitHub's `main` branch an [Deploy docs](../.github/workflows/docs-deploy.yml) runs on **published GitHub releases**, including prereleases, and through **Actions → Deploy docs → Run workflow**. Ordinary pushes and tag creation alone do not deploy. Publish a release only when its corresponding package is available; the workflow does not publish packages or wait for a separate package-publishing workflow. The workflow checks out the release tag or the manually supplied Git ref. It builds an AMD64 image, runs the Dockerfile's type check and production build, starts the resulting container, and checks pages, links, assets, search, and the 404 response. Only then does it publish to GHCR and deploy through the Coolify API. Use an x86-64 Coolify server for this workflow; an ARM server needs a matching runner/image platform. + +## Browser playground + +The WASM guide is in `content/docs/wasm`. The playground is a small static app +in `public/playground`, embedded in the docs with `components/wasm-playground.tsx`. +It loads WASM only when a visitor searches. Keep API internals in the guide and +code example; use scene and catalog language in the results. + +For local work, reuse an existing build without compiling Rust: + +```sh +node docs/scripts/prepare-wasm.mjs +``` + +This copies `crates/wasm/pkg` into the ignored `docs/public/wasm` directory. +If it is missing, download the `wasm-web` artifact from the **WASM checks** workflow +and extract it there, or follow the binding's build instructions. Docs startup +never builds Rust. Open `/superstac/docs/wasm/playground` on the docs dev server. + +The docs deployment workflow runs WASM checks for the same Git ref and downloads +its tested browser package before building the docs image. No generated WASM +binaries are committed. The playground tests run one headless browser against +fixture responses; live provider availability is not a CI requirement. diff --git a/docs/app/layout.tsx b/docs/app/layout.tsx index 19732cd..1b4b555 100644 --- a/docs/app/layout.tsx +++ b/docs/app/layout.tsx @@ -20,7 +20,7 @@ const crimson = localFont({ export const metadata: Metadata = { metadataBase: new URL('https://spatialnode.com'), title: { default: 'SuperSTAC — Many catalogs. One search.', template: '%s | SuperSTAC' }, - description: 'Search across STAC catalogs through one interface, with Python, Rust, or the command line.', + description: 'Search across STAC catalogs through one interface, with JavaScript, Python, Rust, or the command line.', icons: { icon: '/superstac/favicon.svg' }, }; diff --git a/docs/components/wasm-playground.tsx b/docs/components/wasm-playground.tsx new file mode 100644 index 0000000..449aaaf --- /dev/null +++ b/docs/components/wasm-playground.tsx @@ -0,0 +1,21 @@ +'use client'; +import { useEffect, useRef, useState } from 'react'; + +export function WasmPlayground() { + const frame = useRef(null); + const [height, setHeight] = useState(940); + function sendTheme() { + frame.current?.contentWindow?.postMessage({ type: 'superstac-theme', dark: document.documentElement.classList.contains('dark') }, location.origin); + } + useEffect(() => { + function resize(event: MessageEvent) { + if (event.origin !== location.origin || event.source !== frame.current?.contentWindow || event.data?.type !== 'superstac-height') return; + if (Number.isFinite(event.data.height)) setHeight(Math.max(600, Math.min(2400, event.data.height + 4))); + } + window.addEventListener('message', resize); + const observer = new MutationObserver(sendTheme); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class'] }); + return () => { window.removeEventListener('message', resize); observer.disconnect(); }; + }, []); + return