diff --git a/README.md b/README.md index b6198b4..4107dbe 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,8 @@ stack with the realistic sample, start to finish: `gdi-dataset-tool publish ` then makes the dataset visible, and the node serves it at `/fairdp/dataset/` and in the Beacon. There is a scripted equivalent for automation: -`gdi-dataset-tool config init -o tool.toml` writes a commented template, and the minimum is +`gdi-dataset-tool config init` writes a commented template to `~/.config/gdi/tool.toml`, +and the minimum is ```toml country_code = "EE" @@ -341,14 +342,18 @@ gdi-aggregated = "Genome of Europe Aggregated Data" ```bash export GDI_TOOL__PROFILES__DEFAULT__S3__ACCESS_KEY_ID=… GDI_TOOL__PROFILES__DEFAULT__S3__SECRET_ACCESS_KEY=… -gdi-dataset-tool --config tool.toml keys pin-recipient # fetches {service_url}/.well-known/c4gh-recipient and pins it; --file node.pub when handed over offline -gdi-dataset-tool init # scaffolds package.yaml: point it at your VCFs, fill the metadata -gdi-dataset-tool --config tool.toml package package.yaml # build + pack: .tar.c4gh, encrypted to the node; mints your provider key on first use -gdi-dataset-tool --config tool.toml upload .tar.c4gh # into the bucket, hidden; the node picks it up on its next poll -gdi-dataset-tool --config tool.toml publish # flips the sidecar to visible -gdi-dataset-tool --config tool.toml status # the node's verdict, once the operator sets the bucket's write_status = true or gives you a management_url +gdi-dataset-tool keys pin-recipient # fetches {service_url}/.well-known/c4gh-recipient and pins it; --file node.pub when handed over offline +gdi-dataset-tool init # scaffolds package.yaml: point it at your VCFs, fill the metadata +gdi-dataset-tool package package.yaml # build + pack: .tar.c4gh, encrypted to the node; mints your provider key on first use +gdi-dataset-tool upload .tar.c4gh # into the bucket, hidden; the node picks it up on its next poll +gdi-dataset-tool publish # flips the sidecar to visible +gdi-dataset-tool status # the node's verdict, once the operator sets the bucket's write_status = true or gives you a management_url ``` +Your provider key and the pinned node key live in `~/.config/gdi/keys/` and +`~/.config/gdi/recipients/`, or next to the file `--config` names, so keep such a config +out of any directory you share or package from. + Every tool field, including the co-located inbox profile and `keyless = true`: [`tool.example.toml`](tool.example.toml). @@ -465,9 +470,12 @@ keys and no profile. Worth knowing before your first real build: Health is `GET :9090/health/live` and `/health/ready` (`ready: true, degraded: true` means a provider bucket is dark), metrics are Prometheus text at `:9090/metrics`, and a dataset's -state is `GET :9090/datasets/{id}/state`. Below is the shipped dashboard, captured on the -dev stack with the realistic sample under a few minutes of query load. The alert names in -the panel titles are the shipped Prometheus rules: +state is `GET :9090/datasets/{id}/state`. Route traffic only to a ready node: one that +started while Vault was unreachable can answer publicly as if it had no datasets until it is +restarted ([operating.md §1](docs/operating.md#1-health-and-readiness-endpoints)). +Kubernetes readiness probes do that; `docker run` and Compose do not. Below is the shipped +dashboard, captured on the dev stack with the realistic sample under a few minutes of query +load. The alert names in the panel titles are the shipped Prometheus rules: ![The shipped Grafana dashboard: node readiness, datasets by state, request rate, error ratio and latency, on the dev stack under query load](docs/images/grafana-dashboard.png) diff --git a/crates/core/src/validate_pkg.rs b/crates/core/src/validate_pkg.rs index 9ee60c2..4dd2339 100644 --- a/crates/core/src/validate_pkg.rs +++ b/crates/core/src/validate_pkg.rs @@ -979,17 +979,26 @@ pub fn canonical_bcp47(tag: &str) -> Option { Some(tag.to_ascii_lowercase()) } -/// Reject a value not in the allowed set. +/// Reject a value not in the allowed set, naming what is allowed: the IRI a bare last +/// segment such as `PUBLIC` stands for, or else the whole set (at most a handful). /// /// # Errors /// /// Returns [`CoreError::InvalidManifest`] if `value` is not in `allowed`. pub fn validate_enum(field: &str, value: &str, allowed: &[&str]) -> CoreResult<()> { if allowed.contains(&value) { - Ok(()) - } else { - Err(invalid(&format!("{field} value {value:?} is not allowed"))) + return Ok(()); } + let hint = allowed + .iter() + .find(|iri| iri.rsplit(['/', '#']).next() == Some(value)) + .map_or_else( + || format!("allowed: {}", allowed.join(", ")), + |iri| format!("write the full IRI {iri}"), + ); + Err(invalid(&format!( + "{field} value {value:?} is not allowed; {hint}" + ))) } /// Validate `type`: at least one IRI, each from the closed [`DATASET_TYPES`] set. @@ -1092,12 +1101,28 @@ fn reject_disallowed_scheme(field: &str, value: &str, parsed: &url::Url) -> Core let scheme = parsed.scheme(); if !ALLOWED_IRI_SCHEMES.contains(&scheme) { return Err(invalid(&format!( - "{field} value {value:?} uses a disallowed URI scheme {scheme:?} (allowed: {ALLOWED_IRI_SCHEMES:?})" + "{field} value {value:?} uses a disallowed URI scheme {scheme:?} (allowed: {ALLOWED_IRI_SCHEMES:?}){}", + full_address_hint(value) ))); } Ok(()) } +/// The hint for a web address without its scheme, such as `node.example.org` (unparseable) +/// or `localhost:8080` (parsed with the scheme `localhost`). A full URI with another scheme, +/// such as `mailto:x@example.org`, gets none. +fn full_address_hint(value: &str) -> &'static str { + let schemeless = match value.split_once(':') { + None => true, + Some((_, rest)) => rest.starts_with(|c: char| c.is_ascii_digit()), + }; + if schemeless && !value.contains("://") { + "; write a web address in full, with its scheme (https://…)" + } else { + "" + } +} + /// Validate an IRI metadata field: parses as an absolute URL with a scheme and a /// non-empty host (or, for `urn:`/`w3id`-style identifiers, a non-empty path), /// carries no `IRIREF`-forbidden character (see RFC 3987 §2.2), uses an allowed @@ -1116,8 +1141,12 @@ pub fn validate_iri(field: &str, value: &str) -> CoreResult<()> { ))); } reject_iri_unsafe_chars(field, value)?; - let parsed = url::Url::parse(value) - .map_err(|_| invalid(&format!("{field} value {value:?} is not a valid IRI/URL")))?; + let parsed = url::Url::parse(value).map_err(|_| { + invalid(&format!( + "{field} value {value:?} is not a valid IRI/URL{}", + full_address_hint(value) + )) + })?; reject_disallowed_scheme(field, value, &parsed)?; // Require a scheme and either a host (http/https/...) or a non-empty // opaque path (urn:, ...). A relative or scheme-less string fails @@ -1209,8 +1238,12 @@ pub fn validate_url(field: &str, value: &str) -> CoreResult<()> { ))); } reject_iri_unsafe_chars(field, value)?; - let parsed = url::Url::parse(value) - .map_err(|_| invalid(&format!("{field} value {value:?} is not a valid URL")))?; + let parsed = url::Url::parse(value).map_err(|_| { + invalid(&format!( + "{field} value {value:?} is not a valid URL{}", + full_address_hint(value) + )) + })?; // Same scheme allow-list as `validate_iri`: a `hasURL`/reference value is emitted as // an IRI into the public RDF too. reject_disallowed_scheme(field, value, &parsed) @@ -1955,6 +1988,31 @@ mod tests { ); } + /// A web address written without its scheme is told so, whether it fails to parse or + /// parses with its host as the scheme; a full URI with a disallowed scheme is not. + #[test] + fn a_schemeless_address_is_told_to_start_with_https() { + for (field, value) in [ + ("service_url", "localhost:8080"), + ("service_url", "node.example.org"), + ("hasURL", "www.example.org/data"), + ] { + let iri = validate_iri(field, value).unwrap_err().to_string(); + let url = validate_url(field, value).unwrap_err().to_string(); + for err in [iri, url] { + assert!(err.contains("with its scheme"), "{value:?}: {err}"); + } + } + for value in [ + "file:///etc/passwd", + "javascript:alert(1)", + "mailto:x@example.org", + ] { + let err = validate_iri("license", value).unwrap_err().to_string(); + assert!(!err.contains("with its scheme"), "{value:?}: {err}"); + } + } + #[test] fn disallowed_iri_scheme_rejected() { // Schemes that `url::Url::parse` accepts but that must never reach the public FDP @@ -2405,7 +2463,20 @@ mod tests { ) .is_ok() ); - assert!(super::validate_enum("accessRights", "PUBLIC", super::ACCESS_RIGHTS).is_err()); + let bare = super::validate_enum("accessRights", "PUBLIC", super::ACCESS_RIGHTS) + .unwrap_err() + .to_string(); + assert!( + bare.contains(&format!("write the full IRI {}", super::ACCESS_RIGHTS[0])), + "a bare last segment names its IRI: {bare}" + ); + let unknown = super::validate_enum("accessRights", "OPEN", super::ACCESS_RIGHTS) + .unwrap_err() + .to_string(); + assert!( + super::ACCESS_RIGHTS.iter().all(|iri| unknown.contains(iri)), + "an unknown value lists the allowed set: {unknown}" + ); assert!( super::validate_health_category( "http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic" diff --git a/crates/gdi-dataset-tool/src/commands/cmd_build.rs b/crates/gdi-dataset-tool/src/commands/cmd_build.rs index 8d5ee77..c3514a3 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_build.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_build.rs @@ -40,7 +40,7 @@ use gdi_node_standalone_core::{ validate_parquet::{ ParquetCaps, ParquetScan, check_data_file_block_range, validate_parquet_dir, }, - validate_pkg::validate_package, + validate_pkg::{validate_package, validate_package_collect_all}, }; use crate::{ToolError, cli::BuildArgs}; @@ -104,7 +104,9 @@ fn diagnostic_count(diagnostics: &[(String, Diagnostic)], severity: Severity) -> .count() } -/// The build timestamp: `--build-epoch` when pinned, otherwise the wall clock. +/// The build timestamp: `--build-epoch` when pinned, otherwise the wall clock, moved past +/// any millisecond another build sharing the config directory has already used (see +/// [`unique_build_epoch`]). /// /// This is the build's only wall-clock read, so pinning it makes the generated /// `datasetId` — and therefore `manifest.json` — a pure function of the inputs. @@ -112,7 +114,7 @@ fn diagnostic_count(diagnostics: &[(String, Diagnostic)], severity: Severity) -> /// # Errors /// /// Returns a [`ToolError`] when the system clock is before the Unix epoch. -fn resolve_build_epoch(args: &BuildArgs) -> Result { +fn resolve_build_epoch(args: &BuildArgs, config_path: Option<&Path>) -> Result { match args.build_epoch { Some(pinned) => { let resolved = validate_pinned_epoch(pinned, now_unix_millis()?)?; @@ -134,10 +136,68 @@ fn resolve_build_epoch(args: &BuildArgs) -> Result { ); Ok(resolved) } - None => now_unix_millis(), + // A dry run writes nothing durable, so it takes the clock as it is. + None if args.dry_run => now_unix_millis(), + None => Ok(unique_build_epoch(config_path, now_unix_millis()?)), } } +/// The file in the config directory that holds the last build epoch minted there. +const LAST_BUILD_EPOCH_FILE: &str = ".last-build-epoch"; + +/// `now`, or one past the last millisecond a build sharing this config directory used, +/// whichever is later. +/// +/// A dataset id varies only by its millisecond, so builds started together would mint the +/// same id. They take turns under a lock on [`LAST_BUILD_EPOCH_FILE`]; without a config +/// directory, or when the lock fails, the clock stands alone. +fn unique_build_epoch(config_path: Option<&Path>, now: u64) -> u64 { + let Some(dir) = + gdi_node_standalone_core::config::config_base_dir(config_path).filter(|d| d.is_dir()) + else { + return now; + }; + reserve_build_epoch(&dir, now).unwrap_or_else(|e| { + crate::output::warn(&format!( + "warning: could not reserve a build epoch in {}: {e}; using the clock alone", + dir.display() + )); + now + }) +} + +/// How far ahead of the clock a stored build epoch may be and still come from a concurrent +/// build. Further ahead, it is a clock that was wrong, and following it would date every +/// later build in the future. +const CONCURRENT_BUILD_WINDOW_MS: u64 = 60_000; + +/// Take the lock on `dir`'s [`LAST_BUILD_EPOCH_FILE`], and record and return the first +/// millisecond at or after `now` that is later than the one it holds. +fn reserve_build_epoch(dir: &Path, now: u64) -> std::io::Result { + use std::io::{Seek as _, SeekFrom}; + + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(dir.join(LAST_BUILD_EPOCH_FILE))?; + // Held until `file` drops at the end of this function. + file.lock()?; + let mut last = String::new(); + file.read_to_string(&mut last)?; + let epoch = last + .trim() + .parse::() + .ok() + .filter(|&last| last < now.saturating_add(CONCURRENT_BUILD_WINDOW_MS)) + .map_or(now, |last| now.max(last.saturating_add(1))); + file.set_len(0)?; + file.seek(SeekFrom::Start(0))?; + file.write_all(epoch.to_string().as_bytes())?; + Ok(epoch) +} + /// Clock-skew tolerance for a pinned `--build-epoch`: 24h, absorbing timezone mistakes and /// NTP skew. A build-epoch further ahead than this is rejected. const BUILD_EPOCH_FUTURE_SKEW_MS: u64 = 24 * 60 * 60 * 1000; @@ -270,11 +330,35 @@ fn check_converted_output( /// The three-tier obligation's advisories are all "you should fix this", so they count. /// The report's `notes` describe a valid, deliberate configuration (a declared but inert /// `hideLowerCounts`) and never do. +/// +/// The gate stops at its first error, so a failing package is also checked with +/// [`validate_package_collect_all`], and its other errors are printed first, so they can all +/// be fixed in one go. fn report_metadata_diagnostics( package: &PackageYaml, catalogs: Option<&BTreeMap>, ) -> Result { - let report = validate_package(package, catalogs).map_err(|e| ToolError::from(&e))?; + let report = validate_package(package, catalogs).map_err(|first| { + let first_message = first.to_string(); + let others: Vec = validate_package_collect_all(package, catalogs) + .errors + .iter() + .map(ToString::to_string) + .filter(|message| *message != first_message) + .collect(); + for other in &others { + crate::output::always(&format!("error: {other}")); + } + let mut err = ToolError::from(&first); + if !others.is_empty() { + err.message = format!( + "{} (and {} more metadata error(s) above)", + err.message, + others.len() + ); + } + err + })?; for warning in &report.warnings { crate::output::always(&format!("warning: {warning}")); } @@ -461,7 +545,7 @@ pub fn build_staging_dir( .org .as_deref() .ok_or_else(|| ToolError::user("package metadata.org is required"))?; - let epoch_millis = resolve_build_epoch(args)?; + let epoch_millis = resolve_build_epoch(args, config_path)?; let dataset_id = generate_dataset_id(prefix, &country_code, org, epoch_millis) .map_err(|e| ToolError::from(&e))?; @@ -1803,6 +1887,30 @@ config: load_package(&yaml, false).expect("a plain load warns but still builds"); } + /// A package with errors in several sections reports every section's error from one + /// run, not one per run; the gate's own error still decides the failure. + #[test] + fn a_build_reports_the_metadata_errors_of_every_section() { + let dir = tempfile::tempdir().expect("tempdir"); + let yaml = dir.path().join("package.yaml"); + let text = test_util::sample_package_yaml() + .replace( + "\"http://publications.europa.eu/resource/authority/access-right/PUBLIC\"", + "PUBLIC", + ) + .replace("hasEmail: \"mailto:", "hasEmail: \"REPLACE: "); + std::fs::write(&yaml, text).expect("write yaml"); + let package = load_package(&yaml, false).expect("loads"); + + let err = report_metadata_diagnostics(&package, None).expect_err("several errors"); + assert!( + err.message.contains("REPLACE:") + && err.message.contains("more metadata error(s) above"), + "{}", + err.message + ); + } + /// The property an allow-list has and a deny-list cannot: a `##` key nobody has /// invented yet does not survive. /// @@ -2414,4 +2522,54 @@ config: ); assert!(text.contains("FATAL: node key compromised") && text.contains("typo")); } + + /// Builds a script starts together, sharing one config directory, each get their own + /// millisecond, so none mints another's dataset id. + #[test] + fn parallel_builds_sharing_a_config_dir_get_distinct_epochs() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().to_path_buf(); + let threads: Vec<_> = (0..8) + .map(|_| { + let path = path.clone(); + std::thread::spawn(move || reserve_build_epoch(&path, 1_000).expect("reserve")) + }) + .collect(); + let mut epochs: Vec = threads + .into_iter() + .map(|t| t.join().expect("join")) + .collect(); + epochs.sort_unstable(); + assert_eq!(epochs, (1_000..1_008).collect::>()); + } + + /// A later clock is used as it is; an earlier one moves past the last epoch minted. + #[test] + fn a_reserved_epoch_follows_the_clock_and_never_repeats() { + let dir = tempfile::tempdir().expect("tempdir"); + assert_eq!( + reserve_build_epoch(dir.path(), 1_000).expect("reserve"), + 1_000 + ); + assert_eq!( + reserve_build_epoch(dir.path(), 5_000).expect("reserve"), + 5_000 + ); + assert_eq!( + reserve_build_epoch(dir.path(), 4_000).expect("reserve"), + 5_001 + ); + // A stored value from a clock that was far ahead is not followed. + let skewed = tempfile::tempdir().expect("tempdir"); + let ahead = 1_000_000; + assert_eq!( + reserve_build_epoch(skewed.path(), ahead).expect("reserve"), + ahead + ); + let now = ahead - CONCURRENT_BUILD_WINDOW_MS - 1; + assert_eq!( + reserve_build_epoch(skewed.path(), now).expect("reserve"), + now + ); + } } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_check.rs b/crates/gdi-dataset-tool/src/commands/cmd_check.rs index 2ac8f5f..8f6eef4 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_check.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_check.rs @@ -415,6 +415,11 @@ async fn explain_unserved(id: &str, oracle: Option>) -> String { Ok(NodeProbe::Unknown) => { ": the node has never seen this id. Install it first (`upload` or `deploy`).".to_owned() } + Ok(NodeProbe::Failed(status)) => format!( + ": it must be visible and ingested (the management plane at {} answered {status}, \ + so the node's own state could not be read).", + oracle.base + ), Ok(NodeProbe::Unreachable) => format!( ": it must be visible and ingested (the management plane at {} did not answer, \ so the node's own state could not be read).", diff --git a/crates/gdi-dataset-tool/src/commands/cmd_config.rs b/crates/gdi-dataset-tool/src/commands/cmd_config.rs index 98eabd7..689f64b 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_config.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_config.rs @@ -55,8 +55,8 @@ service_url = "REPLACE: https://node.example" [profiles.default.s3] # Non-secret S3 settings only (for credentials, see the top). # bucket = "REPLACE: my-bucket" -# Key prefix inside the bucket; must match the node's [[s3.buckets]].prefix for this -# channel, or the tool writes where the node does not look. +# Key prefix inside the bucket; must match the node's key prefix for this channel (ask +# the node's operator), or the tool writes where the node does not look. # prefix = "gdi-node-storage/" # The node channel that monitors this bucket (catches wrong-bucket publishes). # channel = "REPLACE: primary" diff --git a/crates/gdi-dataset-tool/src/commands/cmd_delete.rs b/crates/gdi-dataset-tool/src/commands/cmd_delete.rs index e0373cb..891775f 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_delete.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_delete.rs @@ -187,22 +187,28 @@ fn ensure_s3_package_exists( /// Turn a [`DeleteRefusal`] into the user-facing error, or `Ok(())` to proceed. /// -/// The two arms carry different remedies on purpose: a confirmed-visible dataset wants +/// The arms carry different remedies on purpose: a confirmed-visible dataset wants /// `unpublish`, while an unconfirmable one wants an oracle (`--management-url`) — telling an /// operator to `unpublish` a dataset whose state nobody can read is advice they cannot act -/// on. +/// on. A visible refusal names where it read the state, since only the oracle lags an +/// `unpublish`. /// /// # Errors /// /// Returns a [`ToolError`] whenever `refusal` is `Some`. fn refusal_error(id: &str, refusal: Option<&DeleteRefusal>) -> Result<(), ToolError> { match refusal { - Some(DeleteRefusal::Visible) => Err(ToolError::user(format!( + Some(DeleteRefusal::Visible(VisibilitySource::Oracle)) => Err(ToolError::user(format!( "dataset {id} reads as visible on the node's state oracle. If you just ran \ `unpublish`, the node has not reconciled it yet (the oracle lags the sidecar \ by a few seconds); wait briefly and retry, or pass --force. Otherwise \ `unpublish` it first." ))), + Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)) => Err(ToolError::user(format!( + "dataset {id} reads as visible in the bucket's state sidecar ({id}.state.json), \ + read because the node's state oracle gave no answer. `unpublish` it first, or \ + pass --force." + ))), Some(DeleteRefusal::Unconfirmable) => Err(ToolError::user(format!( "cannot confirm whether dataset {id} is currently visible: no management-plane \ state oracle is reachable, so the visible-guard cannot run. `delete` is \ @@ -217,12 +223,21 @@ fn refusal_error(id: &str, refusal: Option<&DeleteRefusal>) -> Result<(), ToolEr /// Why a delete was refused, or `None` to proceed. #[derive(Debug, PartialEq, Eq)] enum DeleteRefusal { - /// The dataset is confirmed visible and `--force` was not given. - Visible, + /// The dataset is confirmed visible, by the named source, and `--force` was not given. + Visible(VisibilitySource), /// Visibility could not be determined at all — no oracle, no sidecar. Unconfirmable, } +/// Where a delete read the dataset's visibility. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum VisibilitySource { + /// The node's management-plane state oracle. + Oracle, + /// The bucket's `{id}.state.json` sidecar, read when the node is unreachable. + Sidecar, +} + /// Whether a delete must be refused, and why. /// /// `node_visible` is the authoritative management-plane visibility when the node is @@ -246,10 +261,11 @@ fn delete_refusal( if force { return None; } - match node_visible.or(sidecar_visible) { - Some(true) => Some(DeleteRefusal::Visible), - Some(false) => None, - None => Some(DeleteRefusal::Unconfirmable), + match (node_visible, sidecar_visible) { + (Some(true), _) => Some(DeleteRefusal::Visible(VisibilitySource::Oracle)), + (None, Some(true)) => Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)), + (Some(false), _) | (None, Some(false)) => None, + (None, None) => Some(DeleteRefusal::Unconfirmable), } } @@ -287,13 +303,13 @@ mod tests { // Authoritative node says visible -> refused. assert_eq!( delete_refusal(Some(true), None, false), - Some(DeleteRefusal::Visible) + Some(DeleteRefusal::Visible(VisibilitySource::Oracle)) ); assert_eq!(delete_refusal(Some(false), None, false), None); // Node unreachable: fall back to the S3 bucket sidecar. assert_eq!( delete_refusal(None, Some(true), false), - Some(DeleteRefusal::Visible) + Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)) ); assert_eq!(delete_refusal(None, Some(false), false), None); } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs b/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs index e47fdfa..d81b29e 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs @@ -323,12 +323,17 @@ pub(crate) fn wait_for_ingest( // Set on the poll that saw a pre-existing error and declined to blame this drop for it, // so the timeout can say which of the two ambiguous things happened instead of guessing. let mut stale_error = false; + // Whether every poll so far said the node does not know this id: then the timeout says + // the node never saw the package, not "still processing". + let mut unknown_every_poll = true; crate::output::note(&format!("waiting for the node to ingest {id}...")); loop { // One detailed probe per poll. The plain `probe_node_state` folds `410 Gone` into // the same `None` as "unreachable" and "not seen yet", which would keep `--wait` // polling an id the node has permanently refused and then blame a slow ingest. - match runtime::block_on(crate::state::probe_node_state_detailed(base, id))? { + let probe = runtime::block_on(crate::state::probe_node_state_detailed(base, id))?; + unknown_every_poll &= matches!(probe, crate::state::NodeProbe::Unknown); + match probe { // Tombstoned: terminal, and the node says why. crate::state::NodeProbe::Gone(reason) => { return Err(ToolError::user(format!( @@ -424,6 +429,15 @@ pub(crate) fn wait_for_ingest( timeout.as_secs(), ))); } + if unknown_every_poll { + return Err(ToolError::user(format!( + "timed out after {}s: the node still does not know {id}. Either it has \ + not picked it up yet (raise --wait-timeout), or it watches another inbox, \ + bucket, key prefix or channel than this was written to; check with the \ + node's operator.", + timeout.as_secs() + ))); + } return Err(ToolError::user(format!( "timed out after {}s waiting for the node to ingest {id}. It may still be \ processing. Check with `status {id}`, or raise --wait-timeout.", @@ -909,6 +923,54 @@ mod tests { ); } + /// A wait the node answers "unknown id" to throughout says the node has not seen the + /// package and names the likely mismatch, rather than "it may still be processing". + #[test] + fn a_wait_the_node_never_answers_for_names_the_mismatch() { + use std::io::{Read as _, Write as _}; + + // The management plane: `/version` answers, and the state oracle answers `status`. + let wait_against = |status: &'static str| { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind stub"); + let addr = listener.local_addr().expect("stub addr"); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let mut buf = [0u8; 1024]; + let n = s.read(&mut buf).unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..n]); + let reply = if request.starts_with("GET /version") { + "HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}".to_owned() + } else { + format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\n\r\n") + }; + let _ = s.write_all(reply.as_bytes()); + } + }); + wait_for_ingest( + &format!("http://{addr}"), + "GDI-EE-UTARTU-20260409143052837", + std::time::Duration::from_secs(1), + None, + None, + None, + ) + .expect_err("an id the node never reports cannot finish ingesting") + .message + }; + + let unknown = wait_against("404 Not Found"); + assert!( + unknown.contains("does not know") && unknown.contains("key prefix"), + "must say the node has not seen it, and why it might not: {unknown}" + ); + let failing = wait_against("503 Service Unavailable"); + assert!( + !failing.contains("does not know"), + "a failing oracle says nothing about the id: {failing}" + ); + } + /// The truth table the `--wait` loop hangs on. /// /// `error` is terminal, and the signature is what makes it decidable. Comparing diff --git a/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs b/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs index 0942dd8..a920d7f 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs @@ -459,9 +459,8 @@ fn check_beacon_url(active: &Profile, responds: bool) -> Check { Check::pass( "beacon-url", format!( - "custom [beacon].aggregated_base_path; the default-shaped {url} does not \ - answer here. Do not register it; take the exact URL from the node's \ - `gdi-node-standalone check-config` output" + "{url} does not answer here: if the node is up, its Beacon is at another \ + path. Do not register this one; ask the node's operator for the exact URL" ), ) } @@ -537,8 +536,8 @@ fn check_s3_writable(active: &Profile) -> Check { Ok(()) => Check::pass( "s3-writable", format!( - "{} accepts a probe PUT/DELETE (this must match the node's [[s3.buckets]].prefix \ - for this channel)", + "{} accepts a probe PUT/DELETE (this must match the node's key prefix for \ + this channel)", s3::target_label(cfg) ), ), diff --git a/crates/gdi-dataset-tool/src/commands/cmd_keys.rs b/crates/gdi-dataset-tool/src/commands/cmd_keys.rs index 7b0fc48..2aceab7 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_keys.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_keys.rs @@ -458,6 +458,31 @@ fn primary(identities: &[PathBuf]) -> Result<&PathBuf, ToolError> { /// key cannot be read/parsed, or a newly generated key cannot be written. pub fn load_or_generate_provider_secret( config_path: Option<&Path>, +) -> Result { + load_or_mint_provider_secret(config_path, |path, fingerprint| { + crate::output::warn(&minted_identity_warning(path, fingerprint)); + }) +} + +/// Make sure the primary provider identity exists, creating it (`0o600`) if missing: the +/// `wizard setup` step. Creating it is expected there, so it is announced as a progress +/// line, not as the warning `pack` gives when a key it expected is missing. +/// +/// # Errors +/// +/// As [`load_or_generate_provider_secret`]. +pub fn ensure_provider_secret(config_path: Option<&Path>) -> Result<(), ToolError> { + load_or_mint_provider_secret(config_path, |path, fingerprint| { + crate::output::progress(&created_identity_line(path, fingerprint)); + }) + .map(drop) +} + +/// Load the primary provider identity, or mint it and pass its path and fingerprint to +/// `announce`. +fn load_or_mint_provider_secret( + config_path: Option<&Path>, + announce: fn(&Path, &str), ) -> Result { let identities = resolve_identities(config_path)?; let secret_path = primary(&identities)?; @@ -471,10 +496,7 @@ pub fn load_or_generate_provider_secret( // resets who the node believes authored every package built from here on. It happens // whenever the resolved config dir holds no identity, which a `--config` typo or a // fresh checkout produces just as easily as a genuine first run. - crate::output::warn(&minted_identity_warning( - secret_path, - &public_key_fingerprint(&sk.public_key()), - )); + announce(secret_path, &public_key_fingerprint(&sk.public_key())); Ok(sk) } @@ -485,11 +507,21 @@ pub fn load_or_generate_provider_secret( /// accepted under `writer_policy = enforce`. fn minted_identity_warning(path: &Path, fingerprint: &str) -> String { format!( - "warning: no provider identity at {}; generated a new one ({fingerprint}). This key \ - is the crypt4gh writer provenance of every package built from here on; a node with \ - [ingest].writer_policy = enforce will reject them until this fingerprint is \ - allow-listed. If you expected an existing key, check --config / GDI_CONFIG_DIR \ - before packaging.", + "warning: no provider identity at {}; generated a new one ({fingerprint}). Packages \ + built from now on are written with it, so a node that only accepts listed keys \ + rejects them until it lists this fingerprint. If you expected an existing key, \ + check --config / GDI_CONFIG_DIR.", + path.display() + ) +} + +/// The line `wizard setup` prints when it creates the provider identity, worded for any +/// node. +fn created_identity_line(path: &Path, fingerprint: &str) -> String { + format!( + " created your provider key at {} ({fingerprint}). Every package you build carries \ + it as its writer; if the node accepts packages only from listed keys, give its \ + operator this fingerprint.", path.display() ) } @@ -711,7 +743,7 @@ mod tests { "the fingerprint is the actionable part; it is what gets allow-listed: {msg}" ); assert!( - msg.contains("writer_policy"), + msg.contains("listed keys"), "say why it matters, not just that a key was made: {msg}" ); assert!( @@ -737,6 +769,24 @@ mod tests { ); } + /// `wizard setup` creates the key that a re-run and a later `pack` load. + #[test] + fn setup_creates_the_key_that_pack_then_loads() { + let dir = tempfile::tempdir().expect("tempdir"); + let cfg = dir.path().join("config.toml"); + std::fs::write(&cfg, "").expect("write config"); + + ensure_provider_secret(Some(&cfg)).expect("creates on a first run"); + let created = load_provider_secret_readonly(Some(&cfg)).expect("the key exists"); + ensure_provider_secret(Some(&cfg)).expect("loads on a re-run"); + let loaded = load_or_generate_provider_secret(Some(&cfg)).expect("pack loads it"); + assert_eq!( + public_key_fingerprint(&created.public_key()), + public_key_fingerprint(&loaded.public_key()), + "a re-run of setup must keep the key" + ); + } + #[cfg(unix)] #[test] fn a_world_readable_provider_key_is_reported_on_every_read() { diff --git a/crates/gdi-dataset-tool/src/commands/cmd_list.rs b/crates/gdi-dataset-tool/src/commands/cmd_list.rs index fd65734..685e20d 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_list.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_list.rs @@ -90,7 +90,7 @@ fn warn_if_datasets_sit_outside_the_prefix(active: &gdi_node_standalone_core::co "warning: no datasets {here}, but {total} elsewhere in the bucket ({where_}). \ A prefix set on one side only, or set differently on each, is a desync in which \ every signal is a success: uploads land where the node never lists. Check it against \ - the node's [[s3.buckets]].prefix for this channel" + the node's key prefix for this channel with the node's operator" )); } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_pack.rs b/crates/gdi-dataset-tool/src/commands/cmd_pack.rs index c776477..9fe2de9 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_pack.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_pack.rs @@ -43,6 +43,7 @@ pub fn run( let started = std::time::Instant::now(); let dataset_id = dataset_id_from_staging(&args.staging)?; let output = resolve_output(args.out.as_deref(), &dataset_id); + warn_if_misnamed(&output, &dataset_id); crate::output::note(&format!( "packing {dataset_id}: {} -> {}", args.staging.display(), @@ -106,6 +107,7 @@ pub fn run_package( // 2. Pack the freshly built staging dir. let output = resolve_output(args.out.as_deref(), &built.dataset_id); + warn_if_misnamed(&output, &built.dataset_id); crate::output::note(&format!( "packing {} -> {}", built.dataset_id, @@ -218,7 +220,7 @@ fn manifest_dataset_id(staging: &Path) -> Result { /// except an existing directory receives the package inside it as /// `{datasetId}.tar.c4gh` — else `{datasetId}.tar.c4gh` in the current working directory. fn resolve_output(out_flag: Option<&Path>, dataset_id: &str) -> PathBuf { - let file_name = format!("{dataset_id}.tar.c4gh"); + let file_name = format!("{dataset_id}{}", crate::pkgio::TAR_C4GH_SUFFIX); match out_flag { // An existing directory means "write the package into here" — the same noun // `build -o ` takes, and what the flag's "defaults to the cwd" contract @@ -233,6 +235,26 @@ fn resolve_output(out_flag: Option<&Path>, dataset_id: &str) -> PathBuf { } } +/// Warn when `-o` names the package other than `{dataset_id}.tar.c4gh`. +fn warn_if_misnamed(output: &Path, dataset_id: &str) { + if let Some(warning) = misnamed_package_warning(output, dataset_id) { + crate::output::warn(&warning); + } +} + +/// The warning for a package named other than `{dataset_id}.tar.c4gh`: any name packs and +/// unpacks, but `upload` and `deploy` read the dataset id from the name. +fn misnamed_package_warning(output: &Path, dataset_id: &str) -> Option { + let expected = format!("{dataset_id}{}", crate::pkgio::TAR_C4GH_SUFFIX); + (output.file_name() != Some(std::ffi::OsStr::new(&expected))).then(|| { + format!( + "warning: {} is not named {expected}; rename it before `upload` or `deploy`, \ + which read the dataset id from the file name", + output.display() + ) + }) +} + /// Resolve the encryption recipients: the node recipient (mandatory) followed by /// the provider's own recipient (so the provider can decrypt its own packages). /// @@ -1031,6 +1053,25 @@ mod tests { ); } + /// A package named other than `{id}.tar.c4gh` is warned about at `pack`, not first + /// refused at `upload`. + #[test] + fn a_package_not_named_for_its_id_is_warned_about() { + let dir = tempfile::tempdir().unwrap(); + let warning = misnamed_package_warning(&dir.path().join("wrong.tar.c4gh"), ID) + .expect("a misnamed package is warned about"); + assert!( + warning.starts_with("warning:") + && warning.contains(&format!("is not named {ID}.tar.c4gh")), + "{warning}" + ); + assert_eq!( + misnamed_package_warning(&resolve_output(Some(dir.path()), ID), ID), + None, + "the name `pack` chooses itself is the right one" + ); + } + /// With neither `--recipient` nor a profile, the error must name `--recipient`. A bare /// "no profiles configured" leaves a first-time provider with no next step, and /// `pack --recipient ` needs no profile at all. diff --git a/crates/gdi-dataset-tool/src/commands/cmd_publish.rs b/crates/gdi-dataset-tool/src/commands/cmd_publish.rs index 12a08d9..9449527 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_publish.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_publish.rs @@ -170,7 +170,8 @@ fn publish_result( // path this message is most often printed from would be a worse pointer than none. let text = format!( "{verb} {id}: wrote the {target_state} sidecar on {target}. The node applies it on \ - its next scan; its management-plane /datasets/{id}/state reports when it has" + its next scan; its management-plane /datasets/{id}/state shows {target_state} once \ + it has" ); let payload = serde_json::json!({ "schemaVersion": 1, diff --git a/crates/gdi-dataset-tool/src/commands/cmd_status.rs b/crates/gdi-dataset-tool/src/commands/cmd_status.rs index a4ae358..4da8911 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_status.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_status.rs @@ -147,6 +147,28 @@ fn s3_visibility( } } +/// The node's authoritative state for `id`, and whether its management plane said it does +/// not know the id. Only an explicit management URL can say so: without one the base falls +/// back to the public plane, which answers 404 for every id. +fn probe_node_state( + active: &Profile, + id: &str, + probe_node: bool, + override_base: Option<&str>, +) -> Result<(Option, bool), ToolError> { + if !probe_node { + return Ok((None, false)); + } + let probe = state::resolve_node_probe(active, id, override_base)?; + let not_seen = matches!(probe, Some(state::NodeProbe::Unknown)) + && (override_base.is_some() || active.management_url.is_some()); + let node_state = match probe { + Some(state::NodeProbe::Live(state)) => Some(*state), + _ => None, + }; + Ok((node_state, not_seen)) +} + /// Report on a single dataset id. In `text` mode prints the report (returning /// `None`); in `json` mode builds and returns the [`StatusSummary`] for the caller /// to emit (so `--all` can aggregate them into one JSON array). @@ -163,11 +185,7 @@ fn report_one( // management_url (else service_url). `probe_node` is the up-front reachability // verdict: `--all` sets it false when the state base is down, so this skips a // per-dataset 5s timeout that would only resolve to `None` anyway (the S3 path). - let node_state = if probe_node { - state::resolve_node_state(active, id, override_base)? - } else { - None - }; + let (node_state, not_seen) = probe_node_state(active, id, probe_node, override_base)?; // The provenance of the `state` value reported below: the authoritative node oracle, // or the S3-derived (non-authoritative) fallback. let state_source = if node_state.is_some() { @@ -216,7 +234,15 @@ fn report_one( let sync = status::channel_sync(channel, package_present, drifted); if format == OutputFormat::Text { - print_report(id, &node_status, channel, sync, store, etag.as_deref())?; + print_report( + id, + &node_status, + not_seen, + channel, + sync, + store, + etag.as_deref(), + )?; if diff { print_diff( id, @@ -272,6 +298,7 @@ fn report_one( fn print_report( id: &str, node_status: &NodeStatus, + not_seen: bool, channel: Channel, sync: Sync, store: Option<&Store>, @@ -282,13 +309,25 @@ fn print_report( .map_or_else(String::new, |v| format!("\tvisibility: {}", v.as_str())); println!( "{id}\tstate: {}\tchannel: {}\tsync: {}{vis_label}", - node_status.display_label(), + state_label(node_status, not_seen), channel_label(channel), sync.label() ); Ok(()) } +/// The text report's `state:` value. `not_seen` means the management plane answered that it +/// does not know the id, which the generic `unavailable` would blame on an unreachable plane. +fn state_label(node_status: &NodeStatus, not_seen: bool) -> String { + if not_seen && matches!(node_status, NodeStatus::Unavailable) { + "unknown to the node (not picked up yet, or it watches another location than this \ + profile writes to)" + .to_owned() + } else { + node_status.display_label() + } +} + /// Whether the dataset is drifted: the node's served state disagrees with the S3 /// writeback's freshness (a stale `source_signature` ⇒ the node has not caught up /// with the latest upload, i.e. drift between local-intended and node-served). @@ -839,4 +878,22 @@ mod tests { // No writeback / no etag (non-pending) is not drift. assert!(!is_drifted(&visible, None, Some("etag-1"))); } + + /// A management plane that answered "unknown id" is reported as such, not as the + /// generic `unavailable` that blames an unreachable plane; any real state still wins. + #[test] + fn a_dataset_the_management_plane_does_not_know_is_reported_as_unknown() { + let unknown = state_label(&NodeStatus::Unavailable, true); + assert!(unknown.starts_with("unknown to the node"), "{unknown}"); + assert_eq!( + state_label(&NodeStatus::Unavailable, false), + NodeStatus::Unavailable.display_label(), + "without that answer the generic label stands" + ); + let visible = NodeStatus::State { + state: "visible".to_owned(), + error_message: None, + }; + assert_eq!(state_label(&visible, true), "visible"); + } } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_upload.rs b/crates/gdi-dataset-tool/src/commands/cmd_upload.rs index 70d031e..e1df90b 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_upload.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_upload.rs @@ -171,7 +171,9 @@ fn wait_outcome( payload["status"] = serde_json::Value::String("error".to_owned()); payload["reason"] = serde_json::Value::String(e.to_string()); ( - format!("upload of {id} to {target} rejected by the node: {e}"), + // Not "rejected": a timeout is not a rejection, and a rejection's own + // message already says so. + format!("upload of {id} to {target} not confirmed by the node: {e}"), payload, ) } diff --git a/crates/gdi-dataset-tool/src/s3.rs b/crates/gdi-dataset-tool/src/s3.rs index 9ac97fe..8d70858 100644 --- a/crates/gdi-dataset-tool/src/s3.rs +++ b/crates/gdi-dataset-tool/src/s3.rs @@ -187,6 +187,21 @@ fn conn_params( .filter(|b| !b.is_empty()) .ok_or_else(|| ToolError::user("profile s3.bucket is required for S3 operations"))?; + // `object_store` refuses a plaintext endpoint without `allow_http`, but its error + // (`... builder error`) names neither the scheme nor the setting. + if !s3.allow_http + && endpoint + .trim_start() + .to_ascii_lowercase() + .starts_with("http://") + { + return Err(ToolError::user(format!( + "profile s3.endpoint {endpoint} is plaintext http:// but allow_http is false. \ + Use an https:// endpoint, or set allow_http = true in this profile's s3 block \ + (for a local dev backend such as MinIO or Garage)" + ))); + } + // The prefix must survive `object_store::path::Path` normalization unchanged, as it // must on the node; the rule is imported rather than restated. The node rejects a bad // prefix at boot. Without this the tool would accept every spelling the node refuses, @@ -195,10 +210,9 @@ fn conn_params( // only one of them normalizes is an upload the node never lists. gdi_node_standalone_core::config::validate_key_prefix(&s3.prefix).map_err(|why| { ToolError::user(format!( - "profile s3.prefix {:?} is invalid: {why}. It must match the node's \ - [[s3.buckets]].prefix for this channel exactly; the node refuses to boot on \ - this spelling, and a prefix set on only one side means uploads the node never \ - lists", + "profile s3.prefix {:?} is invalid: {why}. It must match the node's key prefix \ + for this channel exactly (ask the node's operator); the node rejects this \ + spelling, and a prefix set on only one side means uploads the node never lists", s3.prefix )) })?; @@ -404,9 +418,10 @@ const MAX_INFLIGHT_PARTS: usize = 16; /// bounded chunks so an arbitrarily large `.tar.c4gh` is never buffered whole in /// memory and is not subject to the single-PUT 5 GiB ceiling. /// -/// Every failure path aborts the upload, best effort, before returning: a read error, a -/// failed part, and a failed `complete`. That is why this drives the `MultipartUpload` -/// handle itself instead of handing it to `object_store::WriteMultipart`. That writer +/// Every failure path aborts the upload before returning: a read error, a failed part, and +/// a failed `complete`; an abort that fails too is added to the error. That is why this +/// drives the `MultipartUpload` handle itself instead of handing it to +/// `object_store::WriteMultipart`. That writer /// aborts only when `complete()` fails inside `finish()`, has no `Drop` impl, keeps the /// upload handle private, and `finish()` consumes it, so a part failing while the writer is /// still being fed returns `Err` with the handle gone and nothing left that could abort. @@ -441,12 +456,24 @@ async fn stream_file_multipart( progress.finish(); Ok(()) } - Err(e) => { + Err(mut e) => { // Cancel what is still in flight before aborting: a part that lands after the // abort would re-create the upload this is trying to erase. `JoinSet::shutdown` // aborts each task and awaits it, as `WriteMultipart` does. parts.shutdown().await; - let _ = upload.abort().await; + // `NoSuchUpload` means it is gone already, as when the answer to a `complete` + // that succeeded was lost. + if let Err(abort) = upload.abort().await + && !(matches!(abort, object_store::Error::NotFound { .. }) + || abort.to_string().contains("NoSuchUpload")) + { + e.message = format!( + "{}; aborting the multipart upload of {key} failed too ({}), so its parts \ + stay in the bucket until a lifecycle rule expires them or someone aborts it", + e.message, + terse(&abort) + ); + } Err(e) } } @@ -671,12 +698,7 @@ pub async fn probe_writable(store: &Store) -> Result<(), ToolError> { store .put(&key, PutPayload::from(b"probe".to_vec())) .await - .map_err(|e| { - classify_object_store_error( - "S3 bucket is not writable (your token may lack write permission)", - e, - ) - })?; + .map_err(|e| classify_object_store_error("S3 bucket is not writable", e))?; // Best-effort cleanup; a failed delete does not fail the probe. let _ = store.delete(&key).await; Ok(()) @@ -931,9 +953,8 @@ fn generic_error_text(e: &object_store::Error) -> Option { /// Whether an object-store failure is an untyped (`Generic`) 403 / `AccessDenied`. fn is_denied_object_store(e: &object_store::Error) -> bool { - generic_error_text(e).is_some_and(|msg| { - msg.contains("403") || msg.contains("Forbidden") || msg.contains("AccessDenied") - }) + generic_error_text(e) + .is_some_and(|msg| msg.contains("Forbidden") || msg.contains("AccessDenied")) } /// Whether an object-store failure means "that key is not there", however the backend @@ -973,20 +994,26 @@ fn classify_get_error(key: &ObjPath, id: &str, e: object_store::Error) -> ToolEr } /// A short remediation hint for a non-2xx object-store failure, keyed on the HTTP status -/// embedded in the error text (`object_store` 0.13 exposes no typed status). Empty when no +/// embedded in the error text (`object_store` exposes no typed status). Empty when no /// known status is present: connectivity errors already carry rich messages, so this /// targets only genuine non-2xx responses. +/// +/// A status is matched with its reason phrase (`502 Bad Gateway`), never as bare digits, +/// which the request URL (a dataset id is a timestamp) and the elapsed time also contain. fn status_hint(msg: &str) -> &'static str { - if msg.contains("403") || msg.contains("Forbidden") { + if msg.contains("Forbidden") { ". Check the bucket policy and credentials: `gdi-dataset-tool profiles` shows whether \ this profile has both keys (in tool-secrets.toml, or GDI_TOOL__PROFILES____... \ with underscores for hyphens). Then run `gdi-dataset-tool doctor`" - } else if msg.contains("404") || msg.contains("NoSuchBucket") { + } else if msg.contains("404 Not Found") || msg.contains("NoSuchBucket") { ". Check the bucket name and endpoint, then run `gdi-dataset-tool doctor`" - } else if msg.contains("400") || msg.contains("Bad Request") { + } else if msg.contains("Bad Request") { ". Likely a region, path-style or endpoint misconfiguration; run \ `gdi-dataset-tool doctor`" - } else if msg.contains("500") || msg.contains("502") || msg.contains("504") { + } else if msg.contains("500 Internal Server Error") + || msg.contains("502 Bad Gateway") + || msg.contains("504 Gateway Timeout") + { ". Server-side error; retry, then run `gdi-dataset-tool doctor` if it persists" } else { "" @@ -995,20 +1022,23 @@ fn status_hint(msg: &str) -> &'static str { /// Heuristic: whether an `object_store` failure looks transient (retry may fix). /// -/// `object_store` 0.13 has no dedicated transient variant. A throttled or `5xx` request +/// `object_store` has no dedicated transient variant. A throttled or `5xx` request /// surfaces as [`object_store::Error::Generic`] only after the client's own retry budget is /// exhausted, with the HTTP status in the wrapped source, so this keys on that `Display` /// text via [`generic_error_text`]. Kept conservative to keep false positives off the /// transient (exit-3) path. +/// +/// `object_store` retries only `5xx`, `429`, `408` and transport errors, so its retry clause +/// (`after retries, max_retries: …`) alone marks a transient failure. Statuses are +/// otherwise matched by reason phrase, as in [`status_hint`]. fn is_transient_object_store(e: &object_store::Error) -> bool { generic_error_text(e).is_some_and(|msg| { - msg.contains("503") - || msg.contains("429") + msg.contains(", max_retries: ") || msg.contains("Service Unavailable") || msg.contains("Too Many Requests") || msg.contains("SlowDown") || msg.contains("timed out") - || msg.contains("timeout") + || msg.contains("RequestTimeout") }) } @@ -1906,6 +1936,85 @@ mod tests { })); } + /// A plaintext endpoint without `allow_http` is refused before any client is built, + /// with a message naming the setting, rather than surfacing `object_store`'s + /// `builder error` from whichever verb ran first. + #[test] + fn a_plaintext_endpoint_needs_allow_http_and_the_refusal_names_it() { + for endpoint in ["http://localhost:9000", "HTTP://localhost:9000"] { + let s3 = super::ProfileS3 { + endpoint: Some(endpoint.to_owned()), + bucket: Some("b".to_owned()), + ..Default::default() + }; + let Err(err) = super::conn_params(&s3) else { + panic!("{endpoint} without allow_http must be refused") + }; + assert!( + err.to_string().contains("allow_http = true"), + "the refusal must name the setting: {err}" + ); + let s3 = super::ProfileS3 { + allow_http: true, + ..s3 + }; + assert!(super::conn_params(&s3).is_ok(), "allow_http opts in"); + } + } + + /// A failure is classified by what `object_store` reports, never by the digits in the + /// request URL, whose key is a dataset id (a timestamp). A request it retried until the + /// budget ran out is transient; it never retries a `403`. + #[test] + fn a_failed_request_is_classified_by_its_status_not_its_url() { + let key = "GDI-EE-UTARTU-20260409150040312.tar.c4gh"; // gitleaks:allow - dataset id + assert!( + key.contains("500") && key.contains("403"), + "the fixture id must exercise the hazard" + ); + // The texts `object_store` 0.14 writes, with and without its retry clause. + let failure = |retried: bool, cause: &str| object_store::Error::Generic { + store: "S3", + source: format!( + "Error performing PUT http://localhost:9000/b/{key} in 1.5s{} - {cause}", + if retried { + ", after 10 retries, max_retries: 10, retry_timeout: 180s " + } else { + "" + } + ) + .into(), + }; + let exit = |e| classify_object_store_error("uploading k", e).exit_code; + + assert_eq!( + exit(failure(true, "HTTP error: error sending request")), + crate::EXIT_TRANSIENT + ); + assert_eq!( + exit(failure( + true, + "Server returned non-2xx status code: 502 Bad Gateway: " + )), + crate::EXIT_TRANSIENT + ); + assert_eq!( + exit(failure( + false, + "Server returned non-2xx status code: 403 Forbidden: no" + )), + crate::EXIT_AUTH + ); + let no_response = + classify_object_store_error("uploading k", failure(false, "HTTP error: builder error")); + assert_eq!(no_response.exit_code, crate::EXIT_USER); + assert!( + !no_response.message.contains("Server-side"), + "no response, so no server-side hint: {}", + no_response.message + ); + } + #[tokio::test] async fn list_classifies_visible_and_hidden_and_ignores_noise() { let store = mem_store(); @@ -2018,6 +2127,7 @@ mod tests { let s3 = ProfileS3 { bucket: Some("b".to_owned()), endpoint: Some("http://localhost:9000".to_owned()), + allow_http: true, access_key_id: Some("k".to_owned()), ..ProfileS3::default() }; @@ -2145,7 +2255,7 @@ mod tests { async fn a_failed_part_upload_aborts_the_multipart_upload() { let aborted = Arc::new(AtomicBool::new(false)); let store = HookStore::new() - .fail_every_part(Arc::clone(&aborted)) + .fail_every_part(Arc::clone(&aborted), false) .into_store(); let err = upload_package_bytes( @@ -2170,6 +2280,39 @@ mod tests { ); } + /// An abort that fails too is reported, naming the key: nothing else would ever show the + /// parts it leaves behind. + #[tokio::test] + async fn a_failed_abort_is_reported_with_the_key_it_leaves_behind() { + let aborted = Arc::new(AtomicBool::new(false)); + let store = HookStore::new() + .fail_every_part(Arc::clone(&aborted), true) + .into_store(); + let dataset = id(10); + + let err = upload_package_bytes( + &store, + &PackageStore::new(store.clone()), + &dataset, + b"a small package".to_vec(), + false, + ) + .await + .unwrap_err(); + + assert!( + aborted.load(Ordering::SeqCst), + "the abort must still be attempted" + ); + assert!( + err.message.contains(&format!( + "aborting the multipart upload of {dataset}{TAR_C4GH_SUFFIX} failed too" + )) && err.message.contains("lifecycle rule"), + "a failed abort must name the key and how its parts go away: {}", + err.message + ); + } + /// A package spanning several parts must round-trip byte-for-byte. /// /// Every other upload test here fits in a single part, so this is what exercises the @@ -2212,10 +2355,12 @@ mod tests { /// Supply the [`MultipartUpload`] a `put_multipart` hands back. type MultipartHook = Box Box + Send + Sync>; - /// A [`MultipartUpload`] whose every part fails, recording whether it was aborted. + /// A [`MultipartUpload`] whose every part fails, recording whether it was aborted, and + /// whose abort fails too when `abort_fails`. #[derive(Debug)] struct FailingUpload { aborted: Arc, + abort_fails: bool, } #[async_trait] @@ -2238,6 +2383,12 @@ mod tests { async fn abort(&mut self) -> object_store::Result<()> { self.aborted.store(true, Ordering::SeqCst); + if self.abort_fails { + return Err(object_store::Error::Generic { + store: "test", + source: "abort refused".into(), + }); + } Ok(()) } } @@ -2285,11 +2436,13 @@ mod tests { } /// Hand back an upload whose every part fails, flagging `aborted` if the caller - /// aborts it: the seam for "the network dropped mid-upload". - fn fail_every_part(mut self, aborted: Arc) -> Self { + /// aborts it, and failing that abort when `abort_fails`: the seam for "the network + /// dropped mid-upload". + fn fail_every_part(mut self, aborted: Arc, abort_fails: bool) -> Self { self.on_multipart = Some(Box::new(move || { Box::new(FailingUpload { aborted: Arc::clone(&aborted), + abort_fails, }) })); self diff --git a/crates/gdi-dataset-tool/src/state.rs b/crates/gdi-dataset-tool/src/state.rs index 2185ec2..36a598e 100644 --- a/crates/gdi-dataset-tool/src/state.rs +++ b/crates/gdi-dataset-tool/src/state.rs @@ -156,6 +156,9 @@ pub(crate) enum NodeProbe { /// never seen this id" tells the operator to re-install a dataset that is present. /// Callers that only need a verdict fold it into the same fallback as `Unknown`. Unreachable, + /// The oracle answered with another error status, such as `401` or `503`: it is up, but + /// says nothing about the id. + Failed(u16), /// No authoritative view of this id: a `404`, or an unusable body on a `2xx`. Callers /// fall back to S3-ownership routing. Unknown, @@ -197,6 +200,10 @@ pub(crate) async fn probe_node_state_detailed( reason.to_owned() })); } + let status = resp.status(); + if !status.is_success() && status != reqwest::StatusCode::NOT_FOUND { + return Ok(NodeProbe::Failed(status.as_u16())); + } Ok(match probe_parse(resp, &url).await { Some(state) => NodeProbe::Live(Box::new(state)), None => NodeProbe::Unknown, @@ -290,6 +297,24 @@ pub(crate) fn resolve_node_state( } } +/// Like [`resolve_node_state`], without folding the answer: `None` when no oracle base is +/// configured, otherwise what the probe saw. +/// +/// # Errors +/// +/// Returns a [`ToolError`] if the probe runtime cannot be started, or for the reasons +/// [`probe_node_state_detailed`] gives. +pub(crate) fn resolve_node_probe( + active: &Profile, + id: &str, + override_base: Option<&str>, +) -> Result, ToolError> { + match override_base.or_else(|| active.node_state_base()) { + Some(base) => runtime::block_on(probe_node_state_detailed(base, id)).map(Some), + None => Ok(None), + } +} + /// Resolve the channel to write to: prefer the node's authoritative `channel`; on /// no authoritative view fall back to the explicit `--s3` / `--local` override, or /// (with neither) infer from the profile (S3 configured => S3, else inbox). diff --git a/crates/gdi-dataset-tool/src/wizard/mod.rs b/crates/gdi-dataset-tool/src/wizard/mod.rs index e9ac4aa..74503dd 100644 --- a/crates/gdi-dataset-tool/src/wizard/mod.rs +++ b/crates/gdi-dataset-tool/src/wizard/mod.rs @@ -427,11 +427,7 @@ pub fn run( summary_publish = Some( "staging dir deployed to the node's inbox; the dataset lands hidden".to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } else { summary_publish = Some("nothing sent to the node (your choice)".to_owned()); summary_next = @@ -515,11 +511,7 @@ pub fn run( the node polls (allow ~30 s before it even looks)" .to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } Some(PublishRoute::DeployInbox) => { let deploy_args = wizard_deploy_args(package, wait_for_ingest); @@ -528,11 +520,7 @@ pub fn run( summary_publish = Some( "deployed to the node's inbox; the dataset lands hidden".to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } None => { summary_publish = Some("nothing sent to the node (your choice)".to_owned()); @@ -864,6 +852,23 @@ fn route_label(route: PublishRoute, active: Option<&Profile>) -> String { } } +/// The summary's next step after sending a dataset. Without a `management_url`, `status` may +/// never get the node's verdict, so the step says when it can and who else knows. +fn next_after_send(id: &str, management_url: bool) -> String { + if management_url { + format!( + "run `gdi-dataset-tool status {id}` to confirm ingest, then \ + `gdi-dataset-tool publish {id}` to make it visible" + ) + } else { + format!( + "confirm the ingest with `gdi-dataset-tool status {id}` if the node reports \ + status to you, else ask the node's operator; then `gdi-dataset-tool publish \ + {id}` to make it visible" + ) + } +} + /// The `deploy` arguments the wizard uses for `artifact` — the single place both publish /// routes (keyless staging dir, packed `.tar.c4gh`) get them from. /// diff --git a/crates/gdi-dataset-tool/src/wizard/prompts.rs b/crates/gdi-dataset-tool/src/wizard/prompts.rs index f960786..f415d88 100644 --- a/crates/gdi-dataset-tool/src/wizard/prompts.rs +++ b/crates/gdi-dataset-tool/src/wizard/prompts.rs @@ -97,12 +97,10 @@ pub trait Prompter { /// current-thread runtime in a background thread: the CLI core is synchronous and blocked /// reading a key, so there is no ambient runtime to spawn onto. /// -/// Registering the handler also stops SIGINT killing the process outright, which turns -/// Ctrl-C into the ordinary abort the wizard already reports: the blocked read returns -/// `EINTR`, `dialoguer` surfaces it, and the run ends via [`abort`] with "wizard aborted: -/// …" and exit 1, its own cleanup restoring the cursor. This thread gives that path a -/// moment to happen and only forces the exit if it does not, so the exit code cannot race -/// between 1 and 130 for the same keystroke. Showing the cursor first is idempotent. +/// Registering the handler also stops SIGINT killing the process. At a text, yes/no or +/// menu prompt the read then fails, and the run ends via [`abort`] ("wizard cancelled +/// (Ctrl-C)", exit 1). A hidden-input read is restarted instead, so this thread forces exit +/// 130 after a short grace. Showing the cursor first is idempotent. /// /// Cursor visibility is a DEC private mode, not a termios flag, which is why the shell's /// own `tcsetattr` on regaining the foreground restores echo but not this. @@ -161,8 +159,16 @@ pub fn require_tty() -> Result<(), ToolError> { } } -/// Map a dialoguer error (incl. Ctrl-C/Esc) to a clean tool error. +/// Map a dialoguer error (incl. Ctrl-C) to a clean tool error. +/// +/// Ctrl-C interrupts a read with the cursor still on the prompt's line, so this ends that +/// line first, and says what happened rather than quoting the interrupted read. fn abort(e: &dialoguer::Error) -> ToolError { + let dialoguer::Error::IO(io) = e; + if io.kind() == std::io::ErrorKind::Interrupted { + let _ = console::Term::stderr().write_line(""); + return ToolError::user("wizard cancelled (Ctrl-C)"); + } ToolError::user(format!("wizard aborted: {e}")) } @@ -788,6 +794,20 @@ mod tests { #![expect(clippy::unwrap_used, reason = "unwrap is permitted in test code")] use super::*; + /// Ctrl-C reads as a cancel, not as the interrupted read it surfaces as; any other + /// prompt failure keeps its cause. + #[test] + fn ctrl_c_is_a_cancel_and_other_failures_keep_their_cause() { + let interrupted = + dialoguer::Error::IO(std::io::Error::from(std::io::ErrorKind::Interrupted)); + assert_eq!(abort(&interrupted).message, "wizard cancelled (Ctrl-C)"); + let broken = dialoguer::Error::IO(std::io::Error::other("terminal gone")); + assert_eq!( + abort(&broken).message, + "wizard aborted: IO error: terminal gone" + ); + } + #[test] fn scripted_prompter_returns_answers_in_order() { let p = ScriptedPrompter::new() diff --git a/crates/gdi-dataset-tool/src/wizard/setup.rs b/crates/gdi-dataset-tool/src/wizard/setup.rs index 8e8e9fd..4fd0ef8 100644 --- a/crates/gdi-dataset-tool/src/wizard/setup.rs +++ b/crates/gdi-dataset-tool/src/wizard/setup.rs @@ -116,10 +116,13 @@ pub fn run_setup( // re-entry for rotation, and a prompt that forgets the previous value silently narrows // a deliberate choice back to a default. let target = resolve_write_target(config_path)?; - let existing: Profile = ToolConfig::load_file_only(&target) - .ok() + let stored = ToolConfig::load_file_only(&target).ok(); + let existing: Profile = stored + .as_ref() .and_then(|cfg| cfg.profiles.get(&name).cloned()) .unwrap_or_default(); + // The country code is the one recorded answer kept outside the profile. + let existing_cc = stored.and_then(|cfg| cfg.country_code); // Step 2: Node service URL. Blank is a real answer: a provider who prepares packages // for hand-off has no node to name. Demanding a URL would make them invent one, after @@ -319,8 +322,8 @@ pub fn run_setup( .map(|s3| s3.prefix.clone()) .unwrap_or_default(); crate::output::progress( - " must match the node's [[s3.buckets]].prefix exactly; a prefix on one side \ - only is a silent desync.", + " must match the node's key prefix for this bucket exactly; ask the node's \ + operator. A prefix on one side only is a silent desync.", ); let answer = p.input_validated( "Key prefix in the bucket (blank = the whole bucket)", @@ -348,7 +351,7 @@ pub fn run_setup( let channel = { let existing_channel = existing.s3.as_ref().and_then(|s3| s3.channel.clone()); crate::output::progress( - " the node's [[s3.buckets]].name for this bucket; ask the node operator.", + " the node's name for this bucket's channel; ask the node's operator.", ); let answer = p.input_validated( "Channel name (blank leaves the wrong-bucket guard off)", @@ -373,7 +376,7 @@ pub fn run_setup( // Not prompted: this is a fact about the endpoint just entered, not a preference. // An `http://` endpoint is otherwise accepted here and then rejected by an // https-only client. - let allow_http = endpoint.starts_with("http://"); + let allow_http = endpoint.to_ascii_lowercase().starts_with("http://"); ( Some(ProfileS3 { @@ -392,14 +395,16 @@ pub fn run_setup( (None, None) }; - // Step 6: Ensure the provider keypair exists (auto-generates if missing). - cmd_keys::load_or_generate_provider_secret(config_path)?; + // Step 6: Ensure the provider keypair exists (created on a first run). + cmd_keys::ensure_provider_secret(config_path)?; // Step 7: Country code — validated inline (2 uppercase letters) so a bad value is // caught here rather than failing late at the build stage. - let cc_raw = p.input_validated("Two-letter country code (e.g. EE)", None, &|s| { - fields::resolve_country_code(s).map(|_| ()) - })?; + let cc_raw = p.input_validated( + "Two-letter country code (e.g. EE)", + existing_cc.as_deref(), + &|s| fields::resolve_country_code(s).map(|_| ()), + )?; let cc = fields::resolve_country_code(&cc_raw).unwrap_or(cc_raw); // Step 7b: the institute abbreviation — the other identity half of every dataset id @@ -1768,6 +1773,38 @@ mod tests { assert_eq!(cfg.default_profile.as_deref(), Some("other")); } + /// A re-run offers the stored country code like every other recorded answer, so Enter + /// keeps it instead of failing validation on an empty entry. + #[test] + #[serial_test::serial(env)] + fn a_re_run_offers_the_stored_country_code() { + use gdi_node_standalone_core::crypt4gh::{generate_keypair, serialize_public_key}; + let (_sk, pk) = generate_keypair(); + let base = stub_node(&serialize_public_key(&pk)); + let dir = tempfile::tempdir().unwrap(); + let cfg_path = dir.path().join("config.toml"); + let stored = ToolConfig { + country_code: Some("LV".into()), + ..ToolConfig::default() + }; + gdi_node_standalone_core::config::write(&stored, &cfg_path).unwrap(); + + let p = ScriptedPrompter::new() + .with_inputs(vec!["default", &base, "", "", "UTARTU"]) + .with_confirms(vec![ + true, // trust recipient + true, // sync catalogs + false, // S3? + ]) + .with_selects(vec![0]); // header policy: 0 = minimal + let _prev = test_util::EnvGuard::set("GDI_CONFIG_DIR", dir.path()); + + run_setup(&p, Some(&cfg_path), None, None, true).unwrap(); + + let cfg = ToolConfig::load(Some(&cfg_path)).unwrap(); + assert_eq!(cfg.country_code.as_deref(), Some("LV")); + } + /// Adding a second profile must never leave the config with no default. A config with /// two profiles and no `default_profile` selects nothing: every profile-reading command /// fails "no profile selected: pass `--profile` or set `default_profile`", and the diff --git a/crates/gdi-node-standalone/src/init_identity.rs b/crates/gdi-node-standalone/src/init_identity.rs index 8ddf7dd..9934632 100644 --- a/crates/gdi-node-standalone/src/init_identity.rs +++ b/crates/gdi-node-standalone/src/init_identity.rs @@ -167,7 +167,9 @@ pub async fn run( })?; (pem, serialize_public_key(&secret_key.public_key())) } else { - crate::init_identity_file::warn_if_datasets_predate_new_identity(data_dir); + if let Some(notice) = crate::init_identity_file::new_identity_notice(data_dir, None) { + eprintln!("{notice}"); + } let (secret_key, public_key) = generate_keypair(); ( Zeroizing::new(serialize_secret_key(&secret_key)), diff --git a/crates/gdi-node-standalone/src/init_identity_file.rs b/crates/gdi-node-standalone/src/init_identity_file.rs index 6a7125a..ae130e8 100644 --- a/crates/gdi-node-standalone/src/init_identity_file.rs +++ b/crates/gdi-node-standalone/src/init_identity_file.rs @@ -83,7 +83,10 @@ pub fn run( })?; (pem, serialize_public_key(&secret_key.public_key())) } else { - warn_if_datasets_predate_new_identity(&config.service.data_dir); + let kept = kept_identity(config, &target); + if let Some(notice) = new_identity_notice(&config.service.data_dir, kept) { + eprintln!("{notice}"); + } let (secret_key, public_key) = generate_keypair(); ( Zeroizing::new(serialize_secret_key(&secret_key)), @@ -176,22 +179,55 @@ pub(crate) fn data_dir_has_published_datasets(data_dir: &Path) -> bool { .any(|entry| entry.path().join("manifest.json").is_file()) } -/// Warn on stderr when a fresh identity is minted onto a `data_dir` that already holds -/// published datasets. +/// What minting a fresh identity onto `data_dir` should tell the operator, or `None` when +/// no dataset is published there. /// -/// That is the shape of an ephemeral-secrets loss: an in-memory dev `OpenBao` is wiped by a -/// restart, the node loses its identity, and the operator mints a new one over surviving -/// data, leaving the old identity's PME-at-rest parquet unreadable. Minting onto an empty -/// volume stays silent, and the warning is skipped when `--from` imports a specific key. -pub(crate) fn warn_if_datasets_predate_new_identity(data_dir: &Path) { - if data_dir_has_published_datasets(data_dir) { - eprintln!( +/// Without `kept`, it looks like a lost identity: an in-memory dev `OpenBao` wiped by a +/// restart, and a new key minted over data the old one encrypted. With `kept`, another +/// configured key still on disk, it is a file-posture rotation: packages still wrapped to +/// `kept` need it, so it must stay listed. `--from` imports skip this. +pub(crate) fn new_identity_notice(data_dir: &Path, kept: Option<&Path>) -> Option { + if !data_dir_has_published_datasets(data_dir) { + return None; + } + Some(match kept { + Some(kept) => format!( + "note: packages wrapped to {} still need it: list the new key first in \ + [keys].identities and keep {} after it until they are re-keyed.", + kept.display(), + kept.display() + ), + None => format!( "warning: minting a new node identity, but published datasets already exist \ under {}. If the previous identity was lost, PME-at-rest data under those \ datasets stays encrypted to the old key and is unreadable. Proceed only if \ this is a new node; otherwise restore the previous identity first.", data_dir.display() - ); + ), + }) +} + +/// A configured identity other than `target` that is on disk, so it stays when `target` is +/// minted: what makes the mint a rotation. A `--force` over an existing `target` is a +/// replacement, not a rotation. +fn kept_identity<'a>(config: &'a ServiceConfig, target: &Path) -> Option<&'a Path> { + if target.exists() { + return None; + } + config + .keys + .identities + .iter() + .map(PathBuf::as_path) + .find(|path| path.is_file() && !is_same_file(path, target)) +} + +/// Whether `a` and `b` name the same file: compared resolved when both exist, as spelled +/// otherwise. +fn is_same_file(a: &Path, b: &Path) -> bool { + match (a.canonicalize(), b.canonicalize()) { + (Ok(a), Ok(b)) => a == b, + _ => a == b, } } @@ -384,6 +420,43 @@ mod tests { ); } + /// Minting next to a configured key that stays (a rotation) reminds the operator to + /// keep it listed; only a mint with no surviving key warns of a lost identity. + #[test] + fn a_rotation_mint_reminds_rather_than_warns_of_a_lost_identity() { + let tmp = tempfile::tempdir().expect("tmp"); + let data_dir = tmp.path().join("data"); + std::fs::create_dir_all(data_dir.join("ds-1")).expect("mkdir"); + std::fs::write(data_dir.join("ds-1").join("manifest.json"), b"{}").expect("write"); + let current = tmp.path().join("node.c4gh"); + let config = config_with_identity(¤t); + run(&config, false, false, None, None).expect("mint the current key"); + + let kept = kept_identity(&config, &tmp.path().join("node-new.c4gh")); + let rotation = new_identity_notice(&data_dir, kept).expect("datasets exist"); + assert!( + rotation.starts_with("note: packages wrapped to") + && rotation.contains(¤t.display().to_string()), + "{rotation}" + ); + let lost = new_identity_notice(&data_dir, None).expect("datasets exist"); + assert!( + lost.starts_with("warning:") && lost.contains("Proceed only if this is a new node"), + "{lost}" + ); + let mut two_keys = config.clone(); + two_keys + .keys + .identities + .push(tmp.path().join("node-old.c4gh")); + std::fs::write(tmp.path().join("node-old.c4gh"), b"key").expect("write"); + assert_eq!( + kept_identity(&two_keys, ¤t), + None, + "a --force over an existing key is a replacement, not a rotation" + ); + } + #[test] fn refuses_to_overwrite_an_existing_identity() { let tmp = tempfile::tempdir().expect("tmp"); diff --git a/crates/gdi-node-standalone/src/main.rs b/crates/gdi-node-standalone/src/main.rs index 42fb8ce..1ae7dea 100644 --- a/crates/gdi-node-standalone/src/main.rs +++ b/crates/gdi-node-standalone/src/main.rs @@ -1599,10 +1599,11 @@ fn log_startup_summary(config: &ServiceConfig, config_path: &Path, vault_ok: boo /// /// The node does not self-register: an operator hands the beacon URL to the Beacon Network /// aggregator and the `fairdp` URL to the FDP harvester. The aggregated and sensitive beacon -/// collapse to one URL when the node mounts them combined, which is the default; the FDP URL -/// is present only when `[fairdp]` is configured. Each is `base_url` plus the mount path, -/// and must equal what the operator forwards, because the node mints these same IRIs into -/// its FDP output and a mismatched URL breaks the harvester's crawl. +/// share one URL when mounted combined. Mounted split, the default, only the aggregated one +/// is listed: the sensitive one is a zeros-only placeholder. The FDP URL is present only +/// when `[fairdp]` is configured. Each is `base_url` plus the mount path, and must equal what the operator +/// forwards, because the node mints these same IRIs into its FDP output and a mismatched +/// URL breaks the harvester's crawl. fn registration_urls( base_url: &str, aggregated_path: &str, @@ -1618,7 +1619,6 @@ fn registration_urls( )); } else { urls.push(("beacon (aggregated)", format!("{base}{aggregated_path}"))); - urls.push(("beacon (sensitive)", format!("{base}{sensitive_path}"))); } if has_fairdp { urls.push(("fairdp", format!("{base}/fairdp"))); @@ -3144,11 +3144,13 @@ fn check_config(cli_config: Option<&Path>) -> Result<()> { // the config is not loaded yet. Held until return. let _telemetry = logging::init(None, None, "", None, 1.0); let path = ServiceConfig::resolve_path(cli_config); + // The stdout verdict carries the reason; the error only sets the exit code, so stderr + // does not repeat it. let config = match ServiceConfig::load(cli_config) { Ok(config) => config, Err(e) => { println!("config check FAILED ({}): {e}", path.display()); - return Err(anyhow::anyhow!("loading config: {e}")); + return Err(anyhow::anyhow!("config check failed")); } }; match preflight::run(&config) { @@ -3159,7 +3161,7 @@ fn check_config(cli_config: Option<&Path>) -> Result<()> { } Err(e) => { println!("config check FAILED ({}): {e}", path.display()); - Err(anyhow::Error::new(e).context("service config failed startup preflight")) + Err(anyhow::anyhow!("config check failed")) } } } @@ -3419,42 +3421,63 @@ fn print_s3_bucket_summary(config: &gdi_node_standalone_core::config::ServiceCon return; }; let vault_supplies = config.vault.as_ref().is_some_and(|v| v.s3_path.is_some()); - for bucket in &s3.buckets { - // Empty overrides: this is the offline view. `credential_source` is the function - // the live boot uses, so the two cannot disagree about what "config" or "anonymous" - // means. - let local = - gdi_node_standalone::s3::credential_source(bucket, &std::collections::BTreeMap::new()); - let note = if vault_supplies { - " (vault.s3_path is set: Vault may override this at boot)" - } else { - "" - }; - // The prefix decides which objects this channel addresses, and it is invisible - // everywhere else a pre-deploy check looks, so a prefix set on the node and not on - // the writer, or the reverse, reads as "the bucket is empty" with nothing naming the - // cause. Printed as `bucket/prefix`, matching `gdi-dataset-tool`'s `target_label`, - // so the two sides can be compared by eye. - let target = if bucket.prefix.is_empty() { - bucket - .bucket - .clone() - .unwrap_or_else(|| "".to_owned()) - } else { - format!( - "{}/{}", - bucket.bucket.as_deref().unwrap_or(""), - bucket.prefix.trim_end_matches('/') - ) - }; - println!( - " channel {name:<19} target={target} endpoint={endpoint} credentials={local}{note}", - name = bucket.name, - endpoint = bucket.endpoint.as_deref().unwrap_or(""), - ); + for (index, bucket) in s3.buckets.iter().enumerate() { + println!("{}", bucket_summary(index, bucket, vault_supplies)); } } +/// One bucket's `check-config` line, plus a warning when it would run anonymous: right for a +/// public bucket, but also what unexported credentials look like. +#[cfg(feature = "s3")] +fn bucket_summary( + index: usize, + bucket: &gdi_node_standalone_core::config::S3Bucket, + vault_supplies: bool, +) -> String { + // Empty overrides: this is the offline view. `credential_source` is the function + // the live boot uses, so the two cannot disagree about what "config" or "anonymous" + // means. + let local = + gdi_node_standalone::s3::credential_source(bucket, &std::collections::BTreeMap::new()); + let note = if vault_supplies { + " (vault.s3_path is set: Vault may override this at boot)" + } else { + "" + }; + // The prefix decides which objects this channel addresses, and it is invisible + // everywhere else a pre-deploy check looks, so a prefix set on the node and not on + // the writer, or the reverse, reads as "the bucket is empty" with nothing naming the + // cause. Printed as `bucket/prefix`, matching `gdi-dataset-tool`'s `target_label`, + // so the two sides can be compared by eye. + let target = if bucket.prefix.is_empty() { + bucket + .bucket + .clone() + .unwrap_or_else(|| "".to_owned()) + } else { + format!( + "{}/{}", + bucket.bucket.as_deref().unwrap_or(""), + bucket.prefix.trim_end_matches('/') + ) + }; + let warning = if local == "anonymous" && !vault_supplies { + format!( + "\n warning: no credentials, so requests go unsigned. Unless the bucket \ + is public, set GDI_NODE__S3__BUCKETS__{index}__ACCESS_KEY_ID and \ + GDI_NODE__S3__BUCKETS__{index}__SECRET_ACCESS_KEY" + ) + } else { + String::new() + }; + format!( + " channel {name:<19} target={target} endpoint={endpoint} \ + credentials={local}{note}{warning}", + name = bucket.name, + endpoint = bucket.endpoint.as_deref().unwrap_or(""), + ) +} + /// No-op without the `s3` feature: a lite build cannot carry buckets at all. #[cfg(not(feature = "s3"))] fn print_s3_bucket_summary(_config: &gdi_node_standalone_core::config::ServiceConfig) {} @@ -4700,8 +4723,9 @@ fn reload_s3_monitors( // successful reconcile, so say so. `set_channel_health` is otherwise called // only by a monitor, so a channel whose monitor stops would keep its last // `true` forever and read ready while half-blind, the shape - // `register_configured_channels` seeds `false` to avoid at boot. A rotation - // therefore reads `degraded` for one poll, which is the honest state. + // `register_configured_channels` seeds `false` to avoid at boot. The + // replacement reconciles on its first tick, so a rotation reads `degraded` for + // one marker poll, which is the honest state. state.readiness.set_channel_health(&bucket.name, false); // Reached only for an access or behaviour change — credentials, region, // addressing, poll intervals, `write_status` — so the same objects with a @@ -4754,6 +4778,10 @@ fn reload_s3_monitors( if let Some(ingesting) = carried_ingesting { monitor.adopt_ingesting(ingesting); } + // No startup reconcile preceded this monitor, whether it replaces one or + // serves a bucket added live, and its channel reads unhealthy until it + // reconciles. + monitor.reconcile_on_first_tick(); spawn_monitor(&monitor, running); } Err(e) => { @@ -5759,9 +5787,35 @@ mod tests { ); } + /// A bucket with no credentials passes `check-config` with a warning naming the + /// variables; one with credentials, or that Vault may supply, does not warn. + #[cfg(feature = "s3")] + #[test] + fn check_config_warns_of_a_bucket_without_credentials() { + let anonymous = gdi_node_standalone_core::config::S3Bucket { + name: "primary".to_owned(), + ..Default::default() + }; + let line = bucket_summary(1, &anonymous, false); + assert!( + line.contains("credentials=anonymous") + && line.contains("warning: no credentials") + && line.contains("GDI_NODE__S3__BUCKETS__1__ACCESS_KEY_ID"), + "{line}" + ); + assert!(!bucket_summary(1, &anonymous, true).contains("warning")); + let keyed = gdi_node_standalone_core::config::S3Bucket { + access_key_id: Some("id".to_owned()), + secret_access_key: Some("secret".to_owned()), + ..anonymous + }; + assert!(!bucket_summary(1, &keyed, false).contains("warning")); + } + #[test] fn registration_urls_split_beacon_without_fairdp() { - // Distinct mounts yield two beacon URLs; no `[fairdp]` yields no FDP URL. + // Distinct mounts list only the aggregated beacon: the sensitive one is a + // zeros-only placeholder with nothing to register. No `[fairdp]`, no FDP URL. let urls = registration_urls( "https://n.example.org", "/beacon/agg", @@ -5770,16 +5824,10 @@ mod tests { ); assert_eq!( urls, - vec![ - ( - "beacon (aggregated)", - "https://n.example.org/beacon/agg".to_owned() - ), - ( - "beacon (sensitive)", - "https://n.example.org/beacon/sens".to_owned() - ), - ] + vec![( + "beacon (aggregated)", + "https://n.example.org/beacon/agg".to_owned() + )] ); } diff --git a/crates/gdi-node-standalone/src/s3.rs b/crates/gdi-node-standalone/src/s3.rs index 24acf72..67de13b 100644 --- a/crates/gdi-node-standalone/src/s3.rs +++ b/crates/gdi-node-standalone/src/s3.rs @@ -232,6 +232,9 @@ pub struct BucketMonitor { /// leaves the id marked forever, so that dataset would never ingest again until a /// restart. Returning between polls has no such window. retire: Arc, + /// Reconcile on the first tick rather than at the first full poll: a monitor that a + /// reload started has no startup reconcile behind it, and reads unhealthy until one runs. + reconcile_first: bool, } /// The retirement flag and its wakeup, shared by a monitor and its supervisor. @@ -577,6 +580,7 @@ impl BucketMonitor { ingesting: Arc::new(Mutex::new(HashSet::new())), pending_removals: Arc::new(Mutex::new(HashMap::new())), retire: Arc::new(RetireSignal::default()), + reconcile_first: false, } } @@ -640,6 +644,13 @@ impl BucketMonitor { self.ingesting = handle; } + /// Make [`Self::run`] reconcile on its first tick, for a monitor a reload starts. That + /// reconcile is a timer sweep, which processes no package-absence removals. Call it + /// before the monitor is spawned or cloned. + pub fn reconcile_on_first_tick(&mut self) { + self.reconcile_first = true; + } + /// Test-support hook: mark `id` as one this monitor is ingesting, as the download and /// enqueue path does, so a test can reproduce the deleted-while-in-flight state without /// racing a real ingest. Not part of the public API. @@ -687,8 +698,8 @@ impl BucketMonitor { // Monotonic. This is the safety net that catches a marker bump the node missed, so // the wall clock must not disarm it: a backward step from an NTP correction or a // manual set would otherwise read as "not due yet" and postpone the sweep until the - // clock caught back up. - let mut last_full = std::time::Instant::now(); + // clock caught back up. `None` makes the first tick's sweep due at once. + let mut last_full = (!self.reconcile_first).then(std::time::Instant::now); // The startup reconcile has already run for readiness unless the channel was // suppressed at boot, in which case no startup reconcile ran either. Seed the marker // so the first loop iteration does not immediately re-reconcile on an unchanged one. @@ -700,6 +711,10 @@ impl BucketMonitor { self.head_marker().await }; let mut monitor_paused = suppressed_at_boot; + // Whether the last marker check got an answer. The first answer after a failure + // reconciles at once, so a bucket that comes back reads healthy within one marker + // poll rather than one full poll. + let mut marker_answered = last_marker.is_known(); loop { // Wake on the marker poll interval or an operator `SIGUSR1` reconcile trigger, @@ -751,14 +766,16 @@ impl BucketMonitor { monitor_paused = false; } - let due_full = last_full.elapsed() >= full_period; + let due_full = last_full.is_none_or(|t| t.elapsed() >= full_period); let marker = self.head_marker().await; + let answered_again = !marker_answered && marker.is_known(); + marker_answered = marker.is_known(); // Only a definite change opens the removal gate; an `Unknown` from a transient // failure is never a change. A due full poll still reconciles below regardless, // so an `Unknown` never suppresses a needed reconcile either. let marker_changed = marker.opens_removal_gate(&last_marker); - if should_reconcile(triggered, marker_changed, due_full) { + if should_reconcile(triggered, marker_changed, due_full || answered_again) { if triggered { debug!( channel = self.channel(), @@ -766,6 +783,11 @@ impl BucketMonitor { ); } else if marker_changed { debug!(channel = self.channel(), "marker changed; reconciling"); + } else if answered_again && !due_full { + debug!( + channel = self.channel(), + "marker answered again after a failure; reconciling" + ); } else { debug!(channel = self.channel(), "full-poll interval; reconciling"); } @@ -792,7 +814,7 @@ impl BucketMonitor { last_marker = marker; } } - last_full = std::time::Instant::now(); + last_full = Some(std::time::Instant::now()); } } } diff --git a/crates/gdi-node-standalone/tests/it/binary_cli.rs b/crates/gdi-node-standalone/tests/it/binary_cli.rs index 88217ef..b1d68ef 100644 --- a/crates/gdi-node-standalone/tests/it/binary_cli.rs +++ b/crates/gdi-node-standalone/tests/it/binary_cli.rs @@ -119,6 +119,10 @@ fn check_config_rejects_the_shipped_example_and_says_why() { "the verdict must be the PLACEHOLDER rejection, not an incidental failure such as a \ missing file — that is the contract this test exists to pin: {stdout}" ); + assert!( + !stderr.contains("placeholder"), + "the reason is the verdict's, printed once; stderr only ends the run: {stderr}" + ); } #[test] diff --git a/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs b/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs index 345045e..e41b42f 100644 --- a/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs +++ b/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs @@ -902,3 +902,58 @@ async fn a_failing_download_backs_off_instead_of_refetching_every_poll() { "a backing-off dataset must not be re-downloaded on every subsequent poll" ); } + +/// A bucket that answers again after an outage reconciles within one marker poll, so its +/// channel reads healthy without waiting for the full poll. The marker is unchanged and the +/// full poll an hour away, so only its answering again can trigger that. +#[tokio::test] +async fn a_bucket_that_answers_again_reconciles_within_one_marker_poll() { + let dark = Arc::new(AtomicBool::new(false)); + let get_dark = Arc::clone(&dark); + let list_dark = Arc::clone(&dark); + let store = HookStore::new("DarkStore") + .on_get(move |_| { + let fail = get_dark.load(Ordering::SeqCst); + Box::pin(async move { fail.then(injected) }) + }) + .on_list(move |_| { + let fail = list_dark.load(Ordering::SeqCst); + Box::pin(async move { fail.then(injected) }) + }) + .into_store(); + let bucket = S3Bucket { + marker_poll_interval: 1, + full_poll_interval: 3_600, + ..bucket_cfg("primary", false) + }; + let rig = Rig::new(store, bucket); + rig.monitor.reconcile().await; + let health = || { + rig.state + .readiness + .channel_health_snapshot() + .get("primary") + .copied() + }; + let running = tokio::spawn(rig.monitor.clone().run()); + tokio::time::sleep(Duration::from_millis(1_500)).await; + + // The outage, as a failed listing would have flagged it. + dark.store(true, Ordering::SeqCst); + rig.state.readiness.set_channel_health("primary", false); + tokio::time::sleep(Duration::from_millis(2_500)).await; + assert_eq!( + health(), + Some(false), + "nothing reconciles while the bucket is dark" + ); + + dark.store(false, Ordering::SeqCst); + poll_until(Duration::from_secs(10), || health() == Some(true)).await; + + rig.monitor.retire_signal().retire(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .expect("a retired monitor returns") + .expect("the monitor task must not panic"); +} diff --git a/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs b/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs index 2ce1822..9282f92 100644 --- a/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs +++ b/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs @@ -86,3 +86,35 @@ async fn an_unretired_monitor_keeps_running() { "a monitor nobody retired must still be polling" ); } + +/// A monitor that a reload starts reconciles on its first tick, so its channel, marked +/// unhealthy by the reload, reads healthy within one marker poll. The full poll here is an +/// hour, so only the first-tick reconcile can do that within the bound. +#[tokio::test] +async fn a_monitor_started_by_a_reload_reconciles_on_its_first_tick() { + let bucket = S3Bucket { + marker_poll_interval: 1, + full_poll_interval: 3_600, + ..bucket_cfg("primary", false) + }; + let mut rig = Rig::new(Arc::new(InMemory::new()), bucket); + rig.state.readiness.set_channel_health("primary", false); + rig.monitor.reconcile_on_first_tick(); + + let running = tokio::spawn(rig.monitor.clone().run()); + poll_until(Duration::from_secs(10), || { + rig.state + .readiness + .channel_health_snapshot() + .get("primary") + .copied() + == Some(true) + }) + .await; + + rig.monitor.retire_signal().retire(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .expect("a retired monitor returns") + .expect("the monitor task must not panic"); +} diff --git a/docker-compose.yml b/docker-compose.yml index ee1f837..4e2812d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: # Gate on the secrets backend's listener so the node does not race it on a cold # `up`. If it loses that race (Vault unreachable at startup) it latches # keyless-degraded and does not self-heal — see the KeylessDegraded alert and - # docs/operating.md §8. `required: false` keeps this valid under whichever single + # docs/operating.md §1. `required: false` keeps this valid under whichever single # secrets profile is active (the other backend service is absent). openbao: condition: service_healthy diff --git a/docs/deployment.md b/docs/deployment.md index 422aa99..6cc7814 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -237,7 +237,7 @@ push output, or with `docker inspect --format '{{index .RepoDigests 0}}' | --- | --- | --- | | **Minimal** (inbox-only, plaintext at rest) | a local end-to-end trial run | `compose/node.minimal.toml` via `docker compose -f docker-compose.minimal.yml up -d --build`: the container form of the [README's Quickstart 1](../README.md#quickstart-1--see-it-work). Its inbox is inside the container — see [Feeding the minimal stack's inbox](#feeding-the-minimal-stacks-inbox) | | **S3** (S3 ingest, disk key, no Vault or PME) | the common networked public-data node | `compose/node.s3.toml` via `COMPOSE_PROFILES=garage docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d`. Mint the node identity first (below), or the `keyinit` one-shot exits 1 and the stack does not start. [`node.quickstart.toml`](../node.quickstart.toml) is the bare-metal counterpart — the same S3-plus-disk-key posture, though it keeps the default split beacon prefixes where `compose/node.s3.toml` mounts one combined `/beacon/v2`; check `check-config`'s registration URLs against whichever you deploy. The same file, with no profile, runs against a bucket of your own: [Running against your own backends](#running-against-your-own-backends) | -| **Full dev** (S3 + Vault/OpenBao + at-rest PME) | exercising the full secrets path | `compose/node.full.toml` via `docker-compose.yml`. `up -d` alone is not enough; provision the secrets first (below) | +| **Full dev** (S3 + Vault/OpenBao + at-rest PME) | exercising the full secrets path | `compose/node.full.toml` via `docker-compose.yml`. `up -d` alone is not enough; provision the secrets first (below). Its bucket uses `prefix = "gdi-node-storage/"`, so a provider profile uploading to it needs the same `prefix`, or the node never sees the uploads | **On Kubernetes, S3 is the supported ingest channel.** The three shapes above are the bare-metal and Compose ones; [`deploy/kubernetes/`](../deploy/kubernetes/README.md) is the diff --git a/docs/gdi-dataset-tool.md b/docs/gdi-dataset-tool.md index daa5064..86fabd6 100644 --- a/docs/gdi-dataset-tool.md +++ b/docs/gdi-dataset-tool.md @@ -193,9 +193,10 @@ Format: `(GOE|GDI)-CC-ORG-YYYYMMDDHHMMSSmmm` Example: `GDI-EE-EXAMPLE-20260409143052837`. -The tool generates the whole ID at `build`/`package` time. The millisecond timestamp -makes same-org collisions between two builds negligible, and the `upload`/`deploy` guards -catch the remainder. +The tool generates the whole ID at `build`/`package` time. Builds that share a config +directory never reuse a millisecond (the last one is kept in `.last-build-epoch`), so a +script can start several at once. Elsewhere the clock alone decides, and `upload`/`deploy` +refuse the rare collision. A pinned `--build-epoch` is used as given. **Country-code precedence** (lowest → highest): @@ -1476,7 +1477,7 @@ gdi-dataset-tool pack [--recipient ] [-o ] [--force] |------|---------|---------| | `` (positional) | (required) | The `build/{datasetId}/` staging directory. | | `--recipient ` | fetched from profile URL, else `node_recipient_file` | Local crypt4gh recipient file for the node (overrides the profile's URL fetch). | -| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. | +| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. Any other file name gets a warning, because `upload` and `deploy` read the dataset ID from the name. | | `--force` | off | Overwrite an existing output file. | | `--format` | `text` | `text` (the human result line) or `json` (a single machine-readable result object). | @@ -1519,7 +1520,7 @@ gdi-dataset-tool package [--cc ] [--build-out ] \ | `--header-policy ` | profile `header_policy`, else `minimal` | As `build --header-policy`. | | `-j`, `--jobs ` | `0` (one per logical CPU) | Conversion worker-pool size; lower it to cap peak memory and parallelism (a single VCF still uses the whole pool). | | `--recipient ` | fetched from profile URL, else `node_recipient_file` | Node recipient file (overrides the profile's URL fetch). | -| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. | +| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. Any other file name gets a warning, because `upload` and `deploy` read the dataset ID from the name. | | `--force` | off | Overwrite an existing staging dir **and** output file. | | `--keep` | off | Keep the staging directory after a successful pack. It is deleted by default, because it holds plaintext genotype-derived intermediates. | | `--refresh-catalogs` | off | Before building, fetch the node's live catalogs for this build only. It does not write to the profile; use `catalogs --sync` to persist them. | @@ -1946,7 +1947,7 @@ effect. `status ` also works wherever a tool profile is configured. gdi-dataset-tool publish GDI-EE-EXAMPLE-20260409143052837 # published GDI-EE-EXAMPLE-20260409143052837: wrote the visible sidecar on inbox /srv/gdi/inbox. # The node applies it on its next scan; its management-plane -# /datasets/GDI-EE-EXAMPLE-20260409143052837/state reports when it has +# /datasets/GDI-EE-EXAMPLE-20260409143052837/state shows visible once it has ``` > `unpublish` on a node's only remaining dataset does not retract it from a GDI User @@ -2391,10 +2392,15 @@ can select a different key set than the default. `download`, `list`, or an S3-channel lifecycle op) ran without a `[profiles..s3]` block in the config. Add one, or use the inbox channel. -**`S3 bucket is not writable (your token may lack write permission)`** — `doctor`'s write -probe, a PUT of a temporary `.doctor-probe-*` key, was denied. The token is read-only, or -the bucket policy withholds `s3:PutObject`. `upload` and every lifecycle write need a -read/write token. +**`profile s3.endpoint http://... is plaintext http:// but allow_http is false`** — use +the `https://` endpoint, or, for a local dev backend such as MinIO or Garage, set +`allow_http = true` in the profile's `[profiles..s3]` block. `wizard setup` sets it +for you. + +**`S3 bucket is not writable: ...`** — `doctor`'s write probe, a PUT of a temporary +`.doctor-probe-*` key, failed, and the rest of the message says why. Prefixed with +`not authorized:`, the token is read-only or the bucket policy withholds `s3:PutObject`; +`upload` and every lifecycle write need a read/write token. **`not authorized: : ...`** — an operation was denied, and the message names the failing step, such as `uploading ` or `listing bucket`. The command exits `4`, so @@ -2407,6 +2413,12 @@ networked verb when it detects such a twin, so check stderr, or run `gdi-dataset-tool profiles`, which prints the same warning alongside what it loaded. Then run `gdi-dataset-tool doctor`. +**`...; aborting the multipart upload of failed too ...`** — an `upload` failed +part-way and could not be cancelled, usually because the connection or the credentials were +gone. The parts already sent stay in the bucket, billed but unlisted, even after a retry. +Ask the bucket's operator to abort the upload (`aws s3api list-multipart-uploads`, then +`abort-multipart-upload`) or to add a lifecycle rule that expires incomplete uploads. + **`dataset is already live on the node` / `dataset is already present in the bucket`** — the id is already installed. `upload` and `deploy` refuse to re-present it without `--replace`. On S3, `--replace` does not overwrite the immutable package bytes, @@ -2418,7 +2430,9 @@ can be published or unpublished. Ingest it first, and let it leave `processing`. **`dataset reads as visible on the node's state oracle` (delete)** — `unpublish` it first, or pass `--force`. If you have just run `unpublish`, the oracle lags the sidecar by -a few seconds; wait and retry. +a few seconds; wait and retry. When the node's state oracle gives no answer, `delete` checks +the bucket's `.state.json` instead, and says `reads as visible in the bucket's state +sidecar`. **`cannot determine the dataset's channel`** — a lifecycle op could not reach the node, and the profile has neither an S3 block nor an `inbox`. Pass `--s3` or `--local`. diff --git a/docs/images/wizard.svg b/docs/images/wizard.svg index 6102800..c9a108b 100644 --- a/docs/images/wizard.svg +++ b/docs/images/wizard.svg @@ -1 +1 @@ -provider@laptop:~/work$gdi-dataset-toolprovider@laptop:~/work$gdi-dataset-toolwizard==[1/5]Setup:profile&keys====================================================================✔Profilename·defaultthenode'spublicbaseURL.Blankifthereisnonodeyet;youwillhandthepackageoverinstead.✔NodeserviceURL·http://127.0.0.1:38080blankunlessthenodeisco-locatedandyoucanreachitsmanagementplane(typicallyhttp://127.0.0.1:9090).✔NodemanagementURL·http://127.0.0.1:39090recipientfingerprint:sha256:5abc47b88264539793f3e7358c2e759b7fc8d71c5dd8ccb5d2800c64e87512fb✔Trustthisrecipient?·yes✔Synccatalogsfromthenode?·yes✔ConfigureS3upload?·yes✔S3bucketname·gdi-datasets✔S3endpointURL·http://127.0.0.1:3900thenode'sregion:garageforGarage,us-east-1forMinIO/Ceph,thebucket'sownforAWS.✔S3region·garagemustmatchthenode's[[s3.buckets]].prefixexactly;aprefixononesideonlyisasilentdesync.✔Keyprefixinthebucket(blank=thewholebucket)·✔S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)·********✔S3secretaccesskey·********thenode's[[s3.buckets]].nameforthisbucket;askthenodeoperator.✔Channelname(blankleavesthewrong-bucketguardoff)·primary✔Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)·yeswarning:noprovideridentityat/home/provider/.config/gdi/keys/provider.c4gh;generatedanewone(sha256:0cf3186d30246c3c7726c5633222ed4ed4afcc3f0bf5581088fc7e2a1a8fb17b).Thiskeyisthecrypt4ghwriterprovenanceofeverypackagebuiltfromhereon;anodewith[ingest].writer_policy=enforcewillrejectthemuntilthisfingerprintisallow-listed.Ifyouexpectedanexistingkey,check--config/GDI_CONFIG_DIRbeforepackaging.✔Two-lettercountrycode(e.g.EE)·EEtheORGinGDI-EE-<ORG>-...,e.g.EXAMPLE.✔Instituteabbreviationfordatasetids·EXAMPLEthenodedropstheseatingest;`--header-policy`onbuild/packagealwaysoverridestheprofile.✔VCFheaderstoshipinpackages·minimal:structuralkeysonly,nosampleidentifiers(default)ok:profile'default'writtento/home/provider/.config/gdi/tool.tomlstoredtheS3credentialsin/home/provider/.config/gdi/tool-secrets.toml(owner-only),whereeverylaterrunreadsthem;verifythebucketwith`gdi-dataset-tooldoctor`(itdoesaprobePUT/DELETE)==[2/5]Author:describethedataset(package.yaml)==============================================--Sourcedata------------------------------------------------------------------------------------plain.vcfor.vcf.gz;adirectoryoffersitsVCFstopickfrom.✔VCFfileordirectoryofVCFs(Tabcompletes)·data/gdi-sample.GRCh38.vcf.gz✔AddanotherVCFfileordirectory?·nopreview:populationsinthe1sourceVCF(s),fromtheheaders:[1/1]gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalassemblyGRCh38readfromtheVCFheader;confirmit,orpicktheother✔Assembly·GRCh38--Catalogentry:publishedviathenode'sFAIRDataPoint----------------------------------------✔DatasetIDprefix·GDIorg:EXAMPLE(fromtheprofile)✔Catalog·gdi-aggregated?Datasettitle›Synthetic?Datasettitle›Syntheticallele?Datasettitle›Syntheticallelefrequencies,?Datasettitle›Syntheticallelefrequencies,twelve✔Datasettitle·Syntheticallelefrequencies,twelvepopulations?Description›Synthetic?Description›Syntheticaggregated?Description›Syntheticaggregatedallele?Description›Syntheticaggregatedallelefrequencies?Description›Syntheticaggregatedallelefrequenciesfor?Description›Syntheticaggregatedallelefrequenciesfora?Description›Syntheticaggregatedallelefrequenciesforanode✔Description·Syntheticaggregatedallelefrequenciesforanodedemo.✔Adddiscoverykeywords?·yes✔Keywords(comma-separated)·allele-frequency,genomicsnumberOfUniqueIndividuals:recommended;buildwarnswhenitisabsent.✔Recordcohortsize?·yes✔Distinctsequencedsubjects(blanktoskip)·500✔Isthissyntheticdata?·yesthefullpublicnameshowninthecatalogentry:nottheIDabbreviationfromsetup.?Creatingorganisation›Example✔Creatingorganisation·ExampleInstitute✔Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)·Humangenomicoptional:acomplianceclaimaboutTHISdataset;leaveeverythinguntickedtoomitit.✔Conformsto(GDIstandards);Spacetoggles,Enterconfirms·--Access&legal---------------------------------------------------------------------------------?Accessrights›NON_PUBLIC✔Accessrights·PUBLIC?License›Other(enterIRI)✔License·CC01.0:publicdomain(typicalforsyntheticdata)EUELIIRIs,e.g.http://data.europa.eu/eli/reg/2025/327/oj.TheEHDSrowmaybeunticked;buildthenwarns,and`--strict`failsonthatwarning.✔Applicablelegislation(Spacetoggles,Enterconfirms)·EHDS:Regulation(EU)2025/327✔AddanotherlegislationELIorIRI?·no--Beaconprovenance:citedineveryBeaconanswer------------------------------------------------setsbothafSourceandafSourceReference.✔UsethestandardGenomeofEuropeAFprovenance?·yes--Disclosurecontrols----------------------------------------------------------------------------Smallallelecountscanbewithheldatbuildtime(minAlleleCount).TheunitisALLELES,notpeople:ahomozygouscarriercounts2,so5~threepeople.✔Withholdrowswithallelecountbelow(0=publisheverything)·10gdi-sample.GRCh38.vcf.gz[]0B/490.81KiB(0%)0B/sETA0satminAlleleCount10:6411row(s)belowthefloor,10687moreremovedbycollapsing1447variant(s)toTotal;populationsthatsurvive:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,Total--Review:package.yamlasitwillbewritten-----------------------------------------------------metadata:prefix:"GDI"org:"EXAMPLE"catalog:"gdi-aggregated"title:"Syntheticallelefrequencies,twelvepopulations"description:"Syntheticaggregatedallelefrequenciesforanodedemo."type:-"https://publications.europa.eu/resource/authority/dataset-type/SYNTHETIC_DATA"keywords:-"allele-frequency"-"genomics"numberOfUniqueIndividuals:500accessRights:"http://publications.europa.eu/resource/authority/access-right/PUBLIC"applicableLegislation:-"http://data.europa.eu/eli/reg/2025/327/oj"license:"http://publications.europa.eu/resource/authority/licence/CC0"creator:-name:"ExampleInstitute"healthCategory:-"http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic"files:-category:"VCF"reference:"GRCh38"files:-"/home/provider/work/data/gdi-sample.GRCh38.vcf.gz"internal:{}config:mode:aggregatedblockRange:10000000minAlleleCount:10afSource:"TheGenomeofEurope"afSourceReference:"https://genomeofeurope.eu/"---------------------------------------------------------------------------------------------------✔Writepackage.yaml?·Writeitwrotepackage.yaml==[3/5]Build:convert&validate================================================================disclosurepreview:whatthisdatasetwillpublish(1sourceVCF(s)):[1/1]/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalVCFheaders:minimal:structuralkeysonly,nosampleidentifiers✔Publishthesepopulations?·yesconvertingVCF1/1:/home/provider/work/data/gdi-sample.GRCh38.vcf.gznote:themin_allele_countfloorsuppressed6411populationrow(s)whoseACwasbelowitnote:k-anonymitycoherencecollapsed1447variant(s)totheirTotalrowonly,removing10687furtherpopulationrow(s)regardlessoftheirownAC:awithheldpopulationforcesitssiblingstobewithheldtoonote:11of413emittedvariant(s)haveTotalAF=0:nocopyofthealternatealleleinthewholecohort,andarestoredandservedlikeanyotherrow,asavariantpresentwithfrequency0;removemonomorphicsitesbeforebuilding(`bcftoolsview-c1`)ifthatisnotintendednote:dropped1224of1637inputrecords(0non-primarycontig,0nosupportedALT;symbolic/SV/gVCF/missing,0noallelefrequencyintheinput,1224everypopulationrowwithheldbythek-anonymityfloor)/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:populationsemitted(12):EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalbuiltdatasetGDI-EE-EXAMPLE-20260930190939700->./build/GDI-EE-EXAMPLE-20260930190939700==[4/5]Pack:encryptintoonepackage===========================================================packingparquet1/23:allele-freq.chr21.0.br10000000.6e44adfd808b30ac.parquetpackingparquet2/23:allele-freq.chrM.0.br10000000.6e44adfd808b30ac.parquetpackingparquet3/23:allele-freq.chrX.0.br10000000.6e44adfd808b30ac.parquetpackingparquet4/23:allele-freq.chrX.1.br10000000.6e44adfd808b30ac.parquetpackingparquet5/23:allele-freq.chrX.10.br10000000.6e44adfd808b30ac.parquetpackingparquet6/23:allele-freq.chrX.11.br10000000.6e44adfd808b30ac.parquetpackingparquet7/23:allele-freq.chrX.12.br10000000.6e44adfd808b30ac.parquetpackingparquet8/23:allele-freq.chrX.13.br10000000.6e44adfd808b30ac.parquetpackingparquet9/23:allele-freq.chrX.14.br10000000.6e44adfd808b30ac.parquetpackingparquet10/23:allele-freq.chrX.15.br10000000.6e44adfd808b30ac.parquetpackingparquet11/23:allele-freq.chrX.2.br10000000.6e44adfd808b30ac.parquetpackingparquet12/23:allele-freq.chrX.3.br10000000.6e44adfd808b30ac.parquetpackingparquet13/23:allele-freq.chrX.4.br10000000.6e44adfd808b30ac.parquetpackingparquet14/23:allele-freq.chrX.5.br10000000.6e44adfd808b30ac.parquetpackingparquet15/23:allele-freq.chrX.6.br10000000.6e44adfd808b30ac.parquetpackingparquet16/23:allele-freq.chrX.7.br10000000.6e44adfd808b30ac.parquetpackingparquet17/23:allele-freq.chrX.8.br10000000.6e44adfd808b30ac.parquetpackingparquet18/23:allele-freq.chrX.9.br10000000.6e44adfd808b30ac.parquetpackingparquet19/23:allele-freq.chrY.1.br10000000.6e44adfd808b30ac.parquetpackingparquet20/23:allele-freq.chrY.2.br10000000.6e44adfd808b30ac.parquetpackingparquet21/23:allele-freq.chrY.3.br10000000.6e44adfd808b30ac.parquetpackingparquet22/23:allele-freq.chrY.4.br10000000.6e44adfd808b30ac.parquetpackingparquet23/23:allele-freq.chrY.5.br10000000.6e44adfd808b30ac.parquetpackedGDI-EE-EXAMPLE-20260930190939700->./GDI-EE-EXAMPLE-20260930190939700.tar.c4gh./build/GDI-EE-EXAMPLE-20260930190939700;plaintextbuildoutput;the.tar.c4ghholdsthesamecontentencrypted,andarebuildrecreatesit.✔Removethestagingdir?·yesremovedthestagingdir./build/GDI-EE-EXAMPLE-20260930190939700==[5/5]Publish:sendittothenode=============================================================✔Sendthepackagetothenodenow?·UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)uploadedGDI-EE-EXAMPLE-20260930190939700(hidden)toS3bucketgdi-datasets@http://127.0.0.1:3900note:thenodepollsitsbucket,soGDI-EE-EXAMPLE-20260930190939700takesuptoafewminutestoappear(thatgapisexpected).Run`statusGDI-EE-EXAMPLE-20260930190939700`toconfirm,then`publishGDI-EE-EXAMPLE-20260930190939700`tomakeitvisible.Orre-runwith`--wait`toblockuntilthenodehasingestedit.ok:wizardfinisheddatasetGDI-EE-EXAMPLE-20260930190939700package./GDI-EE-EXAMPLE-20260930190939700.tar.c4ghmanifestpackage.yaml(re-runthewizardheretorebuildoredit)shippeduploadedtotheS3bucket;thedatasetlandshiddenandthenodepolls(allow~30sbeforeitevenlooks)nextrun`gdi-dataset-toolstatusGDI-EE-EXAMPLE-20260930190939700`toconfirmingest,then`gdi-dataset-toolpublishGDI-EE-EXAMPLE-20260930190939700`tomakeitvisibleprovider@laptop:~/work$provider@laptop:~/work$gprovider@laptop:~/work$gdprovider@laptop:~/work$gdiprovider@laptop:~/work$gdi-provider@laptop:~/work$gdi-dprovider@laptop:~/work$gdi-daprovider@laptop:~/work$gdi-datprovider@laptop:~/work$gdi-dataprovider@laptop:~/work$gdi-datasprovider@laptop:~/work$gdi-dataseprovider@laptop:~/work$gdi-datasetprovider@laptop:~/work$gdi-dataset-provider@laptop:~/work$gdi-dataset-tprovider@laptop:~/work$gdi-dataset-toprovider@laptop:~/work$gdi-dataset-tooprovider@laptop:~/work$gdi-dataset-toolwprovider@laptop:~/work$gdi-dataset-toolwiprovider@laptop:~/work$gdi-dataset-toolwizprovider@laptop:~/work$gdi-dataset-toolwizaprovider@laptop:~/work$gdi-dataset-toolwizar?Profilename(default)›?NodeserviceURL›?NodeserviceURL›h?NodeserviceURL›ht?NodeserviceURL›htt?NodeserviceURL›http?NodeserviceURL›http:?NodeserviceURL›http:/?NodeserviceURL›http://?NodeserviceURL›http://1?NodeserviceURL›http://12?NodeserviceURL›http://127?NodeserviceURL›http://127.?NodeserviceURL›http://127.0?NodeserviceURL›http://127.0.?NodeserviceURL›http://127.0.0?NodeserviceURL›http://127.0.0.?NodeserviceURL›http://127.0.0.1?NodeserviceURL›http://127.0.0.1:?NodeserviceURL›http://127.0.0.1:3?NodeserviceURL›http://127.0.0.1:38?NodeserviceURL›http://127.0.0.1:380?NodeserviceURL›http://127.0.0.1:3808?NodeserviceURL›http://127.0.0.1:38080?NodemanagementURL›?NodemanagementURL›h?NodemanagementURL›ht?NodemanagementURL›htt?NodemanagementURL›http?NodemanagementURL›http:?NodemanagementURL›http:/?NodemanagementURL›http://?NodemanagementURL›http://1?NodemanagementURL›http://12?NodemanagementURL›http://127?NodemanagementURL›http://127.?NodemanagementURL›http://127.0?NodemanagementURL›http://127.0.?NodemanagementURL›http://127.0.0?NodemanagementURL›http://127.0.0.?NodemanagementURL›http://127.0.0.1?NodemanagementURL›http://127.0.0.1:?NodemanagementURL›http://127.0.0.1:3?NodemanagementURL›http://127.0.0.1:39?NodemanagementURL›http://127.0.0.1:390?NodemanagementURL›http://127.0.0.1:3909?NodemanagementURL›http://127.0.0.1:39090?Trustthisrecipient?(y/n)›yes?Synccatalogsfromthenode?(y/n)›yes?ConfigureS3upload?(y/n)›no?S3bucketname›?S3bucketname›g?S3bucketname›gd?S3bucketname›gdi?S3bucketname›gdi-?S3bucketname›gdi-d?S3bucketname›gdi-da?S3bucketname›gdi-dat?S3bucketname›gdi-data?S3bucketname›gdi-datas?S3bucketname›gdi-datase?S3bucketname›gdi-dataset?S3bucketname›gdi-datasets?S3endpointURL›?S3endpointURL›h?S3endpointURL›ht?S3endpointURL›htt?S3endpointURL›http?S3endpointURL›http:?S3endpointURL›http:/?S3endpointURL›http://?S3endpointURL›http://1?S3endpointURL›http://12?S3endpointURL›http://127?S3endpointURL›http://127.?S3endpointURL›http://127.0?S3endpointURL›http://127.0.?S3endpointURL›http://127.0.0?S3endpointURL›http://127.0.0.?S3endpointURL›http://127.0.0.1?S3endpointURL›http://127.0.0.1:?S3endpointURL›http://127.0.0.1:3?S3endpointURL›http://127.0.0.1:39?S3endpointURL›http://127.0.0.1:390?S3endpointURL›http://127.0.0.1:3900?S3region›?S3region›g?S3region›ga?S3region›gar?S3region›gara?S3region›garag?S3region›garage?Keyprefixinthebucket(blank=thewholebucket)›?S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)›?S3secretaccesskey›?Channelname(blankleavesthewrong-bucketguardoff)›?Channelname(blankleavesthewrong-bucketguardoff)›p?Channelname(blankleavesthewrong-bucketguardoff)›pr?Channelname(blankleavesthewrong-bucketguardoff)›pri?Channelname(blankleavesthewrong-bucketguardoff)›prim?Channelname(blankleavesthewrong-bucketguardoff)›prima?Channelname(blankleavesthewrong-bucketguardoff)›primar?Channelname(blankleavesthewrong-bucketguardoff)›primary?Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)(y/n)›yes?Two-lettercountrycode(e.g.EE)›?Two-lettercountrycode(e.g.EE)›E?Two-lettercountrycode(e.g.EE)›EE?Instituteabbreviationfordatasetids›?Instituteabbreviationfordatasetids›E?Instituteabbreviationfordatasetids›EX?Instituteabbreviationfordatasetids›EXA?Instituteabbreviationfordatasetids›EXAM?Instituteabbreviationfordatasetids›EXAMP?Instituteabbreviationfordatasetids›EXAMPL?Instituteabbreviationfordatasetids›EXAMPLE?VCFheaderstoshipinpackages›❯minimal:structuralkeysonly,nosampleidentifiers(default)with-identifiers:alsothe#CHROMsamplecolumnsand##SAMPLE/##PEDIGREElinesnone:shipnoVCFheadersatall?VCFfileordirectoryofVCFs(Tabcompletes)›?VCFfileordirectoryofVCFs(Tabcompletes)›d?VCFfileordirectoryofVCFs(Tabcompletes)›da?VCFfileordirectoryofVCFs(Tabcompletes)›dat?VCFfileordirectoryofVCFs(Tabcompletes)›data?VCFfileordirectoryofVCFs(Tabcompletes)›data/?VCFfileordirectoryofVCFs(Tabcompletes)›data/g?VCFfileordirectoryofVCFs(Tabcompletes)›data/gd?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-s?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sa?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sam?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-samp?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sampl?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.G?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.GRCh38.vcf.gz?AddanotherVCFfileordirectory?(y/n)›no?Assembly›GRCh37❯GRCh38?DatasetIDprefix›GOE❯GDI?Catalog›❯gdi-aggregatedsynthetic-dataRefreshthelistfromthenode...?Datasettitle›?Datasettitle›S?Datasettitle›Sy?Datasettitle›Syn?Datasettitle›Synt?Datasettitle›Synth?Datasettitle›Synthe?Datasettitle›Synthet?Datasettitle›Syntheti?Datasettitle›Synthetica?Datasettitle›Synthetical?Datasettitle›Syntheticall?Datasettitle›Syntheticalle?Datasettitle›Syntheticallel?Datasettitle›Syntheticallelef?Datasettitle›Syntheticallelefr?Datasettitle›Syntheticallelefre?Datasettitle›Syntheticallelefreq?Datasettitle›Syntheticallelefrequ?Datasettitle›Syntheticallelefreque?Datasettitle›Syntheticallelefrequen?Datasettitle›Syntheticallelefrequenc?Datasettitle›Syntheticallelefrequenci?Datasettitle›Syntheticallelefrequencie?Datasettitle›Syntheticallelefrequencies?Datasettitle›Syntheticallelefrequencies,t?Datasettitle›Syntheticallelefrequencies,tw?Datasettitle›Syntheticallelefrequencies,twe?Datasettitle›Syntheticallelefrequencies,twel?Datasettitle›Syntheticallelefrequencies,twelv?Datasettitle›Syntheticallelefrequencies,twelvep?Datasettitle›Syntheticallelefrequencies,twelvepo?Datasettitle›Syntheticallelefrequencies,twelvepop?Datasettitle›Syntheticallelefrequencies,twelvepopu?Datasettitle›Syntheticallelefrequencies,twelvepopul?Datasettitle›Syntheticallelefrequencies,twelvepopula?Datasettitle›Syntheticallelefrequencies,twelvepopulat?Datasettitle›Syntheticallelefrequencies,twelvepopulati?Datasettitle›Syntheticallelefrequencies,twelvepopulatio?Datasettitle›Syntheticallelefrequencies,twelvepopulation?Datasettitle›Syntheticallelefrequencies,twelvepopulations?Description›?Description›S?Description›Sy?Description›Syn?Description›Synt?Description›Synth?Description›Synthe?Description›Synthet?Description›Syntheti?Description›Synthetica?Description›Syntheticag?Description›Syntheticagg?Description›Syntheticaggr?Description›Syntheticaggre?Description›Syntheticaggreg?Description›Syntheticaggrega?Description›Syntheticaggregat?Description›Syntheticaggregate?Description›Syntheticaggregateda?Description›Syntheticaggregatedal?Description›Syntheticaggregatedall?Description›Syntheticaggregatedalle?Description›Syntheticaggregatedallel?Description›Syntheticaggregatedallelef?Description›Syntheticaggregatedallelefr?Description›Syntheticaggregatedallelefre?Description›Syntheticaggregatedallelefreq?Description›Syntheticaggregatedallelefrequ?Description›Syntheticaggregatedallelefreque?Description›Syntheticaggregatedallelefrequen?Description›Syntheticaggregatedallelefrequenc?Description›Syntheticaggregatedallelefrequenci?Description›Syntheticaggregatedallelefrequencie?Description›Syntheticaggregatedallelefrequenciesf?Description›Syntheticaggregatedallelefrequenciesfo?Description›Syntheticaggregatedallelefrequenciesforan?Description›Syntheticaggregatedallelefrequenciesforano?Description›Syntheticaggregatedallelefrequenciesforanod?Description›Syntheticaggregatedallelefrequenciesforanoded?Description›Syntheticaggregatedallelefrequenciesforanodede?Description›Syntheticaggregatedallelefrequenciesforanodedem?Description›Syntheticaggregatedallelefrequenciesforanodedemo?Description›Syntheticaggregatedallelefrequenciesforanodedemo.?Adddiscoverykeywords?(y/n)›yes?Keywords(comma-separated)(allele-frequency,genomics)›?Recordcohortsize?(y/n)›yes?Distinctsequencedsubjects(blanktoskip)›?Distinctsequencedsubjects(blanktoskip)›5?Distinctsequencedsubjects(blanktoskip)›50?Distinctsequencedsubjects(blanktoskip)›500?Isthissyntheticdata?(y/n)›no?Creatingorganisation›?Creatingorganisation›E?Creatingorganisation›Ex?Creatingorganisation›Exa?Creatingorganisation›Exam?Creatingorganisation›Examp?Creatingorganisation›Exampl?Creatingorganisation›ExampleI?Creatingorganisation›ExampleIn?Creatingorganisation›ExampleIns?Creatingorganisation›ExampleInst?Creatingorganisation›ExampleInsti?Creatingorganisation›ExampleInstit?Creatingorganisation›ExampleInstitu?Creatingorganisation›ExampleInstitut?Creatingorganisation›ExampleInstitute?Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)›⬚Humangenetic⬚Humanepigenomic✔Humangenomic?Conformsto(GDIstandards);Spacetoggles,Enterconfirms›⬚Externallygoverned⬚1+MGcompliant⬚1+MGcohortPUBLIC❯RESTRICTED❯PUBLICRESTRICTED❯CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)CC01.0:publicdomain(typicalforsyntheticdata)CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)❯CC01.0:publicdomain(typicalforsyntheticdata)?Applicablelegislation(Spacetoggles,Enterconfirms)›✔EHDS:Regulation(EU)2025/327⬚GDPR:Regulation(EU)2016/679?AddanotherlegislationELIorIRI?(y/n)›no?UsethestandardGenomeofEuropeAFprovenance?(y/n)›yes?Withholdrowswithallelecountbelow(0=publisheverything)(0)›?Withholdrowswithallelecountbelow(0=publisheverything)(0)›1?Withholdrowswithallelecountbelow(0=publisheverything)(0)›10gdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)4.09MiB/sETA0sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)10.41MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)10.39MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)10.94MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)11.41MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.22KiB/490.81KiB(8%)11.73MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)12.21MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)12.40MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.71MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]102.43KiB/490.81KiB(21%)12.83MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]111.45KiB/490.81KiB(23%)12.84MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)12.84MiB/sETA0s?Writepackage.yaml?›❯WriteitEdititin$EDITORfirstAbort(theanswersarediscarded)?Publishthesepopulations?(y/n)›nogdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)200.03KiB/sETA2sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)1.22MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)4.49MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)6.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)7.97MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.20KiB/490.81KiB(8%)9.17MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]50.99KiB/490.81KiB(10%)10.15MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)11.76MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.89MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)14.09MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)14.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[========>]156.96KiB/490.81KiB(32%)15.15MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=========>]186.63KiB/490.81KiB(38%)15.75MiB/sETA0spacking[=====>]16.50KiB/128.51KiB(13%)27.70MiB/sETA0spacking[=============>]40.50KiB/128.51KiB(32%)25.17MiB/sETA0spacking[================>]48.50KiB/128.51KiB(38%)26.05MiB/sETA0spacking[======================>]65.00KiB/128.51KiB(51%)16.82MiB/sETA0spacking[=============================>]87.50KiB/128.51KiB(68%)16.87MiB/sETA0spacking[=======================================>]117.50KiB/128.51KiB(91%)16.94MiB/sETA0spacking[==========================================>]127.50KiB/128.51KiB(99%)16.96MiB/sETA0spacking[==========================================]132.00KiB/128.51KiB(100%)16.97MiB/sETA0spacking[==========================================]140.00KiB/128.51KiB(100%)16.97MiB/sETA0s?Removethestagingdir?(y/n)›yes?Sendthepackagetothenodenow?›❯UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)Skip:publishelsewhereuploadingGDI-EE-…0939700.tar.c4gh[]0B/149.31KiB(0%)0B/sETA0suploadingGDI-EE-…0939700.tar.c4gh[==============]149.31KiB/149.31KiB(100%)128.86MiB/sETA0s \ No newline at end of file +provider@laptop:~/work$gdi-dataset-toolprovider@laptop:~/work$gdi-dataset-toolwizard==[1/5]Setup:profile&keys====================================================================✔Profilename·defaultthenode'spublicbaseURL.Blankifthereisnonodeyet;youwillhandthepackageoverinstead.✔NodeserviceURL·http://127.0.0.1:38080blankunlessthenodeisco-locatedandyoucanreachitsmanagementplane(typicallyhttp://127.0.0.1:9090).✔NodemanagementURL·http://127.0.0.1:39090recipientfingerprint:sha256:7ea8195a445c0b15bdab652619a0e0f9148af528a4d8407b308b1983264d7e28✔Trustthisrecipient?·yes✔Synccatalogsfromthenode?·yes✔ConfigureS3upload?·yes✔S3bucketname·gdi-datasets✔S3endpointURL·http://127.0.0.1:3900thenode'sregion:garageforGarage,us-east-1forMinIO/Ceph,thebucket'sownforAWS.✔S3region·garagemustmatchthenode'skeyprefixforthisbucketexactly;askthenode'soperator.Aprefixononesideonlyisasilentdesync.✔Keyprefixinthebucket(blank=thewholebucket)·✔S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)·********✔S3secretaccesskey·********thenode'snameforthisbucket'schannel;askthenode'soperator.✔Channelname(blankleavesthewrong-bucketguardoff)·primary✔Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)·yescreatedyourproviderkeyat/home/provider/.config/gdi/keys/provider.c4gh(sha256:ed9dfdad9533205bb97ab69ba226c020ad22e03c5963e4fd5b6f6362f4aba367).Everypackageyoubuildcarriesitasitswriter;ifthenodeacceptspackagesonlyfromlistedkeys,giveitsoperatorthisfingerprint.✔Two-lettercountrycode(e.g.EE)·EEtheORGinGDI-EE-<ORG>-...,e.g.EXAMPLE.✔Instituteabbreviationfordatasetids·EXAMPLEthenodedropstheseatingest;`--header-policy`onbuild/packagealwaysoverridestheprofile.✔VCFheaderstoshipinpackages·minimal:structuralkeysonly,nosampleidentifiers(default)ok:profile'default'writtento/home/provider/.config/gdi/tool.tomlstoredtheS3credentialsin/home/provider/.config/gdi/tool-secrets.toml(owner-only),whereeverylaterrunreadsthem;verifythebucketwith`gdi-dataset-tooldoctor`(itdoesaprobePUT/DELETE)==[2/5]Author:describethedataset(package.yaml)==============================================--Sourcedata------------------------------------------------------------------------------------plain.vcfor.vcf.gz;adirectoryoffersitsVCFstopickfrom.✔VCFfileordirectoryofVCFs(Tabcompletes)·data/gdi-sample.GRCh38.vcf.gz✔AddanotherVCFfileordirectory?·nopreview:populationsinthe1sourceVCF(s),fromtheheaders:[1/1]gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalassemblyGRCh38readfromtheVCFheader;confirmit,orpicktheother✔Assembly·GRCh38--Catalogentry:publishedviathenode'sFAIRDataPoint----------------------------------------✔DatasetIDprefix·GDIorg:EXAMPLE(fromtheprofile)✔Catalog·gdi-aggregated?Datasettitle›Synthetic?Datasettitle›Syntheticallele?Datasettitle›Syntheticallelefrequencies,?Datasettitle›Syntheticallelefrequencies,twelve✔Datasettitle·Syntheticallelefrequencies,twelvepopulations?Description›Synthetic?Description›Syntheticaggregated?Description›Syntheticaggregatedallele?Description›Syntheticaggregatedallelefrequencies?Description›Syntheticaggregatedallelefrequenciesfor?Description›Syntheticaggregatedallelefrequenciesfora?Description›Syntheticaggregatedallelefrequenciesforanode✔Description·Syntheticaggregatedallelefrequenciesforanodedemo.howthedatawasproduced:e.g.sequencing,variantcallingandhowthefrequencieswerecomputed.✔Provenance(blanktoskip)·✔Adddiscoverykeywords?·yes✔Keywords(comma-separated)·allele-frequency,genomicsnumberOfUniqueIndividuals:recommended;buildwarnswhenitisabsent.✔Recordcohortsize?·yes✔Distinctsequencedsubjects(blanktoskip)·500✔Typicalagerangeinyears,e.g.18-90or18+(blanktoskip)·✔Isthissyntheticdata?·yesthefullpublicnameshowninthecatalogentry:nottheIDabbreviationfromsetup.?Creatingorganisation›Example✔Creatingorganisation·ExampleInstitutewhotowritetoaboutthedataset,shownonitscataloguepage:ateamorservicemailboxratherthanaperson's.?Contactname(e.g.Dataaccessteam)›Data?Contactname(e.g.Dataaccessteam)›Dataaccess✔Contactname(e.g.Dataaccessteam)·Dataaccessteam✔Contacte-mail·data@example.org✔Contactwebpage(blanktoskip)·✔Rememberthiscontactpointintheprofileforfuturedatasets?·yes✔Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)·Humangenomicgenomicsbelongsunder"Healthproducts,technologies,data&research";foradiseasecohort,alsotickitsdiseasearea.✔Healththemes(Spacetoggles,Enterconfirms)·Healthproducts,technologies,data&researchoptional:acomplianceclaimaboutTHISdataset;leaveeverythinguntickedtoomitit.✔Conformsto(GDIstandards);Spacetoggles,Enterconfirms·✔Publicationaboutthedataset,asaDOI(blanktoskip)·--Access&legal---------------------------------------------------------------------------------?Accessrights›NON_PUBLIC✔Accessrights·PUBLIC?License›Other(enterIRI)✔License·CC01.0:publicdomain(typicalforsyntheticdata)EUELIIRIs,e.g.http://data.europa.eu/eli/reg/2025/327/oj.TheEHDSrowmaybeunticked;buildthenwarns,and`--strict`failsonthatwarning.✔Applicablelegislation(Spacetoggles,Enterconfirms)·EHDS:Regulation(EU)2025/327✔AddanotherlegislationELIorIRI?·no--Beaconprovenance:citedineveryBeaconanswer------------------------------------------------setsbothafSourceandafSourceReference.✔UsethestandardGenomeofEuropeAFprovenance?·yes--Disclosurecontrols----------------------------------------------------------------------------Smallallelecountscanbewithheldatbuildtime(minAlleleCount).TheunitisALLELES,notpeople:ahomozygouscarriercounts2,so5~threepeople.✔Withholdrowswithallelecountbelow(0=publisheverything)·10gdi-sample.GRCh38.vcf.gz[]0B/490.81KiB(0%)0B/sETA0satminAlleleCount10:6411row(s)belowthefloor,10687moreremovedbycollapsing1447variant(s)toTotal;populationsthatsurvive:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,Totalkeywords:-"allele-frequency"-"genomics"numberOfUniqueIndividuals:500accessRights:"http://publications.europa.eu/resource/authority/access-right/PUBLIC"applicableLegislation:-"http://data.europa.eu/eli/reg/2025/327/oj"license:"http://publications.europa.eu/resource/authority/licence/CC0"creator:-name:"ExampleInstitute"contactPoint:fn:"Dataaccessteam"hasEmail:"mailto:data@example.org"healthCategory:-"http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic"healthTheme:-"https://hdeu-dcat.data.health.europa.eu/resource/authority/health-theme/HEALTH_PRODUCTS"files:-category:"VCF"reference:"GRCh38"files:-"/home/provider/work/data/gdi-sample.GRCh38.vcf.gz"internal:{}config:mode:aggregatedblockRange:10000000minAlleleCount:10afSource:"TheGenomeofEurope"afSourceReference:"https://genomeofeurope.eu/"---------------------------------------------------------------------------------------------------✔Writepackage.yaml?·Writeitwrotepackage.yamlrememberedthecontactpointin/home/provider/.config/gdi/tool.toml:thewizardwillofferitnexttime==[3/5]Build:convert&validate================================================================disclosurepreview:whatthisdatasetwillpublish(1sourceVCF(s)):[1/1]/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalVCFheaders:minimal:structuralkeysonly,nosampleidentifiers✔Publishthesepopulations?·yesnote:recommendedsub-field"hasURL"isabsentonthecontactPointconvertingVCF1/1:/home/provider/work/data/gdi-sample.GRCh38.vcf.gznote:themin_allele_countfloorsuppressed6411populationrow(s)whoseACwasbelowitnote:k-anonymitycoherencecollapsed1447variant(s)totheirTotalrowonly,removing10687furtherpopulationrow(s)regardlessoftheirownAC:awithheldpopulationforcesitssiblingstobewithheldtoonote:11of413emittedvariant(s)haveTotalAF=0:nocopyofthealternatealleleinthewholecohort,andarestoredandservedlikeanyotherrow,asavariantpresentwithfrequency0;removemonomorphicsitesbeforebuilding(`bcftoolsview-c1`)ifthatisnotintendednote:dropped1224of1637inputrecords(0non-primarycontig,0nosupportedALT;symbolic/SV/gVCF/missing,0noallelefrequencyintheinput,1224everypopulationrowwithheldbythek-anonymityfloor)/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:populationsemitted(12):EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalbuiltdatasetGDI-EE-EXAMPLE-20261008135653596->./build/GDI-EE-EXAMPLE-20261008135653596==[4/5]Pack:encryptintoonepackage===========================================================packingparquet1/23:allele-freq.chr21.0.br10000000.6e44adfd808b30ac.parquetpackingparquet2/23:allele-freq.chrM.0.br10000000.6e44adfd808b30ac.parquetpackingparquet3/23:allele-freq.chrX.0.br10000000.6e44adfd808b30ac.parquetpackingparquet4/23:allele-freq.chrX.1.br10000000.6e44adfd808b30ac.parquetpackingparquet5/23:allele-freq.chrX.10.br10000000.6e44adfd808b30ac.parquetpackingparquet6/23:allele-freq.chrX.11.br10000000.6e44adfd808b30ac.parquetpackingparquet7/23:allele-freq.chrX.12.br10000000.6e44adfd808b30ac.parquetpackingparquet8/23:allele-freq.chrX.13.br10000000.6e44adfd808b30ac.parquetpackingparquet9/23:allele-freq.chrX.14.br10000000.6e44adfd808b30ac.parquetpackingparquet10/23:allele-freq.chrX.15.br10000000.6e44adfd808b30ac.parquetpackingparquet11/23:allele-freq.chrX.2.br10000000.6e44adfd808b30ac.parquetpackingparquet12/23:allele-freq.chrX.3.br10000000.6e44adfd808b30ac.parquetpackingparquet13/23:allele-freq.chrX.4.br10000000.6e44adfd808b30ac.parquetpackingparquet14/23:allele-freq.chrX.5.br10000000.6e44adfd808b30ac.parquetpackingparquet15/23:allele-freq.chrX.6.br10000000.6e44adfd808b30ac.parquetpackingparquet16/23:allele-freq.chrX.7.br10000000.6e44adfd808b30ac.parquetpackingparquet17/23:allele-freq.chrX.8.br10000000.6e44adfd808b30ac.parquetpackingparquet18/23:allele-freq.chrX.9.br10000000.6e44adfd808b30ac.parquetpackingparquet19/23:allele-freq.chrY.1.br10000000.6e44adfd808b30ac.parquetpackingparquet20/23:allele-freq.chrY.2.br10000000.6e44adfd808b30ac.parquetpackingparquet21/23:allele-freq.chrY.3.br10000000.6e44adfd808b30ac.parquetpackingparquet22/23:allele-freq.chrY.4.br10000000.6e44adfd808b30ac.parquetpackingparquet23/23:allele-freq.chrY.5.br10000000.6e44adfd808b30ac.parquetpackedGDI-EE-EXAMPLE-20261008135653596->./GDI-EE-EXAMPLE-20261008135653596.tar.c4gh./build/GDI-EE-EXAMPLE-20261008135653596;plaintextbuildoutput;the.tar.c4ghholdsthesamecontentencrypted,andarebuildrecreatesit.✔Removethestagingdir?·yesremovedthestagingdir./build/GDI-EE-EXAMPLE-20261008135653596==[5/5]Publish:sendittothenode=============================================================✔Sendthepackagetothenodenow?·UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)uploadedGDI-EE-EXAMPLE-20261008135653596(hidden)toS3bucketgdi-datasets@http://127.0.0.1:3900note:thenodepollsitsbucket,soGDI-EE-EXAMPLE-20261008135653596takesuptoafewminutestoappear(thatgapisexpected).Run`statusGDI-EE-EXAMPLE-20261008135653596`toconfirm,then`publishGDI-EE-EXAMPLE-20261008135653596`tomakeitvisible.Orre-runwith`--wait`toblockuntilthenodehasingestedit.ok:wizardfinisheddatasetGDI-EE-EXAMPLE-20261008135653596package./GDI-EE-EXAMPLE-20261008135653596.tar.c4ghmanifestpackage.yaml(re-runthewizardheretorebuildoredit)shippeduploadedtotheS3bucket;thedatasetlandshiddenandthenodepolls(allow~30sbeforeitevenlooks)nextrun`gdi-dataset-toolstatusGDI-EE-EXAMPLE-20261008135653596`toconfirmingest,then`gdi-dataset-toolpublishGDI-EE-EXAMPLE-20261008135653596`tomakeitvisibleprovider@laptop:~/work$provider@laptop:~/work$gprovider@laptop:~/work$gdprovider@laptop:~/work$gdiprovider@laptop:~/work$gdi-provider@laptop:~/work$gdi-dprovider@laptop:~/work$gdi-daprovider@laptop:~/work$gdi-datprovider@laptop:~/work$gdi-dataprovider@laptop:~/work$gdi-datasprovider@laptop:~/work$gdi-dataseprovider@laptop:~/work$gdi-datasetprovider@laptop:~/work$gdi-dataset-provider@laptop:~/work$gdi-dataset-tprovider@laptop:~/work$gdi-dataset-toprovider@laptop:~/work$gdi-dataset-tooprovider@laptop:~/work$gdi-dataset-toolwprovider@laptop:~/work$gdi-dataset-toolwiprovider@laptop:~/work$gdi-dataset-toolwizprovider@laptop:~/work$gdi-dataset-toolwizaprovider@laptop:~/work$gdi-dataset-toolwizar?Profilename(default)›?NodeserviceURL›?NodeserviceURL›h?NodeserviceURL›ht?NodeserviceURL›htt?NodeserviceURL›http?NodeserviceURL›http:?NodeserviceURL›http:/?NodeserviceURL›http://?NodeserviceURL›http://1?NodeserviceURL›http://12?NodeserviceURL›http://127?NodeserviceURL›http://127.?NodeserviceURL›http://127.0?NodeserviceURL›http://127.0.?NodeserviceURL›http://127.0.0?NodeserviceURL›http://127.0.0.?NodeserviceURL›http://127.0.0.1?NodeserviceURL›http://127.0.0.1:?NodeserviceURL›http://127.0.0.1:3?NodeserviceURL›http://127.0.0.1:38?NodeserviceURL›http://127.0.0.1:380?NodeserviceURL›http://127.0.0.1:3808?NodeserviceURL›http://127.0.0.1:38080?NodemanagementURL›?NodemanagementURL›h?NodemanagementURL›ht?NodemanagementURL›htt?NodemanagementURL›http?NodemanagementURL›http:?NodemanagementURL›http:/?NodemanagementURL›http://?NodemanagementURL›http://1?NodemanagementURL›http://12?NodemanagementURL›http://127?NodemanagementURL›http://127.?NodemanagementURL›http://127.0?NodemanagementURL›http://127.0.?NodemanagementURL›http://127.0.0?NodemanagementURL›http://127.0.0.?NodemanagementURL›http://127.0.0.1?NodemanagementURL›http://127.0.0.1:?NodemanagementURL›http://127.0.0.1:3?NodemanagementURL›http://127.0.0.1:39?NodemanagementURL›http://127.0.0.1:390?NodemanagementURL›http://127.0.0.1:3909?NodemanagementURL›http://127.0.0.1:39090?Trustthisrecipient?(y/n)›yes?Synccatalogsfromthenode?(y/n)›yes?ConfigureS3upload?(y/n)›no?S3bucketname›?S3bucketname›g?S3bucketname›gd?S3bucketname›gdi?S3bucketname›gdi-?S3bucketname›gdi-d?S3bucketname›gdi-da?S3bucketname›gdi-dat?S3bucketname›gdi-data?S3bucketname›gdi-datas?S3bucketname›gdi-datase?S3bucketname›gdi-dataset?S3bucketname›gdi-datasets?S3endpointURL›?S3endpointURL›h?S3endpointURL›ht?S3endpointURL›htt?S3endpointURL›http?S3endpointURL›http:?S3endpointURL›http:/?S3endpointURL›http://?S3endpointURL›http://1?S3endpointURL›http://12?S3endpointURL›http://127?S3endpointURL›http://127.?S3endpointURL›http://127.0?S3endpointURL›http://127.0.?S3endpointURL›http://127.0.0?S3endpointURL›http://127.0.0.?S3endpointURL›http://127.0.0.1?S3endpointURL›http://127.0.0.1:?S3endpointURL›http://127.0.0.1:3?S3endpointURL›http://127.0.0.1:39?S3endpointURL›http://127.0.0.1:390?S3endpointURL›http://127.0.0.1:3900?S3region›?S3region›g?S3region›ga?S3region›gar?S3region›gara?S3region›garag?S3region›garage?Keyprefixinthebucket(blank=thewholebucket)›?S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)›?S3secretaccesskey›?Channelname(blankleavesthewrong-bucketguardoff)›?Channelname(blankleavesthewrong-bucketguardoff)›p?Channelname(blankleavesthewrong-bucketguardoff)›pr?Channelname(blankleavesthewrong-bucketguardoff)›pri?Channelname(blankleavesthewrong-bucketguardoff)›prim?Channelname(blankleavesthewrong-bucketguardoff)›prima?Channelname(blankleavesthewrong-bucketguardoff)›primar?Channelname(blankleavesthewrong-bucketguardoff)›primary?Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)(y/n)›yes?Two-lettercountrycode(e.g.EE)›?Two-lettercountrycode(e.g.EE)›E?Two-lettercountrycode(e.g.EE)›EE?Instituteabbreviationfordatasetids›?Instituteabbreviationfordatasetids›E?Instituteabbreviationfordatasetids›EX?Instituteabbreviationfordatasetids›EXA?Instituteabbreviationfordatasetids›EXAM?Instituteabbreviationfordatasetids›EXAMP?Instituteabbreviationfordatasetids›EXAMPL?Instituteabbreviationfordatasetids›EXAMPLE?VCFheaderstoshipinpackages›❯minimal:structuralkeysonly,nosampleidentifiers(default)with-identifiers:alsothe#CHROMsamplecolumnsand##SAMPLE/##PEDIGREElinesnone:shipnoVCFheadersatall?VCFfileordirectoryofVCFs(Tabcompletes)›?VCFfileordirectoryofVCFs(Tabcompletes)›d?VCFfileordirectoryofVCFs(Tabcompletes)›da?VCFfileordirectoryofVCFs(Tabcompletes)›dat?VCFfileordirectoryofVCFs(Tabcompletes)›data?VCFfileordirectoryofVCFs(Tabcompletes)›data/?VCFfileordirectoryofVCFs(Tabcompletes)›data/g?VCFfileordirectoryofVCFs(Tabcompletes)›data/gd?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-s?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sa?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sam?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-samp?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sampl?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.G?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.GRCh38.vcf.gz?AddanotherVCFfileordirectory?(y/n)›no?Assembly›GRCh37❯GRCh38?DatasetIDprefix›GOE❯GDI?Catalog›❯gdi-aggregatedsynthetic-dataRefreshthelistfromthenode...?Datasettitle›?Datasettitle›S?Datasettitle›Sy?Datasettitle›Syn?Datasettitle›Synt?Datasettitle›Synth?Datasettitle›Synthe?Datasettitle›Synthet?Datasettitle›Syntheti?Datasettitle›Synthetica?Datasettitle›Synthetical?Datasettitle›Syntheticall?Datasettitle›Syntheticalle?Datasettitle›Syntheticallel?Datasettitle›Syntheticallelef?Datasettitle›Syntheticallelefr?Datasettitle›Syntheticallelefre?Datasettitle›Syntheticallelefreq?Datasettitle›Syntheticallelefrequ?Datasettitle›Syntheticallelefreque?Datasettitle›Syntheticallelefrequen?Datasettitle›Syntheticallelefrequenc?Datasettitle›Syntheticallelefrequenci?Datasettitle›Syntheticallelefrequencie?Datasettitle›Syntheticallelefrequencies?Datasettitle›Syntheticallelefrequencies,t?Datasettitle›Syntheticallelefrequencies,tw?Datasettitle›Syntheticallelefrequencies,twe?Datasettitle›Syntheticallelefrequencies,twel?Datasettitle›Syntheticallelefrequencies,twelv?Datasettitle›Syntheticallelefrequencies,twelvep?Datasettitle›Syntheticallelefrequencies,twelvepo?Datasettitle›Syntheticallelefrequencies,twelvepop?Datasettitle›Syntheticallelefrequencies,twelvepopu?Datasettitle›Syntheticallelefrequencies,twelvepopul?Datasettitle›Syntheticallelefrequencies,twelvepopula?Datasettitle›Syntheticallelefrequencies,twelvepopulat?Datasettitle›Syntheticallelefrequencies,twelvepopulati?Datasettitle›Syntheticallelefrequencies,twelvepopulatio?Datasettitle›Syntheticallelefrequencies,twelvepopulation?Datasettitle›Syntheticallelefrequencies,twelvepopulations?Description›?Description›S?Description›Sy?Description›Syn?Description›Synt?Description›Synth?Description›Synthe?Description›Synthet?Description›Syntheti?Description›Synthetica?Description›Syntheticag?Description›Syntheticagg?Description›Syntheticaggr?Description›Syntheticaggre?Description›Syntheticaggreg?Description›Syntheticaggrega?Description›Syntheticaggregat?Description›Syntheticaggregate?Description›Syntheticaggregateda?Description›Syntheticaggregatedal?Description›Syntheticaggregatedall?Description›Syntheticaggregatedalle?Description›Syntheticaggregatedallel?Description›Syntheticaggregatedallelef?Description›Syntheticaggregatedallelefr?Description›Syntheticaggregatedallelefre?Description›Syntheticaggregatedallelefreq?Description›Syntheticaggregatedallelefrequ?Description›Syntheticaggregatedallelefreque?Description›Syntheticaggregatedallelefrequen?Description›Syntheticaggregatedallelefrequenc?Description›Syntheticaggregatedallelefrequenci?Description›Syntheticaggregatedallelefrequencie?Description›Syntheticaggregatedallelefrequenciesf?Description›Syntheticaggregatedallelefrequenciesfo?Description›Syntheticaggregatedallelefrequenciesforan?Description›Syntheticaggregatedallelefrequenciesforano?Description›Syntheticaggregatedallelefrequenciesforanod?Description›Syntheticaggregatedallelefrequenciesforanoded?Description›Syntheticaggregatedallelefrequenciesforanodede?Description›Syntheticaggregatedallelefrequenciesforanodedem?Description›Syntheticaggregatedallelefrequenciesforanodedemo?Description›Syntheticaggregatedallelefrequenciesforanodedemo.?Provenance(blanktoskip)›?Adddiscoverykeywords?(y/n)›yes?Keywords(comma-separated)(allele-frequency,genomics)›?Recordcohortsize?(y/n)›yes?Distinctsequencedsubjects(blanktoskip)›?Distinctsequencedsubjects(blanktoskip)›5?Distinctsequencedsubjects(blanktoskip)›50?Distinctsequencedsubjects(blanktoskip)›500?Typicalagerangeinyears,e.g.18-90or18+(blanktoskip)›?Isthissyntheticdata?(y/n)›no?Creatingorganisation›?Creatingorganisation›E?Creatingorganisation›Ex?Creatingorganisation›Exa?Creatingorganisation›Exam?Creatingorganisation›Examp?Creatingorganisation›Exampl?Creatingorganisation›ExampleI?Creatingorganisation›ExampleIn?Creatingorganisation›ExampleIns?Creatingorganisation›ExampleInst?Creatingorganisation›ExampleInsti?Creatingorganisation›ExampleInstit?Creatingorganisation›ExampleInstitu?Creatingorganisation›ExampleInstitut?Creatingorganisation›ExampleInstitute?Contactname(e.g.Dataaccessteam)›?Contactname(e.g.Dataaccessteam)›D?Contactname(e.g.Dataaccessteam)›Da?Contactname(e.g.Dataaccessteam)›Dat?Contactname(e.g.Dataaccessteam)›Dataa?Contactname(e.g.Dataaccessteam)›Dataac?Contactname(e.g.Dataaccessteam)›Dataacc?Contactname(e.g.Dataaccessteam)›Dataacce?Contactname(e.g.Dataaccessteam)›Dataacces?Contactname(e.g.Dataaccessteam)›Dataaccesst?Contactname(e.g.Dataaccessteam)›Dataaccesste?Contactname(e.g.Dataaccessteam)›Dataaccesstea?Contactname(e.g.Dataaccessteam)›Dataaccessteam?Contacte-mail›?Contacte-mail›d?Contacte-mail›da?Contacte-mail›dat?Contacte-mail›data?Contacte-mail›data@?Contacte-mail›data@e?Contacte-mail›data@ex?Contacte-mail›data@exa?Contacte-mail›data@exam?Contacte-mail›data@examp?Contacte-mail›data@exampl?Contacte-mail›data@example?Contacte-mail›data@example.?Contacte-mail›data@example.o?Contacte-mail›data@example.or?Contacte-mail›data@example.org?Contactwebpage(blanktoskip)›?Rememberthiscontactpointintheprofileforfuturedatasets?(y/n)›yes?Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)›⬚Humangenetic⬚Humanepigenomic✔Humangenomic?Healththemes(Spacetoggles,Enterconfirms)›⬚Antimicrobialresistance&infectioncontrol⬚Blood-borne&sexuallytransmittedinfections⬚Cancer⬚Climate&planetaryhealth⬚Emergencies,disasters,travel&humanitariansettings⬚Enteric,water-&food-borneinfections⬚Environmental,occupational&radiationhealth(incl.WASH&urban)✔Healthproducts,technologies,data&research⬚Healthsystems,quality,caremodels&determinants⬚Immunization&vaccine-preventablediseases⬚Injuries,envenoming&drowning⬚Life-coursehealth:maternal,newborn,child,adolescent&ageing⬚Mental,neurological&substanceuse⬚Noncommunicablediseases–metabolic&cardiopulmonary⬚Nutrition&foodsecurity⬚Sexual&reproductivehealthandrights⬚Respiratoryinfectiousdiseases⬚Oral,eye&sensoryhealth⬚Neglectedtropical,parasitic&fungalskindiseases⬚Vector-borne&zoonoticviraldiseases?Conformsto(GDIstandards);Spacetoggles,Enterconfirms›⬚Externallygoverned⬚1+MGcompliant⬚1+MGcohort?Publicationaboutthedataset,asaDOI(blanktoskip)›PUBLIC❯RESTRICTED❯PUBLICRESTRICTED❯CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)CC01.0:publicdomain(typicalforsyntheticdata)CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)❯CC01.0:publicdomain(typicalforsyntheticdata)?Applicablelegislation(Spacetoggles,Enterconfirms)›✔EHDS:Regulation(EU)2025/327⬚GDPR:Regulation(EU)2016/679?AddanotherlegislationELIorIRI?(y/n)›no?UsethestandardGenomeofEuropeAFprovenance?(y/n)›yes?Withholdrowswithallelecountbelow(0=publisheverything)(0)›?Withholdrowswithallelecountbelow(0=publisheverything)(0)›1?Withholdrowswithallelecountbelow(0=publisheverything)(0)›10gdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)5.50MiB/sETA0sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)9.69MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)10.32MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)10.93MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)11.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.22KiB/490.81KiB(8%)11.83MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)12.38MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)12.60MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.92MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]102.43KiB/490.81KiB(21%)13.04MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)13.23MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)13.37MiB/sETA0s--Review:package.yamlasitwillbewritten-----------------------------------------------------metadata:prefix:"GDI"org:"EXAMPLE"catalog:"gdi-aggregated"title:"Syntheticallelefrequencies,twelvepopulations"description:"Syntheticaggregatedallelefrequenciesforanodedemo."type:-"https://publications.europa.eu/resource/authority/dataset-type/SYNTHETIC_DATA"?Writepackage.yaml?›❯WriteitEdititin$EDITORfirstAbort(theanswersarediscarded)?Publishthesepopulations?(y/n)›nogdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)487.15KiB/sETA1sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)2.16MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)6.39MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)8.75MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)10.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.20KiB/490.81KiB(8%)11.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)13.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)15.78MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)17.11MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)17.77MiB/sETA0sgdi-sample.GRCh38.vcf.gz[========>]166.86KiB/490.81KiB(34%)18.53MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==========>]196.32KiB/490.81KiB(40%)19.13MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===========>]215.42KiB/490.81KiB(44%)19.46MiB/sETA0spacking[]0B/128.57KiB(0%)0B/sETA0spacking[>]512B/128.57KiB(0%)3.33MiB/sETA0spacking[=====>]17.00KiB/128.57KiB(13%)18.65MiB/sETA0spacking[================>]48.50KiB/128.57KiB(38%)32.62MiB/sETA0spacking[====================>]60.50KiB/128.57KiB(47%)34.89MiB/sETA0spacking[====================>]60.50KiB/128.57KiB(47%)35.34MiB/sETA0spacking[===========================>]79.50KiB/128.57KiB(62%)38.14MiB/sETA0spacking[=============================>]87.50KiB/128.57KiB(68%)38.74MiB/sETA0spacking[=================================>]97.00KiB/128.57KiB(75%)39.20MiB/sETA0spacking[=================================>]101.50KiB/128.57KiB(79%)47.49MiB/sETA0spacking[===================================>]107.00KiB/128.57KiB(83%)48.70MiB/sETA0spacking[=====================================>]112.50KiB/128.57KiB(87%)49.44MiB/sETA0spacking[=======================================>]117.50KiB/128.57KiB(91%)50.32MiB/sETA0spacking[========================================>]122.50KiB/128.57KiB(95%)50.92MiB/sETA0spacking[==========================================]132.00KiB/128.57KiB(100%)51.74MiB/sETA0spacking[==========================================]140.00KiB/128.57KiB(100%)52.45MiB/sETA0s?Removethestagingdir?(y/n)›yes?Sendthepackagetothenodenow?›❯UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)Skip:publishelsewhereuploadingGDI-EE-…5653596.tar.c4gh[]0B/149.31KiB(0%)0B/sETA0suploadingGDI-EE-…5653596.tar.c4gh[==============]149.31KiB/149.31KiB(100%)124.71MiB/sETA0s \ No newline at end of file diff --git a/docs/operating.md b/docs/operating.md index b080172..e6f11d6 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -369,6 +369,9 @@ The ready predicate, all of which must hold: `key_material` read `unavailable`, `/health/ready` stays `503`, encrypted-package ingest is skipped, and `gdi_keyless_degraded` latches to `1`. That does not self-heal, because identities load once at startup, so restart the node once Vault is reachable. + Meanwhile a node whose datasets are encrypted at rest answers publicly as if it had none + (`exists: false`), so route traffic only to a ready node. A Kubernetes readiness probe + does that; `docker run` and Compose do not. A Vault outage after a healthy boot is different: cached DEKs keep serving and ingest retries transiently (§8). - **`key_material`**: the node crypt4gh identity or identities loaded, and under @@ -420,6 +423,11 @@ to `unavailable`, and raises `degraded: true`. Page on `gdi_health_ready{component="s3"} == 0` or the per-bucket `gdi_s3_poll_errors_total`, not on `component="overall"`, which stays `1`. +A bucket that stops answering reads `unavailable` at the next full poll +(`full_poll_interval`, 300 s by default), when its listing fails; each failed listing counts +in `gdi_s3_poll_errors_total`. Once it answers again it reads `ok` within one marker poll +(`marker_poll_interval`, 30 s by default), as does a channel a `SIGHUP` reload restarts. + > **A bucket that is dead from boot delays readiness for the whole node.** The startup > reconcile waits for every bucket to finish or time out before `initial_reconcile` flips > to `done`, which is what makes visibility correct from the first served request. One @@ -1144,6 +1152,7 @@ cause silent drift. The node mints the key and you place it: ```bash # 1. Mint the new key alongside the current one (create-only; never touches the old key). +# With datasets published, it prints a note to keep the old key listed (step 2). gdi-node-standalone --config node.toml identity init --file keys/node-$(date +%Y%m%d).c4gh # 2. Edit [keys].identities: put the new file first, and keep the old one after it. @@ -1225,11 +1234,9 @@ only the header, with no payload re-encryption. > ```bash > gdi-dataset-tool rekey .tar.c4gh --as ~/.config/gdi/keys/provider.c4gh --force > ``` - > Pass `-v` to see which writer key was used. `rekey` announces the fingerprint and - > warns when it mints an ephemeral one, but both go to stderr at verbose level only; at - > default verbosity it reports the recipients alone, so a plain re-key looks identical - > to an `--as` one. On a `warn` or `off` node the plain form is fine, because the - > ephemeral writer is not gated. Full flag reference: + > A plain re-key always warns that it used a fresh ephemeral writer key; an `--as` one + > does not. `-v` also shows the writer key's fingerprint. On a `warn` or `off` node the + > plain form is fine, because the ephemeral writer is not gated. Full flag reference: > [`rekey`](gdi-dataset-tool.md#rekey) in the provider tool doc. 4. Once every package has been re-wrapped, retire the old identity rather than hand-editing Vault. @@ -1429,11 +1436,12 @@ which is this shape: `[[s3.buckets]]` plus `[keys]`, no `[vault]`. See **Where the secrets live (no Vault to hold them).** -- **S3 credentials.** With no `[vault]`, the inline `[[s3.buckets]]` `access_key_id` and - `secret_access_key` are the source; nothing overrides them. Do not commit real keys. - Supply them out-of-band, preferring an env var or a mounted file over an inline literal: - `GDI_NODE__S3__BUCKETS__0__ACCESS_KEY_ID` and `…__SECRET_ACCESS_KEY`, where env always - wins over the file. A process environment is readable via `/proc//environ`, +- **S3 credentials.** With no `[vault]`, each bucket uses the `access_key_id` and + `secret_access_key` of its `[[s3.buckets]]` entry, and + `GDI_NODE__S3__BUCKETS____ACCESS_KEY_ID` / `…__SECRET_ACCESS_KEY` override them when + set. The standard `AWS_*` variables are not read. Do not commit real keys: supply them + through those variables or a `node.toml` mounted from a Secret. A process environment is + readable via `/proc//environ`, `docker inspect` and pod specs; a mounted Secret file is not. On Kubernetes also enable etcd Secret encryption-at-rest (§17). If the bucket is public-read, omit both credentials and the node reads anonymously, holding no S3 secret at all. diff --git a/scripts/demo/drive-wizard.py b/scripts/demo/drive-wizard.py index 1035b85..366546b 100755 --- a/scripts/demo/drive-wizard.py +++ b/scripts/demo/drive-wizard.py @@ -78,14 +78,22 @@ def script(a: argparse.Namespace) -> list[Step]: "Synthetic aggregated allele frequencies for a node demo.", ENTER, ), + ("Provenance (blank to skip)", ENTER), ("Add discovery keywords?", YES), ("Keywords (comma-separated)", ENTER), ("Record cohort size?", YES), ("Distinct sequenced subjects", "500", ENTER), + ("Typical age range in years", ENTER), ("Is this synthetic data?", YES), ("Creating organisation", "Example Institute", ENTER), + ("Contact name", "Data access team", ENTER), + ("Contact e-mail", "data@example.org", ENTER), + ("Contact web page", ENTER), + ("Remember this contact point", YES), ("Health categories", 0.8, ENTER), + ("Health themes", 0.8, ENTER), ("Conforms to", 0.8, ENTER), + ("Publication about the dataset", ENTER), ("Access rights", 0.5, UP, 0.5, ENTER), ("License", 0.5, DOWN, 0.5, ENTER), ("Applicable legislation", 0.8, ENTER),