From 4e8dfece7f6c4d274e73fe8cdfcc9b318dc63e70 Mon Sep 17 00:00:00 2001 From: Erik Jaaniso Date: Thu, 8 Oct 2026 22:20:50 +0300 Subject: [PATCH 1/3] fix(node): reconcile reloaded and returning buckets sooner A monitor started by a reload reconciles on its first tick, and a bucket reconciles as soon as its marker answers again. check-config prints a failure once, warns about a bucket without credentials and no longer lists the placeholder sensitive Beacon. identity init notes a rotation instead of warning of a lost key. The docs correct the readiness, S3 credential, rekey and poll-latency notes. --- .../gdi-node-standalone/src/init_identity.rs | 4 +- .../src/init_identity_file.rs | 95 +++++++++-- crates/gdi-node-standalone/src/main.rs | 154 ++++++++++++------ crates/gdi-node-standalone/src/s3.rs | 32 +++- .../tests/it/binary_cli.rs | 4 + .../tests/it/s3_reconcile/errors.rs | 55 +++++++ .../tests/it/s3_reconcile/reload.rs | 32 ++++ docker-compose.yml | 2 +- docs/deployment.md | 2 +- docs/operating.md | 28 ++-- 10 files changed, 326 insertions(+), 82 deletions(-) diff --git a/crates/gdi-node-standalone/src/init_identity.rs b/crates/gdi-node-standalone/src/init_identity.rs index 8ddf7dd..9934632 100644 --- a/crates/gdi-node-standalone/src/init_identity.rs +++ b/crates/gdi-node-standalone/src/init_identity.rs @@ -167,7 +167,9 @@ pub async fn run( })?; (pem, serialize_public_key(&secret_key.public_key())) } else { - crate::init_identity_file::warn_if_datasets_predate_new_identity(data_dir); + if let Some(notice) = crate::init_identity_file::new_identity_notice(data_dir, None) { + eprintln!("{notice}"); + } let (secret_key, public_key) = generate_keypair(); ( Zeroizing::new(serialize_secret_key(&secret_key)), diff --git a/crates/gdi-node-standalone/src/init_identity_file.rs b/crates/gdi-node-standalone/src/init_identity_file.rs index 6a7125a..ae130e8 100644 --- a/crates/gdi-node-standalone/src/init_identity_file.rs +++ b/crates/gdi-node-standalone/src/init_identity_file.rs @@ -83,7 +83,10 @@ pub fn run( })?; (pem, serialize_public_key(&secret_key.public_key())) } else { - warn_if_datasets_predate_new_identity(&config.service.data_dir); + let kept = kept_identity(config, &target); + if let Some(notice) = new_identity_notice(&config.service.data_dir, kept) { + eprintln!("{notice}"); + } let (secret_key, public_key) = generate_keypair(); ( Zeroizing::new(serialize_secret_key(&secret_key)), @@ -176,22 +179,55 @@ pub(crate) fn data_dir_has_published_datasets(data_dir: &Path) -> bool { .any(|entry| entry.path().join("manifest.json").is_file()) } -/// Warn on stderr when a fresh identity is minted onto a `data_dir` that already holds -/// published datasets. +/// What minting a fresh identity onto `data_dir` should tell the operator, or `None` when +/// no dataset is published there. /// -/// That is the shape of an ephemeral-secrets loss: an in-memory dev `OpenBao` is wiped by a -/// restart, the node loses its identity, and the operator mints a new one over surviving -/// data, leaving the old identity's PME-at-rest parquet unreadable. Minting onto an empty -/// volume stays silent, and the warning is skipped when `--from` imports a specific key. -pub(crate) fn warn_if_datasets_predate_new_identity(data_dir: &Path) { - if data_dir_has_published_datasets(data_dir) { - eprintln!( +/// Without `kept`, it looks like a lost identity: an in-memory dev `OpenBao` wiped by a +/// restart, and a new key minted over data the old one encrypted. With `kept`, another +/// configured key still on disk, it is a file-posture rotation: packages still wrapped to +/// `kept` need it, so it must stay listed. `--from` imports skip this. +pub(crate) fn new_identity_notice(data_dir: &Path, kept: Option<&Path>) -> Option { + if !data_dir_has_published_datasets(data_dir) { + return None; + } + Some(match kept { + Some(kept) => format!( + "note: packages wrapped to {} still need it: list the new key first in \ + [keys].identities and keep {} after it until they are re-keyed.", + kept.display(), + kept.display() + ), + None => format!( "warning: minting a new node identity, but published datasets already exist \ under {}. If the previous identity was lost, PME-at-rest data under those \ datasets stays encrypted to the old key and is unreadable. Proceed only if \ this is a new node; otherwise restore the previous identity first.", data_dir.display() - ); + ), + }) +} + +/// A configured identity other than `target` that is on disk, so it stays when `target` is +/// minted: what makes the mint a rotation. A `--force` over an existing `target` is a +/// replacement, not a rotation. +fn kept_identity<'a>(config: &'a ServiceConfig, target: &Path) -> Option<&'a Path> { + if target.exists() { + return None; + } + config + .keys + .identities + .iter() + .map(PathBuf::as_path) + .find(|path| path.is_file() && !is_same_file(path, target)) +} + +/// Whether `a` and `b` name the same file: compared resolved when both exist, as spelled +/// otherwise. +fn is_same_file(a: &Path, b: &Path) -> bool { + match (a.canonicalize(), b.canonicalize()) { + (Ok(a), Ok(b)) => a == b, + _ => a == b, } } @@ -384,6 +420,43 @@ mod tests { ); } + /// Minting next to a configured key that stays (a rotation) reminds the operator to + /// keep it listed; only a mint with no surviving key warns of a lost identity. + #[test] + fn a_rotation_mint_reminds_rather_than_warns_of_a_lost_identity() { + let tmp = tempfile::tempdir().expect("tmp"); + let data_dir = tmp.path().join("data"); + std::fs::create_dir_all(data_dir.join("ds-1")).expect("mkdir"); + std::fs::write(data_dir.join("ds-1").join("manifest.json"), b"{}").expect("write"); + let current = tmp.path().join("node.c4gh"); + let config = config_with_identity(¤t); + run(&config, false, false, None, None).expect("mint the current key"); + + let kept = kept_identity(&config, &tmp.path().join("node-new.c4gh")); + let rotation = new_identity_notice(&data_dir, kept).expect("datasets exist"); + assert!( + rotation.starts_with("note: packages wrapped to") + && rotation.contains(¤t.display().to_string()), + "{rotation}" + ); + let lost = new_identity_notice(&data_dir, None).expect("datasets exist"); + assert!( + lost.starts_with("warning:") && lost.contains("Proceed only if this is a new node"), + "{lost}" + ); + let mut two_keys = config.clone(); + two_keys + .keys + .identities + .push(tmp.path().join("node-old.c4gh")); + std::fs::write(tmp.path().join("node-old.c4gh"), b"key").expect("write"); + assert_eq!( + kept_identity(&two_keys, ¤t), + None, + "a --force over an existing key is a replacement, not a rotation" + ); + } + #[test] fn refuses_to_overwrite_an_existing_identity() { let tmp = tempfile::tempdir().expect("tmp"); diff --git a/crates/gdi-node-standalone/src/main.rs b/crates/gdi-node-standalone/src/main.rs index 42fb8ce..1ae7dea 100644 --- a/crates/gdi-node-standalone/src/main.rs +++ b/crates/gdi-node-standalone/src/main.rs @@ -1599,10 +1599,11 @@ fn log_startup_summary(config: &ServiceConfig, config_path: &Path, vault_ok: boo /// /// The node does not self-register: an operator hands the beacon URL to the Beacon Network /// aggregator and the `fairdp` URL to the FDP harvester. The aggregated and sensitive beacon -/// collapse to one URL when the node mounts them combined, which is the default; the FDP URL -/// is present only when `[fairdp]` is configured. Each is `base_url` plus the mount path, -/// and must equal what the operator forwards, because the node mints these same IRIs into -/// its FDP output and a mismatched URL breaks the harvester's crawl. +/// share one URL when mounted combined. Mounted split, the default, only the aggregated one +/// is listed: the sensitive one is a zeros-only placeholder. The FDP URL is present only +/// when `[fairdp]` is configured. Each is `base_url` plus the mount path, and must equal what the operator +/// forwards, because the node mints these same IRIs into its FDP output and a mismatched +/// URL breaks the harvester's crawl. fn registration_urls( base_url: &str, aggregated_path: &str, @@ -1618,7 +1619,6 @@ fn registration_urls( )); } else { urls.push(("beacon (aggregated)", format!("{base}{aggregated_path}"))); - urls.push(("beacon (sensitive)", format!("{base}{sensitive_path}"))); } if has_fairdp { urls.push(("fairdp", format!("{base}/fairdp"))); @@ -3144,11 +3144,13 @@ fn check_config(cli_config: Option<&Path>) -> Result<()> { // the config is not loaded yet. Held until return. let _telemetry = logging::init(None, None, "", None, 1.0); let path = ServiceConfig::resolve_path(cli_config); + // The stdout verdict carries the reason; the error only sets the exit code, so stderr + // does not repeat it. let config = match ServiceConfig::load(cli_config) { Ok(config) => config, Err(e) => { println!("config check FAILED ({}): {e}", path.display()); - return Err(anyhow::anyhow!("loading config: {e}")); + return Err(anyhow::anyhow!("config check failed")); } }; match preflight::run(&config) { @@ -3159,7 +3161,7 @@ fn check_config(cli_config: Option<&Path>) -> Result<()> { } Err(e) => { println!("config check FAILED ({}): {e}", path.display()); - Err(anyhow::Error::new(e).context("service config failed startup preflight")) + Err(anyhow::anyhow!("config check failed")) } } } @@ -3419,42 +3421,63 @@ fn print_s3_bucket_summary(config: &gdi_node_standalone_core::config::ServiceCon return; }; let vault_supplies = config.vault.as_ref().is_some_and(|v| v.s3_path.is_some()); - for bucket in &s3.buckets { - // Empty overrides: this is the offline view. `credential_source` is the function - // the live boot uses, so the two cannot disagree about what "config" or "anonymous" - // means. - let local = - gdi_node_standalone::s3::credential_source(bucket, &std::collections::BTreeMap::new()); - let note = if vault_supplies { - " (vault.s3_path is set: Vault may override this at boot)" - } else { - "" - }; - // The prefix decides which objects this channel addresses, and it is invisible - // everywhere else a pre-deploy check looks, so a prefix set on the node and not on - // the writer, or the reverse, reads as "the bucket is empty" with nothing naming the - // cause. Printed as `bucket/prefix`, matching `gdi-dataset-tool`'s `target_label`, - // so the two sides can be compared by eye. - let target = if bucket.prefix.is_empty() { - bucket - .bucket - .clone() - .unwrap_or_else(|| "".to_owned()) - } else { - format!( - "{}/{}", - bucket.bucket.as_deref().unwrap_or(""), - bucket.prefix.trim_end_matches('/') - ) - }; - println!( - " channel {name:<19} target={target} endpoint={endpoint} credentials={local}{note}", - name = bucket.name, - endpoint = bucket.endpoint.as_deref().unwrap_or(""), - ); + for (index, bucket) in s3.buckets.iter().enumerate() { + println!("{}", bucket_summary(index, bucket, vault_supplies)); } } +/// One bucket's `check-config` line, plus a warning when it would run anonymous: right for a +/// public bucket, but also what unexported credentials look like. +#[cfg(feature = "s3")] +fn bucket_summary( + index: usize, + bucket: &gdi_node_standalone_core::config::S3Bucket, + vault_supplies: bool, +) -> String { + // Empty overrides: this is the offline view. `credential_source` is the function + // the live boot uses, so the two cannot disagree about what "config" or "anonymous" + // means. + let local = + gdi_node_standalone::s3::credential_source(bucket, &std::collections::BTreeMap::new()); + let note = if vault_supplies { + " (vault.s3_path is set: Vault may override this at boot)" + } else { + "" + }; + // The prefix decides which objects this channel addresses, and it is invisible + // everywhere else a pre-deploy check looks, so a prefix set on the node and not on + // the writer, or the reverse, reads as "the bucket is empty" with nothing naming the + // cause. Printed as `bucket/prefix`, matching `gdi-dataset-tool`'s `target_label`, + // so the two sides can be compared by eye. + let target = if bucket.prefix.is_empty() { + bucket + .bucket + .clone() + .unwrap_or_else(|| "".to_owned()) + } else { + format!( + "{}/{}", + bucket.bucket.as_deref().unwrap_or(""), + bucket.prefix.trim_end_matches('/') + ) + }; + let warning = if local == "anonymous" && !vault_supplies { + format!( + "\n warning: no credentials, so requests go unsigned. Unless the bucket \ + is public, set GDI_NODE__S3__BUCKETS__{index}__ACCESS_KEY_ID and \ + GDI_NODE__S3__BUCKETS__{index}__SECRET_ACCESS_KEY" + ) + } else { + String::new() + }; + format!( + " channel {name:<19} target={target} endpoint={endpoint} \ + credentials={local}{note}{warning}", + name = bucket.name, + endpoint = bucket.endpoint.as_deref().unwrap_or(""), + ) +} + /// No-op without the `s3` feature: a lite build cannot carry buckets at all. #[cfg(not(feature = "s3"))] fn print_s3_bucket_summary(_config: &gdi_node_standalone_core::config::ServiceConfig) {} @@ -4700,8 +4723,9 @@ fn reload_s3_monitors( // successful reconcile, so say so. `set_channel_health` is otherwise called // only by a monitor, so a channel whose monitor stops would keep its last // `true` forever and read ready while half-blind, the shape - // `register_configured_channels` seeds `false` to avoid at boot. A rotation - // therefore reads `degraded` for one poll, which is the honest state. + // `register_configured_channels` seeds `false` to avoid at boot. The + // replacement reconciles on its first tick, so a rotation reads `degraded` for + // one marker poll, which is the honest state. state.readiness.set_channel_health(&bucket.name, false); // Reached only for an access or behaviour change — credentials, region, // addressing, poll intervals, `write_status` — so the same objects with a @@ -4754,6 +4778,10 @@ fn reload_s3_monitors( if let Some(ingesting) = carried_ingesting { monitor.adopt_ingesting(ingesting); } + // No startup reconcile preceded this monitor, whether it replaces one or + // serves a bucket added live, and its channel reads unhealthy until it + // reconciles. + monitor.reconcile_on_first_tick(); spawn_monitor(&monitor, running); } Err(e) => { @@ -5759,9 +5787,35 @@ mod tests { ); } + /// A bucket with no credentials passes `check-config` with a warning naming the + /// variables; one with credentials, or that Vault may supply, does not warn. + #[cfg(feature = "s3")] + #[test] + fn check_config_warns_of_a_bucket_without_credentials() { + let anonymous = gdi_node_standalone_core::config::S3Bucket { + name: "primary".to_owned(), + ..Default::default() + }; + let line = bucket_summary(1, &anonymous, false); + assert!( + line.contains("credentials=anonymous") + && line.contains("warning: no credentials") + && line.contains("GDI_NODE__S3__BUCKETS__1__ACCESS_KEY_ID"), + "{line}" + ); + assert!(!bucket_summary(1, &anonymous, true).contains("warning")); + let keyed = gdi_node_standalone_core::config::S3Bucket { + access_key_id: Some("id".to_owned()), + secret_access_key: Some("secret".to_owned()), + ..anonymous + }; + assert!(!bucket_summary(1, &keyed, false).contains("warning")); + } + #[test] fn registration_urls_split_beacon_without_fairdp() { - // Distinct mounts yield two beacon URLs; no `[fairdp]` yields no FDP URL. + // Distinct mounts list only the aggregated beacon: the sensitive one is a + // zeros-only placeholder with nothing to register. No `[fairdp]`, no FDP URL. let urls = registration_urls( "https://n.example.org", "/beacon/agg", @@ -5770,16 +5824,10 @@ mod tests { ); assert_eq!( urls, - vec![ - ( - "beacon (aggregated)", - "https://n.example.org/beacon/agg".to_owned() - ), - ( - "beacon (sensitive)", - "https://n.example.org/beacon/sens".to_owned() - ), - ] + vec![( + "beacon (aggregated)", + "https://n.example.org/beacon/agg".to_owned() + )] ); } diff --git a/crates/gdi-node-standalone/src/s3.rs b/crates/gdi-node-standalone/src/s3.rs index 24acf72..67de13b 100644 --- a/crates/gdi-node-standalone/src/s3.rs +++ b/crates/gdi-node-standalone/src/s3.rs @@ -232,6 +232,9 @@ pub struct BucketMonitor { /// leaves the id marked forever, so that dataset would never ingest again until a /// restart. Returning between polls has no such window. retire: Arc, + /// Reconcile on the first tick rather than at the first full poll: a monitor that a + /// reload started has no startup reconcile behind it, and reads unhealthy until one runs. + reconcile_first: bool, } /// The retirement flag and its wakeup, shared by a monitor and its supervisor. @@ -577,6 +580,7 @@ impl BucketMonitor { ingesting: Arc::new(Mutex::new(HashSet::new())), pending_removals: Arc::new(Mutex::new(HashMap::new())), retire: Arc::new(RetireSignal::default()), + reconcile_first: false, } } @@ -640,6 +644,13 @@ impl BucketMonitor { self.ingesting = handle; } + /// Make [`Self::run`] reconcile on its first tick, for a monitor a reload starts. That + /// reconcile is a timer sweep, which processes no package-absence removals. Call it + /// before the monitor is spawned or cloned. + pub fn reconcile_on_first_tick(&mut self) { + self.reconcile_first = true; + } + /// Test-support hook: mark `id` as one this monitor is ingesting, as the download and /// enqueue path does, so a test can reproduce the deleted-while-in-flight state without /// racing a real ingest. Not part of the public API. @@ -687,8 +698,8 @@ impl BucketMonitor { // Monotonic. This is the safety net that catches a marker bump the node missed, so // the wall clock must not disarm it: a backward step from an NTP correction or a // manual set would otherwise read as "not due yet" and postpone the sweep until the - // clock caught back up. - let mut last_full = std::time::Instant::now(); + // clock caught back up. `None` makes the first tick's sweep due at once. + let mut last_full = (!self.reconcile_first).then(std::time::Instant::now); // The startup reconcile has already run for readiness unless the channel was // suppressed at boot, in which case no startup reconcile ran either. Seed the marker // so the first loop iteration does not immediately re-reconcile on an unchanged one. @@ -700,6 +711,10 @@ impl BucketMonitor { self.head_marker().await }; let mut monitor_paused = suppressed_at_boot; + // Whether the last marker check got an answer. The first answer after a failure + // reconciles at once, so a bucket that comes back reads healthy within one marker + // poll rather than one full poll. + let mut marker_answered = last_marker.is_known(); loop { // Wake on the marker poll interval or an operator `SIGUSR1` reconcile trigger, @@ -751,14 +766,16 @@ impl BucketMonitor { monitor_paused = false; } - let due_full = last_full.elapsed() >= full_period; + let due_full = last_full.is_none_or(|t| t.elapsed() >= full_period); let marker = self.head_marker().await; + let answered_again = !marker_answered && marker.is_known(); + marker_answered = marker.is_known(); // Only a definite change opens the removal gate; an `Unknown` from a transient // failure is never a change. A due full poll still reconciles below regardless, // so an `Unknown` never suppresses a needed reconcile either. let marker_changed = marker.opens_removal_gate(&last_marker); - if should_reconcile(triggered, marker_changed, due_full) { + if should_reconcile(triggered, marker_changed, due_full || answered_again) { if triggered { debug!( channel = self.channel(), @@ -766,6 +783,11 @@ impl BucketMonitor { ); } else if marker_changed { debug!(channel = self.channel(), "marker changed; reconciling"); + } else if answered_again && !due_full { + debug!( + channel = self.channel(), + "marker answered again after a failure; reconciling" + ); } else { debug!(channel = self.channel(), "full-poll interval; reconciling"); } @@ -792,7 +814,7 @@ impl BucketMonitor { last_marker = marker; } } - last_full = std::time::Instant::now(); + last_full = Some(std::time::Instant::now()); } } } diff --git a/crates/gdi-node-standalone/tests/it/binary_cli.rs b/crates/gdi-node-standalone/tests/it/binary_cli.rs index 88217ef..b1d68ef 100644 --- a/crates/gdi-node-standalone/tests/it/binary_cli.rs +++ b/crates/gdi-node-standalone/tests/it/binary_cli.rs @@ -119,6 +119,10 @@ fn check_config_rejects_the_shipped_example_and_says_why() { "the verdict must be the PLACEHOLDER rejection, not an incidental failure such as a \ missing file — that is the contract this test exists to pin: {stdout}" ); + assert!( + !stderr.contains("placeholder"), + "the reason is the verdict's, printed once; stderr only ends the run: {stderr}" + ); } #[test] diff --git a/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs b/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs index 345045e..e41b42f 100644 --- a/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs +++ b/crates/gdi-node-standalone/tests/it/s3_reconcile/errors.rs @@ -902,3 +902,58 @@ async fn a_failing_download_backs_off_instead_of_refetching_every_poll() { "a backing-off dataset must not be re-downloaded on every subsequent poll" ); } + +/// A bucket that answers again after an outage reconciles within one marker poll, so its +/// channel reads healthy without waiting for the full poll. The marker is unchanged and the +/// full poll an hour away, so only its answering again can trigger that. +#[tokio::test] +async fn a_bucket_that_answers_again_reconciles_within_one_marker_poll() { + let dark = Arc::new(AtomicBool::new(false)); + let get_dark = Arc::clone(&dark); + let list_dark = Arc::clone(&dark); + let store = HookStore::new("DarkStore") + .on_get(move |_| { + let fail = get_dark.load(Ordering::SeqCst); + Box::pin(async move { fail.then(injected) }) + }) + .on_list(move |_| { + let fail = list_dark.load(Ordering::SeqCst); + Box::pin(async move { fail.then(injected) }) + }) + .into_store(); + let bucket = S3Bucket { + marker_poll_interval: 1, + full_poll_interval: 3_600, + ..bucket_cfg("primary", false) + }; + let rig = Rig::new(store, bucket); + rig.monitor.reconcile().await; + let health = || { + rig.state + .readiness + .channel_health_snapshot() + .get("primary") + .copied() + }; + let running = tokio::spawn(rig.monitor.clone().run()); + tokio::time::sleep(Duration::from_millis(1_500)).await; + + // The outage, as a failed listing would have flagged it. + dark.store(true, Ordering::SeqCst); + rig.state.readiness.set_channel_health("primary", false); + tokio::time::sleep(Duration::from_millis(2_500)).await; + assert_eq!( + health(), + Some(false), + "nothing reconciles while the bucket is dark" + ); + + dark.store(false, Ordering::SeqCst); + poll_until(Duration::from_secs(10), || health() == Some(true)).await; + + rig.monitor.retire_signal().retire(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .expect("a retired monitor returns") + .expect("the monitor task must not panic"); +} diff --git a/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs b/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs index 2ce1822..9282f92 100644 --- a/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs +++ b/crates/gdi-node-standalone/tests/it/s3_reconcile/reload.rs @@ -86,3 +86,35 @@ async fn an_unretired_monitor_keeps_running() { "a monitor nobody retired must still be polling" ); } + +/// A monitor that a reload starts reconciles on its first tick, so its channel, marked +/// unhealthy by the reload, reads healthy within one marker poll. The full poll here is an +/// hour, so only the first-tick reconcile can do that within the bound. +#[tokio::test] +async fn a_monitor_started_by_a_reload_reconciles_on_its_first_tick() { + let bucket = S3Bucket { + marker_poll_interval: 1, + full_poll_interval: 3_600, + ..bucket_cfg("primary", false) + }; + let mut rig = Rig::new(Arc::new(InMemory::new()), bucket); + rig.state.readiness.set_channel_health("primary", false); + rig.monitor.reconcile_on_first_tick(); + + let running = tokio::spawn(rig.monitor.clone().run()); + poll_until(Duration::from_secs(10), || { + rig.state + .readiness + .channel_health_snapshot() + .get("primary") + .copied() + == Some(true) + }) + .await; + + rig.monitor.retire_signal().retire(); + tokio::time::timeout(Duration::from_secs(5), running) + .await + .expect("a retired monitor returns") + .expect("the monitor task must not panic"); +} diff --git a/docker-compose.yml b/docker-compose.yml index ee1f837..4e2812d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: # Gate on the secrets backend's listener so the node does not race it on a cold # `up`. If it loses that race (Vault unreachable at startup) it latches # keyless-degraded and does not self-heal — see the KeylessDegraded alert and - # docs/operating.md §8. `required: false` keeps this valid under whichever single + # docs/operating.md §1. `required: false` keeps this valid under whichever single # secrets profile is active (the other backend service is absent). openbao: condition: service_healthy diff --git a/docs/deployment.md b/docs/deployment.md index 422aa99..6cc7814 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -237,7 +237,7 @@ push output, or with `docker inspect --format '{{index .RepoDigests 0}}' | --- | --- | --- | | **Minimal** (inbox-only, plaintext at rest) | a local end-to-end trial run | `compose/node.minimal.toml` via `docker compose -f docker-compose.minimal.yml up -d --build`: the container form of the [README's Quickstart 1](../README.md#quickstart-1--see-it-work). Its inbox is inside the container — see [Feeding the minimal stack's inbox](#feeding-the-minimal-stacks-inbox) | | **S3** (S3 ingest, disk key, no Vault or PME) | the common networked public-data node | `compose/node.s3.toml` via `COMPOSE_PROFILES=garage docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d`. Mint the node identity first (below), or the `keyinit` one-shot exits 1 and the stack does not start. [`node.quickstart.toml`](../node.quickstart.toml) is the bare-metal counterpart — the same S3-plus-disk-key posture, though it keeps the default split beacon prefixes where `compose/node.s3.toml` mounts one combined `/beacon/v2`; check `check-config`'s registration URLs against whichever you deploy. The same file, with no profile, runs against a bucket of your own: [Running against your own backends](#running-against-your-own-backends) | -| **Full dev** (S3 + Vault/OpenBao + at-rest PME) | exercising the full secrets path | `compose/node.full.toml` via `docker-compose.yml`. `up -d` alone is not enough; provision the secrets first (below) | +| **Full dev** (S3 + Vault/OpenBao + at-rest PME) | exercising the full secrets path | `compose/node.full.toml` via `docker-compose.yml`. `up -d` alone is not enough; provision the secrets first (below). Its bucket uses `prefix = "gdi-node-storage/"`, so a provider profile uploading to it needs the same `prefix`, or the node never sees the uploads | **On Kubernetes, S3 is the supported ingest channel.** The three shapes above are the bare-metal and Compose ones; [`deploy/kubernetes/`](../deploy/kubernetes/README.md) is the diff --git a/docs/operating.md b/docs/operating.md index b080172..e6f11d6 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -369,6 +369,9 @@ The ready predicate, all of which must hold: `key_material` read `unavailable`, `/health/ready` stays `503`, encrypted-package ingest is skipped, and `gdi_keyless_degraded` latches to `1`. That does not self-heal, because identities load once at startup, so restart the node once Vault is reachable. + Meanwhile a node whose datasets are encrypted at rest answers publicly as if it had none + (`exists: false`), so route traffic only to a ready node. A Kubernetes readiness probe + does that; `docker run` and Compose do not. A Vault outage after a healthy boot is different: cached DEKs keep serving and ingest retries transiently (§8). - **`key_material`**: the node crypt4gh identity or identities loaded, and under @@ -420,6 +423,11 @@ to `unavailable`, and raises `degraded: true`. Page on `gdi_health_ready{component="s3"} == 0` or the per-bucket `gdi_s3_poll_errors_total`, not on `component="overall"`, which stays `1`. +A bucket that stops answering reads `unavailable` at the next full poll +(`full_poll_interval`, 300 s by default), when its listing fails; each failed listing counts +in `gdi_s3_poll_errors_total`. Once it answers again it reads `ok` within one marker poll +(`marker_poll_interval`, 30 s by default), as does a channel a `SIGHUP` reload restarts. + > **A bucket that is dead from boot delays readiness for the whole node.** The startup > reconcile waits for every bucket to finish or time out before `initial_reconcile` flips > to `done`, which is what makes visibility correct from the first served request. One @@ -1144,6 +1152,7 @@ cause silent drift. The node mints the key and you place it: ```bash # 1. Mint the new key alongside the current one (create-only; never touches the old key). +# With datasets published, it prints a note to keep the old key listed (step 2). gdi-node-standalone --config node.toml identity init --file keys/node-$(date +%Y%m%d).c4gh # 2. Edit [keys].identities: put the new file first, and keep the old one after it. @@ -1225,11 +1234,9 @@ only the header, with no payload re-encryption. > ```bash > gdi-dataset-tool rekey .tar.c4gh --as ~/.config/gdi/keys/provider.c4gh --force > ``` - > Pass `-v` to see which writer key was used. `rekey` announces the fingerprint and - > warns when it mints an ephemeral one, but both go to stderr at verbose level only; at - > default verbosity it reports the recipients alone, so a plain re-key looks identical - > to an `--as` one. On a `warn` or `off` node the plain form is fine, because the - > ephemeral writer is not gated. Full flag reference: + > A plain re-key always warns that it used a fresh ephemeral writer key; an `--as` one + > does not. `-v` also shows the writer key's fingerprint. On a `warn` or `off` node the + > plain form is fine, because the ephemeral writer is not gated. Full flag reference: > [`rekey`](gdi-dataset-tool.md#rekey) in the provider tool doc. 4. Once every package has been re-wrapped, retire the old identity rather than hand-editing Vault. @@ -1429,11 +1436,12 @@ which is this shape: `[[s3.buckets]]` plus `[keys]`, no `[vault]`. See **Where the secrets live (no Vault to hold them).** -- **S3 credentials.** With no `[vault]`, the inline `[[s3.buckets]]` `access_key_id` and - `secret_access_key` are the source; nothing overrides them. Do not commit real keys. - Supply them out-of-band, preferring an env var or a mounted file over an inline literal: - `GDI_NODE__S3__BUCKETS__0__ACCESS_KEY_ID` and `…__SECRET_ACCESS_KEY`, where env always - wins over the file. A process environment is readable via `/proc//environ`, +- **S3 credentials.** With no `[vault]`, each bucket uses the `access_key_id` and + `secret_access_key` of its `[[s3.buckets]]` entry, and + `GDI_NODE__S3__BUCKETS____ACCESS_KEY_ID` / `…__SECRET_ACCESS_KEY` override them when + set. The standard `AWS_*` variables are not read. Do not commit real keys: supply them + through those variables or a `node.toml` mounted from a Secret. A process environment is + readable via `/proc//environ`, `docker inspect` and pod specs; a mounted Secret file is not. On Kubernetes also enable etcd Secret encryption-at-rest (§17). If the bucket is public-read, omit both credentials and the node reads anonymously, holding no S3 secret at all. From 5d34ca627583aab956063e897398684a0adc32c7 Mon Sep 17 00:00:00 2001 From: Erik Jaaniso Date: Thu, 8 Oct 2026 22:22:04 +0300 Subject: [PATCH 2/3] fix(tool): clearer S3 errors, unique build ids, neutral wording Plaintext endpoints need allow_http, failures are classified by status and retries, and a failed multipart abort is reported. Builds sharing a config dir get distinct ids, a failing build lists every metadata error, and pack warns about a misnamed package. deploy --wait and status say when the node has never seen a dataset, delete says where it read visibility, and provider-facing text no longer names node config keys. URL and enum errors suggest the fix, and the wizard keeps the stored country code and treats Ctrl-C as a cancel. --- crates/core/src/validate_pkg.rs | 91 +++++++- .../src/commands/cmd_build.rs | 170 +++++++++++++- .../src/commands/cmd_check.rs | 5 + .../src/commands/cmd_config.rs | 4 +- .../src/commands/cmd_delete.rs | 38 +++- .../src/commands/cmd_deploy.rs | 64 +++++- .../src/commands/cmd_doctor.rs | 9 +- .../gdi-dataset-tool/src/commands/cmd_keys.rs | 70 +++++- .../gdi-dataset-tool/src/commands/cmd_list.rs | 2 +- .../gdi-dataset-tool/src/commands/cmd_pack.rs | 43 +++- .../src/commands/cmd_publish.rs | 3 +- .../src/commands/cmd_status.rs | 71 +++++- .../src/commands/cmd_upload.rs | 4 +- crates/gdi-dataset-tool/src/s3.rs | 215 +++++++++++++++--- crates/gdi-dataset-tool/src/state.rs | 25 ++ crates/gdi-dataset-tool/src/wizard/mod.rs | 35 +-- crates/gdi-dataset-tool/src/wizard/prompts.rs | 34 ++- crates/gdi-dataset-tool/src/wizard/setup.rs | 59 ++++- docs/gdi-dataset-tool.md | 36 ++- 19 files changed, 847 insertions(+), 131 deletions(-) diff --git a/crates/core/src/validate_pkg.rs b/crates/core/src/validate_pkg.rs index 9ee60c2..4dd2339 100644 --- a/crates/core/src/validate_pkg.rs +++ b/crates/core/src/validate_pkg.rs @@ -979,17 +979,26 @@ pub fn canonical_bcp47(tag: &str) -> Option { Some(tag.to_ascii_lowercase()) } -/// Reject a value not in the allowed set. +/// Reject a value not in the allowed set, naming what is allowed: the IRI a bare last +/// segment such as `PUBLIC` stands for, or else the whole set (at most a handful). /// /// # Errors /// /// Returns [`CoreError::InvalidManifest`] if `value` is not in `allowed`. pub fn validate_enum(field: &str, value: &str, allowed: &[&str]) -> CoreResult<()> { if allowed.contains(&value) { - Ok(()) - } else { - Err(invalid(&format!("{field} value {value:?} is not allowed"))) + return Ok(()); } + let hint = allowed + .iter() + .find(|iri| iri.rsplit(['/', '#']).next() == Some(value)) + .map_or_else( + || format!("allowed: {}", allowed.join(", ")), + |iri| format!("write the full IRI {iri}"), + ); + Err(invalid(&format!( + "{field} value {value:?} is not allowed; {hint}" + ))) } /// Validate `type`: at least one IRI, each from the closed [`DATASET_TYPES`] set. @@ -1092,12 +1101,28 @@ fn reject_disallowed_scheme(field: &str, value: &str, parsed: &url::Url) -> Core let scheme = parsed.scheme(); if !ALLOWED_IRI_SCHEMES.contains(&scheme) { return Err(invalid(&format!( - "{field} value {value:?} uses a disallowed URI scheme {scheme:?} (allowed: {ALLOWED_IRI_SCHEMES:?})" + "{field} value {value:?} uses a disallowed URI scheme {scheme:?} (allowed: {ALLOWED_IRI_SCHEMES:?}){}", + full_address_hint(value) ))); } Ok(()) } +/// The hint for a web address without its scheme, such as `node.example.org` (unparseable) +/// or `localhost:8080` (parsed with the scheme `localhost`). A full URI with another scheme, +/// such as `mailto:x@example.org`, gets none. +fn full_address_hint(value: &str) -> &'static str { + let schemeless = match value.split_once(':') { + None => true, + Some((_, rest)) => rest.starts_with(|c: char| c.is_ascii_digit()), + }; + if schemeless && !value.contains("://") { + "; write a web address in full, with its scheme (https://…)" + } else { + "" + } +} + /// Validate an IRI metadata field: parses as an absolute URL with a scheme and a /// non-empty host (or, for `urn:`/`w3id`-style identifiers, a non-empty path), /// carries no `IRIREF`-forbidden character (see RFC 3987 §2.2), uses an allowed @@ -1116,8 +1141,12 @@ pub fn validate_iri(field: &str, value: &str) -> CoreResult<()> { ))); } reject_iri_unsafe_chars(field, value)?; - let parsed = url::Url::parse(value) - .map_err(|_| invalid(&format!("{field} value {value:?} is not a valid IRI/URL")))?; + let parsed = url::Url::parse(value).map_err(|_| { + invalid(&format!( + "{field} value {value:?} is not a valid IRI/URL{}", + full_address_hint(value) + )) + })?; reject_disallowed_scheme(field, value, &parsed)?; // Require a scheme and either a host (http/https/...) or a non-empty // opaque path (urn:, ...). A relative or scheme-less string fails @@ -1209,8 +1238,12 @@ pub fn validate_url(field: &str, value: &str) -> CoreResult<()> { ))); } reject_iri_unsafe_chars(field, value)?; - let parsed = url::Url::parse(value) - .map_err(|_| invalid(&format!("{field} value {value:?} is not a valid URL")))?; + let parsed = url::Url::parse(value).map_err(|_| { + invalid(&format!( + "{field} value {value:?} is not a valid URL{}", + full_address_hint(value) + )) + })?; // Same scheme allow-list as `validate_iri`: a `hasURL`/reference value is emitted as // an IRI into the public RDF too. reject_disallowed_scheme(field, value, &parsed) @@ -1955,6 +1988,31 @@ mod tests { ); } + /// A web address written without its scheme is told so, whether it fails to parse or + /// parses with its host as the scheme; a full URI with a disallowed scheme is not. + #[test] + fn a_schemeless_address_is_told_to_start_with_https() { + for (field, value) in [ + ("service_url", "localhost:8080"), + ("service_url", "node.example.org"), + ("hasURL", "www.example.org/data"), + ] { + let iri = validate_iri(field, value).unwrap_err().to_string(); + let url = validate_url(field, value).unwrap_err().to_string(); + for err in [iri, url] { + assert!(err.contains("with its scheme"), "{value:?}: {err}"); + } + } + for value in [ + "file:///etc/passwd", + "javascript:alert(1)", + "mailto:x@example.org", + ] { + let err = validate_iri("license", value).unwrap_err().to_string(); + assert!(!err.contains("with its scheme"), "{value:?}: {err}"); + } + } + #[test] fn disallowed_iri_scheme_rejected() { // Schemes that `url::Url::parse` accepts but that must never reach the public FDP @@ -2405,7 +2463,20 @@ mod tests { ) .is_ok() ); - assert!(super::validate_enum("accessRights", "PUBLIC", super::ACCESS_RIGHTS).is_err()); + let bare = super::validate_enum("accessRights", "PUBLIC", super::ACCESS_RIGHTS) + .unwrap_err() + .to_string(); + assert!( + bare.contains(&format!("write the full IRI {}", super::ACCESS_RIGHTS[0])), + "a bare last segment names its IRI: {bare}" + ); + let unknown = super::validate_enum("accessRights", "OPEN", super::ACCESS_RIGHTS) + .unwrap_err() + .to_string(); + assert!( + super::ACCESS_RIGHTS.iter().all(|iri| unknown.contains(iri)), + "an unknown value lists the allowed set: {unknown}" + ); assert!( super::validate_health_category( "http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic" diff --git a/crates/gdi-dataset-tool/src/commands/cmd_build.rs b/crates/gdi-dataset-tool/src/commands/cmd_build.rs index 8d5ee77..c3514a3 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_build.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_build.rs @@ -40,7 +40,7 @@ use gdi_node_standalone_core::{ validate_parquet::{ ParquetCaps, ParquetScan, check_data_file_block_range, validate_parquet_dir, }, - validate_pkg::validate_package, + validate_pkg::{validate_package, validate_package_collect_all}, }; use crate::{ToolError, cli::BuildArgs}; @@ -104,7 +104,9 @@ fn diagnostic_count(diagnostics: &[(String, Diagnostic)], severity: Severity) -> .count() } -/// The build timestamp: `--build-epoch` when pinned, otherwise the wall clock. +/// The build timestamp: `--build-epoch` when pinned, otherwise the wall clock, moved past +/// any millisecond another build sharing the config directory has already used (see +/// [`unique_build_epoch`]). /// /// This is the build's only wall-clock read, so pinning it makes the generated /// `datasetId` — and therefore `manifest.json` — a pure function of the inputs. @@ -112,7 +114,7 @@ fn diagnostic_count(diagnostics: &[(String, Diagnostic)], severity: Severity) -> /// # Errors /// /// Returns a [`ToolError`] when the system clock is before the Unix epoch. -fn resolve_build_epoch(args: &BuildArgs) -> Result { +fn resolve_build_epoch(args: &BuildArgs, config_path: Option<&Path>) -> Result { match args.build_epoch { Some(pinned) => { let resolved = validate_pinned_epoch(pinned, now_unix_millis()?)?; @@ -134,10 +136,68 @@ fn resolve_build_epoch(args: &BuildArgs) -> Result { ); Ok(resolved) } - None => now_unix_millis(), + // A dry run writes nothing durable, so it takes the clock as it is. + None if args.dry_run => now_unix_millis(), + None => Ok(unique_build_epoch(config_path, now_unix_millis()?)), } } +/// The file in the config directory that holds the last build epoch minted there. +const LAST_BUILD_EPOCH_FILE: &str = ".last-build-epoch"; + +/// `now`, or one past the last millisecond a build sharing this config directory used, +/// whichever is later. +/// +/// A dataset id varies only by its millisecond, so builds started together would mint the +/// same id. They take turns under a lock on [`LAST_BUILD_EPOCH_FILE`]; without a config +/// directory, or when the lock fails, the clock stands alone. +fn unique_build_epoch(config_path: Option<&Path>, now: u64) -> u64 { + let Some(dir) = + gdi_node_standalone_core::config::config_base_dir(config_path).filter(|d| d.is_dir()) + else { + return now; + }; + reserve_build_epoch(&dir, now).unwrap_or_else(|e| { + crate::output::warn(&format!( + "warning: could not reserve a build epoch in {}: {e}; using the clock alone", + dir.display() + )); + now + }) +} + +/// How far ahead of the clock a stored build epoch may be and still come from a concurrent +/// build. Further ahead, it is a clock that was wrong, and following it would date every +/// later build in the future. +const CONCURRENT_BUILD_WINDOW_MS: u64 = 60_000; + +/// Take the lock on `dir`'s [`LAST_BUILD_EPOCH_FILE`], and record and return the first +/// millisecond at or after `now` that is later than the one it holds. +fn reserve_build_epoch(dir: &Path, now: u64) -> std::io::Result { + use std::io::{Seek as _, SeekFrom}; + + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(dir.join(LAST_BUILD_EPOCH_FILE))?; + // Held until `file` drops at the end of this function. + file.lock()?; + let mut last = String::new(); + file.read_to_string(&mut last)?; + let epoch = last + .trim() + .parse::() + .ok() + .filter(|&last| last < now.saturating_add(CONCURRENT_BUILD_WINDOW_MS)) + .map_or(now, |last| now.max(last.saturating_add(1))); + file.set_len(0)?; + file.seek(SeekFrom::Start(0))?; + file.write_all(epoch.to_string().as_bytes())?; + Ok(epoch) +} + /// Clock-skew tolerance for a pinned `--build-epoch`: 24h, absorbing timezone mistakes and /// NTP skew. A build-epoch further ahead than this is rejected. const BUILD_EPOCH_FUTURE_SKEW_MS: u64 = 24 * 60 * 60 * 1000; @@ -270,11 +330,35 @@ fn check_converted_output( /// The three-tier obligation's advisories are all "you should fix this", so they count. /// The report's `notes` describe a valid, deliberate configuration (a declared but inert /// `hideLowerCounts`) and never do. +/// +/// The gate stops at its first error, so a failing package is also checked with +/// [`validate_package_collect_all`], and its other errors are printed first, so they can all +/// be fixed in one go. fn report_metadata_diagnostics( package: &PackageYaml, catalogs: Option<&BTreeMap>, ) -> Result { - let report = validate_package(package, catalogs).map_err(|e| ToolError::from(&e))?; + let report = validate_package(package, catalogs).map_err(|first| { + let first_message = first.to_string(); + let others: Vec = validate_package_collect_all(package, catalogs) + .errors + .iter() + .map(ToString::to_string) + .filter(|message| *message != first_message) + .collect(); + for other in &others { + crate::output::always(&format!("error: {other}")); + } + let mut err = ToolError::from(&first); + if !others.is_empty() { + err.message = format!( + "{} (and {} more metadata error(s) above)", + err.message, + others.len() + ); + } + err + })?; for warning in &report.warnings { crate::output::always(&format!("warning: {warning}")); } @@ -461,7 +545,7 @@ pub fn build_staging_dir( .org .as_deref() .ok_or_else(|| ToolError::user("package metadata.org is required"))?; - let epoch_millis = resolve_build_epoch(args)?; + let epoch_millis = resolve_build_epoch(args, config_path)?; let dataset_id = generate_dataset_id(prefix, &country_code, org, epoch_millis) .map_err(|e| ToolError::from(&e))?; @@ -1803,6 +1887,30 @@ config: load_package(&yaml, false).expect("a plain load warns but still builds"); } + /// A package with errors in several sections reports every section's error from one + /// run, not one per run; the gate's own error still decides the failure. + #[test] + fn a_build_reports_the_metadata_errors_of_every_section() { + let dir = tempfile::tempdir().expect("tempdir"); + let yaml = dir.path().join("package.yaml"); + let text = test_util::sample_package_yaml() + .replace( + "\"http://publications.europa.eu/resource/authority/access-right/PUBLIC\"", + "PUBLIC", + ) + .replace("hasEmail: \"mailto:", "hasEmail: \"REPLACE: "); + std::fs::write(&yaml, text).expect("write yaml"); + let package = load_package(&yaml, false).expect("loads"); + + let err = report_metadata_diagnostics(&package, None).expect_err("several errors"); + assert!( + err.message.contains("REPLACE:") + && err.message.contains("more metadata error(s) above"), + "{}", + err.message + ); + } + /// The property an allow-list has and a deny-list cannot: a `##` key nobody has /// invented yet does not survive. /// @@ -2414,4 +2522,54 @@ config: ); assert!(text.contains("FATAL: node key compromised") && text.contains("typo")); } + + /// Builds a script starts together, sharing one config directory, each get their own + /// millisecond, so none mints another's dataset id. + #[test] + fn parallel_builds_sharing_a_config_dir_get_distinct_epochs() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().to_path_buf(); + let threads: Vec<_> = (0..8) + .map(|_| { + let path = path.clone(); + std::thread::spawn(move || reserve_build_epoch(&path, 1_000).expect("reserve")) + }) + .collect(); + let mut epochs: Vec = threads + .into_iter() + .map(|t| t.join().expect("join")) + .collect(); + epochs.sort_unstable(); + assert_eq!(epochs, (1_000..1_008).collect::>()); + } + + /// A later clock is used as it is; an earlier one moves past the last epoch minted. + #[test] + fn a_reserved_epoch_follows_the_clock_and_never_repeats() { + let dir = tempfile::tempdir().expect("tempdir"); + assert_eq!( + reserve_build_epoch(dir.path(), 1_000).expect("reserve"), + 1_000 + ); + assert_eq!( + reserve_build_epoch(dir.path(), 5_000).expect("reserve"), + 5_000 + ); + assert_eq!( + reserve_build_epoch(dir.path(), 4_000).expect("reserve"), + 5_001 + ); + // A stored value from a clock that was far ahead is not followed. + let skewed = tempfile::tempdir().expect("tempdir"); + let ahead = 1_000_000; + assert_eq!( + reserve_build_epoch(skewed.path(), ahead).expect("reserve"), + ahead + ); + let now = ahead - CONCURRENT_BUILD_WINDOW_MS - 1; + assert_eq!( + reserve_build_epoch(skewed.path(), now).expect("reserve"), + now + ); + } } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_check.rs b/crates/gdi-dataset-tool/src/commands/cmd_check.rs index 2ac8f5f..8f6eef4 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_check.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_check.rs @@ -415,6 +415,11 @@ async fn explain_unserved(id: &str, oracle: Option>) -> String { Ok(NodeProbe::Unknown) => { ": the node has never seen this id. Install it first (`upload` or `deploy`).".to_owned() } + Ok(NodeProbe::Failed(status)) => format!( + ": it must be visible and ingested (the management plane at {} answered {status}, \ + so the node's own state could not be read).", + oracle.base + ), Ok(NodeProbe::Unreachable) => format!( ": it must be visible and ingested (the management plane at {} did not answer, \ so the node's own state could not be read).", diff --git a/crates/gdi-dataset-tool/src/commands/cmd_config.rs b/crates/gdi-dataset-tool/src/commands/cmd_config.rs index 98eabd7..689f64b 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_config.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_config.rs @@ -55,8 +55,8 @@ service_url = "REPLACE: https://node.example" [profiles.default.s3] # Non-secret S3 settings only (for credentials, see the top). # bucket = "REPLACE: my-bucket" -# Key prefix inside the bucket; must match the node's [[s3.buckets]].prefix for this -# channel, or the tool writes where the node does not look. +# Key prefix inside the bucket; must match the node's key prefix for this channel (ask +# the node's operator), or the tool writes where the node does not look. # prefix = "gdi-node-storage/" # The node channel that monitors this bucket (catches wrong-bucket publishes). # channel = "REPLACE: primary" diff --git a/crates/gdi-dataset-tool/src/commands/cmd_delete.rs b/crates/gdi-dataset-tool/src/commands/cmd_delete.rs index e0373cb..891775f 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_delete.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_delete.rs @@ -187,22 +187,28 @@ fn ensure_s3_package_exists( /// Turn a [`DeleteRefusal`] into the user-facing error, or `Ok(())` to proceed. /// -/// The two arms carry different remedies on purpose: a confirmed-visible dataset wants +/// The arms carry different remedies on purpose: a confirmed-visible dataset wants /// `unpublish`, while an unconfirmable one wants an oracle (`--management-url`) — telling an /// operator to `unpublish` a dataset whose state nobody can read is advice they cannot act -/// on. +/// on. A visible refusal names where it read the state, since only the oracle lags an +/// `unpublish`. /// /// # Errors /// /// Returns a [`ToolError`] whenever `refusal` is `Some`. fn refusal_error(id: &str, refusal: Option<&DeleteRefusal>) -> Result<(), ToolError> { match refusal { - Some(DeleteRefusal::Visible) => Err(ToolError::user(format!( + Some(DeleteRefusal::Visible(VisibilitySource::Oracle)) => Err(ToolError::user(format!( "dataset {id} reads as visible on the node's state oracle. If you just ran \ `unpublish`, the node has not reconciled it yet (the oracle lags the sidecar \ by a few seconds); wait briefly and retry, or pass --force. Otherwise \ `unpublish` it first." ))), + Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)) => Err(ToolError::user(format!( + "dataset {id} reads as visible in the bucket's state sidecar ({id}.state.json), \ + read because the node's state oracle gave no answer. `unpublish` it first, or \ + pass --force." + ))), Some(DeleteRefusal::Unconfirmable) => Err(ToolError::user(format!( "cannot confirm whether dataset {id} is currently visible: no management-plane \ state oracle is reachable, so the visible-guard cannot run. `delete` is \ @@ -217,12 +223,21 @@ fn refusal_error(id: &str, refusal: Option<&DeleteRefusal>) -> Result<(), ToolEr /// Why a delete was refused, or `None` to proceed. #[derive(Debug, PartialEq, Eq)] enum DeleteRefusal { - /// The dataset is confirmed visible and `--force` was not given. - Visible, + /// The dataset is confirmed visible, by the named source, and `--force` was not given. + Visible(VisibilitySource), /// Visibility could not be determined at all — no oracle, no sidecar. Unconfirmable, } +/// Where a delete read the dataset's visibility. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +enum VisibilitySource { + /// The node's management-plane state oracle. + Oracle, + /// The bucket's `{id}.state.json` sidecar, read when the node is unreachable. + Sidecar, +} + /// Whether a delete must be refused, and why. /// /// `node_visible` is the authoritative management-plane visibility when the node is @@ -246,10 +261,11 @@ fn delete_refusal( if force { return None; } - match node_visible.or(sidecar_visible) { - Some(true) => Some(DeleteRefusal::Visible), - Some(false) => None, - None => Some(DeleteRefusal::Unconfirmable), + match (node_visible, sidecar_visible) { + (Some(true), _) => Some(DeleteRefusal::Visible(VisibilitySource::Oracle)), + (None, Some(true)) => Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)), + (Some(false), _) | (None, Some(false)) => None, + (None, None) => Some(DeleteRefusal::Unconfirmable), } } @@ -287,13 +303,13 @@ mod tests { // Authoritative node says visible -> refused. assert_eq!( delete_refusal(Some(true), None, false), - Some(DeleteRefusal::Visible) + Some(DeleteRefusal::Visible(VisibilitySource::Oracle)) ); assert_eq!(delete_refusal(Some(false), None, false), None); // Node unreachable: fall back to the S3 bucket sidecar. assert_eq!( delete_refusal(None, Some(true), false), - Some(DeleteRefusal::Visible) + Some(DeleteRefusal::Visible(VisibilitySource::Sidecar)) ); assert_eq!(delete_refusal(None, Some(false), false), None); } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs b/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs index e47fdfa..d81b29e 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_deploy.rs @@ -323,12 +323,17 @@ pub(crate) fn wait_for_ingest( // Set on the poll that saw a pre-existing error and declined to blame this drop for it, // so the timeout can say which of the two ambiguous things happened instead of guessing. let mut stale_error = false; + // Whether every poll so far said the node does not know this id: then the timeout says + // the node never saw the package, not "still processing". + let mut unknown_every_poll = true; crate::output::note(&format!("waiting for the node to ingest {id}...")); loop { // One detailed probe per poll. The plain `probe_node_state` folds `410 Gone` into // the same `None` as "unreachable" and "not seen yet", which would keep `--wait` // polling an id the node has permanently refused and then blame a slow ingest. - match runtime::block_on(crate::state::probe_node_state_detailed(base, id))? { + let probe = runtime::block_on(crate::state::probe_node_state_detailed(base, id))?; + unknown_every_poll &= matches!(probe, crate::state::NodeProbe::Unknown); + match probe { // Tombstoned: terminal, and the node says why. crate::state::NodeProbe::Gone(reason) => { return Err(ToolError::user(format!( @@ -424,6 +429,15 @@ pub(crate) fn wait_for_ingest( timeout.as_secs(), ))); } + if unknown_every_poll { + return Err(ToolError::user(format!( + "timed out after {}s: the node still does not know {id}. Either it has \ + not picked it up yet (raise --wait-timeout), or it watches another inbox, \ + bucket, key prefix or channel than this was written to; check with the \ + node's operator.", + timeout.as_secs() + ))); + } return Err(ToolError::user(format!( "timed out after {}s waiting for the node to ingest {id}. It may still be \ processing. Check with `status {id}`, or raise --wait-timeout.", @@ -909,6 +923,54 @@ mod tests { ); } + /// A wait the node answers "unknown id" to throughout says the node has not seen the + /// package and names the likely mismatch, rather than "it may still be processing". + #[test] + fn a_wait_the_node_never_answers_for_names_the_mismatch() { + use std::io::{Read as _, Write as _}; + + // The management plane: `/version` answers, and the state oracle answers `status`. + let wait_against = |status: &'static str| { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind stub"); + let addr = listener.local_addr().expect("stub addr"); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let mut buf = [0u8; 1024]; + let n = s.read(&mut buf).unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..n]); + let reply = if request.starts_with("GET /version") { + "HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}".to_owned() + } else { + format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\n\r\n") + }; + let _ = s.write_all(reply.as_bytes()); + } + }); + wait_for_ingest( + &format!("http://{addr}"), + "GDI-EE-UTARTU-20260409143052837", + std::time::Duration::from_secs(1), + None, + None, + None, + ) + .expect_err("an id the node never reports cannot finish ingesting") + .message + }; + + let unknown = wait_against("404 Not Found"); + assert!( + unknown.contains("does not know") && unknown.contains("key prefix"), + "must say the node has not seen it, and why it might not: {unknown}" + ); + let failing = wait_against("503 Service Unavailable"); + assert!( + !failing.contains("does not know"), + "a failing oracle says nothing about the id: {failing}" + ); + } + /// The truth table the `--wait` loop hangs on. /// /// `error` is terminal, and the signature is what makes it decidable. Comparing diff --git a/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs b/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs index 0942dd8..a920d7f 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_doctor.rs @@ -459,9 +459,8 @@ fn check_beacon_url(active: &Profile, responds: bool) -> Check { Check::pass( "beacon-url", format!( - "custom [beacon].aggregated_base_path; the default-shaped {url} does not \ - answer here. Do not register it; take the exact URL from the node's \ - `gdi-node-standalone check-config` output" + "{url} does not answer here: if the node is up, its Beacon is at another \ + path. Do not register this one; ask the node's operator for the exact URL" ), ) } @@ -537,8 +536,8 @@ fn check_s3_writable(active: &Profile) -> Check { Ok(()) => Check::pass( "s3-writable", format!( - "{} accepts a probe PUT/DELETE (this must match the node's [[s3.buckets]].prefix \ - for this channel)", + "{} accepts a probe PUT/DELETE (this must match the node's key prefix for \ + this channel)", s3::target_label(cfg) ), ), diff --git a/crates/gdi-dataset-tool/src/commands/cmd_keys.rs b/crates/gdi-dataset-tool/src/commands/cmd_keys.rs index 7b0fc48..2aceab7 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_keys.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_keys.rs @@ -458,6 +458,31 @@ fn primary(identities: &[PathBuf]) -> Result<&PathBuf, ToolError> { /// key cannot be read/parsed, or a newly generated key cannot be written. pub fn load_or_generate_provider_secret( config_path: Option<&Path>, +) -> Result { + load_or_mint_provider_secret(config_path, |path, fingerprint| { + crate::output::warn(&minted_identity_warning(path, fingerprint)); + }) +} + +/// Make sure the primary provider identity exists, creating it (`0o600`) if missing: the +/// `wizard setup` step. Creating it is expected there, so it is announced as a progress +/// line, not as the warning `pack` gives when a key it expected is missing. +/// +/// # Errors +/// +/// As [`load_or_generate_provider_secret`]. +pub fn ensure_provider_secret(config_path: Option<&Path>) -> Result<(), ToolError> { + load_or_mint_provider_secret(config_path, |path, fingerprint| { + crate::output::progress(&created_identity_line(path, fingerprint)); + }) + .map(drop) +} + +/// Load the primary provider identity, or mint it and pass its path and fingerprint to +/// `announce`. +fn load_or_mint_provider_secret( + config_path: Option<&Path>, + announce: fn(&Path, &str), ) -> Result { let identities = resolve_identities(config_path)?; let secret_path = primary(&identities)?; @@ -471,10 +496,7 @@ pub fn load_or_generate_provider_secret( // resets who the node believes authored every package built from here on. It happens // whenever the resolved config dir holds no identity, which a `--config` typo or a // fresh checkout produces just as easily as a genuine first run. - crate::output::warn(&minted_identity_warning( - secret_path, - &public_key_fingerprint(&sk.public_key()), - )); + announce(secret_path, &public_key_fingerprint(&sk.public_key())); Ok(sk) } @@ -485,11 +507,21 @@ pub fn load_or_generate_provider_secret( /// accepted under `writer_policy = enforce`. fn minted_identity_warning(path: &Path, fingerprint: &str) -> String { format!( - "warning: no provider identity at {}; generated a new one ({fingerprint}). This key \ - is the crypt4gh writer provenance of every package built from here on; a node with \ - [ingest].writer_policy = enforce will reject them until this fingerprint is \ - allow-listed. If you expected an existing key, check --config / GDI_CONFIG_DIR \ - before packaging.", + "warning: no provider identity at {}; generated a new one ({fingerprint}). Packages \ + built from now on are written with it, so a node that only accepts listed keys \ + rejects them until it lists this fingerprint. If you expected an existing key, \ + check --config / GDI_CONFIG_DIR.", + path.display() + ) +} + +/// The line `wizard setup` prints when it creates the provider identity, worded for any +/// node. +fn created_identity_line(path: &Path, fingerprint: &str) -> String { + format!( + " created your provider key at {} ({fingerprint}). Every package you build carries \ + it as its writer; if the node accepts packages only from listed keys, give its \ + operator this fingerprint.", path.display() ) } @@ -711,7 +743,7 @@ mod tests { "the fingerprint is the actionable part; it is what gets allow-listed: {msg}" ); assert!( - msg.contains("writer_policy"), + msg.contains("listed keys"), "say why it matters, not just that a key was made: {msg}" ); assert!( @@ -737,6 +769,24 @@ mod tests { ); } + /// `wizard setup` creates the key that a re-run and a later `pack` load. + #[test] + fn setup_creates_the_key_that_pack_then_loads() { + let dir = tempfile::tempdir().expect("tempdir"); + let cfg = dir.path().join("config.toml"); + std::fs::write(&cfg, "").expect("write config"); + + ensure_provider_secret(Some(&cfg)).expect("creates on a first run"); + let created = load_provider_secret_readonly(Some(&cfg)).expect("the key exists"); + ensure_provider_secret(Some(&cfg)).expect("loads on a re-run"); + let loaded = load_or_generate_provider_secret(Some(&cfg)).expect("pack loads it"); + assert_eq!( + public_key_fingerprint(&created.public_key()), + public_key_fingerprint(&loaded.public_key()), + "a re-run of setup must keep the key" + ); + } + #[cfg(unix)] #[test] fn a_world_readable_provider_key_is_reported_on_every_read() { diff --git a/crates/gdi-dataset-tool/src/commands/cmd_list.rs b/crates/gdi-dataset-tool/src/commands/cmd_list.rs index fd65734..685e20d 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_list.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_list.rs @@ -90,7 +90,7 @@ fn warn_if_datasets_sit_outside_the_prefix(active: &gdi_node_standalone_core::co "warning: no datasets {here}, but {total} elsewhere in the bucket ({where_}). \ A prefix set on one side only, or set differently on each, is a desync in which \ every signal is a success: uploads land where the node never lists. Check it against \ - the node's [[s3.buckets]].prefix for this channel" + the node's key prefix for this channel with the node's operator" )); } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_pack.rs b/crates/gdi-dataset-tool/src/commands/cmd_pack.rs index c776477..9fe2de9 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_pack.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_pack.rs @@ -43,6 +43,7 @@ pub fn run( let started = std::time::Instant::now(); let dataset_id = dataset_id_from_staging(&args.staging)?; let output = resolve_output(args.out.as_deref(), &dataset_id); + warn_if_misnamed(&output, &dataset_id); crate::output::note(&format!( "packing {dataset_id}: {} -> {}", args.staging.display(), @@ -106,6 +107,7 @@ pub fn run_package( // 2. Pack the freshly built staging dir. let output = resolve_output(args.out.as_deref(), &built.dataset_id); + warn_if_misnamed(&output, &built.dataset_id); crate::output::note(&format!( "packing {} -> {}", built.dataset_id, @@ -218,7 +220,7 @@ fn manifest_dataset_id(staging: &Path) -> Result { /// except an existing directory receives the package inside it as /// `{datasetId}.tar.c4gh` — else `{datasetId}.tar.c4gh` in the current working directory. fn resolve_output(out_flag: Option<&Path>, dataset_id: &str) -> PathBuf { - let file_name = format!("{dataset_id}.tar.c4gh"); + let file_name = format!("{dataset_id}{}", crate::pkgio::TAR_C4GH_SUFFIX); match out_flag { // An existing directory means "write the package into here" — the same noun // `build -o ` takes, and what the flag's "defaults to the cwd" contract @@ -233,6 +235,26 @@ fn resolve_output(out_flag: Option<&Path>, dataset_id: &str) -> PathBuf { } } +/// Warn when `-o` names the package other than `{dataset_id}.tar.c4gh`. +fn warn_if_misnamed(output: &Path, dataset_id: &str) { + if let Some(warning) = misnamed_package_warning(output, dataset_id) { + crate::output::warn(&warning); + } +} + +/// The warning for a package named other than `{dataset_id}.tar.c4gh`: any name packs and +/// unpacks, but `upload` and `deploy` read the dataset id from the name. +fn misnamed_package_warning(output: &Path, dataset_id: &str) -> Option { + let expected = format!("{dataset_id}{}", crate::pkgio::TAR_C4GH_SUFFIX); + (output.file_name() != Some(std::ffi::OsStr::new(&expected))).then(|| { + format!( + "warning: {} is not named {expected}; rename it before `upload` or `deploy`, \ + which read the dataset id from the file name", + output.display() + ) + }) +} + /// Resolve the encryption recipients: the node recipient (mandatory) followed by /// the provider's own recipient (so the provider can decrypt its own packages). /// @@ -1031,6 +1053,25 @@ mod tests { ); } + /// A package named other than `{id}.tar.c4gh` is warned about at `pack`, not first + /// refused at `upload`. + #[test] + fn a_package_not_named_for_its_id_is_warned_about() { + let dir = tempfile::tempdir().unwrap(); + let warning = misnamed_package_warning(&dir.path().join("wrong.tar.c4gh"), ID) + .expect("a misnamed package is warned about"); + assert!( + warning.starts_with("warning:") + && warning.contains(&format!("is not named {ID}.tar.c4gh")), + "{warning}" + ); + assert_eq!( + misnamed_package_warning(&resolve_output(Some(dir.path()), ID), ID), + None, + "the name `pack` chooses itself is the right one" + ); + } + /// With neither `--recipient` nor a profile, the error must name `--recipient`. A bare /// "no profiles configured" leaves a first-time provider with no next step, and /// `pack --recipient ` needs no profile at all. diff --git a/crates/gdi-dataset-tool/src/commands/cmd_publish.rs b/crates/gdi-dataset-tool/src/commands/cmd_publish.rs index 12a08d9..9449527 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_publish.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_publish.rs @@ -170,7 +170,8 @@ fn publish_result( // path this message is most often printed from would be a worse pointer than none. let text = format!( "{verb} {id}: wrote the {target_state} sidecar on {target}. The node applies it on \ - its next scan; its management-plane /datasets/{id}/state reports when it has" + its next scan; its management-plane /datasets/{id}/state shows {target_state} once \ + it has" ); let payload = serde_json::json!({ "schemaVersion": 1, diff --git a/crates/gdi-dataset-tool/src/commands/cmd_status.rs b/crates/gdi-dataset-tool/src/commands/cmd_status.rs index a4ae358..4da8911 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_status.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_status.rs @@ -147,6 +147,28 @@ fn s3_visibility( } } +/// The node's authoritative state for `id`, and whether its management plane said it does +/// not know the id. Only an explicit management URL can say so: without one the base falls +/// back to the public plane, which answers 404 for every id. +fn probe_node_state( + active: &Profile, + id: &str, + probe_node: bool, + override_base: Option<&str>, +) -> Result<(Option, bool), ToolError> { + if !probe_node { + return Ok((None, false)); + } + let probe = state::resolve_node_probe(active, id, override_base)?; + let not_seen = matches!(probe, Some(state::NodeProbe::Unknown)) + && (override_base.is_some() || active.management_url.is_some()); + let node_state = match probe { + Some(state::NodeProbe::Live(state)) => Some(*state), + _ => None, + }; + Ok((node_state, not_seen)) +} + /// Report on a single dataset id. In `text` mode prints the report (returning /// `None`); in `json` mode builds and returns the [`StatusSummary`] for the caller /// to emit (so `--all` can aggregate them into one JSON array). @@ -163,11 +185,7 @@ fn report_one( // management_url (else service_url). `probe_node` is the up-front reachability // verdict: `--all` sets it false when the state base is down, so this skips a // per-dataset 5s timeout that would only resolve to `None` anyway (the S3 path). - let node_state = if probe_node { - state::resolve_node_state(active, id, override_base)? - } else { - None - }; + let (node_state, not_seen) = probe_node_state(active, id, probe_node, override_base)?; // The provenance of the `state` value reported below: the authoritative node oracle, // or the S3-derived (non-authoritative) fallback. let state_source = if node_state.is_some() { @@ -216,7 +234,15 @@ fn report_one( let sync = status::channel_sync(channel, package_present, drifted); if format == OutputFormat::Text { - print_report(id, &node_status, channel, sync, store, etag.as_deref())?; + print_report( + id, + &node_status, + not_seen, + channel, + sync, + store, + etag.as_deref(), + )?; if diff { print_diff( id, @@ -272,6 +298,7 @@ fn report_one( fn print_report( id: &str, node_status: &NodeStatus, + not_seen: bool, channel: Channel, sync: Sync, store: Option<&Store>, @@ -282,13 +309,25 @@ fn print_report( .map_or_else(String::new, |v| format!("\tvisibility: {}", v.as_str())); println!( "{id}\tstate: {}\tchannel: {}\tsync: {}{vis_label}", - node_status.display_label(), + state_label(node_status, not_seen), channel_label(channel), sync.label() ); Ok(()) } +/// The text report's `state:` value. `not_seen` means the management plane answered that it +/// does not know the id, which the generic `unavailable` would blame on an unreachable plane. +fn state_label(node_status: &NodeStatus, not_seen: bool) -> String { + if not_seen && matches!(node_status, NodeStatus::Unavailable) { + "unknown to the node (not picked up yet, or it watches another location than this \ + profile writes to)" + .to_owned() + } else { + node_status.display_label() + } +} + /// Whether the dataset is drifted: the node's served state disagrees with the S3 /// writeback's freshness (a stale `source_signature` ⇒ the node has not caught up /// with the latest upload, i.e. drift between local-intended and node-served). @@ -839,4 +878,22 @@ mod tests { // No writeback / no etag (non-pending) is not drift. assert!(!is_drifted(&visible, None, Some("etag-1"))); } + + /// A management plane that answered "unknown id" is reported as such, not as the + /// generic `unavailable` that blames an unreachable plane; any real state still wins. + #[test] + fn a_dataset_the_management_plane_does_not_know_is_reported_as_unknown() { + let unknown = state_label(&NodeStatus::Unavailable, true); + assert!(unknown.starts_with("unknown to the node"), "{unknown}"); + assert_eq!( + state_label(&NodeStatus::Unavailable, false), + NodeStatus::Unavailable.display_label(), + "without that answer the generic label stands" + ); + let visible = NodeStatus::State { + state: "visible".to_owned(), + error_message: None, + }; + assert_eq!(state_label(&visible, true), "visible"); + } } diff --git a/crates/gdi-dataset-tool/src/commands/cmd_upload.rs b/crates/gdi-dataset-tool/src/commands/cmd_upload.rs index 70d031e..e1df90b 100644 --- a/crates/gdi-dataset-tool/src/commands/cmd_upload.rs +++ b/crates/gdi-dataset-tool/src/commands/cmd_upload.rs @@ -171,7 +171,9 @@ fn wait_outcome( payload["status"] = serde_json::Value::String("error".to_owned()); payload["reason"] = serde_json::Value::String(e.to_string()); ( - format!("upload of {id} to {target} rejected by the node: {e}"), + // Not "rejected": a timeout is not a rejection, and a rejection's own + // message already says so. + format!("upload of {id} to {target} not confirmed by the node: {e}"), payload, ) } diff --git a/crates/gdi-dataset-tool/src/s3.rs b/crates/gdi-dataset-tool/src/s3.rs index 9ac97fe..8d70858 100644 --- a/crates/gdi-dataset-tool/src/s3.rs +++ b/crates/gdi-dataset-tool/src/s3.rs @@ -187,6 +187,21 @@ fn conn_params( .filter(|b| !b.is_empty()) .ok_or_else(|| ToolError::user("profile s3.bucket is required for S3 operations"))?; + // `object_store` refuses a plaintext endpoint without `allow_http`, but its error + // (`... builder error`) names neither the scheme nor the setting. + if !s3.allow_http + && endpoint + .trim_start() + .to_ascii_lowercase() + .starts_with("http://") + { + return Err(ToolError::user(format!( + "profile s3.endpoint {endpoint} is plaintext http:// but allow_http is false. \ + Use an https:// endpoint, or set allow_http = true in this profile's s3 block \ + (for a local dev backend such as MinIO or Garage)" + ))); + } + // The prefix must survive `object_store::path::Path` normalization unchanged, as it // must on the node; the rule is imported rather than restated. The node rejects a bad // prefix at boot. Without this the tool would accept every spelling the node refuses, @@ -195,10 +210,9 @@ fn conn_params( // only one of them normalizes is an upload the node never lists. gdi_node_standalone_core::config::validate_key_prefix(&s3.prefix).map_err(|why| { ToolError::user(format!( - "profile s3.prefix {:?} is invalid: {why}. It must match the node's \ - [[s3.buckets]].prefix for this channel exactly; the node refuses to boot on \ - this spelling, and a prefix set on only one side means uploads the node never \ - lists", + "profile s3.prefix {:?} is invalid: {why}. It must match the node's key prefix \ + for this channel exactly (ask the node's operator); the node rejects this \ + spelling, and a prefix set on only one side means uploads the node never lists", s3.prefix )) })?; @@ -404,9 +418,10 @@ const MAX_INFLIGHT_PARTS: usize = 16; /// bounded chunks so an arbitrarily large `.tar.c4gh` is never buffered whole in /// memory and is not subject to the single-PUT 5 GiB ceiling. /// -/// Every failure path aborts the upload, best effort, before returning: a read error, a -/// failed part, and a failed `complete`. That is why this drives the `MultipartUpload` -/// handle itself instead of handing it to `object_store::WriteMultipart`. That writer +/// Every failure path aborts the upload before returning: a read error, a failed part, and +/// a failed `complete`; an abort that fails too is added to the error. That is why this +/// drives the `MultipartUpload` handle itself instead of handing it to +/// `object_store::WriteMultipart`. That writer /// aborts only when `complete()` fails inside `finish()`, has no `Drop` impl, keeps the /// upload handle private, and `finish()` consumes it, so a part failing while the writer is /// still being fed returns `Err` with the handle gone and nothing left that could abort. @@ -441,12 +456,24 @@ async fn stream_file_multipart( progress.finish(); Ok(()) } - Err(e) => { + Err(mut e) => { // Cancel what is still in flight before aborting: a part that lands after the // abort would re-create the upload this is trying to erase. `JoinSet::shutdown` // aborts each task and awaits it, as `WriteMultipart` does. parts.shutdown().await; - let _ = upload.abort().await; + // `NoSuchUpload` means it is gone already, as when the answer to a `complete` + // that succeeded was lost. + if let Err(abort) = upload.abort().await + && !(matches!(abort, object_store::Error::NotFound { .. }) + || abort.to_string().contains("NoSuchUpload")) + { + e.message = format!( + "{}; aborting the multipart upload of {key} failed too ({}), so its parts \ + stay in the bucket until a lifecycle rule expires them or someone aborts it", + e.message, + terse(&abort) + ); + } Err(e) } } @@ -671,12 +698,7 @@ pub async fn probe_writable(store: &Store) -> Result<(), ToolError> { store .put(&key, PutPayload::from(b"probe".to_vec())) .await - .map_err(|e| { - classify_object_store_error( - "S3 bucket is not writable (your token may lack write permission)", - e, - ) - })?; + .map_err(|e| classify_object_store_error("S3 bucket is not writable", e))?; // Best-effort cleanup; a failed delete does not fail the probe. let _ = store.delete(&key).await; Ok(()) @@ -931,9 +953,8 @@ fn generic_error_text(e: &object_store::Error) -> Option { /// Whether an object-store failure is an untyped (`Generic`) 403 / `AccessDenied`. fn is_denied_object_store(e: &object_store::Error) -> bool { - generic_error_text(e).is_some_and(|msg| { - msg.contains("403") || msg.contains("Forbidden") || msg.contains("AccessDenied") - }) + generic_error_text(e) + .is_some_and(|msg| msg.contains("Forbidden") || msg.contains("AccessDenied")) } /// Whether an object-store failure means "that key is not there", however the backend @@ -973,20 +994,26 @@ fn classify_get_error(key: &ObjPath, id: &str, e: object_store::Error) -> ToolEr } /// A short remediation hint for a non-2xx object-store failure, keyed on the HTTP status -/// embedded in the error text (`object_store` 0.13 exposes no typed status). Empty when no +/// embedded in the error text (`object_store` exposes no typed status). Empty when no /// known status is present: connectivity errors already carry rich messages, so this /// targets only genuine non-2xx responses. +/// +/// A status is matched with its reason phrase (`502 Bad Gateway`), never as bare digits, +/// which the request URL (a dataset id is a timestamp) and the elapsed time also contain. fn status_hint(msg: &str) -> &'static str { - if msg.contains("403") || msg.contains("Forbidden") { + if msg.contains("Forbidden") { ". Check the bucket policy and credentials: `gdi-dataset-tool profiles` shows whether \ this profile has both keys (in tool-secrets.toml, or GDI_TOOL__PROFILES____... \ with underscores for hyphens). Then run `gdi-dataset-tool doctor`" - } else if msg.contains("404") || msg.contains("NoSuchBucket") { + } else if msg.contains("404 Not Found") || msg.contains("NoSuchBucket") { ". Check the bucket name and endpoint, then run `gdi-dataset-tool doctor`" - } else if msg.contains("400") || msg.contains("Bad Request") { + } else if msg.contains("Bad Request") { ". Likely a region, path-style or endpoint misconfiguration; run \ `gdi-dataset-tool doctor`" - } else if msg.contains("500") || msg.contains("502") || msg.contains("504") { + } else if msg.contains("500 Internal Server Error") + || msg.contains("502 Bad Gateway") + || msg.contains("504 Gateway Timeout") + { ". Server-side error; retry, then run `gdi-dataset-tool doctor` if it persists" } else { "" @@ -995,20 +1022,23 @@ fn status_hint(msg: &str) -> &'static str { /// Heuristic: whether an `object_store` failure looks transient (retry may fix). /// -/// `object_store` 0.13 has no dedicated transient variant. A throttled or `5xx` request +/// `object_store` has no dedicated transient variant. A throttled or `5xx` request /// surfaces as [`object_store::Error::Generic`] only after the client's own retry budget is /// exhausted, with the HTTP status in the wrapped source, so this keys on that `Display` /// text via [`generic_error_text`]. Kept conservative to keep false positives off the /// transient (exit-3) path. +/// +/// `object_store` retries only `5xx`, `429`, `408` and transport errors, so its retry clause +/// (`after retries, max_retries: …`) alone marks a transient failure. Statuses are +/// otherwise matched by reason phrase, as in [`status_hint`]. fn is_transient_object_store(e: &object_store::Error) -> bool { generic_error_text(e).is_some_and(|msg| { - msg.contains("503") - || msg.contains("429") + msg.contains(", max_retries: ") || msg.contains("Service Unavailable") || msg.contains("Too Many Requests") || msg.contains("SlowDown") || msg.contains("timed out") - || msg.contains("timeout") + || msg.contains("RequestTimeout") }) } @@ -1906,6 +1936,85 @@ mod tests { })); } + /// A plaintext endpoint without `allow_http` is refused before any client is built, + /// with a message naming the setting, rather than surfacing `object_store`'s + /// `builder error` from whichever verb ran first. + #[test] + fn a_plaintext_endpoint_needs_allow_http_and_the_refusal_names_it() { + for endpoint in ["http://localhost:9000", "HTTP://localhost:9000"] { + let s3 = super::ProfileS3 { + endpoint: Some(endpoint.to_owned()), + bucket: Some("b".to_owned()), + ..Default::default() + }; + let Err(err) = super::conn_params(&s3) else { + panic!("{endpoint} without allow_http must be refused") + }; + assert!( + err.to_string().contains("allow_http = true"), + "the refusal must name the setting: {err}" + ); + let s3 = super::ProfileS3 { + allow_http: true, + ..s3 + }; + assert!(super::conn_params(&s3).is_ok(), "allow_http opts in"); + } + } + + /// A failure is classified by what `object_store` reports, never by the digits in the + /// request URL, whose key is a dataset id (a timestamp). A request it retried until the + /// budget ran out is transient; it never retries a `403`. + #[test] + fn a_failed_request_is_classified_by_its_status_not_its_url() { + let key = "GDI-EE-UTARTU-20260409150040312.tar.c4gh"; // gitleaks:allow - dataset id + assert!( + key.contains("500") && key.contains("403"), + "the fixture id must exercise the hazard" + ); + // The texts `object_store` 0.14 writes, with and without its retry clause. + let failure = |retried: bool, cause: &str| object_store::Error::Generic { + store: "S3", + source: format!( + "Error performing PUT http://localhost:9000/b/{key} in 1.5s{} - {cause}", + if retried { + ", after 10 retries, max_retries: 10, retry_timeout: 180s " + } else { + "" + } + ) + .into(), + }; + let exit = |e| classify_object_store_error("uploading k", e).exit_code; + + assert_eq!( + exit(failure(true, "HTTP error: error sending request")), + crate::EXIT_TRANSIENT + ); + assert_eq!( + exit(failure( + true, + "Server returned non-2xx status code: 502 Bad Gateway: " + )), + crate::EXIT_TRANSIENT + ); + assert_eq!( + exit(failure( + false, + "Server returned non-2xx status code: 403 Forbidden: no" + )), + crate::EXIT_AUTH + ); + let no_response = + classify_object_store_error("uploading k", failure(false, "HTTP error: builder error")); + assert_eq!(no_response.exit_code, crate::EXIT_USER); + assert!( + !no_response.message.contains("Server-side"), + "no response, so no server-side hint: {}", + no_response.message + ); + } + #[tokio::test] async fn list_classifies_visible_and_hidden_and_ignores_noise() { let store = mem_store(); @@ -2018,6 +2127,7 @@ mod tests { let s3 = ProfileS3 { bucket: Some("b".to_owned()), endpoint: Some("http://localhost:9000".to_owned()), + allow_http: true, access_key_id: Some("k".to_owned()), ..ProfileS3::default() }; @@ -2145,7 +2255,7 @@ mod tests { async fn a_failed_part_upload_aborts_the_multipart_upload() { let aborted = Arc::new(AtomicBool::new(false)); let store = HookStore::new() - .fail_every_part(Arc::clone(&aborted)) + .fail_every_part(Arc::clone(&aborted), false) .into_store(); let err = upload_package_bytes( @@ -2170,6 +2280,39 @@ mod tests { ); } + /// An abort that fails too is reported, naming the key: nothing else would ever show the + /// parts it leaves behind. + #[tokio::test] + async fn a_failed_abort_is_reported_with_the_key_it_leaves_behind() { + let aborted = Arc::new(AtomicBool::new(false)); + let store = HookStore::new() + .fail_every_part(Arc::clone(&aborted), true) + .into_store(); + let dataset = id(10); + + let err = upload_package_bytes( + &store, + &PackageStore::new(store.clone()), + &dataset, + b"a small package".to_vec(), + false, + ) + .await + .unwrap_err(); + + assert!( + aborted.load(Ordering::SeqCst), + "the abort must still be attempted" + ); + assert!( + err.message.contains(&format!( + "aborting the multipart upload of {dataset}{TAR_C4GH_SUFFIX} failed too" + )) && err.message.contains("lifecycle rule"), + "a failed abort must name the key and how its parts go away: {}", + err.message + ); + } + /// A package spanning several parts must round-trip byte-for-byte. /// /// Every other upload test here fits in a single part, so this is what exercises the @@ -2212,10 +2355,12 @@ mod tests { /// Supply the [`MultipartUpload`] a `put_multipart` hands back. type MultipartHook = Box Box + Send + Sync>; - /// A [`MultipartUpload`] whose every part fails, recording whether it was aborted. + /// A [`MultipartUpload`] whose every part fails, recording whether it was aborted, and + /// whose abort fails too when `abort_fails`. #[derive(Debug)] struct FailingUpload { aborted: Arc, + abort_fails: bool, } #[async_trait] @@ -2238,6 +2383,12 @@ mod tests { async fn abort(&mut self) -> object_store::Result<()> { self.aborted.store(true, Ordering::SeqCst); + if self.abort_fails { + return Err(object_store::Error::Generic { + store: "test", + source: "abort refused".into(), + }); + } Ok(()) } } @@ -2285,11 +2436,13 @@ mod tests { } /// Hand back an upload whose every part fails, flagging `aborted` if the caller - /// aborts it: the seam for "the network dropped mid-upload". - fn fail_every_part(mut self, aborted: Arc) -> Self { + /// aborts it, and failing that abort when `abort_fails`: the seam for "the network + /// dropped mid-upload". + fn fail_every_part(mut self, aborted: Arc, abort_fails: bool) -> Self { self.on_multipart = Some(Box::new(move || { Box::new(FailingUpload { aborted: Arc::clone(&aborted), + abort_fails, }) })); self diff --git a/crates/gdi-dataset-tool/src/state.rs b/crates/gdi-dataset-tool/src/state.rs index 2185ec2..36a598e 100644 --- a/crates/gdi-dataset-tool/src/state.rs +++ b/crates/gdi-dataset-tool/src/state.rs @@ -156,6 +156,9 @@ pub(crate) enum NodeProbe { /// never seen this id" tells the operator to re-install a dataset that is present. /// Callers that only need a verdict fold it into the same fallback as `Unknown`. Unreachable, + /// The oracle answered with another error status, such as `401` or `503`: it is up, but + /// says nothing about the id. + Failed(u16), /// No authoritative view of this id: a `404`, or an unusable body on a `2xx`. Callers /// fall back to S3-ownership routing. Unknown, @@ -197,6 +200,10 @@ pub(crate) async fn probe_node_state_detailed( reason.to_owned() })); } + let status = resp.status(); + if !status.is_success() && status != reqwest::StatusCode::NOT_FOUND { + return Ok(NodeProbe::Failed(status.as_u16())); + } Ok(match probe_parse(resp, &url).await { Some(state) => NodeProbe::Live(Box::new(state)), None => NodeProbe::Unknown, @@ -290,6 +297,24 @@ pub(crate) fn resolve_node_state( } } +/// Like [`resolve_node_state`], without folding the answer: `None` when no oracle base is +/// configured, otherwise what the probe saw. +/// +/// # Errors +/// +/// Returns a [`ToolError`] if the probe runtime cannot be started, or for the reasons +/// [`probe_node_state_detailed`] gives. +pub(crate) fn resolve_node_probe( + active: &Profile, + id: &str, + override_base: Option<&str>, +) -> Result, ToolError> { + match override_base.or_else(|| active.node_state_base()) { + Some(base) => runtime::block_on(probe_node_state_detailed(base, id)).map(Some), + None => Ok(None), + } +} + /// Resolve the channel to write to: prefer the node's authoritative `channel`; on /// no authoritative view fall back to the explicit `--s3` / `--local` override, or /// (with neither) infer from the profile (S3 configured => S3, else inbox). diff --git a/crates/gdi-dataset-tool/src/wizard/mod.rs b/crates/gdi-dataset-tool/src/wizard/mod.rs index e9ac4aa..74503dd 100644 --- a/crates/gdi-dataset-tool/src/wizard/mod.rs +++ b/crates/gdi-dataset-tool/src/wizard/mod.rs @@ -427,11 +427,7 @@ pub fn run( summary_publish = Some( "staging dir deployed to the node's inbox; the dataset lands hidden".to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } else { summary_publish = Some("nothing sent to the node (your choice)".to_owned()); summary_next = @@ -515,11 +511,7 @@ pub fn run( the node polls (allow ~30 s before it even looks)" .to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } Some(PublishRoute::DeployInbox) => { let deploy_args = wizard_deploy_args(package, wait_for_ingest); @@ -528,11 +520,7 @@ pub fn run( summary_publish = Some( "deployed to the node's inbox; the dataset lands hidden".to_owned(), ); - summary_next = Some(format!( - "run `gdi-dataset-tool status {id}` to confirm ingest, then \ - `gdi-dataset-tool publish {id}` to make it visible", - id = build.dataset_id - )); + summary_next = Some(next_after_send(&build.dataset_id, wait_for_ingest)); } None => { summary_publish = Some("nothing sent to the node (your choice)".to_owned()); @@ -864,6 +852,23 @@ fn route_label(route: PublishRoute, active: Option<&Profile>) -> String { } } +/// The summary's next step after sending a dataset. Without a `management_url`, `status` may +/// never get the node's verdict, so the step says when it can and who else knows. +fn next_after_send(id: &str, management_url: bool) -> String { + if management_url { + format!( + "run `gdi-dataset-tool status {id}` to confirm ingest, then \ + `gdi-dataset-tool publish {id}` to make it visible" + ) + } else { + format!( + "confirm the ingest with `gdi-dataset-tool status {id}` if the node reports \ + status to you, else ask the node's operator; then `gdi-dataset-tool publish \ + {id}` to make it visible" + ) + } +} + /// The `deploy` arguments the wizard uses for `artifact` — the single place both publish /// routes (keyless staging dir, packed `.tar.c4gh`) get them from. /// diff --git a/crates/gdi-dataset-tool/src/wizard/prompts.rs b/crates/gdi-dataset-tool/src/wizard/prompts.rs index f960786..f415d88 100644 --- a/crates/gdi-dataset-tool/src/wizard/prompts.rs +++ b/crates/gdi-dataset-tool/src/wizard/prompts.rs @@ -97,12 +97,10 @@ pub trait Prompter { /// current-thread runtime in a background thread: the CLI core is synchronous and blocked /// reading a key, so there is no ambient runtime to spawn onto. /// -/// Registering the handler also stops SIGINT killing the process outright, which turns -/// Ctrl-C into the ordinary abort the wizard already reports: the blocked read returns -/// `EINTR`, `dialoguer` surfaces it, and the run ends via [`abort`] with "wizard aborted: -/// …" and exit 1, its own cleanup restoring the cursor. This thread gives that path a -/// moment to happen and only forces the exit if it does not, so the exit code cannot race -/// between 1 and 130 for the same keystroke. Showing the cursor first is idempotent. +/// Registering the handler also stops SIGINT killing the process. At a text, yes/no or +/// menu prompt the read then fails, and the run ends via [`abort`] ("wizard cancelled +/// (Ctrl-C)", exit 1). A hidden-input read is restarted instead, so this thread forces exit +/// 130 after a short grace. Showing the cursor first is idempotent. /// /// Cursor visibility is a DEC private mode, not a termios flag, which is why the shell's /// own `tcsetattr` on regaining the foreground restores echo but not this. @@ -161,8 +159,16 @@ pub fn require_tty() -> Result<(), ToolError> { } } -/// Map a dialoguer error (incl. Ctrl-C/Esc) to a clean tool error. +/// Map a dialoguer error (incl. Ctrl-C) to a clean tool error. +/// +/// Ctrl-C interrupts a read with the cursor still on the prompt's line, so this ends that +/// line first, and says what happened rather than quoting the interrupted read. fn abort(e: &dialoguer::Error) -> ToolError { + let dialoguer::Error::IO(io) = e; + if io.kind() == std::io::ErrorKind::Interrupted { + let _ = console::Term::stderr().write_line(""); + return ToolError::user("wizard cancelled (Ctrl-C)"); + } ToolError::user(format!("wizard aborted: {e}")) } @@ -788,6 +794,20 @@ mod tests { #![expect(clippy::unwrap_used, reason = "unwrap is permitted in test code")] use super::*; + /// Ctrl-C reads as a cancel, not as the interrupted read it surfaces as; any other + /// prompt failure keeps its cause. + #[test] + fn ctrl_c_is_a_cancel_and_other_failures_keep_their_cause() { + let interrupted = + dialoguer::Error::IO(std::io::Error::from(std::io::ErrorKind::Interrupted)); + assert_eq!(abort(&interrupted).message, "wizard cancelled (Ctrl-C)"); + let broken = dialoguer::Error::IO(std::io::Error::other("terminal gone")); + assert_eq!( + abort(&broken).message, + "wizard aborted: IO error: terminal gone" + ); + } + #[test] fn scripted_prompter_returns_answers_in_order() { let p = ScriptedPrompter::new() diff --git a/crates/gdi-dataset-tool/src/wizard/setup.rs b/crates/gdi-dataset-tool/src/wizard/setup.rs index 8e8e9fd..4fd0ef8 100644 --- a/crates/gdi-dataset-tool/src/wizard/setup.rs +++ b/crates/gdi-dataset-tool/src/wizard/setup.rs @@ -116,10 +116,13 @@ pub fn run_setup( // re-entry for rotation, and a prompt that forgets the previous value silently narrows // a deliberate choice back to a default. let target = resolve_write_target(config_path)?; - let existing: Profile = ToolConfig::load_file_only(&target) - .ok() + let stored = ToolConfig::load_file_only(&target).ok(); + let existing: Profile = stored + .as_ref() .and_then(|cfg| cfg.profiles.get(&name).cloned()) .unwrap_or_default(); + // The country code is the one recorded answer kept outside the profile. + let existing_cc = stored.and_then(|cfg| cfg.country_code); // Step 2: Node service URL. Blank is a real answer: a provider who prepares packages // for hand-off has no node to name. Demanding a URL would make them invent one, after @@ -319,8 +322,8 @@ pub fn run_setup( .map(|s3| s3.prefix.clone()) .unwrap_or_default(); crate::output::progress( - " must match the node's [[s3.buckets]].prefix exactly; a prefix on one side \ - only is a silent desync.", + " must match the node's key prefix for this bucket exactly; ask the node's \ + operator. A prefix on one side only is a silent desync.", ); let answer = p.input_validated( "Key prefix in the bucket (blank = the whole bucket)", @@ -348,7 +351,7 @@ pub fn run_setup( let channel = { let existing_channel = existing.s3.as_ref().and_then(|s3| s3.channel.clone()); crate::output::progress( - " the node's [[s3.buckets]].name for this bucket; ask the node operator.", + " the node's name for this bucket's channel; ask the node's operator.", ); let answer = p.input_validated( "Channel name (blank leaves the wrong-bucket guard off)", @@ -373,7 +376,7 @@ pub fn run_setup( // Not prompted: this is a fact about the endpoint just entered, not a preference. // An `http://` endpoint is otherwise accepted here and then rejected by an // https-only client. - let allow_http = endpoint.starts_with("http://"); + let allow_http = endpoint.to_ascii_lowercase().starts_with("http://"); ( Some(ProfileS3 { @@ -392,14 +395,16 @@ pub fn run_setup( (None, None) }; - // Step 6: Ensure the provider keypair exists (auto-generates if missing). - cmd_keys::load_or_generate_provider_secret(config_path)?; + // Step 6: Ensure the provider keypair exists (created on a first run). + cmd_keys::ensure_provider_secret(config_path)?; // Step 7: Country code — validated inline (2 uppercase letters) so a bad value is // caught here rather than failing late at the build stage. - let cc_raw = p.input_validated("Two-letter country code (e.g. EE)", None, &|s| { - fields::resolve_country_code(s).map(|_| ()) - })?; + let cc_raw = p.input_validated( + "Two-letter country code (e.g. EE)", + existing_cc.as_deref(), + &|s| fields::resolve_country_code(s).map(|_| ()), + )?; let cc = fields::resolve_country_code(&cc_raw).unwrap_or(cc_raw); // Step 7b: the institute abbreviation — the other identity half of every dataset id @@ -1768,6 +1773,38 @@ mod tests { assert_eq!(cfg.default_profile.as_deref(), Some("other")); } + /// A re-run offers the stored country code like every other recorded answer, so Enter + /// keeps it instead of failing validation on an empty entry. + #[test] + #[serial_test::serial(env)] + fn a_re_run_offers_the_stored_country_code() { + use gdi_node_standalone_core::crypt4gh::{generate_keypair, serialize_public_key}; + let (_sk, pk) = generate_keypair(); + let base = stub_node(&serialize_public_key(&pk)); + let dir = tempfile::tempdir().unwrap(); + let cfg_path = dir.path().join("config.toml"); + let stored = ToolConfig { + country_code: Some("LV".into()), + ..ToolConfig::default() + }; + gdi_node_standalone_core::config::write(&stored, &cfg_path).unwrap(); + + let p = ScriptedPrompter::new() + .with_inputs(vec!["default", &base, "", "", "UTARTU"]) + .with_confirms(vec![ + true, // trust recipient + true, // sync catalogs + false, // S3? + ]) + .with_selects(vec![0]); // header policy: 0 = minimal + let _prev = test_util::EnvGuard::set("GDI_CONFIG_DIR", dir.path()); + + run_setup(&p, Some(&cfg_path), None, None, true).unwrap(); + + let cfg = ToolConfig::load(Some(&cfg_path)).unwrap(); + assert_eq!(cfg.country_code.as_deref(), Some("LV")); + } + /// Adding a second profile must never leave the config with no default. A config with /// two profiles and no `default_profile` selects nothing: every profile-reading command /// fails "no profile selected: pass `--profile` or set `default_profile`", and the diff --git a/docs/gdi-dataset-tool.md b/docs/gdi-dataset-tool.md index daa5064..86fabd6 100644 --- a/docs/gdi-dataset-tool.md +++ b/docs/gdi-dataset-tool.md @@ -193,9 +193,10 @@ Format: `(GOE|GDI)-CC-ORG-YYYYMMDDHHMMSSmmm` Example: `GDI-EE-EXAMPLE-20260409143052837`. -The tool generates the whole ID at `build`/`package` time. The millisecond timestamp -makes same-org collisions between two builds negligible, and the `upload`/`deploy` guards -catch the remainder. +The tool generates the whole ID at `build`/`package` time. Builds that share a config +directory never reuse a millisecond (the last one is kept in `.last-build-epoch`), so a +script can start several at once. Elsewhere the clock alone decides, and `upload`/`deploy` +refuse the rare collision. A pinned `--build-epoch` is used as given. **Country-code precedence** (lowest → highest): @@ -1476,7 +1477,7 @@ gdi-dataset-tool pack [--recipient ] [-o ] [--force] |------|---------|---------| | `` (positional) | (required) | The `build/{datasetId}/` staging directory. | | `--recipient ` | fetched from profile URL, else `node_recipient_file` | Local crypt4gh recipient file for the node (overrides the profile's URL fetch). | -| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. | +| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. Any other file name gets a warning, because `upload` and `deploy` read the dataset ID from the name. | | `--force` | off | Overwrite an existing output file. | | `--format` | `text` | `text` (the human result line) or `json` (a single machine-readable result object). | @@ -1519,7 +1520,7 @@ gdi-dataset-tool package [--cc ] [--build-out ] \ | `--header-policy ` | profile `header_policy`, else `minimal` | As `build --header-policy`. | | `-j`, `--jobs ` | `0` (one per logical CPU) | Conversion worker-pool size; lower it to cap peak memory and parallelism (a single VCF still uses the whole pool). | | `--recipient ` | fetched from profile URL, else `node_recipient_file` | Node recipient file (overrides the profile's URL fetch). | -| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. | +| `-o`, `--output ` | `{datasetId}.tar.c4gh` in cwd | Output path. An existing directory means "into here", as `cp` reads it: the package lands at `/{datasetId}.tar.c4gh`. Any other file name gets a warning, because `upload` and `deploy` read the dataset ID from the name. | | `--force` | off | Overwrite an existing staging dir **and** output file. | | `--keep` | off | Keep the staging directory after a successful pack. It is deleted by default, because it holds plaintext genotype-derived intermediates. | | `--refresh-catalogs` | off | Before building, fetch the node's live catalogs for this build only. It does not write to the profile; use `catalogs --sync` to persist them. | @@ -1946,7 +1947,7 @@ effect. `status ` also works wherever a tool profile is configured. gdi-dataset-tool publish GDI-EE-EXAMPLE-20260409143052837 # published GDI-EE-EXAMPLE-20260409143052837: wrote the visible sidecar on inbox /srv/gdi/inbox. # The node applies it on its next scan; its management-plane -# /datasets/GDI-EE-EXAMPLE-20260409143052837/state reports when it has +# /datasets/GDI-EE-EXAMPLE-20260409143052837/state shows visible once it has ``` > `unpublish` on a node's only remaining dataset does not retract it from a GDI User @@ -2391,10 +2392,15 @@ can select a different key set than the default. `download`, `list`, or an S3-channel lifecycle op) ran without a `[profiles..s3]` block in the config. Add one, or use the inbox channel. -**`S3 bucket is not writable (your token may lack write permission)`** — `doctor`'s write -probe, a PUT of a temporary `.doctor-probe-*` key, was denied. The token is read-only, or -the bucket policy withholds `s3:PutObject`. `upload` and every lifecycle write need a -read/write token. +**`profile s3.endpoint http://... is plaintext http:// but allow_http is false`** — use +the `https://` endpoint, or, for a local dev backend such as MinIO or Garage, set +`allow_http = true` in the profile's `[profiles..s3]` block. `wizard setup` sets it +for you. + +**`S3 bucket is not writable: ...`** — `doctor`'s write probe, a PUT of a temporary +`.doctor-probe-*` key, failed, and the rest of the message says why. Prefixed with +`not authorized:`, the token is read-only or the bucket policy withholds `s3:PutObject`; +`upload` and every lifecycle write need a read/write token. **`not authorized: : ...`** — an operation was denied, and the message names the failing step, such as `uploading ` or `listing bucket`. The command exits `4`, so @@ -2407,6 +2413,12 @@ networked verb when it detects such a twin, so check stderr, or run `gdi-dataset-tool profiles`, which prints the same warning alongside what it loaded. Then run `gdi-dataset-tool doctor`. +**`...; aborting the multipart upload of failed too ...`** — an `upload` failed +part-way and could not be cancelled, usually because the connection or the credentials were +gone. The parts already sent stay in the bucket, billed but unlisted, even after a retry. +Ask the bucket's operator to abort the upload (`aws s3api list-multipart-uploads`, then +`abort-multipart-upload`) or to add a lifecycle rule that expires incomplete uploads. + **`dataset is already live on the node` / `dataset is already present in the bucket`** — the id is already installed. `upload` and `deploy` refuse to re-present it without `--replace`. On S3, `--replace` does not overwrite the immutable package bytes, @@ -2418,7 +2430,9 @@ can be published or unpublished. Ingest it first, and let it leave `processing`. **`dataset reads as visible on the node's state oracle` (delete)** — `unpublish` it first, or pass `--force`. If you have just run `unpublish`, the oracle lags the sidecar by -a few seconds; wait and retry. +a few seconds; wait and retry. When the node's state oracle gives no answer, `delete` checks +the bucket's `.state.json` instead, and says `reads as visible in the bucket's state +sidecar`. **`cannot determine the dataset's channel`** — a lifecycle op could not reach the node, and the profile has neither an S3 block nor an `inbox`. Pass `--s3` or `--local`. From 39186f5e79edfc8126254162b84e2fa7d7223fa2 Mon Sep 17 00:00:00 2001 From: Erik Jaaniso Date: Thu, 8 Oct 2026 22:23:23 +0300 Subject: [PATCH 3/3] docs: re-record the wizard demo and fix the scripted quickstart The demo driver now answers the contact, provenance, age range, health theme and publication prompts. The quickstart keeps keys in the config dir, and the README says to route traffic only to a ready node. --- README.md | 28 ++++++++++++++++++---------- docs/images/wizard.svg | 2 +- scripts/demo/drive-wizard.py | 8 ++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b6198b4..4107dbe 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,8 @@ stack with the realistic sample, start to finish: `gdi-dataset-tool publish ` then makes the dataset visible, and the node serves it at `/fairdp/dataset/` and in the Beacon. There is a scripted equivalent for automation: -`gdi-dataset-tool config init -o tool.toml` writes a commented template, and the minimum is +`gdi-dataset-tool config init` writes a commented template to `~/.config/gdi/tool.toml`, +and the minimum is ```toml country_code = "EE" @@ -341,14 +342,18 @@ gdi-aggregated = "Genome of Europe Aggregated Data" ```bash export GDI_TOOL__PROFILES__DEFAULT__S3__ACCESS_KEY_ID=… GDI_TOOL__PROFILES__DEFAULT__S3__SECRET_ACCESS_KEY=… -gdi-dataset-tool --config tool.toml keys pin-recipient # fetches {service_url}/.well-known/c4gh-recipient and pins it; --file node.pub when handed over offline -gdi-dataset-tool init # scaffolds package.yaml: point it at your VCFs, fill the metadata -gdi-dataset-tool --config tool.toml package package.yaml # build + pack: .tar.c4gh, encrypted to the node; mints your provider key on first use -gdi-dataset-tool --config tool.toml upload .tar.c4gh # into the bucket, hidden; the node picks it up on its next poll -gdi-dataset-tool --config tool.toml publish # flips the sidecar to visible -gdi-dataset-tool --config tool.toml status # the node's verdict, once the operator sets the bucket's write_status = true or gives you a management_url +gdi-dataset-tool keys pin-recipient # fetches {service_url}/.well-known/c4gh-recipient and pins it; --file node.pub when handed over offline +gdi-dataset-tool init # scaffolds package.yaml: point it at your VCFs, fill the metadata +gdi-dataset-tool package package.yaml # build + pack: .tar.c4gh, encrypted to the node; mints your provider key on first use +gdi-dataset-tool upload .tar.c4gh # into the bucket, hidden; the node picks it up on its next poll +gdi-dataset-tool publish # flips the sidecar to visible +gdi-dataset-tool status # the node's verdict, once the operator sets the bucket's write_status = true or gives you a management_url ``` +Your provider key and the pinned node key live in `~/.config/gdi/keys/` and +`~/.config/gdi/recipients/`, or next to the file `--config` names, so keep such a config +out of any directory you share or package from. + Every tool field, including the co-located inbox profile and `keyless = true`: [`tool.example.toml`](tool.example.toml). @@ -465,9 +470,12 @@ keys and no profile. Worth knowing before your first real build: Health is `GET :9090/health/live` and `/health/ready` (`ready: true, degraded: true` means a provider bucket is dark), metrics are Prometheus text at `:9090/metrics`, and a dataset's -state is `GET :9090/datasets/{id}/state`. Below is the shipped dashboard, captured on the -dev stack with the realistic sample under a few minutes of query load. The alert names in -the panel titles are the shipped Prometheus rules: +state is `GET :9090/datasets/{id}/state`. Route traffic only to a ready node: one that +started while Vault was unreachable can answer publicly as if it had no datasets until it is +restarted ([operating.md §1](docs/operating.md#1-health-and-readiness-endpoints)). +Kubernetes readiness probes do that; `docker run` and Compose do not. Below is the shipped +dashboard, captured on the dev stack with the realistic sample under a few minutes of query +load. The alert names in the panel titles are the shipped Prometheus rules: ![The shipped Grafana dashboard: node readiness, datasets by state, request rate, error ratio and latency, on the dev stack under query load](docs/images/grafana-dashboard.png) diff --git a/docs/images/wizard.svg b/docs/images/wizard.svg index 6102800..c9a108b 100644 --- a/docs/images/wizard.svg +++ b/docs/images/wizard.svg @@ -1 +1 @@ -provider@laptop:~/work$gdi-dataset-toolprovider@laptop:~/work$gdi-dataset-toolwizard==[1/5]Setup:profile&keys====================================================================✔Profilename·defaultthenode'spublicbaseURL.Blankifthereisnonodeyet;youwillhandthepackageoverinstead.✔NodeserviceURL·http://127.0.0.1:38080blankunlessthenodeisco-locatedandyoucanreachitsmanagementplane(typicallyhttp://127.0.0.1:9090).✔NodemanagementURL·http://127.0.0.1:39090recipientfingerprint:sha256:5abc47b88264539793f3e7358c2e759b7fc8d71c5dd8ccb5d2800c64e87512fb✔Trustthisrecipient?·yes✔Synccatalogsfromthenode?·yes✔ConfigureS3upload?·yes✔S3bucketname·gdi-datasets✔S3endpointURL·http://127.0.0.1:3900thenode'sregion:garageforGarage,us-east-1forMinIO/Ceph,thebucket'sownforAWS.✔S3region·garagemustmatchthenode's[[s3.buckets]].prefixexactly;aprefixononesideonlyisasilentdesync.✔Keyprefixinthebucket(blank=thewholebucket)·✔S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)·********✔S3secretaccesskey·********thenode's[[s3.buckets]].nameforthisbucket;askthenodeoperator.✔Channelname(blankleavesthewrong-bucketguardoff)·primary✔Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)·yeswarning:noprovideridentityat/home/provider/.config/gdi/keys/provider.c4gh;generatedanewone(sha256:0cf3186d30246c3c7726c5633222ed4ed4afcc3f0bf5581088fc7e2a1a8fb17b).Thiskeyisthecrypt4ghwriterprovenanceofeverypackagebuiltfromhereon;anodewith[ingest].writer_policy=enforcewillrejectthemuntilthisfingerprintisallow-listed.Ifyouexpectedanexistingkey,check--config/GDI_CONFIG_DIRbeforepackaging.✔Two-lettercountrycode(e.g.EE)·EEtheORGinGDI-EE-<ORG>-...,e.g.EXAMPLE.✔Instituteabbreviationfordatasetids·EXAMPLEthenodedropstheseatingest;`--header-policy`onbuild/packagealwaysoverridestheprofile.✔VCFheaderstoshipinpackages·minimal:structuralkeysonly,nosampleidentifiers(default)ok:profile'default'writtento/home/provider/.config/gdi/tool.tomlstoredtheS3credentialsin/home/provider/.config/gdi/tool-secrets.toml(owner-only),whereeverylaterrunreadsthem;verifythebucketwith`gdi-dataset-tooldoctor`(itdoesaprobePUT/DELETE)==[2/5]Author:describethedataset(package.yaml)==============================================--Sourcedata------------------------------------------------------------------------------------plain.vcfor.vcf.gz;adirectoryoffersitsVCFstopickfrom.✔VCFfileordirectoryofVCFs(Tabcompletes)·data/gdi-sample.GRCh38.vcf.gz✔AddanotherVCFfileordirectory?·nopreview:populationsinthe1sourceVCF(s),fromtheheaders:[1/1]gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalassemblyGRCh38readfromtheVCFheader;confirmit,orpicktheother✔Assembly·GRCh38--Catalogentry:publishedviathenode'sFAIRDataPoint----------------------------------------✔DatasetIDprefix·GDIorg:EXAMPLE(fromtheprofile)✔Catalog·gdi-aggregated?Datasettitle›Synthetic?Datasettitle›Syntheticallele?Datasettitle›Syntheticallelefrequencies,?Datasettitle›Syntheticallelefrequencies,twelve✔Datasettitle·Syntheticallelefrequencies,twelvepopulations?Description›Synthetic?Description›Syntheticaggregated?Description›Syntheticaggregatedallele?Description›Syntheticaggregatedallelefrequencies?Description›Syntheticaggregatedallelefrequenciesfor?Description›Syntheticaggregatedallelefrequenciesfora?Description›Syntheticaggregatedallelefrequenciesforanode✔Description·Syntheticaggregatedallelefrequenciesforanodedemo.✔Adddiscoverykeywords?·yes✔Keywords(comma-separated)·allele-frequency,genomicsnumberOfUniqueIndividuals:recommended;buildwarnswhenitisabsent.✔Recordcohortsize?·yes✔Distinctsequencedsubjects(blanktoskip)·500✔Isthissyntheticdata?·yesthefullpublicnameshowninthecatalogentry:nottheIDabbreviationfromsetup.?Creatingorganisation›Example✔Creatingorganisation·ExampleInstitute✔Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)·Humangenomicoptional:acomplianceclaimaboutTHISdataset;leaveeverythinguntickedtoomitit.✔Conformsto(GDIstandards);Spacetoggles,Enterconfirms·--Access&legal---------------------------------------------------------------------------------?Accessrights›NON_PUBLIC✔Accessrights·PUBLIC?License›Other(enterIRI)✔License·CC01.0:publicdomain(typicalforsyntheticdata)EUELIIRIs,e.g.http://data.europa.eu/eli/reg/2025/327/oj.TheEHDSrowmaybeunticked;buildthenwarns,and`--strict`failsonthatwarning.✔Applicablelegislation(Spacetoggles,Enterconfirms)·EHDS:Regulation(EU)2025/327✔AddanotherlegislationELIorIRI?·no--Beaconprovenance:citedineveryBeaconanswer------------------------------------------------setsbothafSourceandafSourceReference.✔UsethestandardGenomeofEuropeAFprovenance?·yes--Disclosurecontrols----------------------------------------------------------------------------Smallallelecountscanbewithheldatbuildtime(minAlleleCount).TheunitisALLELES,notpeople:ahomozygouscarriercounts2,so5~threepeople.✔Withholdrowswithallelecountbelow(0=publisheverything)·10gdi-sample.GRCh38.vcf.gz[]0B/490.81KiB(0%)0B/sETA0satminAlleleCount10:6411row(s)belowthefloor,10687moreremovedbycollapsing1447variant(s)toTotal;populationsthatsurvive:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,Total--Review:package.yamlasitwillbewritten-----------------------------------------------------metadata:prefix:"GDI"org:"EXAMPLE"catalog:"gdi-aggregated"title:"Syntheticallelefrequencies,twelvepopulations"description:"Syntheticaggregatedallelefrequenciesforanodedemo."type:-"https://publications.europa.eu/resource/authority/dataset-type/SYNTHETIC_DATA"keywords:-"allele-frequency"-"genomics"numberOfUniqueIndividuals:500accessRights:"http://publications.europa.eu/resource/authority/access-right/PUBLIC"applicableLegislation:-"http://data.europa.eu/eli/reg/2025/327/oj"license:"http://publications.europa.eu/resource/authority/licence/CC0"creator:-name:"ExampleInstitute"healthCategory:-"http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic"files:-category:"VCF"reference:"GRCh38"files:-"/home/provider/work/data/gdi-sample.GRCh38.vcf.gz"internal:{}config:mode:aggregatedblockRange:10000000minAlleleCount:10afSource:"TheGenomeofEurope"afSourceReference:"https://genomeofeurope.eu/"---------------------------------------------------------------------------------------------------✔Writepackage.yaml?·Writeitwrotepackage.yaml==[3/5]Build:convert&validate================================================================disclosurepreview:whatthisdatasetwillpublish(1sourceVCF(s)):[1/1]/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalVCFheaders:minimal:structuralkeysonly,nosampleidentifiers✔Publishthesepopulations?·yesconvertingVCF1/1:/home/provider/work/data/gdi-sample.GRCh38.vcf.gznote:themin_allele_countfloorsuppressed6411populationrow(s)whoseACwasbelowitnote:k-anonymitycoherencecollapsed1447variant(s)totheirTotalrowonly,removing10687furtherpopulationrow(s)regardlessoftheirownAC:awithheldpopulationforcesitssiblingstobewithheldtoonote:11of413emittedvariant(s)haveTotalAF=0:nocopyofthealternatealleleinthewholecohort,andarestoredandservedlikeanyotherrow,asavariantpresentwithfrequency0;removemonomorphicsitesbeforebuilding(`bcftoolsview-c1`)ifthatisnotintendednote:dropped1224of1637inputrecords(0non-primarycontig,0nosupportedALT;symbolic/SV/gVCF/missing,0noallelefrequencyintheinput,1224everypopulationrowwithheldbythek-anonymityfloor)/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:populationsemitted(12):EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalbuiltdatasetGDI-EE-EXAMPLE-20260930190939700->./build/GDI-EE-EXAMPLE-20260930190939700==[4/5]Pack:encryptintoonepackage===========================================================packingparquet1/23:allele-freq.chr21.0.br10000000.6e44adfd808b30ac.parquetpackingparquet2/23:allele-freq.chrM.0.br10000000.6e44adfd808b30ac.parquetpackingparquet3/23:allele-freq.chrX.0.br10000000.6e44adfd808b30ac.parquetpackingparquet4/23:allele-freq.chrX.1.br10000000.6e44adfd808b30ac.parquetpackingparquet5/23:allele-freq.chrX.10.br10000000.6e44adfd808b30ac.parquetpackingparquet6/23:allele-freq.chrX.11.br10000000.6e44adfd808b30ac.parquetpackingparquet7/23:allele-freq.chrX.12.br10000000.6e44adfd808b30ac.parquetpackingparquet8/23:allele-freq.chrX.13.br10000000.6e44adfd808b30ac.parquetpackingparquet9/23:allele-freq.chrX.14.br10000000.6e44adfd808b30ac.parquetpackingparquet10/23:allele-freq.chrX.15.br10000000.6e44adfd808b30ac.parquetpackingparquet11/23:allele-freq.chrX.2.br10000000.6e44adfd808b30ac.parquetpackingparquet12/23:allele-freq.chrX.3.br10000000.6e44adfd808b30ac.parquetpackingparquet13/23:allele-freq.chrX.4.br10000000.6e44adfd808b30ac.parquetpackingparquet14/23:allele-freq.chrX.5.br10000000.6e44adfd808b30ac.parquetpackingparquet15/23:allele-freq.chrX.6.br10000000.6e44adfd808b30ac.parquetpackingparquet16/23:allele-freq.chrX.7.br10000000.6e44adfd808b30ac.parquetpackingparquet17/23:allele-freq.chrX.8.br10000000.6e44adfd808b30ac.parquetpackingparquet18/23:allele-freq.chrX.9.br10000000.6e44adfd808b30ac.parquetpackingparquet19/23:allele-freq.chrY.1.br10000000.6e44adfd808b30ac.parquetpackingparquet20/23:allele-freq.chrY.2.br10000000.6e44adfd808b30ac.parquetpackingparquet21/23:allele-freq.chrY.3.br10000000.6e44adfd808b30ac.parquetpackingparquet22/23:allele-freq.chrY.4.br10000000.6e44adfd808b30ac.parquetpackingparquet23/23:allele-freq.chrY.5.br10000000.6e44adfd808b30ac.parquetpackedGDI-EE-EXAMPLE-20260930190939700->./GDI-EE-EXAMPLE-20260930190939700.tar.c4gh./build/GDI-EE-EXAMPLE-20260930190939700;plaintextbuildoutput;the.tar.c4ghholdsthesamecontentencrypted,andarebuildrecreatesit.✔Removethestagingdir?·yesremovedthestagingdir./build/GDI-EE-EXAMPLE-20260930190939700==[5/5]Publish:sendittothenode=============================================================✔Sendthepackagetothenodenow?·UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)uploadedGDI-EE-EXAMPLE-20260930190939700(hidden)toS3bucketgdi-datasets@http://127.0.0.1:3900note:thenodepollsitsbucket,soGDI-EE-EXAMPLE-20260930190939700takesuptoafewminutestoappear(thatgapisexpected).Run`statusGDI-EE-EXAMPLE-20260930190939700`toconfirm,then`publishGDI-EE-EXAMPLE-20260930190939700`tomakeitvisible.Orre-runwith`--wait`toblockuntilthenodehasingestedit.ok:wizardfinisheddatasetGDI-EE-EXAMPLE-20260930190939700package./GDI-EE-EXAMPLE-20260930190939700.tar.c4ghmanifestpackage.yaml(re-runthewizardheretorebuildoredit)shippeduploadedtotheS3bucket;thedatasetlandshiddenandthenodepolls(allow~30sbeforeitevenlooks)nextrun`gdi-dataset-toolstatusGDI-EE-EXAMPLE-20260930190939700`toconfirmingest,then`gdi-dataset-toolpublishGDI-EE-EXAMPLE-20260930190939700`tomakeitvisibleprovider@laptop:~/work$provider@laptop:~/work$gprovider@laptop:~/work$gdprovider@laptop:~/work$gdiprovider@laptop:~/work$gdi-provider@laptop:~/work$gdi-dprovider@laptop:~/work$gdi-daprovider@laptop:~/work$gdi-datprovider@laptop:~/work$gdi-dataprovider@laptop:~/work$gdi-datasprovider@laptop:~/work$gdi-dataseprovider@laptop:~/work$gdi-datasetprovider@laptop:~/work$gdi-dataset-provider@laptop:~/work$gdi-dataset-tprovider@laptop:~/work$gdi-dataset-toprovider@laptop:~/work$gdi-dataset-tooprovider@laptop:~/work$gdi-dataset-toolwprovider@laptop:~/work$gdi-dataset-toolwiprovider@laptop:~/work$gdi-dataset-toolwizprovider@laptop:~/work$gdi-dataset-toolwizaprovider@laptop:~/work$gdi-dataset-toolwizar?Profilename(default)›?NodeserviceURL›?NodeserviceURL›h?NodeserviceURL›ht?NodeserviceURL›htt?NodeserviceURL›http?NodeserviceURL›http:?NodeserviceURL›http:/?NodeserviceURL›http://?NodeserviceURL›http://1?NodeserviceURL›http://12?NodeserviceURL›http://127?NodeserviceURL›http://127.?NodeserviceURL›http://127.0?NodeserviceURL›http://127.0.?NodeserviceURL›http://127.0.0?NodeserviceURL›http://127.0.0.?NodeserviceURL›http://127.0.0.1?NodeserviceURL›http://127.0.0.1:?NodeserviceURL›http://127.0.0.1:3?NodeserviceURL›http://127.0.0.1:38?NodeserviceURL›http://127.0.0.1:380?NodeserviceURL›http://127.0.0.1:3808?NodeserviceURL›http://127.0.0.1:38080?NodemanagementURL›?NodemanagementURL›h?NodemanagementURL›ht?NodemanagementURL›htt?NodemanagementURL›http?NodemanagementURL›http:?NodemanagementURL›http:/?NodemanagementURL›http://?NodemanagementURL›http://1?NodemanagementURL›http://12?NodemanagementURL›http://127?NodemanagementURL›http://127.?NodemanagementURL›http://127.0?NodemanagementURL›http://127.0.?NodemanagementURL›http://127.0.0?NodemanagementURL›http://127.0.0.?NodemanagementURL›http://127.0.0.1?NodemanagementURL›http://127.0.0.1:?NodemanagementURL›http://127.0.0.1:3?NodemanagementURL›http://127.0.0.1:39?NodemanagementURL›http://127.0.0.1:390?NodemanagementURL›http://127.0.0.1:3909?NodemanagementURL›http://127.0.0.1:39090?Trustthisrecipient?(y/n)›yes?Synccatalogsfromthenode?(y/n)›yes?ConfigureS3upload?(y/n)›no?S3bucketname›?S3bucketname›g?S3bucketname›gd?S3bucketname›gdi?S3bucketname›gdi-?S3bucketname›gdi-d?S3bucketname›gdi-da?S3bucketname›gdi-dat?S3bucketname›gdi-data?S3bucketname›gdi-datas?S3bucketname›gdi-datase?S3bucketname›gdi-dataset?S3bucketname›gdi-datasets?S3endpointURL›?S3endpointURL›h?S3endpointURL›ht?S3endpointURL›htt?S3endpointURL›http?S3endpointURL›http:?S3endpointURL›http:/?S3endpointURL›http://?S3endpointURL›http://1?S3endpointURL›http://12?S3endpointURL›http://127?S3endpointURL›http://127.?S3endpointURL›http://127.0?S3endpointURL›http://127.0.?S3endpointURL›http://127.0.0?S3endpointURL›http://127.0.0.?S3endpointURL›http://127.0.0.1?S3endpointURL›http://127.0.0.1:?S3endpointURL›http://127.0.0.1:3?S3endpointURL›http://127.0.0.1:39?S3endpointURL›http://127.0.0.1:390?S3endpointURL›http://127.0.0.1:3900?S3region›?S3region›g?S3region›ga?S3region›gar?S3region›gara?S3region›garag?S3region›garage?Keyprefixinthebucket(blank=thewholebucket)›?S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)›?S3secretaccesskey›?Channelname(blankleavesthewrong-bucketguardoff)›?Channelname(blankleavesthewrong-bucketguardoff)›p?Channelname(blankleavesthewrong-bucketguardoff)›pr?Channelname(blankleavesthewrong-bucketguardoff)›pri?Channelname(blankleavesthewrong-bucketguardoff)›prim?Channelname(blankleavesthewrong-bucketguardoff)›prima?Channelname(blankleavesthewrong-bucketguardoff)›primar?Channelname(blankleavesthewrong-bucketguardoff)›primary?Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)(y/n)›yes?Two-lettercountrycode(e.g.EE)›?Two-lettercountrycode(e.g.EE)›E?Two-lettercountrycode(e.g.EE)›EE?Instituteabbreviationfordatasetids›?Instituteabbreviationfordatasetids›E?Instituteabbreviationfordatasetids›EX?Instituteabbreviationfordatasetids›EXA?Instituteabbreviationfordatasetids›EXAM?Instituteabbreviationfordatasetids›EXAMP?Instituteabbreviationfordatasetids›EXAMPL?Instituteabbreviationfordatasetids›EXAMPLE?VCFheaderstoshipinpackages›❯minimal:structuralkeysonly,nosampleidentifiers(default)with-identifiers:alsothe#CHROMsamplecolumnsand##SAMPLE/##PEDIGREElinesnone:shipnoVCFheadersatall?VCFfileordirectoryofVCFs(Tabcompletes)›?VCFfileordirectoryofVCFs(Tabcompletes)›d?VCFfileordirectoryofVCFs(Tabcompletes)›da?VCFfileordirectoryofVCFs(Tabcompletes)›dat?VCFfileordirectoryofVCFs(Tabcompletes)›data?VCFfileordirectoryofVCFs(Tabcompletes)›data/?VCFfileordirectoryofVCFs(Tabcompletes)›data/g?VCFfileordirectoryofVCFs(Tabcompletes)›data/gd?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-s?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sa?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sam?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-samp?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sampl?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.G?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.GRCh38.vcf.gz?AddanotherVCFfileordirectory?(y/n)›no?Assembly›GRCh37❯GRCh38?DatasetIDprefix›GOE❯GDI?Catalog›❯gdi-aggregatedsynthetic-dataRefreshthelistfromthenode...?Datasettitle›?Datasettitle›S?Datasettitle›Sy?Datasettitle›Syn?Datasettitle›Synt?Datasettitle›Synth?Datasettitle›Synthe?Datasettitle›Synthet?Datasettitle›Syntheti?Datasettitle›Synthetica?Datasettitle›Synthetical?Datasettitle›Syntheticall?Datasettitle›Syntheticalle?Datasettitle›Syntheticallel?Datasettitle›Syntheticallelef?Datasettitle›Syntheticallelefr?Datasettitle›Syntheticallelefre?Datasettitle›Syntheticallelefreq?Datasettitle›Syntheticallelefrequ?Datasettitle›Syntheticallelefreque?Datasettitle›Syntheticallelefrequen?Datasettitle›Syntheticallelefrequenc?Datasettitle›Syntheticallelefrequenci?Datasettitle›Syntheticallelefrequencie?Datasettitle›Syntheticallelefrequencies?Datasettitle›Syntheticallelefrequencies,t?Datasettitle›Syntheticallelefrequencies,tw?Datasettitle›Syntheticallelefrequencies,twe?Datasettitle›Syntheticallelefrequencies,twel?Datasettitle›Syntheticallelefrequencies,twelv?Datasettitle›Syntheticallelefrequencies,twelvep?Datasettitle›Syntheticallelefrequencies,twelvepo?Datasettitle›Syntheticallelefrequencies,twelvepop?Datasettitle›Syntheticallelefrequencies,twelvepopu?Datasettitle›Syntheticallelefrequencies,twelvepopul?Datasettitle›Syntheticallelefrequencies,twelvepopula?Datasettitle›Syntheticallelefrequencies,twelvepopulat?Datasettitle›Syntheticallelefrequencies,twelvepopulati?Datasettitle›Syntheticallelefrequencies,twelvepopulatio?Datasettitle›Syntheticallelefrequencies,twelvepopulation?Datasettitle›Syntheticallelefrequencies,twelvepopulations?Description›?Description›S?Description›Sy?Description›Syn?Description›Synt?Description›Synth?Description›Synthe?Description›Synthet?Description›Syntheti?Description›Synthetica?Description›Syntheticag?Description›Syntheticagg?Description›Syntheticaggr?Description›Syntheticaggre?Description›Syntheticaggreg?Description›Syntheticaggrega?Description›Syntheticaggregat?Description›Syntheticaggregate?Description›Syntheticaggregateda?Description›Syntheticaggregatedal?Description›Syntheticaggregatedall?Description›Syntheticaggregatedalle?Description›Syntheticaggregatedallel?Description›Syntheticaggregatedallelef?Description›Syntheticaggregatedallelefr?Description›Syntheticaggregatedallelefre?Description›Syntheticaggregatedallelefreq?Description›Syntheticaggregatedallelefrequ?Description›Syntheticaggregatedallelefreque?Description›Syntheticaggregatedallelefrequen?Description›Syntheticaggregatedallelefrequenc?Description›Syntheticaggregatedallelefrequenci?Description›Syntheticaggregatedallelefrequencie?Description›Syntheticaggregatedallelefrequenciesf?Description›Syntheticaggregatedallelefrequenciesfo?Description›Syntheticaggregatedallelefrequenciesforan?Description›Syntheticaggregatedallelefrequenciesforano?Description›Syntheticaggregatedallelefrequenciesforanod?Description›Syntheticaggregatedallelefrequenciesforanoded?Description›Syntheticaggregatedallelefrequenciesforanodede?Description›Syntheticaggregatedallelefrequenciesforanodedem?Description›Syntheticaggregatedallelefrequenciesforanodedemo?Description›Syntheticaggregatedallelefrequenciesforanodedemo.?Adddiscoverykeywords?(y/n)›yes?Keywords(comma-separated)(allele-frequency,genomics)›?Recordcohortsize?(y/n)›yes?Distinctsequencedsubjects(blanktoskip)›?Distinctsequencedsubjects(blanktoskip)›5?Distinctsequencedsubjects(blanktoskip)›50?Distinctsequencedsubjects(blanktoskip)›500?Isthissyntheticdata?(y/n)›no?Creatingorganisation›?Creatingorganisation›E?Creatingorganisation›Ex?Creatingorganisation›Exa?Creatingorganisation›Exam?Creatingorganisation›Examp?Creatingorganisation›Exampl?Creatingorganisation›ExampleI?Creatingorganisation›ExampleIn?Creatingorganisation›ExampleIns?Creatingorganisation›ExampleInst?Creatingorganisation›ExampleInsti?Creatingorganisation›ExampleInstit?Creatingorganisation›ExampleInstitu?Creatingorganisation›ExampleInstitut?Creatingorganisation›ExampleInstitute?Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)›⬚Humangenetic⬚Humanepigenomic✔Humangenomic?Conformsto(GDIstandards);Spacetoggles,Enterconfirms›⬚Externallygoverned⬚1+MGcompliant⬚1+MGcohortPUBLIC❯RESTRICTED❯PUBLICRESTRICTED❯CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)CC01.0:publicdomain(typicalforsyntheticdata)CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)❯CC01.0:publicdomain(typicalforsyntheticdata)?Applicablelegislation(Spacetoggles,Enterconfirms)›✔EHDS:Regulation(EU)2025/327⬚GDPR:Regulation(EU)2016/679?AddanotherlegislationELIorIRI?(y/n)›no?UsethestandardGenomeofEuropeAFprovenance?(y/n)›yes?Withholdrowswithallelecountbelow(0=publisheverything)(0)›?Withholdrowswithallelecountbelow(0=publisheverything)(0)›1?Withholdrowswithallelecountbelow(0=publisheverything)(0)›10gdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)4.09MiB/sETA0sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)10.41MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)10.39MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)10.94MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)11.41MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.22KiB/490.81KiB(8%)11.73MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)12.21MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)12.40MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.71MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]102.43KiB/490.81KiB(21%)12.83MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]111.45KiB/490.81KiB(23%)12.84MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)12.84MiB/sETA0s?Writepackage.yaml?›❯WriteitEdititin$EDITORfirstAbort(theanswersarediscarded)?Publishthesepopulations?(y/n)›nogdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)200.03KiB/sETA2sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)1.22MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)4.49MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)6.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)7.97MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.20KiB/490.81KiB(8%)9.17MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]50.99KiB/490.81KiB(10%)10.15MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)11.76MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.89MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)14.09MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)14.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[========>]156.96KiB/490.81KiB(32%)15.15MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=========>]186.63KiB/490.81KiB(38%)15.75MiB/sETA0spacking[=====>]16.50KiB/128.51KiB(13%)27.70MiB/sETA0spacking[=============>]40.50KiB/128.51KiB(32%)25.17MiB/sETA0spacking[================>]48.50KiB/128.51KiB(38%)26.05MiB/sETA0spacking[======================>]65.00KiB/128.51KiB(51%)16.82MiB/sETA0spacking[=============================>]87.50KiB/128.51KiB(68%)16.87MiB/sETA0spacking[=======================================>]117.50KiB/128.51KiB(91%)16.94MiB/sETA0spacking[==========================================>]127.50KiB/128.51KiB(99%)16.96MiB/sETA0spacking[==========================================]132.00KiB/128.51KiB(100%)16.97MiB/sETA0spacking[==========================================]140.00KiB/128.51KiB(100%)16.97MiB/sETA0s?Removethestagingdir?(y/n)›yes?Sendthepackagetothenodenow?›❯UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)Skip:publishelsewhereuploadingGDI-EE-…0939700.tar.c4gh[]0B/149.31KiB(0%)0B/sETA0suploadingGDI-EE-…0939700.tar.c4gh[==============]149.31KiB/149.31KiB(100%)128.86MiB/sETA0s \ No newline at end of file +provider@laptop:~/work$gdi-dataset-toolprovider@laptop:~/work$gdi-dataset-toolwizard==[1/5]Setup:profile&keys====================================================================✔Profilename·defaultthenode'spublicbaseURL.Blankifthereisnonodeyet;youwillhandthepackageoverinstead.✔NodeserviceURL·http://127.0.0.1:38080blankunlessthenodeisco-locatedandyoucanreachitsmanagementplane(typicallyhttp://127.0.0.1:9090).✔NodemanagementURL·http://127.0.0.1:39090recipientfingerprint:sha256:7ea8195a445c0b15bdab652619a0e0f9148af528a4d8407b308b1983264d7e28✔Trustthisrecipient?·yes✔Synccatalogsfromthenode?·yes✔ConfigureS3upload?·yes✔S3bucketname·gdi-datasets✔S3endpointURL·http://127.0.0.1:3900thenode'sregion:garageforGarage,us-east-1forMinIO/Ceph,thebucket'sownforAWS.✔S3region·garagemustmatchthenode'skeyprefixforthisbucketexactly;askthenode'soperator.Aprefixononesideonlyisasilentdesync.✔Keyprefixinthebucket(blank=thewholebucket)·✔S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)·********✔S3secretaccesskey·********thenode'snameforthisbucket'schannel;askthenode'soperator.✔Channelname(blankleavesthewrong-bucketguardoff)·primary✔Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)·yescreatedyourproviderkeyat/home/provider/.config/gdi/keys/provider.c4gh(sha256:ed9dfdad9533205bb97ab69ba226c020ad22e03c5963e4fd5b6f6362f4aba367).Everypackageyoubuildcarriesitasitswriter;ifthenodeacceptspackagesonlyfromlistedkeys,giveitsoperatorthisfingerprint.✔Two-lettercountrycode(e.g.EE)·EEtheORGinGDI-EE-<ORG>-...,e.g.EXAMPLE.✔Instituteabbreviationfordatasetids·EXAMPLEthenodedropstheseatingest;`--header-policy`onbuild/packagealwaysoverridestheprofile.✔VCFheaderstoshipinpackages·minimal:structuralkeysonly,nosampleidentifiers(default)ok:profile'default'writtento/home/provider/.config/gdi/tool.tomlstoredtheS3credentialsin/home/provider/.config/gdi/tool-secrets.toml(owner-only),whereeverylaterrunreadsthem;verifythebucketwith`gdi-dataset-tooldoctor`(itdoesaprobePUT/DELETE)==[2/5]Author:describethedataset(package.yaml)==============================================--Sourcedata------------------------------------------------------------------------------------plain.vcfor.vcf.gz;adirectoryoffersitsVCFstopickfrom.✔VCFfileordirectoryofVCFs(Tabcompletes)·data/gdi-sample.GRCh38.vcf.gz✔AddanotherVCFfileordirectory?·nopreview:populationsinthe1sourceVCF(s),fromtheheaders:[1/1]gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalassemblyGRCh38readfromtheVCFheader;confirmit,orpicktheother✔Assembly·GRCh38--Catalogentry:publishedviathenode'sFAIRDataPoint----------------------------------------✔DatasetIDprefix·GDIorg:EXAMPLE(fromtheprofile)✔Catalog·gdi-aggregated?Datasettitle›Synthetic?Datasettitle›Syntheticallele?Datasettitle›Syntheticallelefrequencies,?Datasettitle›Syntheticallelefrequencies,twelve✔Datasettitle·Syntheticallelefrequencies,twelvepopulations?Description›Synthetic?Description›Syntheticaggregated?Description›Syntheticaggregatedallele?Description›Syntheticaggregatedallelefrequencies?Description›Syntheticaggregatedallelefrequenciesfor?Description›Syntheticaggregatedallelefrequenciesfora?Description›Syntheticaggregatedallelefrequenciesforanode✔Description·Syntheticaggregatedallelefrequenciesforanodedemo.howthedatawasproduced:e.g.sequencing,variantcallingandhowthefrequencieswerecomputed.✔Provenance(blanktoskip)·✔Adddiscoverykeywords?·yes✔Keywords(comma-separated)·allele-frequency,genomicsnumberOfUniqueIndividuals:recommended;buildwarnswhenitisabsent.✔Recordcohortsize?·yes✔Distinctsequencedsubjects(blanktoskip)·500✔Typicalagerangeinyears,e.g.18-90or18+(blanktoskip)·✔Isthissyntheticdata?·yesthefullpublicnameshowninthecatalogentry:nottheIDabbreviationfromsetup.?Creatingorganisation›Example✔Creatingorganisation·ExampleInstitutewhotowritetoaboutthedataset,shownonitscataloguepage:ateamorservicemailboxratherthanaperson's.?Contactname(e.g.Dataaccessteam)›Data?Contactname(e.g.Dataaccessteam)›Dataaccess✔Contactname(e.g.Dataaccessteam)·Dataaccessteam✔Contacte-mail·data@example.org✔Contactwebpage(blanktoskip)·✔Rememberthiscontactpointintheprofileforfuturedatasets?·yes✔Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)·Humangenomicgenomicsbelongsunder"Healthproducts,technologies,data&research";foradiseasecohort,alsotickitsdiseasearea.✔Healththemes(Spacetoggles,Enterconfirms)·Healthproducts,technologies,data&researchoptional:acomplianceclaimaboutTHISdataset;leaveeverythinguntickedtoomitit.✔Conformsto(GDIstandards);Spacetoggles,Enterconfirms·✔Publicationaboutthedataset,asaDOI(blanktoskip)·--Access&legal---------------------------------------------------------------------------------?Accessrights›NON_PUBLIC✔Accessrights·PUBLIC?License›Other(enterIRI)✔License·CC01.0:publicdomain(typicalforsyntheticdata)EUELIIRIs,e.g.http://data.europa.eu/eli/reg/2025/327/oj.TheEHDSrowmaybeunticked;buildthenwarns,and`--strict`failsonthatwarning.✔Applicablelegislation(Spacetoggles,Enterconfirms)·EHDS:Regulation(EU)2025/327✔AddanotherlegislationELIorIRI?·no--Beaconprovenance:citedineveryBeaconanswer------------------------------------------------setsbothafSourceandafSourceReference.✔UsethestandardGenomeofEuropeAFprovenance?·yes--Disclosurecontrols----------------------------------------------------------------------------Smallallelecountscanbewithheldatbuildtime(minAlleleCount).TheunitisALLELES,notpeople:ahomozygouscarriercounts2,so5~threepeople.✔Withholdrowswithallelecountbelow(0=publisheverything)·10gdi-sample.GRCh38.vcf.gz[]0B/490.81KiB(0%)0B/sETA0satminAlleleCount10:6411row(s)belowthefloor,10687moreremovedbycollapsing1447variant(s)toTotal;populationsthatsurvive:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,Totalkeywords:-"allele-frequency"-"genomics"numberOfUniqueIndividuals:500accessRights:"http://publications.europa.eu/resource/authority/access-right/PUBLIC"applicableLegislation:-"http://data.europa.eu/eli/reg/2025/327/oj"license:"http://publications.europa.eu/resource/authority/licence/CC0"creator:-name:"ExampleInstitute"contactPoint:fn:"Dataaccessteam"hasEmail:"mailto:data@example.org"healthCategory:-"http://data.gdi.eu/core/p2/HealthCategoryHumanGenomic"healthTheme:-"https://hdeu-dcat.data.health.europa.eu/resource/authority/health-theme/HEALTH_PRODUCTS"files:-category:"VCF"reference:"GRCh38"files:-"/home/provider/work/data/gdi-sample.GRCh38.vcf.gz"internal:{}config:mode:aggregatedblockRange:10000000minAlleleCount:10afSource:"TheGenomeofEurope"afSourceReference:"https://genomeofeurope.eu/"---------------------------------------------------------------------------------------------------✔Writepackage.yaml?·Writeitwrotepackage.yamlrememberedthecontactpointin/home/provider/.config/gdi/tool.toml:thewizardwillofferitnexttime==[3/5]Build:convert&validate================================================================disclosurepreview:whatthisdatasetwillpublish(1sourceVCF(s)):[1/1]/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalVCFheaders:minimal:structuralkeysonly,nosampleidentifiers✔Publishthesepopulations?·yesnote:recommendedsub-field"hasURL"isabsentonthecontactPointconvertingVCF1/1:/home/provider/work/data/gdi-sample.GRCh38.vcf.gznote:themin_allele_countfloorsuppressed6411populationrow(s)whoseACwasbelowitnote:k-anonymitycoherencecollapsed1447variant(s)totheirTotalrowonly,removing10687furtherpopulationrow(s)regardlessoftheirownAC:awithheldpopulationforcesitssiblingstobewithheldtoonote:11of413emittedvariant(s)haveTotalAF=0:nocopyofthealternatealleleinthewholecohort,andarestoredandservedlikeanyotherrow,asavariantpresentwithfrequency0;removemonomorphicsitesbeforebuilding(`bcftoolsview-c1`)ifthatisnotintendednote:dropped1224of1637inputrecords(0non-primarycontig,0nosupportedALT;symbolic/SV/gVCF/missing,0noallelefrequencyintheinput,1224everypopulationrowwithheldbythek-anonymityfloor)/home/provider/work/data/gdi-sample.GRCh38.vcf.gz:populationsemitted(12):EE,EE_F,EE_M,F,FI,FI_F,FI_M,LV,LV_F,LV_M,M,TotalbuiltdatasetGDI-EE-EXAMPLE-20261008135653596->./build/GDI-EE-EXAMPLE-20261008135653596==[4/5]Pack:encryptintoonepackage===========================================================packingparquet1/23:allele-freq.chr21.0.br10000000.6e44adfd808b30ac.parquetpackingparquet2/23:allele-freq.chrM.0.br10000000.6e44adfd808b30ac.parquetpackingparquet3/23:allele-freq.chrX.0.br10000000.6e44adfd808b30ac.parquetpackingparquet4/23:allele-freq.chrX.1.br10000000.6e44adfd808b30ac.parquetpackingparquet5/23:allele-freq.chrX.10.br10000000.6e44adfd808b30ac.parquetpackingparquet6/23:allele-freq.chrX.11.br10000000.6e44adfd808b30ac.parquetpackingparquet7/23:allele-freq.chrX.12.br10000000.6e44adfd808b30ac.parquetpackingparquet8/23:allele-freq.chrX.13.br10000000.6e44adfd808b30ac.parquetpackingparquet9/23:allele-freq.chrX.14.br10000000.6e44adfd808b30ac.parquetpackingparquet10/23:allele-freq.chrX.15.br10000000.6e44adfd808b30ac.parquetpackingparquet11/23:allele-freq.chrX.2.br10000000.6e44adfd808b30ac.parquetpackingparquet12/23:allele-freq.chrX.3.br10000000.6e44adfd808b30ac.parquetpackingparquet13/23:allele-freq.chrX.4.br10000000.6e44adfd808b30ac.parquetpackingparquet14/23:allele-freq.chrX.5.br10000000.6e44adfd808b30ac.parquetpackingparquet15/23:allele-freq.chrX.6.br10000000.6e44adfd808b30ac.parquetpackingparquet16/23:allele-freq.chrX.7.br10000000.6e44adfd808b30ac.parquetpackingparquet17/23:allele-freq.chrX.8.br10000000.6e44adfd808b30ac.parquetpackingparquet18/23:allele-freq.chrX.9.br10000000.6e44adfd808b30ac.parquetpackingparquet19/23:allele-freq.chrY.1.br10000000.6e44adfd808b30ac.parquetpackingparquet20/23:allele-freq.chrY.2.br10000000.6e44adfd808b30ac.parquetpackingparquet21/23:allele-freq.chrY.3.br10000000.6e44adfd808b30ac.parquetpackingparquet22/23:allele-freq.chrY.4.br10000000.6e44adfd808b30ac.parquetpackingparquet23/23:allele-freq.chrY.5.br10000000.6e44adfd808b30ac.parquetpackedGDI-EE-EXAMPLE-20261008135653596->./GDI-EE-EXAMPLE-20261008135653596.tar.c4gh./build/GDI-EE-EXAMPLE-20261008135653596;plaintextbuildoutput;the.tar.c4ghholdsthesamecontentencrypted,andarebuildrecreatesit.✔Removethestagingdir?·yesremovedthestagingdir./build/GDI-EE-EXAMPLE-20261008135653596==[5/5]Publish:sendittothenode=============================================================✔Sendthepackagetothenodenow?·UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)uploadedGDI-EE-EXAMPLE-20261008135653596(hidden)toS3bucketgdi-datasets@http://127.0.0.1:3900note:thenodepollsitsbucket,soGDI-EE-EXAMPLE-20261008135653596takesuptoafewminutestoappear(thatgapisexpected).Run`statusGDI-EE-EXAMPLE-20261008135653596`toconfirm,then`publishGDI-EE-EXAMPLE-20261008135653596`tomakeitvisible.Orre-runwith`--wait`toblockuntilthenodehasingestedit.ok:wizardfinisheddatasetGDI-EE-EXAMPLE-20261008135653596package./GDI-EE-EXAMPLE-20261008135653596.tar.c4ghmanifestpackage.yaml(re-runthewizardheretorebuildoredit)shippeduploadedtotheS3bucket;thedatasetlandshiddenandthenodepolls(allow~30sbeforeitevenlooks)nextrun`gdi-dataset-toolstatusGDI-EE-EXAMPLE-20261008135653596`toconfirmingest,then`gdi-dataset-toolpublishGDI-EE-EXAMPLE-20261008135653596`tomakeitvisibleprovider@laptop:~/work$provider@laptop:~/work$gprovider@laptop:~/work$gdprovider@laptop:~/work$gdiprovider@laptop:~/work$gdi-provider@laptop:~/work$gdi-dprovider@laptop:~/work$gdi-daprovider@laptop:~/work$gdi-datprovider@laptop:~/work$gdi-dataprovider@laptop:~/work$gdi-datasprovider@laptop:~/work$gdi-dataseprovider@laptop:~/work$gdi-datasetprovider@laptop:~/work$gdi-dataset-provider@laptop:~/work$gdi-dataset-tprovider@laptop:~/work$gdi-dataset-toprovider@laptop:~/work$gdi-dataset-tooprovider@laptop:~/work$gdi-dataset-toolwprovider@laptop:~/work$gdi-dataset-toolwiprovider@laptop:~/work$gdi-dataset-toolwizprovider@laptop:~/work$gdi-dataset-toolwizaprovider@laptop:~/work$gdi-dataset-toolwizar?Profilename(default)›?NodeserviceURL›?NodeserviceURL›h?NodeserviceURL›ht?NodeserviceURL›htt?NodeserviceURL›http?NodeserviceURL›http:?NodeserviceURL›http:/?NodeserviceURL›http://?NodeserviceURL›http://1?NodeserviceURL›http://12?NodeserviceURL›http://127?NodeserviceURL›http://127.?NodeserviceURL›http://127.0?NodeserviceURL›http://127.0.?NodeserviceURL›http://127.0.0?NodeserviceURL›http://127.0.0.?NodeserviceURL›http://127.0.0.1?NodeserviceURL›http://127.0.0.1:?NodeserviceURL›http://127.0.0.1:3?NodeserviceURL›http://127.0.0.1:38?NodeserviceURL›http://127.0.0.1:380?NodeserviceURL›http://127.0.0.1:3808?NodeserviceURL›http://127.0.0.1:38080?NodemanagementURL›?NodemanagementURL›h?NodemanagementURL›ht?NodemanagementURL›htt?NodemanagementURL›http?NodemanagementURL›http:?NodemanagementURL›http:/?NodemanagementURL›http://?NodemanagementURL›http://1?NodemanagementURL›http://12?NodemanagementURL›http://127?NodemanagementURL›http://127.?NodemanagementURL›http://127.0?NodemanagementURL›http://127.0.?NodemanagementURL›http://127.0.0?NodemanagementURL›http://127.0.0.?NodemanagementURL›http://127.0.0.1?NodemanagementURL›http://127.0.0.1:?NodemanagementURL›http://127.0.0.1:3?NodemanagementURL›http://127.0.0.1:39?NodemanagementURL›http://127.0.0.1:390?NodemanagementURL›http://127.0.0.1:3909?NodemanagementURL›http://127.0.0.1:39090?Trustthisrecipient?(y/n)›yes?Synccatalogsfromthenode?(y/n)›yes?ConfigureS3upload?(y/n)›no?S3bucketname›?S3bucketname›g?S3bucketname›gd?S3bucketname›gdi?S3bucketname›gdi-?S3bucketname›gdi-d?S3bucketname›gdi-da?S3bucketname›gdi-dat?S3bucketname›gdi-data?S3bucketname›gdi-datas?S3bucketname›gdi-datase?S3bucketname›gdi-dataset?S3bucketname›gdi-datasets?S3endpointURL›?S3endpointURL›h?S3endpointURL›ht?S3endpointURL›htt?S3endpointURL›http?S3endpointURL›http:?S3endpointURL›http:/?S3endpointURL›http://?S3endpointURL›http://1?S3endpointURL›http://12?S3endpointURL›http://127?S3endpointURL›http://127.?S3endpointURL›http://127.0?S3endpointURL›http://127.0.?S3endpointURL›http://127.0.0?S3endpointURL›http://127.0.0.?S3endpointURL›http://127.0.0.1?S3endpointURL›http://127.0.0.1:?S3endpointURL›http://127.0.0.1:3?S3endpointURL›http://127.0.0.1:39?S3endpointURL›http://127.0.0.1:390?S3endpointURL›http://127.0.0.1:3900?S3region›?S3region›g?S3region›ga?S3region›gar?S3region›gara?S3region›garag?S3region›garage?Keyprefixinthebucket(blank=thewholebucket)›?S3accesskeyid(blanktoskip;fillintool-secrets.tomllater)›?S3secretaccesskey›?Channelname(blankleavesthewrong-bucketguardoff)›?Channelname(blankleavesthewrong-bucketguardoff)›p?Channelname(blankleavesthewrong-bucketguardoff)›pr?Channelname(blankleavesthewrong-bucketguardoff)›pri?Channelname(blankleavesthewrong-bucketguardoff)›prim?Channelname(blankleavesthewrong-bucketguardoff)›prima?Channelname(blankleavesthewrong-bucketguardoff)›primar?Channelname(blankleavesthewrong-bucketguardoff)›primary?Path-styleS3URLs?(yes:RGW/MinIO/Garage;no:virtual-hosted)(y/n)›yes?Two-lettercountrycode(e.g.EE)›?Two-lettercountrycode(e.g.EE)›E?Two-lettercountrycode(e.g.EE)›EE?Instituteabbreviationfordatasetids›?Instituteabbreviationfordatasetids›E?Instituteabbreviationfordatasetids›EX?Instituteabbreviationfordatasetids›EXA?Instituteabbreviationfordatasetids›EXAM?Instituteabbreviationfordatasetids›EXAMP?Instituteabbreviationfordatasetids›EXAMPL?Instituteabbreviationfordatasetids›EXAMPLE?VCFheaderstoshipinpackages›❯minimal:structuralkeysonly,nosampleidentifiers(default)with-identifiers:alsothe#CHROMsamplecolumnsand##SAMPLE/##PEDIGREElinesnone:shipnoVCFheadersatall?VCFfileordirectoryofVCFs(Tabcompletes)›?VCFfileordirectoryofVCFs(Tabcompletes)›d?VCFfileordirectoryofVCFs(Tabcompletes)›da?VCFfileordirectoryofVCFs(Tabcompletes)›dat?VCFfileordirectoryofVCFs(Tabcompletes)›data?VCFfileordirectoryofVCFs(Tabcompletes)›data/?VCFfileordirectoryofVCFs(Tabcompletes)›data/g?VCFfileordirectoryofVCFs(Tabcompletes)›data/gd?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-s?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sa?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sam?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-samp?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sampl?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.G?VCFfileordirectoryofVCFs(Tabcompletes)›data/gdi-sample.GRCh38.vcf.gz?AddanotherVCFfileordirectory?(y/n)›no?Assembly›GRCh37❯GRCh38?DatasetIDprefix›GOE❯GDI?Catalog›❯gdi-aggregatedsynthetic-dataRefreshthelistfromthenode...?Datasettitle›?Datasettitle›S?Datasettitle›Sy?Datasettitle›Syn?Datasettitle›Synt?Datasettitle›Synth?Datasettitle›Synthe?Datasettitle›Synthet?Datasettitle›Syntheti?Datasettitle›Synthetica?Datasettitle›Synthetical?Datasettitle›Syntheticall?Datasettitle›Syntheticalle?Datasettitle›Syntheticallel?Datasettitle›Syntheticallelef?Datasettitle›Syntheticallelefr?Datasettitle›Syntheticallelefre?Datasettitle›Syntheticallelefreq?Datasettitle›Syntheticallelefrequ?Datasettitle›Syntheticallelefreque?Datasettitle›Syntheticallelefrequen?Datasettitle›Syntheticallelefrequenc?Datasettitle›Syntheticallelefrequenci?Datasettitle›Syntheticallelefrequencie?Datasettitle›Syntheticallelefrequencies?Datasettitle›Syntheticallelefrequencies,t?Datasettitle›Syntheticallelefrequencies,tw?Datasettitle›Syntheticallelefrequencies,twe?Datasettitle›Syntheticallelefrequencies,twel?Datasettitle›Syntheticallelefrequencies,twelv?Datasettitle›Syntheticallelefrequencies,twelvep?Datasettitle›Syntheticallelefrequencies,twelvepo?Datasettitle›Syntheticallelefrequencies,twelvepop?Datasettitle›Syntheticallelefrequencies,twelvepopu?Datasettitle›Syntheticallelefrequencies,twelvepopul?Datasettitle›Syntheticallelefrequencies,twelvepopula?Datasettitle›Syntheticallelefrequencies,twelvepopulat?Datasettitle›Syntheticallelefrequencies,twelvepopulati?Datasettitle›Syntheticallelefrequencies,twelvepopulatio?Datasettitle›Syntheticallelefrequencies,twelvepopulation?Datasettitle›Syntheticallelefrequencies,twelvepopulations?Description›?Description›S?Description›Sy?Description›Syn?Description›Synt?Description›Synth?Description›Synthe?Description›Synthet?Description›Syntheti?Description›Synthetica?Description›Syntheticag?Description›Syntheticagg?Description›Syntheticaggr?Description›Syntheticaggre?Description›Syntheticaggreg?Description›Syntheticaggrega?Description›Syntheticaggregat?Description›Syntheticaggregate?Description›Syntheticaggregateda?Description›Syntheticaggregatedal?Description›Syntheticaggregatedall?Description›Syntheticaggregatedalle?Description›Syntheticaggregatedallel?Description›Syntheticaggregatedallelef?Description›Syntheticaggregatedallelefr?Description›Syntheticaggregatedallelefre?Description›Syntheticaggregatedallelefreq?Description›Syntheticaggregatedallelefrequ?Description›Syntheticaggregatedallelefreque?Description›Syntheticaggregatedallelefrequen?Description›Syntheticaggregatedallelefrequenc?Description›Syntheticaggregatedallelefrequenci?Description›Syntheticaggregatedallelefrequencie?Description›Syntheticaggregatedallelefrequenciesf?Description›Syntheticaggregatedallelefrequenciesfo?Description›Syntheticaggregatedallelefrequenciesforan?Description›Syntheticaggregatedallelefrequenciesforano?Description›Syntheticaggregatedallelefrequenciesforanod?Description›Syntheticaggregatedallelefrequenciesforanoded?Description›Syntheticaggregatedallelefrequenciesforanodede?Description›Syntheticaggregatedallelefrequenciesforanodedem?Description›Syntheticaggregatedallelefrequenciesforanodedemo?Description›Syntheticaggregatedallelefrequenciesforanodedemo.?Provenance(blanktoskip)›?Adddiscoverykeywords?(y/n)›yes?Keywords(comma-separated)(allele-frequency,genomics)›?Recordcohortsize?(y/n)›yes?Distinctsequencedsubjects(blanktoskip)›?Distinctsequencedsubjects(blanktoskip)›5?Distinctsequencedsubjects(blanktoskip)›50?Distinctsequencedsubjects(blanktoskip)›500?Typicalagerangeinyears,e.g.18-90or18+(blanktoskip)›?Isthissyntheticdata?(y/n)›no?Creatingorganisation›?Creatingorganisation›E?Creatingorganisation›Ex?Creatingorganisation›Exa?Creatingorganisation›Exam?Creatingorganisation›Examp?Creatingorganisation›Exampl?Creatingorganisation›ExampleI?Creatingorganisation›ExampleIn?Creatingorganisation›ExampleIns?Creatingorganisation›ExampleInst?Creatingorganisation›ExampleInsti?Creatingorganisation›ExampleInstit?Creatingorganisation›ExampleInstitu?Creatingorganisation›ExampleInstitut?Creatingorganisation›ExampleInstitute?Contactname(e.g.Dataaccessteam)›?Contactname(e.g.Dataaccessteam)›D?Contactname(e.g.Dataaccessteam)›Da?Contactname(e.g.Dataaccessteam)›Dat?Contactname(e.g.Dataaccessteam)›Dataa?Contactname(e.g.Dataaccessteam)›Dataac?Contactname(e.g.Dataaccessteam)›Dataacc?Contactname(e.g.Dataaccessteam)›Dataacce?Contactname(e.g.Dataaccessteam)›Dataacces?Contactname(e.g.Dataaccessteam)›Dataaccesst?Contactname(e.g.Dataaccessteam)›Dataaccesste?Contactname(e.g.Dataaccessteam)›Dataaccesstea?Contactname(e.g.Dataaccessteam)›Dataaccessteam?Contacte-mail›?Contacte-mail›d?Contacte-mail›da?Contacte-mail›dat?Contacte-mail›data?Contacte-mail›data@?Contacte-mail›data@e?Contacte-mail›data@ex?Contacte-mail›data@exa?Contacte-mail›data@exam?Contacte-mail›data@examp?Contacte-mail›data@exampl?Contacte-mail›data@example?Contacte-mail›data@example.?Contacte-mail›data@example.o?Contacte-mail›data@example.or?Contacte-mail›data@example.org?Contactwebpage(blanktoskip)›?Rememberthiscontactpointintheprofileforfuturedatasets?(y/n)›yes?Healthcategories(EHDSArt.51;Spacetoggles,Enterconfirms)›⬚Humangenetic⬚Humanepigenomic✔Humangenomic?Healththemes(Spacetoggles,Enterconfirms)›⬚Antimicrobialresistance&infectioncontrol⬚Blood-borne&sexuallytransmittedinfections⬚Cancer⬚Climate&planetaryhealth⬚Emergencies,disasters,travel&humanitariansettings⬚Enteric,water-&food-borneinfections⬚Environmental,occupational&radiationhealth(incl.WASH&urban)✔Healthproducts,technologies,data&research⬚Healthsystems,quality,caremodels&determinants⬚Immunization&vaccine-preventablediseases⬚Injuries,envenoming&drowning⬚Life-coursehealth:maternal,newborn,child,adolescent&ageing⬚Mental,neurological&substanceuse⬚Noncommunicablediseases–metabolic&cardiopulmonary⬚Nutrition&foodsecurity⬚Sexual&reproductivehealthandrights⬚Respiratoryinfectiousdiseases⬚Oral,eye&sensoryhealth⬚Neglectedtropical,parasitic&fungalskindiseases⬚Vector-borne&zoonoticviraldiseases?Conformsto(GDIstandards);Spacetoggles,Enterconfirms›⬚Externallygoverned⬚1+MGcompliant⬚1+MGcohort?Publicationaboutthedataset,asaDOI(blanktoskip)›PUBLIC❯RESTRICTED❯PUBLICRESTRICTED❯CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)CC01.0:publicdomain(typicalforsyntheticdata)CCBY4.0:reusewithattribution(typicalforpublicaggregateddata)❯CC01.0:publicdomain(typicalforsyntheticdata)?Applicablelegislation(Spacetoggles,Enterconfirms)›✔EHDS:Regulation(EU)2025/327⬚GDPR:Regulation(EU)2016/679?AddanotherlegislationELIorIRI?(y/n)›no?UsethestandardGenomeofEuropeAFprovenance?(y/n)›yes?Withholdrowswithallelecountbelow(0=publisheverything)(0)›?Withholdrowswithallelecountbelow(0=publisheverything)(0)›1?Withholdrowswithallelecountbelow(0=publisheverything)(0)›10gdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)5.50MiB/sETA0sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)9.69MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)10.32MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)10.93MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)11.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.22KiB/490.81KiB(8%)11.83MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)12.38MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]72.44KiB/490.81KiB(15%)12.60MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)12.92MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]102.43KiB/490.81KiB(21%)13.04MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)13.23MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)13.37MiB/sETA0s--Review:package.yamlasitwillbewritten-----------------------------------------------------metadata:prefix:"GDI"org:"EXAMPLE"catalog:"gdi-aggregated"title:"Syntheticallelefrequencies,twelvepopulations"description:"Syntheticaggregatedallelefrequenciesforanodedemo."type:-"https://publications.europa.eu/resource/authority/dataset-type/SYNTHETIC_DATA"?Writepackage.yaml?›❯WriteitEdititin$EDITORfirstAbort(theanswersarediscarded)?Publishthesepopulations?(y/n)›nogdi-sample.GRCh38.vcf.gz[>]2.50KiB/490.81KiB(1%)487.15KiB/sETA1sgdi-sample.GRCh38.vcf.gz[>]10.88KiB/490.81KiB(2%)2.16MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]20.82KiB/490.81KiB(4%)6.39MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=>]31.17KiB/490.81KiB(6%)8.75MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]40.98KiB/490.81KiB(8%)10.45MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==>]41.20KiB/490.81KiB(8%)11.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===>]61.25KiB/490.81KiB(12%)13.68MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=====>]92.13KiB/490.81KiB(19%)15.78MiB/sETA0sgdi-sample.GRCh38.vcf.gz[======>]121.22KiB/490.81KiB(25%)17.11MiB/sETA0sgdi-sample.GRCh38.vcf.gz[=======>]140.05KiB/490.81KiB(29%)17.77MiB/sETA0sgdi-sample.GRCh38.vcf.gz[========>]166.86KiB/490.81KiB(34%)18.53MiB/sETA0sgdi-sample.GRCh38.vcf.gz[==========>]196.32KiB/490.81KiB(40%)19.13MiB/sETA0sgdi-sample.GRCh38.vcf.gz[===========>]215.42KiB/490.81KiB(44%)19.46MiB/sETA0spacking[]0B/128.57KiB(0%)0B/sETA0spacking[>]512B/128.57KiB(0%)3.33MiB/sETA0spacking[=====>]17.00KiB/128.57KiB(13%)18.65MiB/sETA0spacking[================>]48.50KiB/128.57KiB(38%)32.62MiB/sETA0spacking[====================>]60.50KiB/128.57KiB(47%)34.89MiB/sETA0spacking[====================>]60.50KiB/128.57KiB(47%)35.34MiB/sETA0spacking[===========================>]79.50KiB/128.57KiB(62%)38.14MiB/sETA0spacking[=============================>]87.50KiB/128.57KiB(68%)38.74MiB/sETA0spacking[=================================>]97.00KiB/128.57KiB(75%)39.20MiB/sETA0spacking[=================================>]101.50KiB/128.57KiB(79%)47.49MiB/sETA0spacking[===================================>]107.00KiB/128.57KiB(83%)48.70MiB/sETA0spacking[=====================================>]112.50KiB/128.57KiB(87%)49.44MiB/sETA0spacking[=======================================>]117.50KiB/128.57KiB(91%)50.32MiB/sETA0spacking[========================================>]122.50KiB/128.57KiB(95%)50.92MiB/sETA0spacking[==========================================]132.00KiB/128.57KiB(100%)51.74MiB/sETA0spacking[==========================================]140.00KiB/128.57KiB(100%)52.45MiB/sETA0s?Removethestagingdir?(y/n)›yes?Sendthepackagetothenodenow?›❯UploadtoS3(S3bucketgdi-datasets@http://127.0.0.1:3900)Skip:publishelsewhereuploadingGDI-EE-…5653596.tar.c4gh[]0B/149.31KiB(0%)0B/sETA0suploadingGDI-EE-…5653596.tar.c4gh[==============]149.31KiB/149.31KiB(100%)124.71MiB/sETA0s \ No newline at end of file diff --git a/scripts/demo/drive-wizard.py b/scripts/demo/drive-wizard.py index 1035b85..366546b 100755 --- a/scripts/demo/drive-wizard.py +++ b/scripts/demo/drive-wizard.py @@ -78,14 +78,22 @@ def script(a: argparse.Namespace) -> list[Step]: "Synthetic aggregated allele frequencies for a node demo.", ENTER, ), + ("Provenance (blank to skip)", ENTER), ("Add discovery keywords?", YES), ("Keywords (comma-separated)", ENTER), ("Record cohort size?", YES), ("Distinct sequenced subjects", "500", ENTER), + ("Typical age range in years", ENTER), ("Is this synthetic data?", YES), ("Creating organisation", "Example Institute", ENTER), + ("Contact name", "Data access team", ENTER), + ("Contact e-mail", "data@example.org", ENTER), + ("Contact web page", ENTER), + ("Remember this contact point", YES), ("Health categories", 0.8, ENTER), + ("Health themes", 0.8, ENTER), ("Conforms to", 0.8, ENTER), + ("Publication about the dataset", ENTER), ("Access rights", 0.5, UP, 0.5, ENTER), ("License", 0.5, DOWN, 0.5, ENTER), ("Applicable legislation", 0.8, ENTER),