diff --git a/.github/workflows/ci-nix.yml b/.github/workflows/ci-nix.yml index b85cac0..77153a3 100644 --- a/.github/workflows/ci-nix.yml +++ b/.github/workflows/ci-nix.yml @@ -1,9 +1,9 @@ name: "CI Nix" on: - # Run only when pushing to master branch, and making PRs + # Run only when pushing to main branch, and making PRs push: branches: - - master + - main pull_request: jobs: build: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cfa36..66905ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -on: [push] +on: [push, pull_request] name: CI diff --git a/Cargo.toml b/Cargo.toml index 7617cb6..df0e237 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ nu-command = { version = "0.116.0", features = ["plugin"] } nu-plugin = "0.116.0" nu-plugin-engine = { version = "0.116.0", features = ["local-socket"] } nu-protocol = { version = "0.116.0", features = ["plugin"] } -object_store = { version = "0.14", features = ["aws"] } +object_store = { version = "0.14.2", features = ["aws", "gcp"] } tokio = { version = "1.53", features = ["full"] } url = "2.5" diff --git a/README.md b/README.md index 447e249..7092295 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ Provides uniform access to cloud storage services for nushell. - `cloud rm` - Remove a file from cloud sotrage - `cloud save` - Save a file to cloud storage - AWS S3 support +- Google Cloud Storage support - Coming Soon: Azure support -- Coming Soon: Google cloud support ## Installation diff --git a/flake.lock b/flake.lock index bfe8608..d97f21b 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1758277210, - "narHash": "sha256-iCGWf/LTy+aY0zFu8q12lK8KuZp7yvdhStehhyX1v8w=", + "lastModified": 1790463110, + "narHash": "sha256-hKlVl12B1dF0Q5vd9dY3lIJM5mFGWYSlXwSLAqHZ1+s=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8eaee110344796db060382e15d3af0a9fc396e0e", + "rev": "e158d9ed9b51c98974c5e66e1ba1c9e0255fecaa", "type": "github" }, "original": { @@ -49,11 +49,11 @@ ] }, "locked": { - "lastModified": 1758508617, - "narHash": "sha256-kx2uELmVnAbiekj/YFfWR26OXqXedImkhe2ocnbumTA=", + "lastModified": 1790584267, + "narHash": "sha256-NvjQLSztzad7ulLbvrYFKpz4pRrxZ34zKM7mTGEK1wk=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "d2bac276ac7e669a1f09c48614538a37e3eb6d0f", + "rev": "3f4df219c0d9aa82710232324856755d51ef0b69", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 32848c0..c30c6ed 100644 --- a/flake.nix +++ b/flake.nix @@ -2,5 +2,16 @@ inputs.rust-flake.url = "github:KaiSforza/rust-flake"; outputs = inputs: let inherit (inputs.rust-flake.lib) rust-flake rust-flakes; - in rust-flakes [ (rust-flake { root = ./.; }) ]; + in rust-flakes [ + (rust-flake { + root = ./.; + # Tests construct object_store/reqwest TLS clients, which load the + # system CA store when the client is built. The nix build sandbox + # has no system CA store, so provide one for the test phase. + pkg-overrides = final: prev: { + nativeCheckInputs = [ final.cacert ]; + SSL_CERT_FILE = "${final.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; + }) + ]; } diff --git a/src/cache.rs b/src/cache.rs index 8f774ad..5947d2f 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -28,6 +28,7 @@ pub enum ObjectStoreCacheKey { Memory, Local, AmazonS3 { bucket: String, region: String }, + GoogleCloudStorage { bucket: String }, } impl From<&NuObjectStore> for ObjectStoreCacheKey { @@ -39,7 +40,11 @@ impl From<&NuObjectStore> for ObjectStoreCacheKey { bucket: bucket.to_owned(), region: region.to_owned(), }, - NuObjectStore::GoogleCloudStorage(_) => unimplemented!(), + NuObjectStore::GoogleCloudStorage { bucket, .. } => { + ObjectStoreCacheKey::GoogleCloudStorage { + bucket: bucket.to_owned(), + } + } NuObjectStore::MicrosoftAzure(_) => unimplemented!(), NuObjectStore::Http(_) => unimplemented!(), } diff --git a/src/command/ls.rs b/src/command/ls.rs index 2d63734..d8b06c3 100644 --- a/src/command/ls.rs +++ b/src/command/ls.rs @@ -31,11 +31,18 @@ impl PluginCommand for Ls { } fn examples(&self) -> Vec> { - vec![Example { - description: "List the files in a s3 bucket.", - example: "cloud ls s3://mybucket", - result: None, - }] + vec![ + Example { + description: "List the files in a s3 bucket.", + example: "cloud ls s3://mybucket", + result: None, + }, + Example { + description: "List the files in a gcs bucket.", + example: "cloud ls gs://mybucket", + result: None, + }, + ] } fn run( diff --git a/src/command/open.rs b/src/command/open.rs index d7494fd..5fb5203 100644 --- a/src/command/open.rs +++ b/src/command/open.rs @@ -34,11 +34,18 @@ impl PluginCommand for Open { } fn examples(&self) -> Vec> { - vec![Example { - description: "Load a file from s3.", - example: "cloud open s3://mybucket/file.txt", - result: None, - }] + vec![ + Example { + description: "Load a file from s3.", + example: "cloud open s3://mybucket/file.txt", + result: None, + }, + Example { + description: "Load a file from gcs.", + example: "cloud open gs://mybucket/file.txt", + result: None, + }, + ] } fn run( diff --git a/src/command/rm.rs b/src/command/rm.rs index 55b6a89..7d85bc9 100644 --- a/src/command/rm.rs +++ b/src/command/rm.rs @@ -31,11 +31,18 @@ impl PluginCommand for Remove { } fn examples(&self) -> Vec> { - vec![Example { - description: "Remove a file from s3.", - example: "cloud rm s3://mybucket/file.txt", - result: None, - }] + vec![ + Example { + description: "Remove a file from s3.", + example: "cloud rm s3://mybucket/file.txt", + result: None, + }, + Example { + description: "Remove a file from gcs.", + example: "cloud rm gs://mybucket/file.txt", + result: None, + }, + ] } fn run( diff --git a/src/command/save.rs b/src/command/save.rs index 9e3b7b9..016e7c8 100644 --- a/src/command/save.rs +++ b/src/command/save.rs @@ -36,11 +36,18 @@ impl PluginCommand for Save { } fn examples(&self) -> Vec> { - vec![Example { - description: "Save a csv file to s3.", - example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save s3://mybucket/file.csv", - result: None, - }] + vec![ + Example { + description: "Save a csv file to s3.", + example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save s3://mybucket/file.csv", + result: None, + }, + Example { + description: "Save a csv file to gcs.", + example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save gs://mybucket/file.csv", + result: None, + }, + ] } fn description(&self) -> &str { diff --git a/src/providers/gcs.rs b/src/providers/gcs.rs new file mode 100644 index 0000000..4db85dd --- /dev/null +++ b/src/providers/gcs.rs @@ -0,0 +1,142 @@ +use std::sync::Arc; + +use nu_plugin::EngineInterface; +use nu_protocol::{ShellError, Span, Spanned}; +use object_store::ObjectStore; +use object_store::gcp::GoogleCloudStorageBuilder; +use url::Url; + +use crate::cache::{Cache, ObjectStoreCacheKey}; + +use super::NuObjectStore; + +pub async fn build_object_store( + engine: &EngineInterface, + cache: &Cache, + url: &Spanned, +) -> Result { + let bucket = parse_bucket(&url.item).ok_or_else(|| { + ShellError::Generic(nu_protocol::shell_error::generic::GenericError::new( + format!( + "Could not determine Google Cloud Storage bucket name from url {}", + url.item + ), + "", + url.span, + )) + })?; + + let cache_key = ObjectStoreCacheKey::GoogleCloudStorage { + bucket: bucket.clone(), + }; + + if let Some(object_store) = cache.get_store(&cache_key).await { + Ok(object_store) + } else { + let store = build_store(GoogleCloudStorageBuilder::from_env(), &url.item, url.span)?; + + let object_store = NuObjectStore::GoogleCloudStorage { store, bucket }; + + cache + .put_store(engine, cache_key, object_store.clone()) + .await?; + Ok(object_store) + } +} + +fn build_store( + builder: GoogleCloudStorageBuilder, + url: &Url, + span: Span, +) -> Result, ShellError> { + let gcs = builder.with_url(url.to_string()).build().map_err(|e| { + ShellError::Generic(nu_protocol::shell_error::generic::GenericError::new( + format!("Could not create Google Cloud Storage client: {e}"), + "", + span, + )) + })?; + + Ok(Arc::new(gcs)) +} + +fn parse_bucket(url: &Url) -> Option { + if url.scheme() != "gs" { + return None; + } + + url.host_str() + .filter(|bucket| !bucket.is_empty()) + .map(ToString::to_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true}"#; + + #[test] + fn test_parse_bucket() { + assert_eq!( + parse_bucket(&Url::parse("gs://my-bucket/path/to/file.txt").unwrap()), + Some("my-bucket".to_string()) + ); + assert_eq!( + parse_bucket(&Url::parse("gs://my-bucket").unwrap()), + Some("my-bucket".to_string()) + ); + assert_eq!( + parse_bucket(&Url::parse("gs:///path/to/file.txt").unwrap()), + None + ); + assert_eq!( + parse_bucket(&Url::parse("s3://my-bucket/path/to/file.txt").unwrap()), + None + ); + } + + #[test] + fn test_build_store_without_network_access() { + let url = Url::parse("gs://test-bucket/path/to/file.txt").unwrap(); + let builder = GoogleCloudStorageBuilder::new().with_service_account_key(FAKE_KEY); + let store = build_store(builder, &url, Span::test_data()).unwrap(); + + let object_store = NuObjectStore::GoogleCloudStorage { + store, + bucket: "test-bucket".to_string(), + }; + assert_eq!( + ObjectStoreCacheKey::from(&object_store), + ObjectStoreCacheKey::GoogleCloudStorage { + bucket: "test-bucket".to_string() + } + ); + } + + #[test] + fn test_build_store_invalid_service_account_key() { + let url = Url::parse("gs://test-bucket/path/to/file.txt").unwrap(); + let builder = GoogleCloudStorageBuilder::new().with_service_account_key("not json"); + let error = build_store(builder, &url, Span::test_data()).unwrap_err(); + + assert!( + error + .to_string() + .contains("Could not create Google Cloud Storage client") + ); + } + + #[test] + fn test_build_store_missing_bucket_name() { + let url = Url::parse("gs://").unwrap(); + let builder = GoogleCloudStorageBuilder::new().with_service_account_key(FAKE_KEY); + let error = build_store(builder, &url, Span::test_data()).unwrap_err(); + + assert!( + error + .to_string() + .contains("Could not create Google Cloud Storage client") + ); + } +} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 7661a75..71e300f 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,4 +1,5 @@ mod aws; +mod gcs; mod local; mod mem; @@ -18,8 +19,10 @@ pub enum NuObjectStore { bucket: String, region: String, }, - #[allow(dead_code)] - GoogleCloudStorage(Arc), + GoogleCloudStorage { + store: Arc, + bucket: String, + }, #[allow(dead_code)] MicrosoftAzure(Arc), #[allow(dead_code)] @@ -31,7 +34,7 @@ impl NuObjectStore { NuObjectStore::Local(store) => store.as_ref(), NuObjectStore::Memory(store) => store.as_ref(), NuObjectStore::AmazonS3 { store, .. } => store.as_ref(), - NuObjectStore::GoogleCloudStorage(store) => store.as_ref(), + NuObjectStore::GoogleCloudStorage { store, .. } => store.as_ref(), NuObjectStore::MicrosoftAzure(store) => store.as_ref(), NuObjectStore::Http(store) => store.as_ref(), } @@ -59,6 +62,9 @@ pub async fn parse_url( let object_store = match scheme { ObjectStoreScheme::AmazonS3 => aws::build_object_store(engine, cache, url).await?, + ObjectStoreScheme::GoogleCloudStorage => { + gcs::build_object_store(engine, cache, url).await? + } ObjectStoreScheme::Local => local::build_object_store(engine, cache).await?, ObjectStoreScheme::Memory => mem::build_object_store(engine, cache).await?, _ => { @@ -72,3 +78,31 @@ pub async fn parse_url( Ok((object_store, path)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gs_url_maps_to_google_cloud_storage() { + let url = Url::parse("gs://test-bucket/path/to/file.txt").unwrap(); + let (scheme, path) = ObjectStoreScheme::parse(&url).unwrap(); + + assert!(matches!(scheme, ObjectStoreScheme::GoogleCloudStorage)); + assert_eq!(path.as_ref(), "path/to/file.txt"); + } + + #[test] + fn test_gs_url_without_host_is_not_supported() { + let url = Url::parse("gs://").unwrap(); + assert!(ObjectStoreScheme::parse(&url).is_err()); + } + + #[test] + fn test_s3_url_does_not_map_to_google_cloud_storage() { + let url = Url::parse("s3://test-bucket/path/to/file.txt").unwrap(); + let (scheme, _) = ObjectStoreScheme::parse(&url).unwrap(); + + assert!(!matches!(scheme, ObjectStoreScheme::GoogleCloudStorage)); + } +}