feat: add support for Google Cloud Storage

.github/workflows/ci-nix.yml

@@ -1,9 +1,9 @@
 name: "CI Nix"
 on:
-  # Run only when pushing to master branch, and making PRs
+  # Run only when pushing to main branch, and making PRs
   push:
     branches:
-      - master
+      - main
   pull_request:
 jobs:
   build:

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-on: [push]
+on: [push, pull_request]
 
 name: CI
 

Cargo.toml

@@ -18,11 +18,11 @@
 log = "0.4"
 mimalloc = "0.1.52"
 mime_guess = "2.0.5"
 nu-command = { version = "0.116.0", features = ["plugin"] }
 nu-plugin = "0.116.0"
 nu-plugin-engine = { version = "0.116.0", features = ["local-socket"] }
 nu-protocol = { version = "0.116.0", features = ["plugin"] }
-object_store = { version = "0.14", features = ["aws"] }
+object_store = { version = "0.14.2", features = ["aws", "gcp"] }
 tokio = { version = "1.53", features = ["full"] }
 url = "2.5"
 

README.md

@@ -8,8 +8,8 @@ Provides uniform access to cloud storage services for nushell.
 - `cloud rm` - Remove a file from cloud sotrage
 - `cloud save` - Save a file to cloud storage
 - AWS S3 support
+- Google Cloud Storage support
 - Coming Soon: Azure support
-- Coming Soon: Google cloud support
 
 ## Installation
 

flake.nix

@@ -2,5 +2,16 @@
   inputs.rust-flake.url = "github:KaiSforza/rust-flake";
   outputs = inputs:
     let inherit (inputs.rust-flake.lib) rust-flake rust-flakes;
-    in rust-flakes [ (rust-flake { root = ./.; }) ];
+    in rust-flakes [
+      (rust-flake {
+        root = ./.;
+        # Tests construct object_store/reqwest TLS clients, which load the
+        # system CA store when the client is built. The nix build sandbox
+        # has no system CA store, so provide one for the test phase.
+        pkg-overrides = final: prev: {
+          nativeCheckInputs = [ final.cacert ];
+          SSL_CERT_FILE = "${final.cacert}/etc/ssl/certs/ca-bundle.crt";
+        };
+      })
+    ];
 }

src/cache.rs

@@ -28,6 +28,7 @@ pub enum ObjectStoreCacheKey {
     Memory,
     Local,
     AmazonS3 { bucket: String, region: String },
+    GoogleCloudStorage { bucket: String },
 }
 
 impl From<&NuObjectStore> for ObjectStoreCacheKey {
@@ -39,7 +40,11 @@ impl From<&NuObjectStore> for ObjectStoreCacheKey {
                 bucket: bucket.to_owned(),
                 region: region.to_owned(),
             },
-            NuObjectStore::GoogleCloudStorage(_) => unimplemented!(),
+            NuObjectStore::GoogleCloudStorage { bucket, .. } => {
+                ObjectStoreCacheKey::GoogleCloudStorage {
+                    bucket: bucket.to_owned(),
+                }
+            }
             NuObjectStore::MicrosoftAzure(_) => unimplemented!(),
             NuObjectStore::Http(_) => unimplemented!(),
         }

src/command/ls.rs

@@ -31,11 +31,18 @@ impl PluginCommand for Ls {
     }
 
     fn examples(&self) -> Vec<Example<'_>> {
-        vec![Example {
-            description: "List the files in a s3 bucket.",
-            example: "cloud ls s3://mybucket",
-            result: None,
-        }]
+        vec![
+            Example {
+                description: "List the files in a s3 bucket.",
+                example: "cloud ls s3://mybucket",
+                result: None,
+            },
+            Example {
+                description: "List the files in a gcs bucket.",
+                example: "cloud ls gs://mybucket",
+                result: None,
+            },
+        ]
     }
 
     fn run(

src/command/open.rs

@@ -34,11 +34,18 @@ impl PluginCommand for Open {
     }
 
     fn examples(&self) -> Vec<Example<'_>> {
-        vec![Example {
-            description: "Load a file from s3.",
-            example: "cloud open s3://mybucket/file.txt",
-            result: None,
-        }]
+        vec![
+            Example {
+                description: "Load a file from s3.",
+                example: "cloud open s3://mybucket/file.txt",
+                result: None,
+            },
+            Example {
+                description: "Load a file from gcs.",
+                example: "cloud open gs://mybucket/file.txt",
+                result: None,
+            },
+        ]
     }
 
     fn run(

src/command/rm.rs

@@ -31,11 +31,18 @@ impl PluginCommand for Remove {
     }
 
     fn examples(&self) -> Vec<Example<'_>> {
-        vec![Example {
-            description: "Remove a file from s3.",
-            example: "cloud rm s3://mybucket/file.txt",
-            result: None,
-        }]
+        vec![
+            Example {
+                description: "Remove a file from s3.",
+                example: "cloud rm s3://mybucket/file.txt",
+                result: None,
+            },
+            Example {
+                description: "Remove a file from gcs.",
+                example: "cloud rm gs://mybucket/file.txt",
+                result: None,
+            },
+        ]
     }
 
     fn run(

src/command/save.rs

@@ -36,11 +36,18 @@ impl PluginCommand for Save {
     }
 
     fn examples(&self) -> Vec<Example<'_>> {
-        vec![Example {
-            description: "Save a csv file to s3.",
-            example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save s3://mybucket/file.csv",
-            result: None,
-        }]
+        vec![
+            Example {
+                description: "Save a csv file to s3.",
+                example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save s3://mybucket/file.csv",
+                result: None,
+            },
+            Example {
+                description: "Save a csv file to gcs.",
+                example: "[[a b]; [1 1] [1 2] [2 1] [2 2] [3 1] [3 2]] | to csv | cloud save gs://mybucket/file.csv",
+                result: None,
+            },
+        ]
     }
 
     fn description(&self) -> &str {

src/providers/gcs.rs

@@ -0,0 +1,142 @@
+use std::sync::Arc;
+
+use nu_plugin::EngineInterface;
+use nu_protocol::{ShellError, Span, Spanned};
+use object_store::ObjectStore;
+use object_store::gcp::GoogleCloudStorageBuilder;
+use url::Url;
+
+use crate::cache::{Cache, ObjectStoreCacheKey};
+
+use super::NuObjectStore;
+
+pub async fn build_object_store(
+    engine: &EngineInterface,
+    cache: &Cache,
+    url: &Spanned<Url>,
+) -> Result<NuObjectStore, ShellError> {
+    let bucket = parse_bucket(&url.item).ok_or_else(|| {
+        ShellError::Generic(nu_protocol::shell_error::generic::GenericError::new(
+            format!(
+                "Could not determine Google Cloud Storage bucket name from url {}",
+                url.item
+            ),
+            "",
+            url.span,
+        ))
+    })?;
+
+    let cache_key = ObjectStoreCacheKey::GoogleCloudStorage {
+        bucket: bucket.clone(),
+    };
+
+    if let Some(object_store) = cache.get_store(&cache_key).await {
+        Ok(object_store)
+    } else {
+        let store = build_store(GoogleCloudStorageBuilder::from_env(), &url.item, url.span)?;
+
+        let object_store = NuObjectStore::GoogleCloudStorage { store, bucket };
+
+        cache
+            .put_store(engine, cache_key, object_store.clone())
+            .await?;
+        Ok(object_store)
+    }
+}
+
+fn build_store(
+    builder: GoogleCloudStorageBuilder,
+    url: &Url,
+    span: Span,
+) -> Result<Arc<dyn ObjectStore>, ShellError> {
+    let gcs = builder.with_url(url.to_string()).build().map_err(|e| {
+        ShellError::Generic(nu_protocol::shell_error::generic::GenericError::new(
+            format!("Could not create Google Cloud Storage client: {e}"),
+            "",
+            span,
+        ))
+    })?;
+
+    Ok(Arc::new(gcs))
+}
+
+fn parse_bucket(url: &Url) -> Option<String> {
+    if url.scheme() != "gs" {
+        return None;
+    }
+
+    url.host_str()
+        .filter(|bucket| !bucket.is_empty())
+        .map(ToString::to_string)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true}"#;
+
+    #[test]
+    fn test_parse_bucket() {
+        assert_eq!(
+            parse_bucket(&Url::parse("gs://my-bucket/path/to/file.txt").unwrap()),
+            Some("my-bucket".to_string())
+        );
+        assert_eq!(
+            parse_bucket(&Url::parse("gs://my-bucket").unwrap()),
+            Some("my-bucket".to_string())
+        );
+        assert_eq!(
+            parse_bucket(&Url::parse("gs:///path/to/file.txt").unwrap()),
+            None
+        );
+        assert_eq!(
+            parse_bucket(&Url::parse("s3://my-bucket/path/to/file.txt").unwrap()),
+            None
+        );
+    }
+
+    #[test]
+    fn test_build_store_without_network_access() {
+        let url = Url::parse("gs://test-bucket/path/to/file.txt").unwrap();
+        let builder = GoogleCloudStorageBuilder::new().with_service_account_key(FAKE_KEY);
+        let store = build_store(builder, &url, Span::test_data()).unwrap();
+
+        let object_store = NuObjectStore::GoogleCloudStorage {
+            store,
+            bucket: "test-bucket".to_string(),
+        };
+        assert_eq!(
+            ObjectStoreCacheKey::from(&object_store),
+            ObjectStoreCacheKey::GoogleCloudStorage {
+                bucket: "test-bucket".to_string()
+            }
+        );
+    }
+
+    #[test]
+    fn test_build_store_invalid_service_account_key() {
+        let url = Url::parse("gs://test-bucket/path/to/file.txt").unwrap();
+        let builder = GoogleCloudStorageBuilder::new().with_service_account_key("not json");
+        let error = build_store(builder, &url, Span::test_data()).unwrap_err();
+
+        assert!(
+            error
+                .to_string()
+                .contains("Could not create Google Cloud Storage client")
+        );
+    }
+
+    #[test]
+    fn test_build_store_missing_bucket_name() {
+        let url = Url::parse("gs://").unwrap();
+        let builder = GoogleCloudStorageBuilder::new().with_service_account_key(FAKE_KEY);
+        let error = build_store(builder, &url, Span::test_data()).unwrap_err();
+
+        assert!(
+            error
+                .to_string()
+                .contains("Could not create Google Cloud Storage client")
+        );
+    }
+}