Allow external API clients to request a gzipped response

Cargo.toml

@@ -460,10 +460,10 @@ crossterm = { version = "0.29.0", features = ["event-stream"] }
 # NOTE: if you change the pinned revision of the `crucible` dependencies, you
 # must also update the references in package-manifest.toml to match the new
 # revision.
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "7103cd3a3d7b0112d2949dd135db06fef0c156bb" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "7103cd3a3d7b0112d2949dd135db06fef0c156bb" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "7103cd3a3d7b0112d2949dd135db06fef0c156bb" }
-crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "7103cd3a3d7b0112d2949dd135db06fef0c156bb" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "3df8a24e046bcfdb0d69f93075e8e1d2d0e9889d" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "3df8a24e046bcfdb0d69f93075e8e1d2d0e9889d" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "3df8a24e046bcfdb0d69f93075e8e1d2d0e9889d" }
+crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "3df8a24e046bcfdb0d69f93075e8e1d2d0e9889d" }
 # NOTE: See above!
 csv = "1.3.1"
 curve25519-dalek = "4"
@@ -485,9 +485,9 @@ dns-server = { path = "dns-server" }
 dns-server-api = { path = "dns-server-api" }
 dns-service-client = { path = "clients/dns-service-client" }
 dpd-client = { git = "https://github.com/oxidecomputer/dendrite", rev = "b31ba90d14636ef6bba9dbb56d756efa249d4b4e" }
-dropshot = { version = "0.16.6", features = [ "usdt-probes" ] }
-dropshot-api-manager = "0.6.0"
-dropshot-api-manager-types = "0.6.0"
+dropshot = { version = "0.17.0", features = [ "usdt-probes" ] }
+dropshot-api-manager = "0.7.0"
+dropshot-api-manager-types = "0.7.0"
 dyn-clone = "1.0.20"
 either = "1.15.0"
 ereport-types = { path = "ereport/types" }
@@ -703,15 +703,14 @@ proc-macro2 = "1.0"
 progenitor = "0.13.0"
 progenitor-client = "0.13.0"
 progenitor-extras = "0.1.1"
-progenitor-client010 = { package = "progenitor-client", version = "0.10.0" }
 # NOTE: if you change the pinned revision of the `bhyve_api` and propolis
 # dependencies, you must also update the references in package-manifest.toml to
 # match the new revision.
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "368a2225b79328514ce0ea9181d8f874019edaa2" }
-propolis-api-types-versions = { git = "https://github.com/oxidecomputer/propolis", rev = "368a2225b79328514ce0ea9181d8f874019edaa2" }
-propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "368a2225b79328514ce0ea9181d8f874019edaa2" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "368a2225b79328514ce0ea9181d8f874019edaa2" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "368a2225b79328514ce0ea9181d8f874019edaa2" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "3278ded80bb93538468313f046be906df4bd5748" }
+propolis-api-types-versions = { git = "https://github.com/oxidecomputer/propolis", rev = "3278ded80bb93538468313f046be906df4bd5748" }
+propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "3278ded80bb93538468313f046be906df4bd5748" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "3278ded80bb93538468313f046be906df4bd5748" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "3278ded80bb93538468313f046be906df4bd5748" }
 # NOTE: see above!
 proptest = "1.7.0"
 qorb = "0.4.1"
@@ -871,8 +870,8 @@ newtype-uuid = { version = "1.3.2", default-features = false }
 newtype-uuid-macros = "0.1.0"
 omicron-uuid-kinds = { path = "uuid-kinds", features = ["serde", "schemars08", "uuid-v4"] }
 
-scim2-rs = { git = "https://github.com/oxidecomputer/scim2-rs", rev = "c78005db837a71f94c3b8efac9a64cfbdb2d527f" }
-scim2-test-client = { git = "https://github.com/oxidecomputer/scim2-rs", rev = "c78005db837a71f94c3b8efac9a64cfbdb2d527f" }
+scim2-rs = { git = "https://github.com/oxidecomputer/scim2-rs", rev = "f3406edbe24d6c96e07a4a1459598671f04b8b59" }
+scim2-test-client = { git = "https://github.com/oxidecomputer/scim2-rs", rev = "f3406edbe24d6c96e07a4a1459598671f04b8b59" }
 
 # NOTE: The test profile inherits from the dev profile, so settings under
 # profile.dev get inherited. AVOID setting anything under profile.test: that
@@ -1049,6 +1048,8 @@ opt-level = 3
 # drift = { path = "../drift" }
 # dropshot = { path = "../dropshot/dropshot" }
 # dropshot_endpoint = { path = "../dropshot/dropshot_endpoint" }
+# dropshot-api-manager = { path = "../dropshot-api-manager/crates/dropshot-api-manager" }
+# dropshot-api-manager-types = { path = "../dropshot-api-manager/crates/dropshot-api-manager-types" }
 # progenitor = { path = "../progenitor/progenitor" }
 # progenitor-client = { path = "../progenitor/progenitor-client" }
 # steno = { path = "../steno" }

common/Cargo.toml

@@ -51,7 +51,6 @@ tokio = { workspace = true, features = ["full"] }
 uuid.workspace = true
 parse-display.workspace = true
 progenitor-client.workspace = true
-progenitor-client010.workspace = true
 progenitor-extras.workspace = true
 omicron-workspace-hack.workspace = true
 regress.workspace = true

common/src/api/external/error.rs

@@ -547,42 +547,6 @@ pub trait ClientError: std::fmt::Debug {
 // external client, others may require, for example, retries with an alternate
 // service instance or additional interpretation to sanitize the output error.
 // This should be removed to avoid leaking data.
-impl<T: ClientError> From<progenitor_client010::Error<T>> for Error {
-    fn from(e: progenitor_client010::Error<T>) -> Self {
-        match e {
-            // For most error variants, we delegate to the display impl for the
-            // Progenitor error type, but we pick apart an error response more
-            // carefully.
-            progenitor_client010::Error::InvalidRequest(_)
-            | progenitor_client010::Error::CommunicationError(_)
-            | progenitor_client010::Error::InvalidResponsePayload(..)
-            | progenitor_client010::Error::UnexpectedResponse(_)
-            | progenitor_client010::Error::InvalidUpgrade(_)
-            | progenitor_client010::Error::ResponseBodyError(_)
-            | progenitor_client010::Error::PreHookError(_)
-            | progenitor_client010::Error::PostHookError(_) => {
-                Error::internal_error(&e.to_string())
-            }
-            // This error represents an expected error from the remote service.
-            progenitor_client010::Error::ErrorResponse(rv) => {
-                let message = rv.message();
-
-                match rv.status() {
-                    http::StatusCode::SERVICE_UNAVAILABLE => {
-                        Error::unavail(&message)
-                    }
-                    status if status.is_client_error() => {
-                        Error::invalid_request(&message)
-                    }
-                    _ => Error::internal_error(&message),
-                }
-            }
-        }
-    }
-}
-
-// Equivalent From impl for progenitor-client 0.13. This coexists with the
-// progenitor_client010 impl above during the cross-repo upgrade window.
 impl<T: ClientError> From<progenitor_client::Error<T>> for Error {
     fn from(e: progenitor_client::Error<T>) -> Self {
         match e {

common/src/progenitor_operation_retry.rs

@@ -25,14 +25,14 @@ pub enum ProgenitorOperationRetryError<E> {
 
     /// The retry loop progenitor operation saw a permanent client error
     #[error("permanent error")]
-    ProgenitorError(#[source] progenitor_client010::Error<E>),
+    ProgenitorError(#[source] progenitor_client::Error<E>),
 }
 
 impl<E> ProgenitorOperationRetryError<E> {
     pub fn is_not_found(&self) -> bool {
         match &self {
             ProgenitorOperationRetryError::ProgenitorError(e) => match e {
-                progenitor_client010::Error::ErrorResponse(rv) => {
+                progenitor_client::Error::ErrorResponse(rv) => {
                     match rv.status() {
                         http::StatusCode::NOT_FOUND => true,
 
@@ -70,7 +70,7 @@ pub struct ProgenitorOperationRetry<
     T,
     E: std::fmt::Debug,
     F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T, progenitor_client010::Error<E>>>,
+    Fut: Future<Output = Result<T, progenitor_client::Error<E>>>,
     BF: FnMut() -> BFut,
     BFut: Future<Output = Result<bool, Error>>,
 > {
@@ -85,7 +85,7 @@ impl<T, E, F, Fut, BF, BFut> ProgenitorOperationRetry<T, E, F, Fut, BF, BFut>
 where
     E: std::fmt::Debug + 'static,
     F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T, progenitor_client010::Error<E>>>,
+    Fut: Future<Output = Result<T, progenitor_client::Error<E>>>,
     BF: FnMut() -> BFut,
     BFut: Future<Output = Result<bool, Error>>,
 {
@@ -121,7 +121,7 @@ where
                     }
 
                     match f.await {
-                        Err(progenitor_client010::Error::CommunicationError(e)) => {
+                        Err(progenitor_client::Error::CommunicationError(e)) => {
                             warn!(
                                 log,
                                 "saw transient communication error, retrying...";
@@ -130,12 +130,12 @@ where
 
                             Err(BackoffError::transient(
                                 ProgenitorOperationRetryError::ProgenitorError(
-                                    progenitor_client010::Error::CommunicationError(e)
+                                    progenitor_client::Error::CommunicationError(e)
                                 )
                             ))
                         }
 
-                        Err(progenitor_client010::Error::ErrorResponse(
+                        Err(progenitor_client::Error::ErrorResponse(
                             response_value,
                         )) => {
                             match response_value.status() {
@@ -144,7 +144,7 @@ where
                                 | http::StatusCode::TOO_MANY_REQUESTS => {
                                     Err(BackoffError::transient(
                                         ProgenitorOperationRetryError::ProgenitorError(
-                                            progenitor_client010::Error::ErrorResponse(
+                                            progenitor_client::Error::ErrorResponse(
                                                 response_value
                                             )
                                         )
@@ -154,7 +154,7 @@ where
                                 // Anything else is a permanent error
                                 _ => Err(BackoffError::Permanent(
                                     ProgenitorOperationRetryError::ProgenitorError(
-                                        progenitor_client010::Error::ErrorResponse(
+                                        progenitor_client::Error::ErrorResponse(
                                             response_value
                                         )
                                     )

dns-server/src/lib.rs

@@ -154,7 +154,7 @@ impl TransientServer {
                 bind_address: "[::1]:0".parse().unwrap(),
                 default_request_body_max_bytes: 4 * 1024 * 1024,
                 default_handler_task_mode: dropshot::HandlerTaskMode::Detached,
-                log_headers: vec![],
+                ..Default::default()
             },
         )
         .await?;

dns-server/tests/basic_test.rs

@@ -732,7 +732,7 @@ fn test_config(
         bind_address: "[::1]:0".to_string().parse().unwrap(),
         default_request_body_max_bytes: 1024,
         default_handler_task_mode: HandlerTaskMode::Detached,
-        log_headers: vec![],
+        ..Default::default()
     };
 
     Ok((tmp_dir, config_storage, config_dropshot, logctx))

dns-server/tests/cross_version_test.rs

@@ -275,7 +275,7 @@ fn test_config(
         bind_address: "[::1]:0".to_string().parse().unwrap(),
         default_request_body_max_bytes: 1024,
         default_handler_task_mode: HandlerTaskMode::Detached,
-        log_headers: vec![],
+        ..Default::default()
     };
 
     Ok((tmp_dir, config_storage, config_dropshot, logctx))

gateway/src/lib.rs

@@ -94,7 +94,7 @@ fn start_dropshot_server(
                 bind_address: SocketAddr::V6(addr),
                 default_request_body_max_bytes,
                 default_handler_task_mode: HandlerTaskMode::Detached,
-                log_headers: vec![],
+                ..Default::default()
             };
 
             let http_server = dropshot::ServerBuilder::new(

installinator-api/src/lib.rs

@@ -125,6 +125,6 @@ pub fn default_config(bind_address: std::net::SocketAddr) -> ConfigDropshot {
         bind_address,
         default_request_body_max_bytes: 1024,
         default_handler_task_mode: HandlerTaskMode::Detached,
-        log_headers: vec![],
+        ..Default::default()
     }
 }

internal-dns/resolver/src/resolver.rs

@@ -520,7 +520,7 @@ mod test {
                     bind_address: "[::1]:0".parse().unwrap(),
                     default_request_body_max_bytes: 8 * 1024,
                     default_handler_task_mode: HandlerTaskMode::Detached,
-                    log_headers: vec![],
+                    ..Default::default()
                 },
             )
             .await

nexus/Cargo.toml

@@ -98,7 +98,6 @@ ref-cast.workspace = true
 rdb-types.workspace = true
 regex.workspace = true
 reqwest = { workspace = true, features = ["http2", "json"] }
-reqwest012 = { workspace = true }
 ring.workspace = true
 samael.workspace = true
 schemars = { workspace = true, features = ["chrono", "uuid1"] }
@@ -168,6 +167,7 @@ camino-tempfile.workspace = true
 criterion.workspace = true
 diesel.workspace = true
 dns-server.workspace = true
+flate2.workspace = true
 expectorate.workspace = true
 gateway-messages.workspace = true
 gateway-test-utils.workspace = true

nexus/examples/config-second.toml

@@ -55,6 +55,7 @@ external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 # used by `omicron-dev run-all`
 bind_address = "127.0.0.1:12222"
 default_request_body_max_bytes = 1048576
+compression = "gzip"
 # To have Nexus's external HTTP endpoint use TLS, uncomment the line below.  You
 # will also need to provide an initial TLS certificate during rack
 # initialization.  If you're using this config file, you're probably running a

nexus/examples/config.toml

@@ -41,6 +41,7 @@ external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 # IP Address and TCP port on which to listen for the external API
 bind_address = "127.0.0.1:12220"
 default_request_body_max_bytes = 1048576
+compression = "gzip"
 # To have Nexus's external HTTP endpoint use TLS, uncomment the line below.  You
 # will also need to provide an initial TLS certificate during rack
 # initialization.  If you're using this config file, you're probably running a

nexus/src/app/background/init.rs

@@ -1530,7 +1530,7 @@ pub mod test {
                 bind_address: "[::1]:0".parse().unwrap(),
                 default_request_body_max_bytes: 8 * 1024,
                 default_handler_task_mode: HandlerTaskMode::Detached,
-                log_headers: vec![],
+                ..Default::default()
             },
         )
         .await

nexus/src/app/crucible.rs

@@ -74,11 +74,9 @@ impl super::Nexus {
         &self,
         dataset: &db::model::CrucibleDataset,
     ) -> CrucibleAgentClient {
-        // Use reqwest012_client because the rev-pinned crucible-agent-client
-        // is still on reqwest 0.12.
         CrucibleAgentClient::new_with_client(
             &format!("http://{}", dataset.address()),
-            self.reqwest012_client.clone(),
+            self.reqwest_client.clone(),
         )
     }
 

nexus/src/app/disk.rs

@@ -564,11 +564,9 @@ impl super::Nexus {
             // that user's program can act accordingly. In a way, the user's
             // program is an externally driven saga instead.
 
-            // Use reqwest012_client because the rev-pinned
-            // crucible-pantry-client is still on reqwest 0.12.
             let client = crucible_pantry_client::Client::new_with_client(
                 &format!("http://{}", endpoint),
-                self.reqwest012_client.clone(),
+                self.reqwest_client.clone(),
             );
             let request = crucible_pantry_client::types::BulkWriteRequest {
                 offset: param.offset,

nexus/src/app/mod.rs

@@ -223,18 +223,8 @@ pub struct Nexus {
     ///
     /// (This does not need to be in an `Arc` because `reqwest::Client` uses
     /// `Arc` internally.)
-    ///
-    /// Currently unused because all `new_with_client` call sites use
-    /// `reqwest012_client` for cross-repo dependencies that are still on
-    /// reqwest 0.12. This field will be used again once rev pins are updated.
-    #[allow(dead_code)]
     reqwest_client: reqwest::Client,
 
-    /// `reqwest012::Client` for cross-repo dependencies where the rev-pinned
-    /// dependency is still on reqwest 0.12. Remove once all rev pins are
-    /// updated.
-    reqwest012_client: reqwest012::Client,
-
     /// Client to the timeseries database.
     timeseries_client: oximeter_db::Client,
 
@@ -436,14 +426,6 @@ impl Nexus {
             .build()
             .map_err(|e| InlineErrorChain::new(&e).to_string())?;
 
-        // reqwest 0.12 client for cross-repo dependencies still on reqwest
-        // 0.12. Remove once all rev pins are updated.
-        let reqwest012_client = reqwest012::ClientBuilder::new()
-            .connect_timeout(std::time::Duration::from_secs(15))
-            .timeout(std::time::Duration::from_secs(15))
-            .build()
-            .map_err(|e| InlineErrorChain::new(&e).to_string())?;
-
         // Client to the ClickHouse database.
         let timeseries_client = match &config.pkg.timeseries_db.address {
             None => {
@@ -544,7 +526,6 @@ impl Nexus {
             producer_server: std::sync::Mutex::new(None),
             populate_status,
             reqwest_client,
-            reqwest012_client,
             timeseries_client,
             webhook_delivery_client,
             tunables: config.pkg.tunables.clone(),

nexus/test-utils/src/http_testing.rs

@@ -97,6 +97,8 @@ impl<'a> RequestBuilder<'a> {
                 http::header::DATE,
                 http::header::LOCATION,
                 http::header::SET_COOKIE,
+                http::header::TRANSFER_ENCODING,
+                http::header::VARY,
                 http::header::HeaderName::from_static("x-request-id"),
             ]),
             expected_response_headers: http::HeaderMap::default(),

nexus/tests/config.test.toml

@@ -51,6 +51,7 @@ external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 # concurrently.
 bind_address = "127.0.0.1:0"
 default_request_body_max_bytes = 1048576
+compression = "gzip"
 
 [deployment.dropshot_internal]
 bind_address = "127.0.0.1:0"