fix: Bound the wait for a procedure instance slot and report it

crates/client-api/src/routes/database.rs

@@ -139,6 +139,10 @@ fn map_procedure_error(e: ProcedureCallError, procedure: &str) -> (StatusCode, S
             log::info!("Procedure {procedure} could not run because the module is out of energy");
             StatusCode::PAYMENT_REQUIRED
         }
+        ProcedureCallError::PoolTimeout(_) => {
+            log::info!("Procedure {procedure} could not run because no procedure instance became free in time");
+            StatusCode::SERVICE_UNAVAILABLE
+        }
         ProcedureCallError::InternalError(_) => {
             // TODO: May need to split this from module errors vs host errors
             log::info!("Internal error while invoking procedure {procedure}: {e:#}");
@@ -298,6 +302,9 @@ async fn handle_http_route_impl<S: ControlStateDelegate + NodeDelegate>(
         Err(spacetimedb::host::module_host::HttpHandlerCallError::NoSuchModule(_)) => {
             return Err(NO_SUCH_DATABASE.into());
         }
+        Err(err @ spacetimedb::host::module_host::HttpHandlerCallError::PoolTimeout(_)) => {
+            return Err((StatusCode::SERVICE_UNAVAILABLE, err.to_string()).into());
+        }
         Err(spacetimedb::host::module_host::HttpHandlerCallError::InternalError(err)) => {
             return Err((StatusCode::INTERNAL_SERVER_ERROR, err).into());
         }

crates/core/src/config.rs

@@ -171,9 +171,11 @@ impl<'de> serde::Deserialize<'de> for ConfigFile {
             module_http: config.module_http,
             wasm: WasmConfig {
                 procedure_instance_pool_size: config.wasm.procedure_instance_pool_size,
+                procedure_queue_timeout: config.wasm.procedure_queue_timeout,
             },
             v8: V8Config {
                 procedure_instance_pool_size: config.v8.procedure_instance_pool_size,
+                procedure_queue_timeout: config.v8.procedure_queue_timeout,
                 heap_policy: config.v8_heap_policy,
             },
         })
@@ -233,12 +235,16 @@ impl Default for ModuleHttpConfig {
 #[derive(Clone, Copy, Debug)]
 pub struct WasmConfig {
     pub procedure_instance_pool_size: NonZeroUsize,
+    /// How long a call may wait for a free procedure instance before it fails.
+    /// `None` waits forever.
+    pub procedure_queue_timeout: Option<Duration>,
 }
 
 impl Default for WasmConfig {
     fn default() -> Self {
         Self {
             procedure_instance_pool_size: default_wasm_procedure_instance_pool_size(),
+            procedure_queue_timeout: default_procedure_queue_timeout(),
         }
     }
 }
@@ -251,26 +257,33 @@ struct WasmConfigToml {
         deserialize_with = "de_nz_usize"
     )]
     pub procedure_instance_pool_size: NonZeroUsize,
+    #[serde(default = "default_procedure_queue_timeout", deserialize_with = "de_nz_duration")]
+    pub procedure_queue_timeout: Option<Duration>,
 }
 
 impl Default for WasmConfigToml {
     fn default() -> Self {
         Self {
             procedure_instance_pool_size: default_wasm_procedure_instance_pool_size(),
+            procedure_queue_timeout: default_procedure_queue_timeout(),
         }
     }
 }
 
 #[derive(Clone, Copy, Debug)]
 pub struct V8Config {
     pub procedure_instance_pool_size: NonZeroUsize,
+    /// How long a call may wait for a free procedure instance before it fails.
+    /// `None` waits forever.
+    pub procedure_queue_timeout: Option<Duration>,
     pub heap_policy: V8HeapPolicyConfig,
 }
 
 impl Default for V8Config {
     fn default() -> Self {
         Self {
             procedure_instance_pool_size: default_v8_procedure_instance_pool_size(),
+            procedure_queue_timeout: default_procedure_queue_timeout(),
             heap_policy: V8HeapPolicyConfig::default(),
         }
     }
@@ -291,12 +304,15 @@ struct V8ConfigToml {
         deserialize_with = "de_nz_usize"
     )]
     pub procedure_instance_pool_size: NonZeroUsize,
+    #[serde(default = "default_procedure_queue_timeout", deserialize_with = "de_nz_duration")]
+    pub procedure_queue_timeout: Option<Duration>,
 }
 
 impl Default for V8ConfigToml {
     fn default() -> Self {
         Self {
             procedure_instance_pool_size: default_v8_procedure_instance_pool_size(),
+            procedure_queue_timeout: default_procedure_queue_timeout(),
         }
     }
 }
@@ -382,6 +398,10 @@ fn default_wasm_procedure_instance_pool_size() -> NonZeroUsize {
     std::thread::available_parallelism().unwrap_or_else(|_| NonZeroUsize::new(1).unwrap())
 }
 
+fn default_procedure_queue_timeout() -> Option<Duration> {
+    Some(Duration::from_secs(30))
+}
+
 fn de_nz_usize<'de, D>(deserializer: D) -> Result<NonZeroUsize, D::Error>
 where
     D: serde::Deserializer<'de>,