danhansen · Oct 2, 2026 · Oct 2, 2026 · Oct 16, 2026 · greptile-apps · Oct 16, 2026
@@ -200,9 +200,7 @@ if(NEMO_SPEECH_BUILD_S2S AND NOT NEMO_SPEECH_GGML_PATCHED AND NOT NEMO_SPEECH_LL
     message(FATAL_ERROR "NEMO_SPEECH_BUILD_S2S requires NEMO_SPEECH_GGML_PATCHED=ON")
 endif()
 
-set(GGML_DEPENDENCIES ggml ggml-base ggml-cpu)
 if(GGML_CUDA)
-    list(APPEND GGML_DEPENDENCIES ggml-cuda)
     # libggml-cuda.so has libcuda.so.1 (the CUDA driver) as a NEEDED entry.
     # The driver is provided at runtime by NVIDIA's container toolkit and is
     # absent from the devel image - only the stub at
@@ -217,47 +215,41 @@ if(GGML_CUDA)
     # can't capture (e.g. shapes that change between calls).
     set(GGML_CUDA_GRAPHS ON CACHE BOOL "Enable ggml-cuda CUDA Graphs")
 endif()
-if(GGML_VULKAN)
-    list(APPEND GGML_DEPENDENCIES ggml-vulkan)
-endif()
-if(GGML_METAL)
-    list(APPEND GGML_DEPENDENCIES ggml-metal)
-endif()
 
 # Tiled CPU GEMM (llamafile tinyBLAS) for F16/F32/Q8_0 matmuls. Stock ggml leaves it off; without
 # it CPU matmuls run one dot product per output.
 set(GGML_LLAMAFILE ON CACHE BOOL "ggml: use llamafile SGEMM")
 
 include(cmake/llama_cpp.cmake)
+# Rebuild ggml's cached backend list when options change between configurations.
+unset(GGML_AVAILABLE_BACKENDS CACHE)
 add_subdirectory("${NEMO_SPEECH_LLAMA_CPP_DIR}/ggml" "${CMAKE_BINARY_DIR}/ggml" EXCLUDE_FROM_ALL)
 
 # ggml is an implementation dependency, so install only the runtime libraries
 # required by our shared libraries. Its C++ headers, CMake package, pkg-config
 # metadata, and tools are not part of the public NeMo-Speech.cpp SDK.
 set_property(TARGET ggml PROPERTY PUBLIC_HEADER "")
-install(TARGETS ggml ggml-base ggml-cpu
+install(TARGETS ggml ggml-base
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-if(TARGET ggml-blas)
-    install(TARGETS ggml-blas
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
-if(GGML_CUDA)
-    install(TARGETS ggml-cuda
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
-if(GGML_VULKAN)
-    install(TARGETS ggml-vulkan
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
-if(GGML_METAL)
-    install(TARGETS ggml-metal
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
+# ggml is EXCLUDE_FROM_ALL, so install its backends explicitly.
+# Match ggml's plugin destination; linked backends go beside the other libraries.
+if(GGML_BACKEND_DL)
+    if(GGML_BACKEND_DIR)
+        set(_nemo_backend_install_dir "${GGML_BACKEND_DIR}")
+    else()
+        set(_nemo_backend_install_dir "${CMAKE_INSTALL_BINDIR}")
+    endif()
+else()
+    set(_nemo_backend_install_dir "${CMAKE_INSTALL_LIBDIR}")
+endif()
+foreach(backend IN LISTS GGML_AVAILABLE_BACKENDS)
+    if(TARGET ${backend})
+        install(TARGETS ${backend}
+            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+            LIBRARY DESTINATION ${_nemo_backend_install_dir})
+    endif()
+endforeach()
 
 # In-tree cuBLAS shim backed by native GEMM kernels (no cuBLASLt). ggml-cuda
 # links real cuBLAS at build time. At runtime the loader resolves its cuBLAS

@@ -118,6 +118,28 @@ adapters and separate `riva_server` executable are needed.
 
 The preset selects which components and ggml backend are compiled.
 
+For portable CPU binaries on platforms supported by ggml's multi-variant build,
+enable ggml's dynamically loaded backends instead of tuning for the build host:
+
+```bash
+scripts/configure.sh cpu-asr \
+  -DGGML_NATIVE=OFF \
+  -DGGML_BACKEND_DL=ON \
+  -DGGML_CPU_ALL_VARIANTS=ON
+cmake --build --preset cpu-asr
+cmake --install build/cpu-asr --prefix "$PWD/install"
+```
+
+ggml selects a compatible CPU plugin at runtime. `GGML_BACKEND_DL` makes all
+enabled backends plugins; CPU variants can coexist with GPU backend plugins.
+Keep the installed plugins beside the executable, including when embedding the
+SDK in an application. An explicit `GGML_BACKEND_DIR` sets their installation
+directory and adds it to ggml's search paths. Absolute paths are not updated
+when relocating the package.
+For CUDA TTS, keep `GGML_BACKEND_DL=OFF`; its optimized paths still call
+CUDA backend functions directly.
+The default directly linked backend configuration remains unchanged.
+
 Additional CMake definitions can follow the preset:
 
 ```bash

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include "ggml-backend.h"
+
+namespace nemo_speech::common {
+
+inline bool
+is_cpu_backend(ggml_backend_t backend) {
+    auto* dev = backend ? ggml_backend_get_device(backend) : nullptr;
+    return dev && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
+}
+
+inline void
+set_cpu_backend_n_threads(ggml_backend_t backend, int n_threads) {
+    if (!is_cpu_backend(backend)) {
+        return;
+    }
+    // CPU plugin functions must be accessed through the backend registry.
+    auto* reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
+    auto* set_n_threads = (ggml_backend_set_n_threads_t)ggml_backend_reg_get_proc_address(
+        reg, "ggml_backend_set_n_threads");
+    if (set_n_threads) {
+        set_n_threads(backend, n_threads);
+    }
+}
+
+}  // namespace nemo_speech::common
@@ -25,7 +25,13 @@ target_include_directories(nemo_speech_runtime_ggml PRIVATE
     ${CMAKE_SOURCE_DIR}/src/common
 )
 
-target_link_libraries(nemo_speech_runtime_ggml PUBLIC ${GGML_DEPENDENCIES})
+# ggml propagates linked backends and builds plugins when GGML_BACKEND_DL is enabled.
+target_link_libraries(nemo_speech_runtime_ggml PUBLIC ggml)
+
+# ggml exports this definition only for directly linked backends.
+if(GGML_VULKAN)
+    target_compile_definitions(nemo_speech_runtime_ggml PRIVATE GGML_USE_VULKAN)
+endif()
 
 # Consumers use the patched operations only when patches/ was applied; runtime.h
 # derives NEMO_SPEECH_CUDA_FAST_PATHS from this and GGML_USE_CUDA.
@@ -38,6 +44,6 @@ endif()
 # here so tests/ and riva_server inherit it and ld can resolve libggml-cuda's
 # NEEDED libcuda.so.1 at link time. CUDAToolkit was found in the top-level
 # CMakeLists.txt under the same GGML_CUDA gate.
-if(GGML_CUDA)
+if(GGML_CUDA AND NOT GGML_BACKEND_DL)
     target_link_libraries(nemo_speech_runtime_ggml PUBLIC CUDA::cuda_driver)
 endif()
@@ -57,6 +57,10 @@ BackendManager::init_backends() {
     // Vulkan reads this setting during device initialization.
     set_environment_default("GGML_VK_DISABLE_GRAPH_OPTIMIZE", "1");
 #endif  // GGML_USE_VULKAN
+#ifdef GGML_BACKEND_DL
+    static std::once_flag load_backends;
+    std::call_once(load_backends, [] { ggml_backend_load_all(); });
+#endif
 
     ggml_time_init();
 
@@ -153,7 +157,7 @@ BackendManager::init_backends() {
             ++extra_bufts;
         }
     }
-    buft_list.emplace_back(cpu_dev, ggml_backend_cpu_buffer_type());
+    buft_list.emplace_back(cpu_dev, ggml_backend_dev_buffer_type(cpu_dev));
 
     for (const auto& buft : buft_list) {
         ggml_backend_dev_t dev = buft.first;

@@ -12,6 +12,7 @@
 #include <string>
 #include <utility>
 
+#include "ggml_backend_utils.h"
 #include "graph.h"
 #include "nvtx_utils.h"
 
@@ -988,9 +989,7 @@ local_transformer_graph_eval(
         }
     }
 
-    if (ggml_backend_is_cpu(model.backend)) {
-        ggml_backend_cpu_set_n_threads(model.backend, threads);
-    }
+    nemo_speech::common::set_cpu_backend_n_threads(model.backend, threads);
 
     ggml_status status = GGML_STATUS_FAILED;
     {
@@ -1109,9 +1108,7 @@ local_transformer_graph_eval_cuda(
         }
     }
 
-    if (ggml_backend_is_cpu(model.backend)) {
-        ggml_backend_cpu_set_n_threads(model.backend, threads);
-    }
+    nemo_speech::common::set_cpu_backend_n_threads(model.backend, threads);
 
     ggml_status status = GGML_STATUS_FAILED;
     {

@@ -17,6 +17,7 @@
 
 #include "decoder.h"
 #include "encoder.h"
+#include "ggml_backend_utils.h"
 #include "graph.h"
 #include "lt.h"
 #include "model_logging.h"
@@ -1471,9 +1472,7 @@ compute_graph(
         }
     }
 
-    if (ggml_backend_is_cpu(backend)) {
-        ggml_backend_cpu_set_n_threads(backend, threads);
-    }
+    nemo_speech::common::set_cpu_backend_n_threads(backend, threads);
 
     ggml_status status = GGML_STATUS_FAILED;
     {

@@ -4,7 +4,6 @@
 
 #include "ggml-alloc.h"
 #include "ggml-backend.h"
-#include "ggml-cpu.h"
 #include "ggml.h"
 #include "gguf.h"
 #if defined(MAGPIETTS_CUDA_SAMPLING)

@@ -20,8 +20,8 @@
 #include "ggml-cuda.h"
 #endif
 #include "ggml-backend.h"
-#include "ggml-cpu.h"
 #include "ggml.h"
+#include "ggml_backend_utils.h"
 #include "gguf.h"
 #include "model_logging.h"
 #include "nvtx_utils.h"
@@ -486,7 +486,7 @@ nc_pack_fused_conv_weights(nc_model& model, bool verbose) {
 // backend.
 static bool
 nc_cpu_gemm_transposed(const nc_model& model, const nc_conv& c) {
-    return model.backend && ggml_backend_is_cpu(model.backend) && c.w &&
+    return model.backend && nemo_speech::common::is_cpu_backend(model.backend) && c.w &&
            c.w->type == GGML_TYPE_F16 && ggml_is_contiguous(c.w) && c.stride > 0 &&
            c.w->ne[0] % c.stride == 0;
 }
@@ -500,7 +500,7 @@ nc_cpu_gemm_transposed(const nc_model& model, const nc_conv& c) {
 //    conv_transpose_1d re-permutes the whole weight on a single thread on every call.
 static void
 nc_prepare_cpu_conv_weights(nc_model& model, bool verbose) {
-    if (!model.backend || !ggml_backend_is_cpu(model.backend)) {
+    if (!model.backend || !nemo_speech::common::is_cpu_backend(model.backend)) {
         return;
     }
     std::vector<nc_conv*> pending;
@@ -1538,9 +1538,7 @@ decode_eval(
         ggml_backend_tensor_set(inp, latent.data(), 0, latent.size() * sizeof(float));
     }
 
-    if (ggml_backend_is_cpu(model.backend)) {
-        ggml_backend_cpu_set_n_threads(model.backend, threads);
-    }
+    nemo_speech::common::set_cpu_backend_n_threads(model.backend, threads);
 
     ggml_status status = GGML_STATUS_FAILED;
     {
@@ -1694,9 +1692,7 @@ decode_eval_stream(
     }
     // Streaming history is device-resident and updated inside the graph.
 
-    if (ggml_backend_is_cpu(model.backend)) {
-        ggml_backend_cpu_set_n_threads(model.backend, threads);
-    }
+    nemo_speech::common::set_cpu_backend_n_threads(model.backend, threads);
 
     ggml_status status = GGML_STATUS_FAILED;
     {
@@ -1852,7 +1848,7 @@ NanoCodecModel::loaded() const {
 
 bool
 NanoCodecModel::onAccelerator() const {
-    return loaded() && !ggml_backend_is_cpu(impl_->model.backend);
+    return loaded() && !nemo_speech::common::is_cpu_backend(impl_->model.backend);
 }
 
 const NanoCodecHParams&

@@ -18,6 +18,7 @@
 #include "engine_registry.h"
 #include "ggml_log_filter.h"
 #include "json.h"
+#include "runtime.h"
 
 namespace {
 
@@ -141,6 +142,16 @@ main() {
         require(nemo_speech::audio::is_wav_path("VOICE.WAVE"), "case-insensitive WAV path");
         require(!nemo_speech::audio::is_wav_path("voice.mp3"), "codec path rejection");
 
+        ggml_runtime::BackendManager backends({});
+        auto* cpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+        require(cpu != nullptr, "CPU backend discovery");
+        bool found_cpu_buffer = false;
+        for (const auto& entry : backends.get_buft_list()) {
+            if (entry.first == cpu && entry.second == ggml_backend_dev_buffer_type(cpu))
+                found_cpu_buffer = true;
+        }
+        require(found_cpu_buffer, "CPU device buffer selection");
+
         nemo_speech::EngineRegistry registry;
         require(!registry.ready(), "empty engine registry readiness");
         require(registry.capabilities().empty(), "empty engine registry capabilities");

@@ -4,6 +4,7 @@
 
 import os
 import pathlib
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -43,6 +44,17 @@ def main() -> None:
         run(*install_command)
         install_prefix.rename(prefix)
         cache = build / "CMakeCache.txt"
+        bin_directory = prefix / cache_value(cache, "CMAKE_INSTALL_BINDIR")
+        lib_directory = prefix / cache_value(cache, "CMAKE_INSTALL_LIBDIR")
+        if cache_bool(cache, "GGML_BACKEND_DL"):
+            plugin_directory = cache_value(cache, "GGML_BACKEND_DIR")
+            plugin_root = prefix / plugin_directory if plugin_directory else bin_directory
+            for backend in cache_value(cache, "GGML_AVAILABLE_BACKENDS").split(";"):
+                if backend:
+                    plugins = list(plugin_root.glob(f"*{backend}*"))
+                    assert any(
+                        p.is_file() for p in plugins
+                    ), f"missing installed backend: {backend}"
         expected = {
             "ASR": cache_bool(cache, "NEMO_SPEECH_BUILD_ASR"),
             "Diarization": cache_bool(cache, "NEMO_SPEECH_BUILD_DIAR"),
@@ -79,22 +91,30 @@ def main() -> None:
         consumer_environment = os.environ.copy()
         if os.name == "nt":
             consumer_environment["PATH"] = (
-                str(prefix / "bin") + os.pathsep + consumer_environment.get("PATH", "")
+                str(bin_directory) + os.pathsep + consumer_environment.get("PATH", "")
             )
         subprocess.run([str(consumer_binary)], check=True, env=consumer_environment)
 
+        # Resolve relative plugin directories from the relocated installation prefix.
+        if cache_bool(cache, "GGML_BACKEND_DL"):
+            runtime_test = bin_directory / f"test_shared_utilities{suffix}"
+            shutil.copy2(build / "bin" / config / runtime_test.name, runtime_test)
+            library_path = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH"
+            consumer_environment[library_path] = str(lib_directory)
+            subprocess.run([str(runtime_test)], check=True, env=consumer_environment, cwd=prefix)
+
         include = prefix / "include"
         assert not (include / "ggml.h").exists(), "internal ggml headers leaked into SDK"
         assert not (include / "llama.h").exists(), "internal llama headers leaked into SDK"
-        assert not (prefix / "bin" / "convert_hf_to_gguf.py").exists()
+        assert not (bin_directory / "convert_hf_to_gguf.py").exists()
 
-        if cache_bool(cache, "GGML_BLAS"):
+        if cache_bool(cache, "GGML_BLAS") and not cache_bool(cache, "GGML_BACKEND_DL"):
             if os.name == "nt":
-                blas_libraries = list((prefix / "bin").glob("*ggml-blas*.dll"))
+                blas_libraries = list(bin_directory.glob("*ggml-blas*.dll"))
             elif sys.platform == "darwin":
-                blas_libraries = list((prefix / "lib").glob("libggml-blas*.dylib"))
+                blas_libraries = list(lib_directory.glob("libggml-blas*.dylib"))
             else:
-                blas_libraries = list((prefix / "lib").glob("libggml-blas.so*"))
+                blas_libraries = list(lib_directory.glob("libggml-blas.so*"))
             assert blas_libraries, "installed runtime is missing the ggml BLAS backend"