feat(c-api): add stream-based save and load

bindings/c/CMakeLists.txt

@@ -40,6 +40,7 @@ set(SVS_C_API_SOURCES
     src/index_builder.hpp
     src/leanvec_training_data.hpp
     src/storage.hpp
+    src/stream.hpp
     src/threadpool.hpp
     src/types_support.hpp
 
@@ -144,7 +145,7 @@ if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC)
     else()
         # Links to LTO-enabled static library, requires GCC/G++ 11.2
         if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "11.2" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.3")
-            set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-lto-nightly-2026-10-06-1613.tar.gz"
+            set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-lto-nightly-2026-10-07-1413.tar.gz"
                 CACHE STRING "URL to download SVS shared library")
         else()
             # The fallback is correct but slower, so nothing downstream fails and CI

bindings/c/README.md

@@ -22,8 +22,8 @@ C applications and any language with C FFI support.
 The API is built around a small set of opaque handles and a builder pattern:
 configure an *algorithm*, optional *storage* and *thread pool*, hand them to an
 *index builder*, then use the resulting *index* to run TopK searches (with
-optional ID filtering), save/load the index, and — for dynamic indices — add or
-delete points at runtime.
+optional ID filtering), save/load the index to disk or a caller-supplied stream,
+and — for dynamic indices — add or delete points at runtime.
 
 For the design rationale, naming conventions, and full API reference see
 [docs/C_API_Design.md](docs/C_API_Design.md).
@@ -181,16 +181,16 @@ cleanup:
 
 ## Samples
 
-Runnable sample applications live in [samples/](samples/):
+Runnable sample applications live in [`examples/c/`](../../examples/c/):
 
-- [`simple.c`](samples/simple.c) – minimal static index build + search with a
+- [`simple.c`](../../examples/c/simple.c) – minimal static index build + search with a
   custom thread pool
-- [`dynamic.c`](samples/dynamic.c) – dynamic index with add / delete /
+- [`dynamic.c`](../../examples/c/dynamic.c) – dynamic index with add / delete /
   consolidate
-- [`save_load.c`](samples/save_load.c) – persisting and reloading indices from
+- [`save_load.c`](../../examples/c/save_load.c) – persisting and reloading indices from
   disk
-
-Additional integration examples: [`examples/c/`](../../examples/c/).
+- [`save_load_stream.c`](../../examples/c/save_load_stream.c) – stream-based index save
+  and load
 
 ## Further Reading
 

bindings/c/docs/C_API_Design.md

@@ -44,6 +44,7 @@
   - [6. Allocator Configuration](#6-allocator-configuration)
   - [7. Search Parameters](#7-search-parameters)
   - [8. ID Filter (optional)](#8-id-filter-optional)
+  - [9. Stream Interface](#9-stream-interface)
 - [API Overview](#api-overview)
   - [Headers](#headers)
   - [Types](#types)
@@ -198,6 +199,10 @@ one of the two optional slots is used per name:
   (e.g. `svs_algorithm_create_vamana` creates a Vamana algorithm;
   `svs_index_build_dynamic` builds a dynamic index).
 
+When both specializations apply (e.g. a save or load variant), `_stream` and
+`_stream_dynamic` are stacked at the end: `svs_index_save_stream`,
+`svs_index_load_stream_dynamic`.
+
 **Examples:**
 
 | Function | Breakdown | Description |
@@ -209,6 +214,8 @@ one of the two optional slots is used per name:
 | `svs_index_build_dynamic()` | `svs` + `index` + `build` + `dynamic` | Build a dynamic index |
 | `svs_index_dynamic_add_points()` | `svs` + `index` + `dynamic` + `add_points` | Add points to a dynamic index |
 | `svs_index_builder_set_threadpool()` | `svs` + `index_builder` + `set_threadpool` | Configure builder thread pool |
+| `svs_index_save_stream()` | `svs` + `index` + `save` + `stream` | Save index to a caller-supplied stream |
+| `svs_index_load_stream_dynamic()` | `svs` + `index` + `load` + `stream_dynamic` | Load a dynamic index from a stream |
 
 ### Examples by Category
 
@@ -225,6 +232,7 @@ typedef enum svs_error_code svs_error_code_t;
 // Interface pointer types
 typedef struct svs_threadpool_interface* svs_threadpool_i;
 typedef struct svs_id_filter_interface*  svs_id_filter_i;
+typedef struct svs_stream_interface* svs_stream_i;
 ```
 
 ## Core Components
@@ -500,6 +508,64 @@ Providing a non-zero `filter_rate` lets the search account for the expected
 selectivity; if the observed hit rate ends up lower than the reported estimate the
 function returns an empty result set for that query.
 
+### 9. Stream Interface
+
+Enables caller-supplied byte streams for index save and load operations, eliminating
+the need for intermediate disk storage. Like the thread pool and allocator, the stream
+interface is a versioned ops table plus an opaque `self` pointer.
+
+```c
+struct svs_stream_interface_ops {
+    uint32_t version;          // Set by SVS_INIT_STREAM_OPS
+    size_t struct_size;        // Set by SVS_INIT_STREAM_OPS
+    size_t (*read)(void* self, void* buf, size_t n, svs_error_h out_err);
+    bool (*write)(void* self, const void* buf, size_t n, svs_error_h out_err);
+};
+
+struct svs_stream_interface {
+    struct svs_stream_interface_ops* ops;
+    void* self;                // User-defined state
+};
+typedef struct svs_stream_interface* svs_stream_i;
+
+// Initialisation macros
+static svs_stream_ops_t my_stream_ops =
+    SVS_INIT_STREAM_OPS(my_read_func, my_write_func);
+static svs_stream_t my_stream = SVS_MAKE_INTERFACE(user_state, my_stream_ops);
+```
+
+**Callback contracts:**
+
+- **`read`** — Read at most `n` bytes into `buf`. Return the number of bytes read; return
+  0 to signal end of stream. Short reads are not errors; the library will call again as
+  needed. To report a read error, set `out_err` via `svs_error_set()` and return 0 — this
+  aborts the load with that error code, unlike returning 0 with no error set, which is a
+  clean end of stream. This differs from `write`, which signals failure through its return
+  value. Required for load operations; may be NULL for write-only streams.
+- **`write`** — Write exactly `n` bytes from `buf`. Return `true` on success. A partial
+  write must be reported as failure. Required for save operations; may be NULL for
+  read-only streams.
+
+**Threading:** Both callbacks are invoked serially from the thread that called the streaming
+save or load function. No synchronization between concurrent stream operations is required.
+
+**Lifetime:** The operations table is copied by value; `self` is retained as a bare pointer
+and only needs to remain valid until the streaming function returns. Data is copied out of
+the stream during load, so the stream buffer need not persist after the call completes.
+
+**Encodings:** SVS supports two mutually exclusive stream encodings identified by an 8-byte magic
+at offset zero: the native stream encoding (`"SVS_STRM"`) and a tar-like directory archive. The load
+functions accept both transparently; detection is handled by SVS internals. `svs_index_save_stream`
+produces only the native encoding by design, so the save and load halves are deliberately asymmetric.
+An index written to disk via `svs_index_save` cannot be streamed, because the C layer does not
+expose the machinery to pack a directory archive into stream form. Streaming is therefore
+self-sufficient only for indexes that were themselves stream-saved.
+
+**Read-ahead behavior:** The native stream encoding carries no total length. During load, the
+library may read up to 64 KiB past the logical end of the index data, so callers embedding an
+index inside a larger stream must frame the payload themselves (e.g. with a length prefix) and
+bound the stream reads to that frame.
+
 ## API Overview
 
 A concise map of the public surface. See [svs/c/svs_c.h](../include/svs/c/svs_c.h)
@@ -518,8 +584,8 @@ for full signatures, parameters, and Doxygen documentation.
 - **Enums** (`_t`): `svs_error_code_t`, `svs_distance_metric_t`,
   `svs_algorithm_type_t`, `svs_data_type_t`, `svs_storage_kind_t`,
   `svs_threadpool_kind_t`, `svs_allocator_kind_t`
-- **Custom interfaces**: `svs_threadpool_i`, `svs_allocator_i`, and `svs_id_filter_i`
-  (versioned ops-table + `self` pointer; build with `SVS_INIT_*_OPS()` /
+- **Custom interfaces**: `svs_threadpool_i`, `svs_allocator_i`, `svs_id_filter_i`, and
+  `svs_stream_i` (versioned ops-table + `self` pointer; build with `SVS_INIT_*_OPS()` /
   `SVS_MAKE_INTERFACE()`)
 - **Value structs**: `svs_search_results_t` (CSR result buffer),
   `svs_memory_breakdown_t`
@@ -535,7 +601,7 @@ for full signatures, parameters, and Doxygen documentation.
 | **Search params** | `svs_search_params_create_vamana`, `svs_search_params_free` |
 | **Builder** | `svs_index_builder_create`, `svs_index_builder_set_{storage,threadpool,threadpool_custom,allocator,allocator_custom}`, `svs_index_builder_free` |
 | **Memory estimation** | `svs_index_builder_estimate_memory`, `svs_index_builder_estimate_memory_dynamic`, `svs_index_builder_estimate_search_memory`, `svs_index_builder_estimate_search_memory_dynamic`, `svs_index_builder_get_default_blocksize_bytes` |
-| **Index lifecycle** | `svs_index_build`, `svs_index_build_dynamic`, `svs_index_load`, `svs_index_load_dynamic`, `svs_index_save`, `svs_index_free` |
+| **Index lifecycle** | `svs_index_build`, `svs_index_build_dynamic`, `svs_index_load`, `svs_index_load_dynamic`, `svs_index_load_stream`, `svs_index_load_stream_dynamic`, `svs_index_save`, `svs_index_save_stream`, `svs_index_free` |
 | **Dynamic ops** | `svs_index_dynamic_{add_points,delete_points,has_id,consolidate,compact}` |
 | **Introspection** | `svs_index_get_num_threads` / `set_num_threads`, `svs_index_get_distance`, `svs_index_reconstruct`, `svs_index_get_memory_usage`, `svs_index_get_memory_breakdown` |
 | **Search** | `svs_index_search_topk` (+ deprecated `svs_index_search`), `svs_search_results_free` |
@@ -558,8 +624,8 @@ for full signatures, parameters, and Doxygen documentation.
 
 - See the top-level [../README.md](../README.md) for a quick start, build/consume
   instructions, and a complete end-to-end usage example.
-- See [../samples/](../samples/) for runnable sample applications:
-  - `simple.c` – minimal static index build + search with a custom thread pool
-  - `dynamic.c` – dynamic index with add / delete / consolidate
-  - `save_load.c` – persisting and reloading indices from disk
-- See [examples/c/](../../../examples/c/) for additional usage examples
+- See [examples/c/](../../../examples/c/) for runnable sample applications:
+  - [`simple.c`](../../../examples/c/simple.c) – minimal static index build + search with a custom thread pool
+  - [`dynamic.c`](../../../examples/c/dynamic.c) – dynamic index with add / delete / consolidate
+  - [`save_load.c`](../../../examples/c/save_load.c) – persisting and reloading indices from disk
+  - [`save_load_stream.c`](../../../examples/c/save_load_stream.c) – stream-based index save and load

bindings/c/include/svs/c/svs_c.h

@@ -294,6 +294,65 @@ struct svs_id_filter_interface {
     void* self;
 };
 
+/// @brief Operations table for a caller-supplied byte stream.
+/// @remarks Access is strictly sequential: the library never repositions the stream. Both
+/// callbacks are invoked serially from the thread that called the streaming save or load
+/// function, so no synchronization is required
+/// @remarks Exactly one direction is required per operation: @ref svs_index_save_stream
+/// needs @p write, the load functions need @p read. The unused callback may be NULL.
+/// @var svs_stream_interface_ops::version
+///   Interface version, set by @ref SVS_INIT_STREAM_OPS.
+/// @var svs_stream_interface_ops::struct_size
+///   Size of this structure, set by @ref SVS_INIT_STREAM_OPS.
+/// @var svs_stream_interface_ops::read
+///   Reads at most @p n bytes into @p buf.
+///   @param self Pointer to the stream instance.
+///   @param buf Destination buffer.
+///   @param n Maximum number of bytes to read.
+///   @param out_err Handle to capture any error that occurs during the read. Returning 0
+///   with an error set on @p out_err via svs_error_set() reports a failed read and aborts
+///   the load with that error code; returning 0 without setting one is a clean end of
+///   stream. This differs from @p write, which signals failure through its return value.
+///   @return The number of bytes read; 0 signals end of stream unless @p out_err carries an
+///   error. A short read is not an error and the library will call again.
+/// @var svs_stream_interface_ops::write
+///   Writes exactly @p n bytes from @p buf.
+///   @param self Pointer to the stream instance.
+///   @param buf Source buffer.
+///   @param n Number of bytes to write.
+///   @param out_err Handle to capture any error that occurs during the write. User code may
+///   call svs_error_set() to set the error code and message if an error occurs.
+///   @return True on success. A partial write must be reported as failure.
+struct svs_stream_interface_ops {
+    uint32_t version;
+    size_t struct_size;
+    size_t (*read)(void* self, void* buf, size_t n, svs_error_h out_err);
+    bool (*write)(void* self, const void* buf, size_t n, svs_error_h out_err);
+};
+
+/// @brief Macro to create a user-defined stream interface operations structure
+/// @param read_func Function pointer that reads at most @p n bytes into @p buf, or NULL for
+/// a write-only stream
+/// @param write_func Function pointer that writes exactly @p n bytes from @p buf, or NULL
+/// for a read-only stream
+#define SVS_INIT_STREAM_OPS(read_func, write_func)                                   \
+    {                                                                                \
+        .version = SVS_C_API_VERSION,                                                \
+        .struct_size = sizeof(struct svs_stream_interface_ops), .read = (read_func), \
+        .write = (write_func)                                                        \
+    }
+
+/// @brief Structure representing a caller-supplied byte stream
+/// @var svs_stream_interface::ops
+///   Function pointers for the stream operations.
+/// @var svs_stream_interface::self
+///   Pointer to the user-defined stream instance. This pointer is passed to the function
+///   pointers in @p ops when they are called.
+struct svs_stream_interface {
+    struct svs_stream_interface_ops* ops;
+    void* self;
+};
+
 /// @brief Macro to create a user-defined interface implementation structure
 /// @param user_ptr Pointer to the user-defined object
 /// @param vtable Function pointers for the interface operations
@@ -498,6 +557,10 @@ typedef struct svs_id_filter_interface_ops svs_id_filter_ops_t;
 typedef struct svs_id_filter_interface svs_id_filter_t;
 typedef struct svs_id_filter_interface* svs_id_filter_i;
 
+typedef struct svs_stream_interface_ops svs_stream_ops_t;
+typedef struct svs_stream_interface svs_stream_t;
+typedef struct svs_stream_interface* svs_stream_i;
+
 typedef struct svs_search_results svs_search_results_t;
 typedef struct svs_memory_breakdown svs_memory_breakdown_t;
 
@@ -996,6 +1059,44 @@ SVS_API svs_index_h svs_index_load_dynamic(
     svs_error_h out_err /*=NULL*/
 );
 
+/// @brief Load an index from a caller-supplied stream
+/// @param builder The index builder handle (used for configuration)
+/// @param stream The stream interface to read the index from
+/// @param out_err An optional error handle to capture errors
+/// @return A handle to the loaded index
+/// @remarks The operations table is copied, but @p stream->self is retained as-is. It only
+///   needs to remain valid until this function returns, because index data is copied out of
+///   the stream rather than referenced.
+/// @remarks Accepts both the native stream encoding produced by @ref svs_index_save_stream
+///   and a packed directory archive. The encoding is detected from the stream itself.
+/// @remarks The native encoding carries no total length, so a load may read up to 64 KiB
+///   past the end of the index. Callers embedding it in a larger stream must frame the
+///   payload (e.g. with a length prefix) and bound reads to that frame.
+SVS_API svs_index_h svs_index_load_stream(
+    svs_index_builder_h builder, svs_stream_i stream, svs_error_h out_err /*=NULL*/
+);
+
+/// @brief Load a dynamic index from a caller-supplied stream
+/// @param builder The index builder handle (used for configuration)
+/// @param stream The stream interface to read the index from
+/// @param blocksize_bytes The block size in bytes for dynamic index loading (0 for default)
+/// @param out_err An optional error handle to capture errors
+/// @return A handle to the loaded dynamic index
+/// @remarks The operations table is copied, but @p stream->self is retained as-is. It only
+///   needs to remain valid until this function returns, because index data is copied out of
+///   the stream rather than referenced.
+/// @remarks Accepts both the native stream encoding produced by @ref svs_index_save_stream
+///   and a packed directory archive. The encoding is detected from the stream itself.
+/// @remarks The native encoding carries no total length, so a load may read up to 64 KiB
+///   past the end of the index. Callers embedding it in a larger stream must frame the
+///   payload (e.g. with a length prefix) and bound reads to that frame.
+SVS_API svs_index_h svs_index_load_stream_dynamic(
+    svs_index_builder_h builder,
+    svs_stream_i stream,
+    size_t blocksize_bytes /*=0*/,
+    svs_error_h out_err /*=NULL*/
+);
+
 /// @brief Convert an index using new builder configuration
 /// @param builder The index builder handle (used for configuration)
 /// @param src_index The source index handle to convert from
@@ -1111,6 +1212,19 @@ static inline bool svs_index_search(
 SVS_API bool
 svs_index_save(svs_index_h index, const char* directory, svs_error_h out_err /*=NULL*/);
 
+/// @brief Save the index to a caller-supplied stream
+/// @param index The index handle
+/// @param stream The stream interface to write the index to
+/// @param out_err An optional error handle to capture errors
+/// @return true on success, false on failure
+/// @remarks The operations table is copied, but @p stream->self is retained as-is. It only
+///   needs to remain valid until this function returns.
+/// @remarks Produces the native stream encoding only. An index previously written with
+///   @ref svs_index_save cannot be converted to a stream through this API.
+SVS_API bool svs_index_save_stream(
+    svs_index_h index, svs_stream_i stream, svs_error_h out_err /*=NULL*/
+);
+
 /// @brief Add points to a dynamic index
 /// @param index The dynamic index handle
 /// @param new_points Pointer to the new vector data (float array)