feat(transaction): add the AddBase action and the conflict substrate

protos/transaction/common.proto

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+syntax = "proto3";
+
+package lance.table;
+
+/*
+ * Shared transaction building blocks, referenced by both the legacy
+ * operations in transaction.proto and the action-based transactions in
+ * actions.proto.
+ *
+ * These types are transaction *machinery* (metadata merges, conflict-detection
+ * filters), not persisted table state, so they live here rather than in
+ * table.proto. Kept in a dedicated file so both transaction models can import
+ * them without a circular dependency (transaction.proto imports actions.proto,
+ * so actions.proto must not import transaction.proto).
+ */
+
+/*
+ * An entry for a map update.
+ *
+ * If `value` is not set, the key is removed from the map.
+ */
+message UpdateMapEntry {
+  // The key of the map entry to update.
+  string key = 1;
+  // The value to set for the key. Absent removes the key.
+  optional string value = 2;
+}
+
+/*
+ * A set of updates to apply to a string map (table config, table metadata,
+ * schema metadata, or per-field metadata).
+ */
+message UpdateMap {
+  repeated UpdateMapEntry update_entries = 1;
+  /*
+   * If true, the map is replaced entirely with these entries. If false, the
+   * entries are merged into the existing map.
+   */
+  bool replace = 2;
+}
+
+/*
+ * Exact set of key hashes for conflict detection.
+ *
+ * Used when the number of inserted rows is small.
+ */
+message ExactKeySetFilter {
+  // 64-bit hashes of the inserted row keys.
+  repeated uint64 key_hashes = 1;
+}
+
+/*
+ * Bloom filter for key existence tests.
+ *
+ * Used when the number of inserted rows is large.
+ */
+message BloomFilter {
+  // Bitset backing the bloom filter (SBBF format).
+  bytes bitmap = 1;
+  // Number of bits in the bitmap.
+  uint32 num_bits = 2;
+  /*
+   * Number of items the filter was sized for. Used for intersection validation:
+   * filters with different sizes cannot be compared. Default: 8192.
+   */
+  uint64 number_of_items = 3;
+  /*
+   * False positive probability the filter was sized for. Used for intersection
+   * validation: filters with different parameters cannot be compared.
+   * Default: 0.00057.
+   */
+  double probability = 4;
+}
+
+/*
+ * A filter for checking key existence in the set of rows inserted by a
+ * merge-insert operation.
+ *
+ * Only created when the merge-insert's ON columns match the schema's unenforced
+ * primary key; its presence indicates strict primary-key conflict detection.
+ * Conflict detection intersects two filters, so the underlying representation
+ * is either an exact set (small row counts) or a Bloom filter (large counts).
+ */
+message KeyExistenceFilter {
+  // Field ids of the columns participating in the key (the unenforced primary key).
+  repeated int32 field_ids = 1;
+  // The underlying data structure storing the key hashes.
+  oneof data {
+    // Exact set of key hashes (small number of rows).
+    ExactKeySetFilter exact = 2;
+    // Bloom filter (large number of rows).
+    BloomFilter bloom = 3;
+  }
+}

protos/transaction.proto → protos/transaction/transaction.proto

@@ -5,6 +5,8 @@ syntax = "proto3";
 
 import "file.proto";
 import "table.proto";
+import "transaction/actions.proto";
+import "transaction/common.proto";
 import "google/protobuf/any.proto";
 
 package lance.table;
@@ -187,46 +189,6 @@ message Transaction {
     optional string branch_name = 5;
   }
 
-  // Exact set of key hashes for conflict detection.
-  // Used when the number of inserted rows is small.
-  message ExactKeySetFilter {
-    // 64-bit hashes of the inserted row keys.
-    repeated uint64 key_hashes = 1;
-  }
-
-  // Bloom filter for key existence tests.
-  // Used when the number of rows is large.
-  message BloomFilter {
-    // Bitset backing the bloom filter (SBBF format).
-    bytes bitmap = 1;
-    // Number of bits in the bitmap.
-    uint32 num_bits = 2;
-    // Number of items the filter was sized for.
-    // Used for intersection validation (filters with different sizes cannot be compared).
-    // Default: 8192
-    uint64 number_of_items = 3;
-    // False positive probability the filter was sized for.
-    // Used for intersection validation (filters with different parameters cannot be compared).
-    // Default: 0.00057
-    double probability = 4;
-  }
-
-  // A filter for checking key existence in set of rows inserted by a merge insert operation.
-  // Only created when the merge insert's ON columns match the schema's unenforced primary key.
-  // The presence of this filter indicates strict primary key conflict detection should be used.
-  // Can use either an exact set (for small row counts) or a Bloom filter (for large row counts).
-  message KeyExistenceFilter {
-    // Field IDs of columns participating in the key (must match unenforced primary key).
-    repeated int32 field_ids = 1;
-    // The underlying data structure storing the key hashes.
-    oneof data {
-      // Exact set of key hashes (used for small number of rows).
-      ExactKeySetFilter exact = 2;
-      // Bloom filter (used for large number of rows).
-      BloomFilter bloom = 3;
-    }
-  }
-
   // Serialized as sorted distinct local physical row offsets within the fragment (0-based).
   message UInt32List {
     repeated uint32 values = 1;
@@ -271,21 +233,6 @@ message Transaction {
     REWRITE_COLUMNS = 1;
   }
 
-  // An entry for a map update. If value is not set, the key will be removed from the map.
-  message UpdateMapEntry {
-    // The key of the map entry to update.
-    string key = 1;
-    // The value to set for the key.
-    optional string value = 2;
-  }
-
-  message UpdateMap {
-    repeated UpdateMapEntry update_entries = 1;
-    // If true, the map will be replaced entirely with the new entries.
-    // If false, the new entries will be merged with the existing map.
-    bool replace = 2;
-  }
-
   // An operation that updates the table config, table metadata, schema metadata,
   // or field metadata.
   message UpdateConfig {
@@ -369,6 +316,9 @@ message Transaction {
     Clone clone = 113;
     UpdateBases update_bases = 114;
     DataOverlay data_overlay = 115;
+    // Action-based transaction (Transaction V2). See actions.proto.
+    // DRAFT: currently rejected on load; no write path.
+    UserOperation user_operation = 116;
   }
 
   // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops.

python/src/transaction.rs

@@ -13,7 +13,7 @@ use lance::dataset::transaction::{
 use lance::datatypes::Schema;
 use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
 use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata};
-use pyo3::exceptions::PyValueError;
+use pyo3::exceptions::{PyNotImplementedError, PyValueError};
 use pyo3::types::PySet;
 use pyo3::{Bound, FromPyObject, PyAny, PyResult, Python};
 use pyo3::{intern, prelude::*};
@@ -732,6 +732,11 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> {
                     base_op.call0()
                 }
             }
+            // Action-based operations have no Python surface yet. An explicit arm
+            // keeps them off the `todo!()` below, which would panic.
+            Operation::UserOperation(..) => Err(PyNotImplementedError::new_err(
+                "action-based transactions are not supported from the Python binding",
+            )),
             _ => todo!(),
         }
     }

rust/lance-index/src/lib.rs

@@ -79,9 +79,7 @@ pub struct IndexMetadata {
     pub distance_type: String,
 }
 
-pub fn is_system_index(index_meta: &lance_table::format::IndexMetadata) -> bool {
-    index_meta.name == FRAG_REUSE_INDEX_NAME || index_meta.name == MEM_WAL_INDEX_NAME
-}
+pub use lance_table::system_index::is_system_index;
 
 pub fn infer_system_index_type(
     index_meta: &lance_table::format::IndexMetadata,

rust/lance-table/build.rs

@@ -19,7 +19,9 @@ fn main() -> Result<()> {
     prost_build.compile_protos(
         &[
             "./protos/table.proto",
-            "./protos/transaction.proto",
+            "./protos/transaction/common.proto",
+            "./protos/transaction/actions.proto",
+            "./protos/transaction/transaction.proto",
             "./protos/rowids.proto",
         ],
         &["./protos"],

rust/lance-table/src/format.rs

@@ -6,6 +6,7 @@ use uuid::Uuid;
 
 mod fragment;
 mod index;
+pub mod key_existence;
 mod manifest;
 pub mod overlay;
 mod transaction;
@@ -17,8 +18,8 @@ pub use fragment::*;
 pub use index::{IndexFile, IndexMetadata, index_metadata_codec, list_index_files_with_sizes};
 
 pub use manifest::{
-    BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, SelfDescribingFileReader,
-    WriterVersion, is_detached_version,
+    BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, ManifestBuildConfig,
+    SelfDescribingFileReader, WriterVersion, is_detached_version,
 };
 pub use transaction::Transaction;