From 88cf999cca32f84394ffa9c96fe6e04df8150608 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 23 Jul 2026 12:02:28 -0700 Subject: [PATCH 01/15] feat(transaction): draft action-based transaction wire format (Transaction V2) Draft the full action vocabulary for action-based transactions (Transaction V2) directly in canonical `transaction.proto`, so it can drive the OSS-1529 squash/merge spike and the OSS-757 PMC vote. A `UserOperation` (a new `Transaction.operation` oneof arm, field 116) is an ordered list of `UserAction` steps, each expanding to granular `Action` deltas. Actions record the *change* to the manifest (not a post-image), which is what makes compound commits and branch merge fall out uniformly. Minted identifiers (field/fragment/base ids) carry a `Local` token via a single `Ref { committed | local }` so they relocate on merge/rebase; reference-stable changes key off stable coordinates. Computed conflict footprints and large derivable row-level deltas stay off the wire. Library support is intentionally READ-side fail-closed only: a transaction carrying a `UserOperation` is rejected on load with a clear "not supported" error, and there is no write path, no `apply`, no translation, and no conflict resolution yet. This keeps older writers safe (a concurrent V2 commit in the conflict window aborts an in-flight commit rather than being silently skipped). Co-Authored-By: Claude Opus 4.8 (1M context) --- protos/transaction.proto | 324 ++++++++++++++++++++++ rust/lance-table/src/transaction/proto.rs | 52 ++++ 2 files changed, 376 insertions(+) diff --git a/protos/transaction.proto b/protos/transaction.proto index 6e01831b4af..99ae27fd7f9 100644 --- a/protos/transaction.proto +++ b/protos/transaction.proto @@ -407,6 +407,327 @@ message Transaction { repeated BasePath new_bases = 1; } + // ========================================================================== + // Action-based transactions (Transaction V2) — DRAFT (OSS-1530). + // + // A `UserOperation` replaces the single legacy `Operation` with an ordered + // list of granular `Action`s that commit atomically as one manifest change. + // Actions are *deltas* (the change to the manifest), not post-images, which is + // what makes compound commits and branch merge fall out uniformly. See the + // design principles in OSS-1530. + // + // STATUS: this is the wire draft for the OSS-1529 spike and the OSS-757 PMC + // vote. The messages and field numbers below are NOT yet a stable contract. + // Current library support is READ-side fail-closed only: a transaction + // carrying a `UserOperation` is rejected on load, and there is no write path. + // `apply`, translation, and conflict resolution are intentionally absent. + // + // The schema is designed to be additive-safe (principle 9): message + // identities and the core mutation fields are pinned; anything discovered + // during implementation should arrive as an *added* optional field. Two + // classes of information deliberately stay off the wire: computed conflict + // *footprints* (a pure function of each action, principle 7) and large + // derivable row-level deltas (e.g. a deletion's affected rows, an update's + // matched offsets) — both are recomputed at conflict time, not serialized. + // ========================================================================== + + // A reference to a counter-allocated identifier (field id, fragment id, or + // base id), which may not be committed yet. + // + // `committed` is an already-assigned id. `local` is a placeholder token minted + // by an `Add*` action earlier in the same `UserOperation`; it resolves to a + // freshly-allocated committed id at apply time, and re-resolves against the + // *target's* counters on merge/rebase (this is what lets two independent + // `AddField`s on divergent branches become two distinct fields). `local` + // tokens are scoped to a single `UserOperation` and must be distinct within it + // (validated by the writer once the write path exists). + message Ref { + oneof kind { + uint64 committed = 1; + uint32 local = 2; + } + } + + // A user-facing, composable transaction: an ordered list of user actions that + // commit atomically as a single manifest change. + message UserOperation { + // Human-readable description, e.g. "INSERT INTO t VALUES (1)". + string description = 1; + // Unique identifier for this operation (matches Transaction.uuid semantics). + string uuid = 2; + // The dataset version this operation was planned against. + uint64 read_version = 3; + // The ordered list of user actions applied by this operation. + repeated UserAction actions = 4; + } + + // A single user-recognizable step within a UserOperation (e.g. "append batch", + // "rebuild index"), carrying a description and the granular actions it expands + // to. The description keeps the transaction history human-readable; when a + // range of transactions is squashed, each original UserOperation collapses + // into one UserAction so the readable sequence survives. Action lists are + // flattened when applied to the manifest. + message UserAction { + // Human-readable description of this step. + string description = 1; + // The granular manifest changes this step expands to. + repeated Action actions = 2; + } + + // A single granular change (a *delta*) or assertion. Legacy operations + // decompose into an ordered list of these. + // + // TODO(OSS-1529): the `data_change` markers below are drafted per-action (for + // maximum flexibility); the spike should validate per-action vs. per-UserAction + // granularity before the vote. + message Action { + oneof action { + // --- Minting actions (allocate a new counter-based id via a Local token) + AddFragment add_fragment = 1; + AddField add_field = 3; + AddBase add_base = 4; + // --- Fragment / data-file deltas + AddDataFile add_data_file = 2; + TombstoneFieldData tombstone_field_data = 5; + RemoveFragment remove_fragment = 6; + // --- Reference-stable changes (committed coordinates; post-image at rest) + SetDeletionFile set_deletion_file = 7; + ConfigUpdate config_update = 8; + AddOverlays add_overlays = 9; + RefreshRowVersionMetadata refresh_row_version_metadata = 10; + UpdateMergedGenerations update_merged_generations = 11; + // --- Schema (field-level; no wholesale SetSchema) + DropField drop_field = 12; + AlterField alter_field = 13; + // --- Index segments (segment = UUID; a logical index = the segments that + // share a `name`; per-segment config duplication is a pre-existing + // limitation this project does not fix) + AddIndexSegment add_index_segment = 14; + RemoveIndexSegment remove_index_segment = 15; + AdjustIndexCoverage adjust_index_coverage = 16; + // --- Wholesale-within-operation + ResetTable reset_table = 17; + ReserveFragmentIds reserve_fragment_ids = 18; + // --- Assertions (preconditions, not deltas) + AssertUniqueKeys assert_unique_keys = 19; + } + } + + // Mint a new, empty fragment. Its data files arrive via AddDataFile actions + // referencing this fragment's `local` token; its deletion vector (if any) via + // SetDeletionFile. + message AddFragment { + // Placeholder token for the fragment id, resolved to a committed id at apply. + uint32 local = 1; + // Number of physical rows (including rows later tombstoned). + uint64 physical_rows = 2; + // Stable-row-id and row-version sequences, carried exactly as on + // DataFragment. Absent on datasets without stable row ids. + oneof row_id_sequence { + bytes inline_row_ids = 3; + ExternalFile external_row_ids = 4; + } + oneof last_updated_at_version_sequence { + bytes inline_last_updated_at_versions = 5; + ExternalFile external_last_updated_at_versions = 6; + } + oneof created_at_version_sequence { + bytes inline_created_at_versions = 7; + ExternalFile external_created_at_versions = 8; + } + // false => rearrangement only (rows unchanged, e.g. compaction); CDC and + // streaming consumers may skip it. Absent is treated as true (real change). + optional bool data_change = 9; + } + + // Add a data file to a fragment. + message AddDataFile { + // The fragment to add the file to (Committed, or a same-op Local fragment). + Ref fragment = 1; + // The data file. Its `fields` are ignored/left unset (-1) and stamped in at + // apply once `field_ids` resolve; the authority for the column->field + // mapping is `field_ids` below. + DataFile file = 2; + // One entry per column in `file`, in order. Committed for existing fields, + // Local for fields minted by an AddField in the same operation. + repeated Ref field_ids = 3; + optional bool data_change = 4; + } + + // Mint a new schema field. A struct/list column that introduces several fields + // is expressed as several ordered AddField actions (parent before children, + // children referencing the parent via a Local ref). + message AddField { + // Placeholder token for the new field id, resolved at apply. + uint32 local = 1; + // The parent field (Committed for reparenting under an existing field, Local + // for a sibling minted in the same operation). Absent => top-level column. + optional Ref parent = 2; + // Field definition; its `id` and `parent_id` are ignored (see `local`, + // `parent`). + lance.file.Field def = 3; + } + + // Mint a new base path. + message AddBase { + // Placeholder token for the base id, resolved at apply. + uint32 local = 1; + // Base path; its `id` is ignored and stamped in at apply. + BasePath base = 2; + } + + // Tombstone the data-file binding of one or more committed fields: their slot + // in whatever data file currently backs them is set to -2, and any file left + // with no live fields is pruned at apply. This is how a column's data is + // dropped or superseded (data files have no id of their own; a live field is + // backed by exactly one file). A column re-encode is TombstoneFieldData(X) + // followed by AddDataFile(new file, field X). + message TombstoneFieldData { + Ref fragment = 1; + // Committed field ids whose current backing is tombstoned. + repeated uint64 field_ids = 2; + optional bool data_change = 3; + } + + // Remove a fragment entirely (all rows deleted, or replaced by compaction). + message RemoveFragment { + Ref fragment = 1; + optional bool data_change = 2; + } + + // Set (replace) a fragment's deletion file. Reference-stable: the fragment id + // is committed and physical row offsets are stable, so the post-image deletion + // file is safe to store; the newly-deleted rows (the delta used for rebase and + // conflict) are derived by diffing against the read-version deletion file and + // are NOT serialized here. + message SetDeletionFile { + uint64 fragment = 1; + DeletionFile deletion_file = 2; + optional bool data_change = 3; + } + + // Apply config / metadata updates. Reference-stable key-merges over stable + // coordinates (config keys, field ids). Mirrors the new-style fields of the + // legacy UpdateConfig operation. + message ConfigUpdate { + optional UpdateMap config = 1; + optional UpdateMap table_metadata = 2; + optional UpdateMap schema_metadata = 3; + // Per-field metadata updates. Keyed by Ref so a field minted in the same + // operation (Local) can receive metadata. + repeated FieldMetadata field_metadata = 4; + + message FieldMetadata { + Ref field = 1; + UpdateMap updates = 2; + } + } + + // Append overlay files to fragments (see DataOverlayFile in table.proto). + // Reference-stable: overlays target committed fragments/offsets/fields and are + // appended, never replacing existing overlays. + message AddOverlays { + Ref fragment = 1; + repeated DataOverlayFile overlays = 2; + optional bool data_change = 3; + } + + // Refresh row-version (created-at / last-updated-at) metadata for fragments + // whose columns were merged in place, mirroring the implicit refresh Merge + // performs on stable-row-id datasets. + message RefreshRowVersionMetadata { + repeated uint64 fragment_ids = 1; + } + + // Mark MemWAL shard generations as merged. Covers UpdateMemWalState and the + // merged-generations bookkeeping of Update. + message UpdateMergedGenerations { + repeated MergedGeneration merged_generations = 1; + } + + // Remove a field from the schema (and, at apply, prune any data file left with + // no live fields). References an existing committed field id. + message DropField { + uint64 field = 1; + } + + // Alter facets of an existing field in place, preserving its identity (id). + // Each facet is optional: present means "change this facet", absent means + // "leave unchanged". Keying conflict on (field id, facet) lets independent + // facet changes to the same field commute (e.g. a cast concurrent with a + // nullability change). A cast additionally needs TombstoneFieldData + a new + // AddDataFile to rewrite the data. New facets are added as optional fields. + message AlterField { + uint64 field = 1; + optional string name = 2; + // New Arrow logical type (see Field.logical_type). The cast. + optional string logical_type = 3; + optional bool nullable = 4; + } + + // Add an index segment. A brand-new index is expressed as its first segment. + // The set of segments sharing `name` constitutes one logical index. + message AddIndexSegment { + UUID uuid = 1; + string name = 2; + // Indexed field ids (Committed, or Local for same-op minted fields). + repeated Ref fields = 3; + google.protobuf.Any index_details = 4; + optional int32 index_version = 5; + // Fragments covered by this segment (resolved to a fragment_bitmap at apply, + // and remapped under fragment relocation). Committed or same-op Local. + repeated Ref covered_fragments = 6; + // Index files with sizes, when produced by the writer (e.g. a compaction + // that rewrites the segment). Empty when unavailable. + repeated IndexFile files = 7; + optional bool data_change = 8; + } + + // Remove an index segment by uuid. + message RemoveIndexSegment { + UUID uuid = 1; + optional bool data_change = 2; + } + + // Adjust the fragment coverage of an existing index segment without rewriting + // it. NOTE: coverage representation is the acknowledged-open area of OSS-1530; + // this shape is provisional pending the OSS-1529 spike (which will settle how + // much coverage change is derivable from fragment relocation vs. stated here). + message AdjustIndexCoverage { + UUID uuid = 1; + repeated Ref add = 2; + repeated uint64 remove = 3; + } + + // Reset the table to an empty state, in preparation for a fresh schema and + // data written by later actions in the same operation (the decomposition of a + // full Overwrite / CREATE OR REPLACE). Drops the entire schema, schema + // metadata, all fragments, and all indices; preserves table config, table + // metadata, and base paths (change those via ConfigUpdate / AddBase in the + // same operation). Its write footprint spans the whole schema, all fragments, + // and all indices, so it conflicts with any concurrent change to them + // (DROP TABLE-style exclusivity). + message ResetTable {} + + // Reserve a contiguous range of fragment ids from the counter, for a later + // (possibly distributed) writer to populate. Fragments written against a + // reserved range reference those ids as Committed. + message ReserveFragmentIds { + uint32 count = 1; + } + + // An assertion (precondition), not a delta: the keys this operation inserts + // must not collide with keys inserted by a concurrent commit. Carries a bloom + // / exact-set filter of the inserted key hashes (not derivable from any + // post-image); conflict = the two filters intersect. Home for merge-insert's + // strict primary-key conflict detection. + message AssertUniqueKeys { + // Field ids of the key columns (unenforced primary key). Committed, or Local + // for same-op minted fields. + repeated Ref key_fields = 1; + KeyExistenceFilter filter = 2; + } + // The operation of this transaction. oneof operation { Append append = 100; @@ -425,6 +746,9 @@ message Transaction { Clone clone = 113; UpdateBases update_bases = 114; DataOverlay data_overlay = 115; + // Action-based transaction (Transaction V2). See UserOperation above. + // DRAFT: currently rejected on load; no write path. + UserOperation user_operation = 116; } // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index c51c8f16719..baa43aeabc8 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -401,6 +401,19 @@ impl TryFrom for Transaction { .map(DataOverlayGroup::try_from) .collect::>>()?, }, + Some(pb::transaction::Operation::UserOperation(_)) => { + // Action-based transactions (Transaction V2) are a draft wire + // format (OSS-1530). This version of Lance recognizes the message + // but has no support for it: reject on load, fail-closed. Because + // load_and_sort_new_transactions collects transactions with + // try_collect, a concurrent V2 commit in the conflict window + // aborts the whole commit rather than being silently skipped. + // Do NOT make this parsing lenient. + return Err(Error::not_supported( + "action-based transactions (Transaction V2) are not supported \ + by this version of Lance; please upgrade", + )); + } None => { return Err(Error::internal( "Transaction message did not contain an operation".to_string(), @@ -854,4 +867,43 @@ mod tests { other => panic!("expected DataOverlay, got {other:?}"), } } + + #[test] + fn test_user_operation_rejected_on_load() { + // Action-based transactions (Transaction V2) are a draft wire format that + // this version of Lance does not support. Loading one must fail closed + // (never be silently skipped or leniently parsed), so that a concurrent + // V2 commit in the conflict window aborts an in-flight commit. + let message = pb::Transaction { + read_version: 1, + uuid: Uuid::new_v4().to_string(), + operation: Some(pb::transaction::Operation::UserOperation( + pb::transaction::UserOperation { + description: "INSERT INTO t VALUES (1)".to_string(), + uuid: Uuid::new_v4().to_string(), + read_version: 1, + actions: vec![pb::transaction::UserAction { + description: "append batch".to_string(), + actions: vec![pb::transaction::Action { + action: Some(pb::transaction::action::Action::AddFragment( + pb::transaction::AddFragment { + local: 0, + physical_rows: 1, + data_change: Some(true), + ..Default::default() + }, + )), + }], + }], + }, + )), + ..Default::default() + }; + + let err = Transaction::try_from(message).unwrap_err(); + assert!( + matches!(err, Error::NotSupported { .. }), + "expected NotSupported, got: {err:?}" + ); + } } From e26f922fa2a235428fd44d06a43001c4ad27b4aa Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 23 Jul 2026 14:25:12 -0700 Subject: [PATCH 02/15] refactor(transaction): split V2 action protos into transaction/ directory Reorganize the action-based transaction (Transaction V2) wire draft into its own files under protos/transaction/ so the design reads clearly and the shared building blocks have a proper home: - protos/transaction/actions.proto: the V2 vocabulary (Ref, UserOperation, UserAction, Action + action messages), as top-level messages. The design rationale is summarized in an in-tree file header (deltas vs post-images, minting vs reference-stable, Ref/Local resolution, field-level schema, index segments, what stays off the wire) so it stands on its own. - protos/transaction/common.proto: UpdateMap/UpdateMapEntry and KeyExistenceFilter/ExactKeySetFilter/BloomFilter, promoted from nested Transaction messages to top-level so both the legacy operations and the V2 actions can reference them without a circular import. Wire-compatible: field numbers unchanged and none of these types are Any-packed, so the fully-qualified name change is invisible on the wire. - protos/transaction.proto moves to protos/transaction/transaction.proto and keeps only the Transaction envelope, legacy operations, and the user_operation oneof arm. Comments use block style for IDE folding. Rust references to the promoted types are repointed from pb::transaction::X to pb::X (mechanical, compiler-checked); the hand-written dataset::transaction domain types are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- protos/transaction.proto | 757 ------------------- protos/transaction/actions.proto | 474 ++++++++++++ protos/transaction/common.proto | 97 +++ protos/transaction/transaction.proto | 377 +++++++++ rust/lance-table/build.rs | 4 +- rust/lance-table/src/format/key_existence.rs | 44 +- rust/lance-table/src/transaction/proto.rs | 44 +- 7 files changed, 988 insertions(+), 809 deletions(-) delete mode 100644 protos/transaction.proto create mode 100644 protos/transaction/actions.proto create mode 100644 protos/transaction/common.proto create mode 100644 protos/transaction/transaction.proto diff --git a/protos/transaction.proto b/protos/transaction.proto deleted file mode 100644 index 99ae27fd7f9..00000000000 --- a/protos/transaction.proto +++ /dev/null @@ -1,757 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -syntax = "proto3"; - -import "file.proto"; -import "table.proto"; -import "google/protobuf/any.proto"; - -package lance.table; - -/* A transaction represents the changes to a dataset. - * - * This has two purposes: - * 1. When retrying a commit, the transaction can be used to re-build an updated - * manifest. - * 2. When there's a conflict, this can be used to determine whether the other - * transaction is compatible with this one. - */ -message Transaction { - /* The version of the dataset this transaction was built from. - * - * For example, for a delete transaction this means the version of the dataset - * that was read from while evaluating the deletion predicate. - */ - uint64 read_version = 1; - - // The UUID that unique identifies a transaction. - string uuid = 2; - - // Optional version tag. - string tag = 3; - - /* Optional properties for the transaction - * __lance_commit_message is a reserved key - */ - map transaction_properties = 4; - - // Add new rows to the dataset. - message Append { - /* The new fragments to append. - * - * Fragment IDs are not yet assigned. - */ - repeated DataFragment fragments = 1; - } - - // Mark rows as deleted. - message Delete { - /* The fragments to update - * - * The fragment IDs will match existing fragments in the dataset. - */ - repeated DataFragment updated_fragments = 1; - // The fragments to delete entirely. - repeated uint64 deleted_fragment_ids = 2; - /* The predicate that was evaluated - * - * This may be used to determine whether the delete would have affected - * files written by a concurrent transaction. - */ - string predicate = 3; - } - - // Create or overwrite the entire dataset. - message Overwrite { - /* The new fragments - * - * Fragment IDs are not yet assigned. - */ - repeated DataFragment fragments = 1; - // The new schema - repeated lance.file.Field schema = 2; - // Schema metadata. - map schema_metadata = 3; - // Key-value pairs to merge with existing config. - map config_upsert_values = 4; - // The base paths to be added for the initial dataset creation - repeated BasePath initial_bases = 5; - } - - /* Add or replace a new secondary index. - * - * This is also used to remove an index (we are replacing it with nothing) - * - * - new_indices: the modified indices, empty if dropping indices only - * - removed_indices: the indices that are being replaced - */ - message CreateIndex { - repeated IndexMetadata new_indices = 1; - repeated IndexMetadata removed_indices = 2; - } - - /* An operation that rewrites but does not change the data in the table. These - * kinds of operations just rearrange data. - */ - message Rewrite { - /* The old fragments that are being replaced - * - * DEPRECATED: use groups instead. - * - * These should all have existing fragment IDs. - */ - repeated DataFragment old_fragments = 1; - /* The new fragments - * - * DEPRECATED: use groups instead. - * - * These fragments IDs are not yet assigned. - */ - repeated DataFragment new_fragments = 2; - - /* During a rewrite an index may be rewritten. We only serialize the UUID - * since a rewrite should not change the other index parameters. - */ - message RewrittenIndex { - // The id of the index that will be replaced - UUID old_id = 1; - // the id of the new index - UUID new_id = 2; - // the new index details - google.protobuf.Any new_index_details = 3; - // the version of the new index - uint32 new_index_version = 4; - /* Files in the new index with their sizes. - * Empty if file sizes are not available (e.g. older writers). - */ - repeated IndexFile new_index_files = 5; - } - - // A group of rewrite files that are all part of the same rewrite. - message RewriteGroup { - /* The old fragment that is being replaced - * - * This should have an existing fragment ID. - */ - repeated DataFragment old_fragments = 1; - /* The new fragment - * - * The ID should have been reserved by an earlier - * reserve operation - */ - repeated DataFragment new_fragments = 2; - } - - // Groups of files that have been rewritten - repeated RewriteGroup groups = 3; - // Indices that have been rewritten - repeated RewrittenIndex rewritten_indices = 4; - } - - // An operation that merges in a new column, altering the schema. - message Merge { - /* The updated fragments - * - * These should all have existing fragment IDs. - */ - repeated DataFragment fragments = 1; - // The new schema - repeated lance.file.Field schema = 2; - // Schema metadata. - map schema_metadata = 3; - /* Set when this merge makes no nullability-affecting schema change: it - * introduces no field that data staged against an earlier schema could - * not safely omit. Without the assertion (including transactions written - * before this field existed) the merge conservatively conflicts with - * concurrent value-writes, which can only cause a retry. - */ - bool preserves_nullability = 4; - } - - // An operation that projects a subset of columns, altering the schema. - message Project { - // The new schema - repeated lance.file.Field schema = 1; - /* Set when this projection makes no nullability-affecting schema change, - * as a rename or a drop does not. Without the assertion (including - * transactions written before this field existed) the projection - * conservatively conflicts with concurrent value-writes, which can only - * cause a retry. A nullability tightening must not set this. - */ - bool preserves_nullability = 2; - } - - // An operation that restores a dataset to a previous version. - message Restore { - // The version to restore to - uint64 version = 1; - } - - /* An operation that reserves fragment ids for future use in - * a rewrite operation. - */ - message ReserveFragments { - uint32 num_fragments = 1; - } - - // An operation that clones a dataset. - message Clone { - /* - true: Performs a metadata-only clone (copies manifest without data files). - * The cloned dataset references original data through `base_paths`, - * suitable for experimental scenarios or rapid metadata migration. - * - false: Performs a full deep clone using the underlying object storage's native - * copy API (e.g., S3 CopyObject, GCS rewrite). This leverages server-side - * bulk copy operations to bypass download/upload bottlenecks, achieving - * near-linear speedup for large datasets (typically 3-10x faster than - * manual file transfers). The operation maintains atomicity and data - * integrity guarantees provided by the storage backend. - */ - bool is_shallow = 1; - /* the reference name in the source dataset - * in most cases it should be the branch or tag name in the source dataset - */ - optional string ref_name = 2; - // the version of the source dataset for cloning - uint64 ref_version = 3; - // the absolute base path of the source dataset for cloning - string ref_path = 4; - // if the target dataset is a branch, this is the branch name of the target dataset - optional string branch_name = 5; - } - - /* Exact set of key hashes for conflict detection. - * Used when the number of inserted rows is small. - */ - message ExactKeySetFilter { - // 64-bit hashes of the inserted row keys. - repeated uint64 key_hashes = 1; - } - - /* Bloom filter for key existence tests. - * Used when the number of rows is large. - */ - message BloomFilter { - // Bitset backing the bloom filter (SBBF format). - bytes bitmap = 1; - // Number of bits in the bitmap. - uint32 num_bits = 2; - /* Number of items the filter was sized for. - * Used for intersection validation (filters with different sizes cannot be compared). - * Default: 8192 - */ - uint64 number_of_items = 3; - /* False positive probability the filter was sized for. - * Used for intersection validation (filters with different parameters cannot be compared). - * Default: 0.00057 - */ - double probability = 4; - } - - /* A filter for checking key existence in set of rows inserted by a merge insert operation. - * Only created when the merge insert's ON columns match the schema's unenforced primary key. - * The presence of this filter indicates strict primary key conflict detection should be used. - * Can use either an exact set (for small row counts) or a Bloom filter (for large row counts). - */ - message KeyExistenceFilter { - // Field IDs of columns participating in the key (must match unenforced primary key). - repeated int32 field_ids = 1; - // The underlying data structure storing the key hashes. - oneof data { - // Exact set of key hashes (used for small number of rows). - ExactKeySetFilter exact = 2; - // Bloom filter (used for large number of rows). - BloomFilter bloom = 3; - } - } - - // Serialized as sorted distinct local physical row offsets within the fragment (0-based). - message UInt32List { - repeated uint32 values = 1; - } - - // An operation that updates rows but does not add or remove rows. - message Update { - /* The fragments that have been removed. These are fragments where all rows - * have been updated and moved to a new fragment. - */ - repeated uint64 removed_fragment_ids = 1; - // The fragments that have been updated. - repeated DataFragment updated_fragments = 2; - // The new fragments where updated rows have been moved to. - repeated DataFragment new_fragments = 3; - // The ids of the fields that have been modified. - repeated uint32 fields_modified = 4; - /// SSTables to mark as compacted after this transaction. - repeated CompactedSsTable compacted_sstables = 5; - /* The fields that used to judge whether to preserve the new frag's id into - * the frag bitmap of the specified indices. - */ - repeated uint32 fields_for_preserving_frag_bitmap = 6; - // The mode of update - UpdateMode update_mode = 7; - /* Filter for checking existence of keys in newly inserted rows, used for conflict detection. - * Only tracks keys from INSERT operations during merge insert, not updates. - */ - optional KeyExistenceFilter inserted_rows = 8; - /* Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns). - * Deprecated: use updated_fragment_offset_bitmaps (field 10) instead. - */ - map updated_fragment_offsets = 9; - /* Per-fragment matched offsets as portable RoaringBitmap bytes (replaces field 9). - * Writers emit field 10 only. Readers prefer field 10; fall back to field 9 for - * manifests written before this change. - */ - map updated_fragment_offset_bitmaps = 10; - } - - // The mode of update operation - enum UpdateMode { - - /* rows are deleted in current fragments and rewritten in new fragments. - * This is most optimal when the majority of columns are being rewritten - * or only a few rows are being updated. - */ - REWRITE_ROWS = 0; - - /* within each fragment, columns are fully rewritten and inserted as new data files. - * Old versions of columns are tombstoned. This is most optimal when most rows are affected - * but a small subset of columns are affected. - */ - REWRITE_COLUMNS = 1; - } - - // An entry for a map update. If value is not set, the key will be removed from the map. - message UpdateMapEntry { - // The key of the map entry to update. - string key = 1; - // The value to set for the key. - optional string value = 2; - } - - message UpdateMap { - repeated UpdateMapEntry update_entries = 1; - /* If true, the map will be replaced entirely with the new entries. - * If false, the new entries will be merged with the existing map. - */ - bool replace = 2; - } - - /* An operation that updates the table config, table metadata, schema metadata, - * or field metadata. - */ - message UpdateConfig { - UpdateMap config_updates = 6; - UpdateMap table_metadata_updates = 7; - UpdateMap schema_metadata_updates = 8; - map field_metadata_updates = 9; - - // Deprecated ------------------------------- - map upsert_values = 1; - repeated string delete_keys = 2; - map schema_metadata = 3; - map field_metadata = 4; - - message FieldMetadataUpdate { - map metadata = 5; - } - } - - message DataReplacementGroup { - uint64 fragment_id = 1; - DataFile new_file = 2; - } - - // An operation that replaces the data in a region of the table with new data. - message DataReplacement { - repeated DataReplacementGroup replacements = 1; - } - - /* Overlay files to append to a single fragment, in order (the last entry is - * newest). The overlays are appended to the fragment's existing `overlays` - * list; they do not replace it, so overlays written by concurrent commits are - * preserved. - */ - message DataOverlayGroup { - uint64 fragment_id = 1; - /* Each DataOverlayFile.committed_version is left 0 by the writer and stamped - * to the new dataset version at commit time (re-stamped on retry), in the - * same way as the created-at / last-updated-at version sequences. The fields - * touched are read from each overlay's `data_file.fields`. - */ - repeated DataOverlayFile overlays = 2; - } - - /* Attach overlay files to fragments, supplying new values for a subset of - * (row offset, field) cells without rewriting the fragments' base data files. - * See the DataOverlayFile message in table.proto for resolution, coverage, and - * versioning rules, and the Data Overlay Files and Transactions specifications - * for the (intentionally permissive) conflict semantics. - */ - message DataOverlay { - repeated DataOverlayGroup groups = 1; - } - - /* Update SSTable compaction progress in the MemWAL index. - * This operation is used during merge-insert to atomically record which - * SSTables have been compacted into the base table. - */ - message UpdateMemWalState { - // SSTables being marked as compacted. - repeated CompactedSsTable compacted_sstables = 1; - } - - // An operation that updates base paths in the dataset. - message UpdateBases { - // The new base paths to add to the manifest. - repeated BasePath new_bases = 1; - } - - // ========================================================================== - // Action-based transactions (Transaction V2) — DRAFT (OSS-1530). - // - // A `UserOperation` replaces the single legacy `Operation` with an ordered - // list of granular `Action`s that commit atomically as one manifest change. - // Actions are *deltas* (the change to the manifest), not post-images, which is - // what makes compound commits and branch merge fall out uniformly. See the - // design principles in OSS-1530. - // - // STATUS: this is the wire draft for the OSS-1529 spike and the OSS-757 PMC - // vote. The messages and field numbers below are NOT yet a stable contract. - // Current library support is READ-side fail-closed only: a transaction - // carrying a `UserOperation` is rejected on load, and there is no write path. - // `apply`, translation, and conflict resolution are intentionally absent. - // - // The schema is designed to be additive-safe (principle 9): message - // identities and the core mutation fields are pinned; anything discovered - // during implementation should arrive as an *added* optional field. Two - // classes of information deliberately stay off the wire: computed conflict - // *footprints* (a pure function of each action, principle 7) and large - // derivable row-level deltas (e.g. a deletion's affected rows, an update's - // matched offsets) — both are recomputed at conflict time, not serialized. - // ========================================================================== - - // A reference to a counter-allocated identifier (field id, fragment id, or - // base id), which may not be committed yet. - // - // `committed` is an already-assigned id. `local` is a placeholder token minted - // by an `Add*` action earlier in the same `UserOperation`; it resolves to a - // freshly-allocated committed id at apply time, and re-resolves against the - // *target's* counters on merge/rebase (this is what lets two independent - // `AddField`s on divergent branches become two distinct fields). `local` - // tokens are scoped to a single `UserOperation` and must be distinct within it - // (validated by the writer once the write path exists). - message Ref { - oneof kind { - uint64 committed = 1; - uint32 local = 2; - } - } - - // A user-facing, composable transaction: an ordered list of user actions that - // commit atomically as a single manifest change. - message UserOperation { - // Human-readable description, e.g. "INSERT INTO t VALUES (1)". - string description = 1; - // Unique identifier for this operation (matches Transaction.uuid semantics). - string uuid = 2; - // The dataset version this operation was planned against. - uint64 read_version = 3; - // The ordered list of user actions applied by this operation. - repeated UserAction actions = 4; - } - - // A single user-recognizable step within a UserOperation (e.g. "append batch", - // "rebuild index"), carrying a description and the granular actions it expands - // to. The description keeps the transaction history human-readable; when a - // range of transactions is squashed, each original UserOperation collapses - // into one UserAction so the readable sequence survives. Action lists are - // flattened when applied to the manifest. - message UserAction { - // Human-readable description of this step. - string description = 1; - // The granular manifest changes this step expands to. - repeated Action actions = 2; - } - - // A single granular change (a *delta*) or assertion. Legacy operations - // decompose into an ordered list of these. - // - // TODO(OSS-1529): the `data_change` markers below are drafted per-action (for - // maximum flexibility); the spike should validate per-action vs. per-UserAction - // granularity before the vote. - message Action { - oneof action { - // --- Minting actions (allocate a new counter-based id via a Local token) - AddFragment add_fragment = 1; - AddField add_field = 3; - AddBase add_base = 4; - // --- Fragment / data-file deltas - AddDataFile add_data_file = 2; - TombstoneFieldData tombstone_field_data = 5; - RemoveFragment remove_fragment = 6; - // --- Reference-stable changes (committed coordinates; post-image at rest) - SetDeletionFile set_deletion_file = 7; - ConfigUpdate config_update = 8; - AddOverlays add_overlays = 9; - RefreshRowVersionMetadata refresh_row_version_metadata = 10; - UpdateMergedGenerations update_merged_generations = 11; - // --- Schema (field-level; no wholesale SetSchema) - DropField drop_field = 12; - AlterField alter_field = 13; - // --- Index segments (segment = UUID; a logical index = the segments that - // share a `name`; per-segment config duplication is a pre-existing - // limitation this project does not fix) - AddIndexSegment add_index_segment = 14; - RemoveIndexSegment remove_index_segment = 15; - AdjustIndexCoverage adjust_index_coverage = 16; - // --- Wholesale-within-operation - ResetTable reset_table = 17; - ReserveFragmentIds reserve_fragment_ids = 18; - // --- Assertions (preconditions, not deltas) - AssertUniqueKeys assert_unique_keys = 19; - } - } - - // Mint a new, empty fragment. Its data files arrive via AddDataFile actions - // referencing this fragment's `local` token; its deletion vector (if any) via - // SetDeletionFile. - message AddFragment { - // Placeholder token for the fragment id, resolved to a committed id at apply. - uint32 local = 1; - // Number of physical rows (including rows later tombstoned). - uint64 physical_rows = 2; - // Stable-row-id and row-version sequences, carried exactly as on - // DataFragment. Absent on datasets without stable row ids. - oneof row_id_sequence { - bytes inline_row_ids = 3; - ExternalFile external_row_ids = 4; - } - oneof last_updated_at_version_sequence { - bytes inline_last_updated_at_versions = 5; - ExternalFile external_last_updated_at_versions = 6; - } - oneof created_at_version_sequence { - bytes inline_created_at_versions = 7; - ExternalFile external_created_at_versions = 8; - } - // false => rearrangement only (rows unchanged, e.g. compaction); CDC and - // streaming consumers may skip it. Absent is treated as true (real change). - optional bool data_change = 9; - } - - // Add a data file to a fragment. - message AddDataFile { - // The fragment to add the file to (Committed, or a same-op Local fragment). - Ref fragment = 1; - // The data file. Its `fields` are ignored/left unset (-1) and stamped in at - // apply once `field_ids` resolve; the authority for the column->field - // mapping is `field_ids` below. - DataFile file = 2; - // One entry per column in `file`, in order. Committed for existing fields, - // Local for fields minted by an AddField in the same operation. - repeated Ref field_ids = 3; - optional bool data_change = 4; - } - - // Mint a new schema field. A struct/list column that introduces several fields - // is expressed as several ordered AddField actions (parent before children, - // children referencing the parent via a Local ref). - message AddField { - // Placeholder token for the new field id, resolved at apply. - uint32 local = 1; - // The parent field (Committed for reparenting under an existing field, Local - // for a sibling minted in the same operation). Absent => top-level column. - optional Ref parent = 2; - // Field definition; its `id` and `parent_id` are ignored (see `local`, - // `parent`). - lance.file.Field def = 3; - } - - // Mint a new base path. - message AddBase { - // Placeholder token for the base id, resolved at apply. - uint32 local = 1; - // Base path; its `id` is ignored and stamped in at apply. - BasePath base = 2; - } - - // Tombstone the data-file binding of one or more committed fields: their slot - // in whatever data file currently backs them is set to -2, and any file left - // with no live fields is pruned at apply. This is how a column's data is - // dropped or superseded (data files have no id of their own; a live field is - // backed by exactly one file). A column re-encode is TombstoneFieldData(X) - // followed by AddDataFile(new file, field X). - message TombstoneFieldData { - Ref fragment = 1; - // Committed field ids whose current backing is tombstoned. - repeated uint64 field_ids = 2; - optional bool data_change = 3; - } - - // Remove a fragment entirely (all rows deleted, or replaced by compaction). - message RemoveFragment { - Ref fragment = 1; - optional bool data_change = 2; - } - - // Set (replace) a fragment's deletion file. Reference-stable: the fragment id - // is committed and physical row offsets are stable, so the post-image deletion - // file is safe to store; the newly-deleted rows (the delta used for rebase and - // conflict) are derived by diffing against the read-version deletion file and - // are NOT serialized here. - message SetDeletionFile { - uint64 fragment = 1; - DeletionFile deletion_file = 2; - optional bool data_change = 3; - } - - // Apply config / metadata updates. Reference-stable key-merges over stable - // coordinates (config keys, field ids). Mirrors the new-style fields of the - // legacy UpdateConfig operation. - message ConfigUpdate { - optional UpdateMap config = 1; - optional UpdateMap table_metadata = 2; - optional UpdateMap schema_metadata = 3; - // Per-field metadata updates. Keyed by Ref so a field minted in the same - // operation (Local) can receive metadata. - repeated FieldMetadata field_metadata = 4; - - message FieldMetadata { - Ref field = 1; - UpdateMap updates = 2; - } - } - - // Append overlay files to fragments (see DataOverlayFile in table.proto). - // Reference-stable: overlays target committed fragments/offsets/fields and are - // appended, never replacing existing overlays. - message AddOverlays { - Ref fragment = 1; - repeated DataOverlayFile overlays = 2; - optional bool data_change = 3; - } - - // Refresh row-version (created-at / last-updated-at) metadata for fragments - // whose columns were merged in place, mirroring the implicit refresh Merge - // performs on stable-row-id datasets. - message RefreshRowVersionMetadata { - repeated uint64 fragment_ids = 1; - } - - // Mark MemWAL shard generations as merged. Covers UpdateMemWalState and the - // merged-generations bookkeeping of Update. - message UpdateMergedGenerations { - repeated MergedGeneration merged_generations = 1; - } - - // Remove a field from the schema (and, at apply, prune any data file left with - // no live fields). References an existing committed field id. - message DropField { - uint64 field = 1; - } - - // Alter facets of an existing field in place, preserving its identity (id). - // Each facet is optional: present means "change this facet", absent means - // "leave unchanged". Keying conflict on (field id, facet) lets independent - // facet changes to the same field commute (e.g. a cast concurrent with a - // nullability change). A cast additionally needs TombstoneFieldData + a new - // AddDataFile to rewrite the data. New facets are added as optional fields. - message AlterField { - uint64 field = 1; - optional string name = 2; - // New Arrow logical type (see Field.logical_type). The cast. - optional string logical_type = 3; - optional bool nullable = 4; - } - - // Add an index segment. A brand-new index is expressed as its first segment. - // The set of segments sharing `name` constitutes one logical index. - message AddIndexSegment { - UUID uuid = 1; - string name = 2; - // Indexed field ids (Committed, or Local for same-op minted fields). - repeated Ref fields = 3; - google.protobuf.Any index_details = 4; - optional int32 index_version = 5; - // Fragments covered by this segment (resolved to a fragment_bitmap at apply, - // and remapped under fragment relocation). Committed or same-op Local. - repeated Ref covered_fragments = 6; - // Index files with sizes, when produced by the writer (e.g. a compaction - // that rewrites the segment). Empty when unavailable. - repeated IndexFile files = 7; - optional bool data_change = 8; - } - - // Remove an index segment by uuid. - message RemoveIndexSegment { - UUID uuid = 1; - optional bool data_change = 2; - } - - // Adjust the fragment coverage of an existing index segment without rewriting - // it. NOTE: coverage representation is the acknowledged-open area of OSS-1530; - // this shape is provisional pending the OSS-1529 spike (which will settle how - // much coverage change is derivable from fragment relocation vs. stated here). - message AdjustIndexCoverage { - UUID uuid = 1; - repeated Ref add = 2; - repeated uint64 remove = 3; - } - - // Reset the table to an empty state, in preparation for a fresh schema and - // data written by later actions in the same operation (the decomposition of a - // full Overwrite / CREATE OR REPLACE). Drops the entire schema, schema - // metadata, all fragments, and all indices; preserves table config, table - // metadata, and base paths (change those via ConfigUpdate / AddBase in the - // same operation). Its write footprint spans the whole schema, all fragments, - // and all indices, so it conflicts with any concurrent change to them - // (DROP TABLE-style exclusivity). - message ResetTable {} - - // Reserve a contiguous range of fragment ids from the counter, for a later - // (possibly distributed) writer to populate. Fragments written against a - // reserved range reference those ids as Committed. - message ReserveFragmentIds { - uint32 count = 1; - } - - // An assertion (precondition), not a delta: the keys this operation inserts - // must not collide with keys inserted by a concurrent commit. Carries a bloom - // / exact-set filter of the inserted key hashes (not derivable from any - // post-image); conflict = the two filters intersect. Home for merge-insert's - // strict primary-key conflict detection. - message AssertUniqueKeys { - // Field ids of the key columns (unenforced primary key). Committed, or Local - // for same-op minted fields. - repeated Ref key_fields = 1; - KeyExistenceFilter filter = 2; - } - - // The operation of this transaction. - oneof operation { - Append append = 100; - Delete delete = 101; - Overwrite overwrite = 102; - CreateIndex create_index = 103; - Rewrite rewrite = 104; - Merge merge = 105; - Restore restore = 106; - ReserveFragments reserve_fragments = 107; - Update update = 108; - Project project = 109; - UpdateConfig update_config = 110; - DataReplacement data_replacement = 111; - UpdateMemWalState update_mem_wal_state = 112; - Clone clone = 113; - UpdateBases update_bases = 114; - DataOverlay data_overlay = 115; - // Action-based transaction (Transaction V2). See UserOperation above. - // DRAFT: currently rejected on load; no write path. - UserOperation user_operation = 116; - } - - // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. - reserved 200, 202; - reserved "blob_append", "blob_overwrite"; -} diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto new file mode 100644 index 00000000000..1c6cda7c754 --- /dev/null +++ b/protos/transaction/actions.proto @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +syntax = "proto3"; + +import "file.proto"; +import "table.proto"; +import "transaction/common.proto"; +import "google/protobuf/any.proto"; + +package lance.table; + +/* + * Action-based transactions (Transaction V2) — DRAFT. + * + * A `UserOperation` replaces the single legacy `Operation` (see + * transaction.proto) with an ordered list of granular `Action`s that commit + * atomically as one manifest change. This file is the wire draft only. + * + * STATUS + * ------ + * The messages and field numbers here are NOT yet a stable contract. Library + * support is read-side fail-closed only: a transaction carrying a + * `UserOperation` is rejected on load, and there is no write path. Apply, + * id translation, and conflict resolution are intentionally absent. + * + * DESIGN RATIONALE + * ---------------- + * The reasoning below is the durable record of why the wire format is shaped + * this way; it is deliberately kept in-tree rather than in an external tracker. + * + * 1. Actions are deltas, not post-images. Each action records *the change* to + * the manifest (add this file, tombstone this field), not the resulting + * state. Deltas are what make two things fall out uniformly with no special + * cases: composing several actions into one atomic commit, and merging or + * rebasing a branch (replay the deltas against the target). A few actions + * that carry non-derivable preconditions are the exception (see 8). + * + * 2. Minting vs. reference-stable actions. Minting actions allocate a new + * counter-based id (a field id, fragment id, or base id). Because the final + * id is not known until commit — and differs when the same action is + * replayed onto a different target during merge/rebase — a minting action + * names its new id with a `Local` token (see `Ref`) rather than a concrete + * value. Reference-stable actions touch already-committed ids at stable + * coordinates (a fragment's deletion vector, a config key); their target is + * unambiguous, so they may safely carry a post-image at rest. + * + * 3. One reference type, `Ref = Committed | Local`. `Committed` is a concrete, + * already-assigned id. `Local` is a placeholder minted by an earlier `Add*` + * action in the same operation. `Local` resolves to a freshly-allocated + * committed id at apply time, and re-resolves against the *target's* + * counters on merge/rebase — which is exactly what lets two independent + * `AddField`s on divergent branches become two distinct fields on merge. + * + * 4. Relocation by counter watermark. When an operation is replayed onto a + * newer version, minting actions are re-applied against the target's + * counters and their `Local` refs re-resolve; ids assigned above the + * read-version watermark are known to be freshly minted and are rewritten + * consistently across every action that referenced them. + * + * 5. Field-level schema. Schema changes are expressed per field + * (`AddField` / `DropField` / `AlterField`), never as a wholesale + * replacement, so concurrent schema edits to disjoint fields commute. + * + * 6. Identity-preserving `AlterField`. Altering a field is distinct from + * dropping it and adding a new one: the field keeps its id. Each mutable + * facet (name, type, nullability) is an independent optional, so two + * concurrent alters of *different* facets of the same field commute + * (e.g. a widening cast alongside a non-null -> nullable relaxation). + * + * 7. Index segments, not indices. The format has no first-class "index" + * separate from its segments; a new index is written as its first segment, + * and a logical index is the set of segments sharing a `name`. The actions + * mirror that: add/remove/adjust operate on a segment (by UUID). The + * resulting per-segment config duplication is a pre-existing limitation this + * draft does not attempt to fix. + * + * 8. Assertions carry only non-derivable preconditions. Most conflict checks + * are computed from the actions themselves. The exception is a fact that + * cannot be reconstructed from any post-image — e.g. the set of keys a + * merge-insert inserted — which is carried explicitly as an assertion + * (`AssertUniqueKeys`) and checked against concurrent commits. + * + * WHAT STAYS OFF THE WIRE + * ----------------------- + * Two classes of information are deliberately not serialized, and are + * recomputed at conflict time instead: + * - Conflict *footprints* (which fields/fragments/keys an action touches). + * These are a pure function of each action, so storing them would only risk + * drift. + * - Large derivable row-level deltas, e.g. a deletion's affected rows or an + * update's matched row offsets. These can be recomputed by diffing against + * the read-version state, and serializing them would bloat transactions + * with potentially huge integer lists. + * + * ADDITIVE-SAFETY + * --------------- + * The schema is meant to evolve additively: message identities and the core + * mutation fields are pinned, and anything discovered during implementation + * should arrive as an *added* optional field rather than a reshaping of what is + * here. + */ + +/* + * A reference to a counter-allocated identifier (field id, fragment id, or base + * id) that may not be committed yet. + * + * `committed` is an already-assigned id. `local` is a placeholder token minted + * by an `Add*` action earlier in the same `UserOperation`; it resolves to a + * freshly-allocated committed id at apply, and re-resolves against the target's + * counters on merge/rebase. `local` tokens are scoped to a single + * `UserOperation` and must be distinct within it (validated by the writer once + * the write path exists). + */ +message Ref { + oneof kind { + uint64 committed = 1; + uint32 local = 2; + } +} + +/* + * A user-facing, composable transaction: an ordered list of user actions that + * commit atomically as a single manifest change. + */ +message UserOperation { + // Human-readable description, e.g. "INSERT INTO t VALUES (1)". + string description = 1; + // Unique identifier for this operation (matches Transaction.uuid semantics). + string uuid = 2; + // The dataset version this operation was planned against. + uint64 read_version = 3; + // The ordered list of user actions applied by this operation. + repeated UserAction actions = 4; +} + +/* + * A single user-recognizable step within a UserOperation (e.g. "append batch", + * "rebuild index"). + * + * The description keeps the transaction history human-readable. When a range of + * transactions is squashed, each original UserOperation collapses into one + * UserAction so the readable sequence survives; the action lists are flattened + * when applied to the manifest. + */ +message UserAction { + // Human-readable description of this step. + string description = 1; + // The granular manifest changes this step expands to. + repeated Action actions = 2; +} + +/* + * A single granular change (a delta) or assertion. Legacy operations decompose + * into an ordered list of these. + * + * TODO: the `data_change` markers on the actions below are drafted per-action + * for maximum flexibility; revisit whether per-action or per-UserAction + * granularity is the right unit before the format stabilizes. + */ +message Action { + oneof action { + // -- Minting actions (allocate a new counter-based id via a Local token) -- + AddFragment add_fragment = 1; + AddField add_field = 3; + AddBase add_base = 4; + // -- Fragment / data-file deltas -- + AddDataFile add_data_file = 2; + TombstoneFieldData tombstone_field_data = 5; + RemoveFragment remove_fragment = 6; + // -- Reference-stable changes (committed coordinates; post-image at rest) -- + SetDeletionFile set_deletion_file = 7; + ConfigUpdate config_update = 8; + AddOverlays add_overlays = 9; + RefreshRowVersionMetadata refresh_row_version_metadata = 10; + UpdateCompactedSsTables update_compacted_sstables = 11; + // -- Schema (field-level; no wholesale SetSchema) -- + DropField drop_field = 12; + AlterField alter_field = 13; + // -- Index segments -- + AddIndexSegment add_index_segment = 14; + RemoveIndexSegment remove_index_segment = 15; + AdjustIndexCoverage adjust_index_coverage = 16; + // -- Wholesale-within-operation -- + ResetTable reset_table = 17; + ReserveFragmentIds reserve_fragment_ids = 18; + // -- Assertions (preconditions, not deltas) -- + AssertUniqueKeys assert_unique_keys = 19; + } +} + +/* + * Mint a new, empty fragment. + * + * Its data files arrive via AddDataFile actions referencing this fragment's + * `local` token; its deletion vector (if any) via SetDeletionFile. + */ +message AddFragment { + // Placeholder token for the fragment id, resolved to a committed id at apply. + uint32 local = 1; + // Number of physical rows (including rows later tombstoned). + uint64 physical_rows = 2; + /* + * Stable-row-id and row-version sequences, carried exactly as on + * DataFragment. Absent on datasets without stable row ids. + */ + oneof row_id_sequence { + bytes inline_row_ids = 3; + ExternalFile external_row_ids = 4; + } + oneof last_updated_at_version_sequence { + bytes inline_last_updated_at_versions = 5; + ExternalFile external_last_updated_at_versions = 6; + } + oneof created_at_version_sequence { + bytes inline_created_at_versions = 7; + ExternalFile external_created_at_versions = 8; + } + /* + * false => rearrangement only (rows unchanged, e.g. compaction); CDC and + * streaming consumers may skip it. Absent is treated as true (real change). + */ + optional bool data_change = 9; +} + +/* + * Add a data file to a fragment. + */ +message AddDataFile { + // The fragment to add the file to (Committed, or a same-op Local fragment). + Ref fragment = 1; + /* + * The data file. Its `fields` are left unset (-1) and stamped in at apply + * once `field_ids` resolve; `field_ids` below is the authority for the + * column -> field mapping. + */ + DataFile file = 2; + /* + * One entry per column in `file`, in order. Committed for existing fields, + * Local for fields minted by an AddField in the same operation. + */ + repeated Ref field_ids = 3; + optional bool data_change = 4; +} + +/* + * Mint a new schema field. + * + * A struct/list column that introduces several fields is expressed as several + * ordered AddField actions (parent before children, children referencing the + * parent via a Local ref). + */ +message AddField { + // Placeholder token for the new field id, resolved at apply. + uint32 local = 1; + /* + * The parent field: Committed to reparent under an existing field, Local for + * a sibling minted in the same operation. Absent => top-level column. + */ + optional Ref parent = 2; + // Field definition; its `id` and `parent_id` are ignored (see `local`, `parent`). + lance.file.Field def = 3; +} + +/* + * Mint a new base path. + */ +message AddBase { + // Placeholder token for the base id, resolved at apply. + uint32 local = 1; + // Base path; its `id` is ignored and stamped in at apply. + BasePath base = 2; +} + +/* + * Tombstone the data-file binding of one or more committed fields. + * + * Each field's slot in whatever data file currently backs it is set to -2, and + * any file left with no live fields is pruned at apply. This is how a column's + * data is dropped or superseded (data files have no id of their own; a live + * field is backed by exactly one file). A column re-encode is + * TombstoneFieldData(X) followed by AddDataFile(new file, field X). + */ +message TombstoneFieldData { + Ref fragment = 1; + // Committed field ids whose current backing is tombstoned. + repeated uint64 field_ids = 2; + optional bool data_change = 3; +} + +/* + * Remove a fragment entirely (all rows deleted, or replaced by compaction). + */ +message RemoveFragment { + Ref fragment = 1; + optional bool data_change = 2; +} + +/* + * Set (replace) a fragment's deletion file. + * + * Reference-stable: the fragment id is committed and physical row offsets are + * stable, so the post-image deletion file is safe to store. The newly-deleted + * rows (the delta used for rebase and conflict) are derived by diffing against + * the read-version deletion file and are not serialized here. + */ +message SetDeletionFile { + uint64 fragment = 1; + DeletionFile deletion_file = 2; + optional bool data_change = 3; +} + +/* + * Apply config / metadata updates. + * + * Reference-stable key-merges over stable coordinates (config keys, field ids). + * Mirrors the new-style fields of the legacy UpdateConfig operation. + */ +message ConfigUpdate { + optional UpdateMap config = 1; + optional UpdateMap table_metadata = 2; + optional UpdateMap schema_metadata = 3; + /* + * Per-field metadata updates, keyed by Ref so a field minted in the same + * operation (Local) can receive metadata. + */ + repeated FieldMetadata field_metadata = 4; + + message FieldMetadata { + Ref field = 1; + UpdateMap updates = 2; + } +} + +/* + * Append overlay files to fragments (see DataOverlayFile in table.proto). + * + * Reference-stable: overlays target committed fragments/offsets/fields and are + * appended, never replacing existing overlays. + */ +message AddOverlays { + Ref fragment = 1; + repeated DataOverlayFile overlays = 2; + optional bool data_change = 3; +} + +/* + * Refresh row-version (created-at / last-updated-at) metadata for fragments + * whose columns were merged in place, mirroring the implicit refresh Merge + * performs on stable-row-id datasets. + */ +message RefreshRowVersionMetadata { + repeated uint64 fragment_ids = 1; +} + +/* + * Mark MemWAL SSTables as compacted into the base table. Covers + * UpdateMemWalState and the compaction bookkeeping of Update. + */ +message UpdateCompactedSsTables { + repeated CompactedSsTable compacted_sstables = 1; +} + +/* + * Remove a field from the schema (and, at apply, prune any data file left with + * no live fields). References an existing committed field id. + */ +message DropField { + uint64 field = 1; +} + +/* + * Alter facets of an existing field in place, preserving its identity (id). + * + * Each facet is optional: present means "change this facet", absent means + * "leave unchanged". Keying conflict on (field id, facet) lets independent + * facet changes to the same field commute (e.g. a cast concurrent with a + * nullability change). A cast additionally needs TombstoneFieldData plus a new + * AddDataFile to rewrite the data. New facets are added as optional fields. + */ +message AlterField { + uint64 field = 1; + optional string name = 2; + // New Arrow logical type (see Field.logical_type). The cast. + optional string logical_type = 3; + optional bool nullable = 4; +} + +/* + * Add an index segment. + * + * A brand-new index is expressed as its first segment; the set of segments + * sharing `name` constitutes one logical index. + */ +message AddIndexSegment { + UUID uuid = 1; + string name = 2; + // Indexed field ids (Committed, or Local for same-op minted fields). + repeated Ref fields = 3; + google.protobuf.Any index_details = 4; + optional int32 index_version = 5; + /* + * Fragments covered by this segment (resolved to a fragment_bitmap at apply, + * and remapped under fragment relocation). Committed or same-op Local. + */ + repeated Ref covered_fragments = 6; + /* + * Index files with sizes, when produced by the writer (e.g. a compaction that + * rewrites the segment). Empty when unavailable. + */ + repeated IndexFile files = 7; + optional bool data_change = 8; +} + +/* + * Remove an index segment by uuid. + */ +message RemoveIndexSegment { + UUID uuid = 1; + optional bool data_change = 2; +} + +/* + * Adjust the fragment coverage of an existing index segment without rewriting + * it. + * + * NOTE: coverage representation is an acknowledged open design area; this shape + * is provisional pending how much coverage change is derivable from fragment + * relocation versus stated explicitly here. + */ +message AdjustIndexCoverage { + UUID uuid = 1; + repeated Ref add = 2; + repeated uint64 remove = 3; +} + +/* + * Reset the table to an empty state, in preparation for a fresh schema and data + * written by later actions in the same operation (the decomposition of a full + * Overwrite / CREATE OR REPLACE). + * + * Drops the entire schema, schema metadata, all fragments, and all indices; + * preserves table config, table metadata, and base paths (change those via + * ConfigUpdate / AddBase in the same operation). Its write footprint spans the + * whole schema, all fragments, and all indices, so it conflicts with any + * concurrent change to them (DROP TABLE-style exclusivity). + */ +message ResetTable {} + +/* + * Reserve a contiguous range of fragment ids from the counter, for a later + * (possibly distributed) writer to populate. Fragments written against a + * reserved range reference those ids as Committed. + */ +message ReserveFragmentIds { + uint32 count = 1; +} + +/* + * An assertion (precondition), not a delta: the keys this operation inserts + * must not collide with keys inserted by a concurrent commit. + * + * Carries a bloom / exact-set filter of the inserted key hashes (not derivable + * from any post-image); conflict = the two filters intersect. Home for + * merge-insert's strict primary-key conflict detection. + */ +message AssertUniqueKeys { + /* + * Field ids of the key columns (unenforced primary key). Committed, or Local + * for same-op minted fields. + */ + repeated Ref key_fields = 1; + KeyExistenceFilter filter = 2; +} diff --git a/protos/transaction/common.proto b/protos/transaction/common.proto new file mode 100644 index 00000000000..ad290bb485a --- /dev/null +++ b/protos/transaction/common.proto @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +syntax = "proto3"; + +package lance.table; + +/* + * Shared transaction building blocks, referenced by both the legacy + * operations in transaction.proto and the action-based transactions in + * actions.proto. + * + * These types are transaction *machinery* (metadata merges, conflict-detection + * filters), not persisted table state, so they live here rather than in + * table.proto. Kept in a dedicated file so both transaction models can import + * them without a circular dependency (transaction.proto imports actions.proto, + * so actions.proto must not import transaction.proto). + */ + +/* + * An entry for a map update. + * + * If `value` is not set, the key is removed from the map. + */ +message UpdateMapEntry { + // The key of the map entry to update. + string key = 1; + // The value to set for the key. Absent removes the key. + optional string value = 2; +} + +/* + * A set of updates to apply to a string map (table config, table metadata, + * schema metadata, or per-field metadata). + */ +message UpdateMap { + repeated UpdateMapEntry update_entries = 1; + /* + * If true, the map is replaced entirely with these entries. If false, the + * entries are merged into the existing map. + */ + bool replace = 2; +} + +/* + * Exact set of key hashes for conflict detection. + * + * Used when the number of inserted rows is small. + */ +message ExactKeySetFilter { + // 64-bit hashes of the inserted row keys. + repeated uint64 key_hashes = 1; +} + +/* + * Bloom filter for key existence tests. + * + * Used when the number of inserted rows is large. + */ +message BloomFilter { + // Bitset backing the bloom filter (SBBF format). + bytes bitmap = 1; + // Number of bits in the bitmap. + uint32 num_bits = 2; + /* + * Number of items the filter was sized for. Used for intersection validation: + * filters with different sizes cannot be compared. Default: 8192. + */ + uint64 number_of_items = 3; + /* + * False positive probability the filter was sized for. Used for intersection + * validation: filters with different parameters cannot be compared. + * Default: 0.00057. + */ + double probability = 4; +} + +/* + * A filter for checking key existence in the set of rows inserted by a + * merge-insert operation. + * + * Only created when the merge-insert's ON columns match the schema's unenforced + * primary key; its presence indicates strict primary-key conflict detection. + * Conflict detection intersects two filters, so the underlying representation + * is either an exact set (small row counts) or a Bloom filter (large counts). + */ +message KeyExistenceFilter { + // Field ids of the columns participating in the key (the unenforced primary key). + repeated int32 field_ids = 1; + // The underlying data structure storing the key hashes. + oneof data { + // Exact set of key hashes (small number of rows). + ExactKeySetFilter exact = 2; + // Bloom filter (large number of rows). + BloomFilter bloom = 3; + } +} diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto new file mode 100644 index 00000000000..42981b5424a --- /dev/null +++ b/protos/transaction/transaction.proto @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +syntax = "proto3"; + +import "file.proto"; +import "table.proto"; +import "transaction/actions.proto"; +import "transaction/common.proto"; +import "google/protobuf/any.proto"; + +package lance.table; + +/* A transaction represents the changes to a dataset. + * + * This has two purposes: + * 1. When retrying a commit, the transaction can be used to re-build an updated + * manifest. + * 2. When there's a conflict, this can be used to determine whether the other + * transaction is compatible with this one. + */ +message Transaction { + /* The version of the dataset this transaction was built from. + * + * For example, for a delete transaction this means the version of the dataset + * that was read from while evaluating the deletion predicate. + */ + uint64 read_version = 1; + + // The UUID that unique identifies a transaction. + string uuid = 2; + + // Optional version tag. + string tag = 3; + + /* Optional properties for the transaction + * __lance_commit_message is a reserved key + */ + map transaction_properties = 4; + + // Add new rows to the dataset. + message Append { + /* The new fragments to append. + * + * Fragment IDs are not yet assigned. + */ + repeated DataFragment fragments = 1; + } + + // Mark rows as deleted. + message Delete { + /* The fragments to update + * + * The fragment IDs will match existing fragments in the dataset. + */ + repeated DataFragment updated_fragments = 1; + // The fragments to delete entirely. + repeated uint64 deleted_fragment_ids = 2; + /* The predicate that was evaluated + * + * This may be used to determine whether the delete would have affected + * files written by a concurrent transaction. + */ + string predicate = 3; + } + + // Create or overwrite the entire dataset. + message Overwrite { + /* The new fragments + * + * Fragment IDs are not yet assigned. + */ + repeated DataFragment fragments = 1; + // The new schema + repeated lance.file.Field schema = 2; + // Schema metadata. + map schema_metadata = 3; + // Key-value pairs to merge with existing config. + map config_upsert_values = 4; + // The base paths to be added for the initial dataset creation + repeated BasePath initial_bases = 5; + } + + /* Add or replace a new secondary index. + * + * This is also used to remove an index (we are replacing it with nothing) + * + * - new_indices: the modified indices, empty if dropping indices only + * - removed_indices: the indices that are being replaced + */ + message CreateIndex { + repeated IndexMetadata new_indices = 1; + repeated IndexMetadata removed_indices = 2; + } + + /* An operation that rewrites but does not change the data in the table. These + * kinds of operations just rearrange data. + */ + message Rewrite { + /* The old fragments that are being replaced + * + * DEPRECATED: use groups instead. + * + * These should all have existing fragment IDs. + */ + repeated DataFragment old_fragments = 1; + /* The new fragments + * + * DEPRECATED: use groups instead. + * + * These fragments IDs are not yet assigned. + */ + repeated DataFragment new_fragments = 2; + + /* During a rewrite an index may be rewritten. We only serialize the UUID + * since a rewrite should not change the other index parameters. + */ + message RewrittenIndex { + // The id of the index that will be replaced + UUID old_id = 1; + // the id of the new index + UUID new_id = 2; + // the new index details + google.protobuf.Any new_index_details = 3; + // the version of the new index + uint32 new_index_version = 4; + /* Files in the new index with their sizes. + * Empty if file sizes are not available (e.g. older writers). + */ + repeated IndexFile new_index_files = 5; + } + + // A group of rewrite files that are all part of the same rewrite. + message RewriteGroup { + /* The old fragment that is being replaced + * + * This should have an existing fragment ID. + */ + repeated DataFragment old_fragments = 1; + /* The new fragment + * + * The ID should have been reserved by an earlier + * reserve operation + */ + repeated DataFragment new_fragments = 2; + } + + // Groups of files that have been rewritten + repeated RewriteGroup groups = 3; + // Indices that have been rewritten + repeated RewrittenIndex rewritten_indices = 4; + } + + // An operation that merges in a new column, altering the schema. + message Merge { + /* The updated fragments + * + * These should all have existing fragment IDs. + */ + repeated DataFragment fragments = 1; + // The new schema + repeated lance.file.Field schema = 2; + // Schema metadata. + map schema_metadata = 3; + /* Set when this merge makes no nullability-affecting schema change: it + * introduces no field that data staged against an earlier schema could + * not safely omit. Without the assertion (including transactions written + * before this field existed) the merge conservatively conflicts with + * concurrent value-writes, which can only cause a retry. + */ + bool preserves_nullability = 4; + } + + // An operation that projects a subset of columns, altering the schema. + message Project { + // The new schema + repeated lance.file.Field schema = 1; + /* Set when this projection makes no nullability-affecting schema change, + * as a rename or a drop does not. Without the assertion (including + * transactions written before this field existed) the projection + * conservatively conflicts with concurrent value-writes, which can only + * cause a retry. A nullability tightening must not set this. + */ + bool preserves_nullability = 2; + } + + // An operation that restores a dataset to a previous version. + message Restore { + // The version to restore to + uint64 version = 1; + } + + /* An operation that reserves fragment ids for future use in + * a rewrite operation. + */ + message ReserveFragments { + uint32 num_fragments = 1; + } + + // An operation that clones a dataset. + message Clone { + /* - true: Performs a metadata-only clone (copies manifest without data files). + * The cloned dataset references original data through `base_paths`, + * suitable for experimental scenarios or rapid metadata migration. + * - false: Performs a full deep clone using the underlying object storage's native + * copy API (e.g., S3 CopyObject, GCS rewrite). This leverages server-side + * bulk copy operations to bypass download/upload bottlenecks, achieving + * near-linear speedup for large datasets (typically 3-10x faster than + * manual file transfers). The operation maintains atomicity and data + * integrity guarantees provided by the storage backend. + */ + bool is_shallow = 1; + /* the reference name in the source dataset + * in most cases it should be the branch or tag name in the source dataset + */ + optional string ref_name = 2; + // the version of the source dataset for cloning + uint64 ref_version = 3; + // the absolute base path of the source dataset for cloning + string ref_path = 4; + // if the target dataset is a branch, this is the branch name of the target dataset + optional string branch_name = 5; + } + + // Serialized as sorted distinct local physical row offsets within the fragment (0-based). + message UInt32List { + repeated uint32 values = 1; + } + + // An operation that updates rows but does not add or remove rows. + message Update { + /* The fragments that have been removed. These are fragments where all rows + * have been updated and moved to a new fragment. + */ + repeated uint64 removed_fragment_ids = 1; + // The fragments that have been updated. + repeated DataFragment updated_fragments = 2; + // The new fragments where updated rows have been moved to. + repeated DataFragment new_fragments = 3; + // The ids of the fields that have been modified. + repeated uint32 fields_modified = 4; + /// SSTables to mark as compacted after this transaction. + repeated CompactedSsTable compacted_sstables = 5; + /* The fields that used to judge whether to preserve the new frag's id into + * the frag bitmap of the specified indices. + */ + repeated uint32 fields_for_preserving_frag_bitmap = 6; + // The mode of update + UpdateMode update_mode = 7; + /* Filter for checking existence of keys in newly inserted rows, used for conflict detection. + * Only tracks keys from INSERT operations during merge insert, not updates. + */ + optional KeyExistenceFilter inserted_rows = 8; + /* Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns). + * Deprecated: use updated_fragment_offset_bitmaps (field 10) instead. + */ + map updated_fragment_offsets = 9; + /* Per-fragment matched offsets as portable RoaringBitmap bytes (replaces field 9). + * Writers emit field 10 only. Readers prefer field 10; fall back to field 9 for + * manifests written before this change. + */ + map updated_fragment_offset_bitmaps = 10; + } + + // The mode of update operation + enum UpdateMode { + + /* rows are deleted in current fragments and rewritten in new fragments. + * This is most optimal when the majority of columns are being rewritten + * or only a few rows are being updated. + */ + REWRITE_ROWS = 0; + + /* within each fragment, columns are fully rewritten and inserted as new data files. + * Old versions of columns are tombstoned. This is most optimal when most rows are affected + * but a small subset of columns are affected. + */ + REWRITE_COLUMNS = 1; + } + + /* An operation that updates the table config, table metadata, schema metadata, + * or field metadata. + */ + message UpdateConfig { + UpdateMap config_updates = 6; + UpdateMap table_metadata_updates = 7; + UpdateMap schema_metadata_updates = 8; + map field_metadata_updates = 9; + + // Deprecated ------------------------------- + map upsert_values = 1; + repeated string delete_keys = 2; + map schema_metadata = 3; + map field_metadata = 4; + + message FieldMetadataUpdate { + map metadata = 5; + } + } + + message DataReplacementGroup { + uint64 fragment_id = 1; + DataFile new_file = 2; + } + + // An operation that replaces the data in a region of the table with new data. + message DataReplacement { + repeated DataReplacementGroup replacements = 1; + } + + /* Overlay files to append to a single fragment, in order (the last entry is + * newest). The overlays are appended to the fragment's existing `overlays` + * list; they do not replace it, so overlays written by concurrent commits are + * preserved. + */ + message DataOverlayGroup { + uint64 fragment_id = 1; + /* Each DataOverlayFile.committed_version is left 0 by the writer and stamped + * to the new dataset version at commit time (re-stamped on retry), in the + * same way as the created-at / last-updated-at version sequences. The fields + * touched are read from each overlay's `data_file.fields`. + */ + repeated DataOverlayFile overlays = 2; + } + + /* Attach overlay files to fragments, supplying new values for a subset of + * (row offset, field) cells without rewriting the fragments' base data files. + * See the DataOverlayFile message in table.proto for resolution, coverage, and + * versioning rules, and the Data Overlay Files and Transactions specifications + * for the (intentionally permissive) conflict semantics. + */ + message DataOverlay { + repeated DataOverlayGroup groups = 1; + } + + /* Update SSTable compaction progress in the MemWAL index. + * This operation is used during merge-insert to atomically record which + * SSTables have been compacted into the base table. + */ + message UpdateMemWalState { + // SSTables being marked as compacted. + repeated CompactedSsTable compacted_sstables = 1; + } + + // An operation that updates base paths in the dataset. + message UpdateBases { + // The new base paths to add to the manifest. + repeated BasePath new_bases = 1; + } + + // The operation of this transaction. + oneof operation { + Append append = 100; + Delete delete = 101; + Overwrite overwrite = 102; + CreateIndex create_index = 103; + Rewrite rewrite = 104; + Merge merge = 105; + Restore restore = 106; + ReserveFragments reserve_fragments = 107; + Update update = 108; + Project project = 109; + UpdateConfig update_config = 110; + DataReplacement data_replacement = 111; + UpdateMemWalState update_mem_wal_state = 112; + Clone clone = 113; + UpdateBases update_bases = 114; + DataOverlay data_overlay = 115; + // Action-based transaction (Transaction V2). See actions.proto. + // DRAFT: currently rejected on load; no write path. + UserOperation user_operation = 116; + } + + // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. + reserved 200, 202; + reserved "blob_append", "blob_overwrite"; +} diff --git a/rust/lance-table/build.rs b/rust/lance-table/build.rs index 03216636b30..103109adfdb 100644 --- a/rust/lance-table/build.rs +++ b/rust/lance-table/build.rs @@ -19,7 +19,9 @@ fn main() -> Result<()> { prost_build.compile_protos( &[ "./protos/table.proto", - "./protos/transaction.proto", + "./protos/transaction/common.proto", + "./protos/transaction/actions.proto", + "./protos/transaction/transaction.proto", "./protos/rowids.proto", ], &["./protos"], diff --git a/rust/lance-table/src/format/key_existence.rs b/rust/lance-table/src/format/key_existence.rs index 210ef5f3836..da88deb62ed 100644 --- a/rust/lance-table/src/format/key_existence.rs +++ b/rust/lance-table/src/format/key_existence.rs @@ -131,18 +131,16 @@ impl KeyExistenceFilterBuilder { } } -impl From<&KeyExistenceFilterBuilder> for pb::transaction::KeyExistenceFilter { +impl From<&KeyExistenceFilterBuilder> for pb::KeyExistenceFilter { fn from(builder: &KeyExistenceFilterBuilder) -> Self { Self { field_ids: builder.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Bloom( - pb::transaction::BloomFilter { - bitmap: builder.sbbf.to_bytes(), - num_bits: (builder.sbbf.size_bytes() as u32) * 8, - number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, - probability: BLOOM_FILTER_DEFAULT_PROBABILITY, - }, - )), + data: Some(pb::key_existence_filter::Data::Bloom(pb::BloomFilter { + bitmap: builder.sbbf.to_bytes(), + num_bits: (builder.sbbf.size_bytes() as u32) * 8, + number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, + probability: BLOOM_FILTER_DEFAULT_PROBABILITY, + })), } } } @@ -212,13 +210,13 @@ impl KeyExistenceFilter { } } -impl From<&KeyExistenceFilter> for pb::transaction::KeyExistenceFilter { +impl From<&KeyExistenceFilter> for pb::KeyExistenceFilter { fn from(filter: &KeyExistenceFilter) -> Self { match &filter.filter { FilterType::ExactSet(hashes) => Self { field_ids: filter.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Exact( - pb::transaction::ExactKeySetFilter { + data: Some(pb::key_existence_filter::Data::Exact( + pb::ExactKeySetFilter { key_hashes: hashes.iter().copied().collect(), }, )), @@ -230,28 +228,26 @@ impl From<&KeyExistenceFilter> for pb::transaction::KeyExistenceFilter { probability, } => Self { field_ids: filter.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Bloom( - pb::transaction::BloomFilter { - bitmap: bitmap.clone(), - num_bits: *num_bits, - number_of_items: *number_of_items, - probability: *probability, - }, - )), + data: Some(pb::key_existence_filter::Data::Bloom(pb::BloomFilter { + bitmap: bitmap.clone(), + num_bits: *num_bits, + number_of_items: *number_of_items, + probability: *probability, + })), }, } } } -impl TryFrom<&pb::transaction::KeyExistenceFilter> for KeyExistenceFilter { +impl TryFrom<&pb::KeyExistenceFilter> for KeyExistenceFilter { type Error = lance_core::Error; - fn try_from(message: &pb::transaction::KeyExistenceFilter) -> Result { + fn try_from(message: &pb::KeyExistenceFilter) -> Result { let filter = match message.data.as_ref() { - Some(pb::transaction::key_existence_filter::Data::Exact(exact)) => { + Some(pb::key_existence_filter::Data::Exact(exact)) => { FilterType::ExactSet(exact.key_hashes.iter().copied().collect()) } - Some(pb::transaction::key_existence_filter::Data::Bloom(b)) => { + Some(pb::key_existence_filter::Data::Bloom(b)) => { // Use defaults for backwards compatibility let number_of_items = if b.number_of_items == 0 { BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index baa43aeabc8..6aee192f00a 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -665,20 +665,12 @@ impl From<&Transaction> for pb::Transaction { schema_metadata_updates, field_metadata_updates, } => pb::transaction::Operation::UpdateConfig(pb::transaction::UpdateConfig { - config_updates: config_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), - table_metadata_updates: table_metadata_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), - schema_metadata_updates: schema_metadata_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), + config_updates: config_updates.as_ref().map(pb::UpdateMap::from), + table_metadata_updates: table_metadata_updates.as_ref().map(pb::UpdateMap::from), + schema_metadata_updates: schema_metadata_updates.as_ref().map(pb::UpdateMap::from), field_metadata_updates: field_metadata_updates .iter() - .map(|(field_id, update_map)| { - (*field_id, pb::transaction::UpdateMap::from(update_map)) - }) + .map(|(field_id, update_map)| (*field_id, pb::UpdateMap::from(update_map))) .collect(), // Leave old fields empty - we only write new-style fields upsert_values: Default::default(), @@ -777,13 +769,13 @@ impl From<&RewriteGroup> for pb::transaction::rewrite::RewriteGroup { } } -impl From<&UpdateMap> for pb::transaction::UpdateMap { +impl From<&UpdateMap> for pb::UpdateMap { fn from(update_map: &UpdateMap) -> Self { Self { update_entries: update_map .update_entries .iter() - .map(|entry| pb::transaction::UpdateMapEntry { + .map(|entry| pb::UpdateMapEntry { key: entry.key.clone(), value: entry.value.clone(), }) @@ -793,8 +785,8 @@ impl From<&UpdateMap> for pb::transaction::UpdateMap { } } -impl From<&pb::transaction::UpdateMap> for UpdateMap { - fn from(pb_update_map: &pb::transaction::UpdateMap) -> Self { +impl From<&pb::UpdateMap> for UpdateMap { + fn from(pb_update_map: &pb::UpdateMap) -> Self { Self { update_entries: pb_update_map .update_entries @@ -878,21 +870,19 @@ mod tests { read_version: 1, uuid: Uuid::new_v4().to_string(), operation: Some(pb::transaction::Operation::UserOperation( - pb::transaction::UserOperation { + pb::UserOperation { description: "INSERT INTO t VALUES (1)".to_string(), uuid: Uuid::new_v4().to_string(), read_version: 1, - actions: vec![pb::transaction::UserAction { + actions: vec![pb::UserAction { description: "append batch".to_string(), - actions: vec![pb::transaction::Action { - action: Some(pb::transaction::action::Action::AddFragment( - pb::transaction::AddFragment { - local: 0, - physical_rows: 1, - data_change: Some(true), - ..Default::default() - }, - )), + actions: vec![pb::Action { + action: Some(pb::action::Action::AddFragment(pb::AddFragment { + local: 0, + physical_rows: 1, + data_change: Some(true), + ..Default::default() + })), }], }], }, From 5c7052b8a13e9ef9a2d52d92ec1e8285b8637b99 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 23 Jul 2026 14:50:54 -0700 Subject: [PATCH 03/15] docs(transaction): clarify draft action semantics from self-review Refinements to the Transaction V2 wire draft (protos/transaction/actions.proto), no behavior change: - AddFragment / SetDeletionFile: resolve a contradiction. AddFragment's doc implied a freshly-minted (Local) fragment could take a deletion file via SetDeletionFile, but SetDeletionFile.fragment is a committed-only uint64. Clarify that a new fragment has no deletion vector and deletions arrive in a later operation once the id is committed, and document why SetDeletionFile takes no Ref. - data_change: document the marker on every carrier (previously only on AddFragment), cross-referencing the canonical definition. Spell out its non-obvious meaning on AddIndexSegment / RemoveIndexSegment, where it refers to the indexed data rather than table rows. - AssertUniqueKeys: note that key_fields is authoritative and the embedded filter.field_ids (an artifact of the shared KeyExistenceFilter type) is ignored. - AdjustIndexCoverage: rename bare add / remove fields to add_fragments / remove_fragments for self-documentation and consistency with AddIndexSegment.covered_fragments. Co-Authored-By: Claude Opus 4.8 (1M context) --- protos/transaction/actions.proto | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 1c6cda7c754..949d8c95ef1 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -193,7 +193,9 @@ message Action { * Mint a new, empty fragment. * * Its data files arrive via AddDataFile actions referencing this fragment's - * `local` token; its deletion vector (if any) via SetDeletionFile. + * `local` token. A freshly-minted fragment has no deletion vector; deletions + * are applied by a later operation via SetDeletionFile, once the fragment has a + * committed id. */ message AddFragment { // Placeholder token for the fragment id, resolved to a committed id at apply. @@ -240,6 +242,7 @@ message AddDataFile { * Local for fields minted by an AddField in the same operation. */ repeated Ref field_ids = 3; + // Data-change marker; see AddFragment.data_change. optional bool data_change = 4; } @@ -285,6 +288,7 @@ message TombstoneFieldData { Ref fragment = 1; // Committed field ids whose current backing is tombstoned. repeated uint64 field_ids = 2; + // Data-change marker; see AddFragment.data_change. optional bool data_change = 3; } @@ -293,6 +297,7 @@ message TombstoneFieldData { */ message RemoveFragment { Ref fragment = 1; + // Data-change marker; see AddFragment.data_change. optional bool data_change = 2; } @@ -305,8 +310,12 @@ message RemoveFragment { * the read-version deletion file and are not serialized here. */ message SetDeletionFile { + // The fragment, by committed id: unlike the sibling fragment actions this + // takes no Ref, because a fragment minted in the same operation has no rows + // to delete yet (see AddFragment). uint64 fragment = 1; DeletionFile deletion_file = 2; + // Data-change marker; see AddFragment.data_change. optional bool data_change = 3; } @@ -341,6 +350,7 @@ message ConfigUpdate { message AddOverlays { Ref fragment = 1; repeated DataOverlayFile overlays = 2; + // Data-change marker; see AddFragment.data_change. optional bool data_change = 3; } @@ -409,6 +419,8 @@ message AddIndexSegment { * rewrites the segment). Empty when unavailable. */ repeated IndexFile files = 7; + // Data-change marker; see AddFragment.data_change. false marks a segment + // rebuild/compaction that does not reflect any change to the indexed data. optional bool data_change = 8; } @@ -417,6 +429,8 @@ message AddIndexSegment { */ message RemoveIndexSegment { UUID uuid = 1; + // Data-change marker; see AddFragment.data_change. false marks removal of a + // segment superseded by compaction, not a change to the indexed data. optional bool data_change = 2; } @@ -430,8 +444,10 @@ message RemoveIndexSegment { */ message AdjustIndexCoverage { UUID uuid = 1; - repeated Ref add = 2; - repeated uint64 remove = 3; + // Fragments to add to the segment's coverage (Committed or same-op Local). + repeated Ref add_fragments = 2; + // Committed fragment ids to drop from the segment's coverage. + repeated uint64 remove_fragments = 3; } /* @@ -467,7 +483,9 @@ message ReserveFragmentIds { message AssertUniqueKeys { /* * Field ids of the key columns (unenforced primary key). Committed, or Local - * for same-op minted fields. + * for same-op minted fields. This is the authoritative key-column list; + * `filter.field_ids` (an artifact of the shared KeyExistenceFilter type) is + * ignored here and left empty. */ repeated Ref key_fields = 1; KeyExistenceFilter filter = 2; From dcb938a2575cd921d3ef1623ade60978722ae072 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 19 Aug 2026 15:09:23 -0700 Subject: [PATCH 04/15] feat(transaction): complete the action wire format Brings the drafted protos to the shape the implementation settled on, so the whole format lands in one PR and one vote: - `UserOperation` becomes `CompositeOperation`, which names what distinguishes it from the other operations -- a composite of granular actions committed atomically -- and loses its `description`, since the per-step `UserAction.description` is what keeps history readable. `UserAction` keeps its name: it is the grouping a user recognizes. - The field actions reference fields by `Ref`, so an action can name a field the same operation is minting. - `AddIndexSegment`, `RemoveIndexSegment`, `AdjustIndexCoverage` and `UpdateCompactedSsTables` join the action set. - The index actions carry the name of the logical index the segment belongs to, which is what conflict detection compares. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 88 ++++++++++++++++++----- protos/transaction/transaction.proto | 2 +- rust/lance-table/src/transaction/proto.rs | 9 ++- 3 files changed, 74 insertions(+), 25 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 949d8c95ef1..2d644e52a80 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -13,7 +13,7 @@ package lance.table; /* * Action-based transactions (Transaction V2) — DRAFT. * - * A `UserOperation` replaces the single legacy `Operation` (see + * A `CompositeOperation` replaces the single legacy `Operation` (see * transaction.proto) with an ordered list of granular `Action`s that commit * atomically as one manifest change. This file is the wire draft only. * @@ -21,7 +21,7 @@ package lance.table; * ------ * The messages and field numbers here are NOT yet a stable contract. Library * support is read-side fail-closed only: a transaction carrying a - * `UserOperation` is rejected on load, and there is no write path. Apply, + * `CompositeOperation` is rejected on load, and there is no write path. Apply, * id translation, and conflict resolution are intentionally absent. * * DESIGN RATIONALE @@ -106,10 +106,10 @@ package lance.table; * id) that may not be committed yet. * * `committed` is an already-assigned id. `local` is a placeholder token minted - * by an `Add*` action earlier in the same `UserOperation`; it resolves to a + * by an `Add*` action earlier in the same `CompositeOperation`; it resolves to a * freshly-allocated committed id at apply, and re-resolves against the target's * counters on merge/rebase. `local` tokens are scoped to a single - * `UserOperation` and must be distinct within it (validated by the writer once + * `CompositeOperation` and must be distinct within it (validated by the writer once * the write path exists). */ message Ref { @@ -123,23 +123,21 @@ message Ref { * A user-facing, composable transaction: an ordered list of user actions that * commit atomically as a single manifest change. */ -message UserOperation { - // Human-readable description, e.g. "INSERT INTO t VALUES (1)". - string description = 1; +message CompositeOperation { // Unique identifier for this operation (matches Transaction.uuid semantics). - string uuid = 2; + string uuid = 1; // The dataset version this operation was planned against. - uint64 read_version = 3; + uint64 read_version = 2; // The ordered list of user actions applied by this operation. - repeated UserAction actions = 4; + repeated UserAction actions = 3; } /* - * A single user-recognizable step within a UserOperation (e.g. "append batch", + * A single user-recognizable step within a CompositeOperation (e.g. "append batch", * "rebuild index"). * * The description keeps the transaction history human-readable. When a range of - * transactions is squashed, each original UserOperation collapses into one + * transactions is squashed, each original CompositeOperation collapses into one * UserAction so the readable sequence survives; the action lists are flattened * when applied to the manifest. */ @@ -173,7 +171,9 @@ message Action { ConfigUpdate config_update = 8; AddOverlays add_overlays = 9; RefreshRowVersionMetadata refresh_row_version_metadata = 10; - UpdateCompactedSsTables update_compacted_sstables = 11; + // Named `update_compacted_ss_tables` rather than `..._sstables` so the + // generated oneof variant matches the message name. + UpdateCompactedSsTables update_compacted_ss_tables = 11; // -- Schema (field-level; no wholesale SetSchema) -- DropField drop_field = 12; AlterField alter_field = 13; @@ -286,8 +286,8 @@ message AddBase { */ message TombstoneFieldData { Ref fragment = 1; - // Committed field ids whose current backing is tombstoned. - repeated uint64 field_ids = 2; + // The fields whose current backing is tombstoned. + repeated Ref field_ids = 2; // Data-change marker; see AddFragment.data_change. optional bool data_change = 3; } @@ -376,7 +376,7 @@ message UpdateCompactedSsTables { * no live fields). References an existing committed field id. */ message DropField { - uint64 field = 1; + Ref field = 1; } /* @@ -389,7 +389,7 @@ message DropField { * AddDataFile to rewrite the data. New facets are added as optional fields. */ message AlterField { - uint64 field = 1; + Ref field = 1; optional string name = 2; // New Arrow logical type (see Field.logical_type). The cast. optional string logical_type = 3; @@ -411,9 +411,15 @@ message AddIndexSegment { optional int32 index_version = 5; /* * Fragments covered by this segment (resolved to a fragment_bitmap at apply, - * and remapped under fragment relocation). Committed or same-op Local. + * and remapped under fragment relocation). + * + * Absent means the coverage is unknown, which is what the system indices + * (MemWAL, fragment reuse) carry and what pre-bitmap segments read back as. + * That is a different statement from a present-but-empty list, which says the + * segment covers no fragment at all -- the query path serves a segment of + * unknown coverage and skips one that covers nothing. */ - repeated Ref covered_fragments = 6; + optional FragmentCoverage covered_fragments = 6; /* * Index files with sizes, when produced by the writer (e.g. a compaction that * rewrites the segment). Empty when unavailable. @@ -422,6 +428,37 @@ message AddIndexSegment { // Data-change marker; see AddFragment.data_change. false marks a segment // rebuild/compaction that does not reflect any change to the indexed data. optional bool data_change = 8; + /* + * The base path this segment's files live under, for a segment imported from + * another dataset. Committed, or Local for a base minted by an AddBase in the + * same operation. Absent => the dataset's own index directory. + */ + optional Ref base = 9; + /* + * When this segment was built, in milliseconds since the Unix epoch. Absent + * for a segment whose build time was not recorded. + */ + optional uint64 created_at = 10; + /* + * The dataset version whose data this segment reflects. + * + * This is a correctness gate, not provenance: an overlay committed at or + * before it is treated as already folded into the index. A segment merged + * from several older ones reflects only as much as its oldest input, so this + * is genuinely below the version the operation reads and cannot be derived + * from it. Absent means the version the operation reads, which is what a + * freshly built segment reflects. It may never exceed that version. + */ + optional uint64 dataset_version = 11; +} + +/* + * A list of fragments, wrapped so that "no coverage recorded" is distinguishable + * from "covers nothing" (a bare `repeated` field cannot tell them apart). + */ +message FragmentCoverage { + // Committed, or Local for a fragment minted in the same operation. + repeated Ref fragments = 1; } /* @@ -432,6 +469,12 @@ message RemoveIndexSegment { // Data-change marker; see AddFragment.data_change. false marks removal of a // segment superseded by compaction, not a change to the indexed data. optional bool data_change = 2; + /* + * The logical index the segment belongs to. Carried so apply can check the + * removal was planned against the segment it actually names; a mismatch means + * the operation was built against a different set of segments. + */ + string name = 3; } /* @@ -448,6 +491,13 @@ message AdjustIndexCoverage { repeated Ref add_fragments = 2; // Committed fragment ids to drop from the segment's coverage. repeated uint64 remove_fragments = 3; + /* + * The logical index the segment belongs to. Conflict detection compares + * coverage across the segments of one index, so it needs the index this + * adjustment widens, not just the segment. Apply also checks the named + * segment really carries this name. + */ + string name = 4; } /* diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto index 42981b5424a..4078027f464 100644 --- a/protos/transaction/transaction.proto +++ b/protos/transaction/transaction.proto @@ -368,7 +368,7 @@ message Transaction { DataOverlay data_overlay = 115; // Action-based transaction (Transaction V2). See actions.proto. // DRAFT: currently rejected on load; no write path. - UserOperation user_operation = 116; + CompositeOperation composite_operation = 116; } // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 6aee192f00a..c7f9e148a23 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -401,7 +401,7 @@ impl TryFrom for Transaction { .map(DataOverlayGroup::try_from) .collect::>>()?, }, - Some(pb::transaction::Operation::UserOperation(_)) => { + Some(pb::transaction::Operation::CompositeOperation(_)) => { // Action-based transactions (Transaction V2) are a draft wire // format (OSS-1530). This version of Lance recognizes the message // but has no support for it: reject on load, fail-closed. Because @@ -861,7 +861,7 @@ mod tests { } #[test] - fn test_user_operation_rejected_on_load() { + fn test_composite_operation_rejected_on_load() { // Action-based transactions (Transaction V2) are a draft wire format that // this version of Lance does not support. Loading one must fail closed // (never be silently skipped or leniently parsed), so that a concurrent @@ -869,9 +869,8 @@ mod tests { let message = pb::Transaction { read_version: 1, uuid: Uuid::new_v4().to_string(), - operation: Some(pb::transaction::Operation::UserOperation( - pb::UserOperation { - description: "INSERT INTO t VALUES (1)".to_string(), + operation: Some(pb::transaction::Operation::CompositeOperation( + pb::CompositeOperation { uuid: Uuid::new_v4().to_string(), read_version: 1, actions: vec![pb::UserAction { From 100eaaae3a2ddeb3c36f17381b835146bf8d2018 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 2 Sep 2026 16:34:17 -0700 Subject: [PATCH 05/15] feat(transaction): let a fragment be added at a reserved id `ReserveFragmentIds` documents that "fragments written against a reserved range reference those ids as Committed", but `AddFragment` carried only a `local` token, so nothing could name a reserved id and the reservation was unreachable from the action vocabulary. Widen `AddFragment.local` into `Ref id`, which spans both forms. A writer that has to know a fragment's id before it commits -- because it bakes row addresses into an index it writes in the same commit -- takes the id from a reservation and names it as `Committed`. Field 1 changes type rather than being deprecated in place. The actions schema is used exclusively by the draft Transaction V2 format, which is read-side fail-closed with no write path, so there are no encoded messages to stay compatible with. --- protos/transaction/actions.proto | 34 ++++++++++++++++++----- rust/lance-table/src/transaction/proto.rs | 4 ++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 2d644e52a80..283e7c82f6a 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -45,6 +45,16 @@ package lance.table; * coordinates (a fragment's deletion vector, a config key); their target is * unambiguous, so they may safely carry a post-image at rest. * + * The exception is a writer that has to know an id *before* it commits, + * because it bakes that id into a file it writes in the same commit — an + * index whose payload holds row addresses or row ids cannot be written + * against a `Local` token. Such a writer takes ids from a range an earlier + * commit reserved (`ReserveFragmentIds`) and names them as + * `Committed`. The reservation is what makes that safe: the ids are the + * writer's alone, so they stay valid at every later version and nothing has + * to be re-resolved when the operation is replayed. It costs an extra + * commit, which is why it is the exception and not the rule. + * * 3. One reference type, `Ref = Committed | Local`. `Committed` is a concrete, * already-assigned id. `Local` is a placeholder minted by an earlier `Add*` * action in the same operation. `Local` resolves to a freshly-allocated @@ -190,16 +200,26 @@ message Action { } /* - * Mint a new, empty fragment. + * Add a new, empty fragment. * - * Its data files arrive via AddDataFile actions referencing this fragment's - * `local` token. A freshly-minted fragment has no deletion vector; deletions - * are applied by a later operation via SetDeletionFile, once the fragment has a - * committed id. + * Its data files arrive via AddDataFile actions referencing this fragment by + * the same `id`. A newly added fragment has no deletion vector; deletions are + * applied by a later operation via SetDeletionFile. */ message AddFragment { - // Placeholder token for the fragment id, resolved to a committed id at apply. - uint32 local = 1; + /* + * The id to add the fragment at. + * + * Local is the usual form: a placeholder token resolved to a freshly + * allocated id at apply. Committed names an id out of a range an earlier + * commit reserved with ReserveFragmentIds, for a writer that must know the + * fragment's id before committing (see rationale 2); apply rejects an id + * that is not reserved or is already in use. + * + * AddField and AddBase have no Committed form, because no reservation + * mechanism exists for field or base ids. + */ + Ref id = 1; // Number of physical rows (including rows later tombstoned). uint64 physical_rows = 2; /* diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index c7f9e148a23..7d42e26eda4 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -877,7 +877,9 @@ mod tests { description: "append batch".to_string(), actions: vec![pb::Action { action: Some(pb::action::Action::AddFragment(pb::AddFragment { - local: 0, + id: Some(pb::Ref { + kind: Some(pb::r#ref::Kind::Local(0)), + }), physical_rows: 1, data_change: Some(true), ..Default::default() From 0f53512727ea74bb7680cac028fbb69143fed78b Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 2 Sep 2026 16:34:41 -0700 Subject: [PATCH 06/15] feat(transaction): add the ReserveRowIds wire message The counterpart of `ReserveFragmentIds` for the row id space. Reserving a fragment id fixes the high half of a row address, which is enough for a dataset without stable row ids; with them, an index records row ids instead, and those come off their own counter with no way to reserve from it. Drafted only here -- the apply side lands with the action implementation. --- protos/transaction/actions.proto | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 283e7c82f6a..c1faf245997 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -49,7 +49,7 @@ package lance.table; * because it bakes that id into a file it writes in the same commit — an * index whose payload holds row addresses or row ids cannot be written * against a `Local` token. Such a writer takes ids from a range an earlier - * commit reserved (`ReserveFragmentIds`) and names them as + * commit reserved (`ReserveFragmentIds`, `ReserveRowIds`) and names them as * `Committed`. The reservation is what makes that safe: the ids are the * writer's alone, so they stay valid at every later version and nothing has * to be re-resolved when the operation is replayed. It costs an extra @@ -194,6 +194,7 @@ message Action { // -- Wholesale-within-operation -- ResetTable reset_table = 17; ReserveFragmentIds reserve_fragment_ids = 18; + ReserveRowIds reserve_row_ids = 20; // -- Assertions (preconditions, not deltas) -- AssertUniqueKeys assert_unique_keys = 19; } @@ -542,6 +543,27 @@ message ReserveFragmentIds { uint32 count = 1; } +/* + * Reserve a contiguous range of stable row ids from the counter, for a later + * writer to populate. + * + * The counterpart of ReserveFragmentIds for the row id space, and needed + * alongside it: reserving a fragment id fixes the high half of a row address, + * but on a dataset with stable row ids an index records row ids instead, and + * those come off their own counter. + * + * The reserved range is the `count` ids ending at the committed manifest's + * next_row_id, so the reserving writer learns which ids it got by reading that + * back. A fragment written against the range carries those ids in its row id + * sequence, which apply then leaves as it finds them. + * + * An error on a dataset that does not use stable row ids, which has no row id + * counter to reserve from. + */ +message ReserveRowIds { + uint64 count = 1; +} + /* * An assertion (precondition), not a delta: the keys this operation inserts * must not collide with keys inserted by a concurrent commit. From e7836c977b9084f36d19b9d029ce3efec080c6d9 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 8 Sep 2026 11:41:58 -0700 Subject: [PATCH 07/15] style(protos): block comments in the action protos The multi-line comment style landed with its own pre-commit hook (#8991) while this draft was out for review, and these files predate it. No wording changes; the hook did the rewrite. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 22 +++++++++++++--------- protos/transaction/transaction.proto | 5 +++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index c1faf245997..fd87fdcb021 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -181,8 +181,9 @@ message Action { ConfigUpdate config_update = 8; AddOverlays add_overlays = 9; RefreshRowVersionMetadata refresh_row_version_metadata = 10; - // Named `update_compacted_ss_tables` rather than `..._sstables` so the - // generated oneof variant matches the message name. + /* Named `update_compacted_ss_tables` rather than `..._sstables` so the + * generated oneof variant matches the message name. + */ UpdateCompactedSsTables update_compacted_ss_tables = 11; // -- Schema (field-level; no wholesale SetSchema) -- DropField drop_field = 12; @@ -331,9 +332,10 @@ message RemoveFragment { * the read-version deletion file and are not serialized here. */ message SetDeletionFile { - // The fragment, by committed id: unlike the sibling fragment actions this - // takes no Ref, because a fragment minted in the same operation has no rows - // to delete yet (see AddFragment). + /* The fragment, by committed id: unlike the sibling fragment actions this + * takes no Ref, because a fragment minted in the same operation has no rows + * to delete yet (see AddFragment). + */ uint64 fragment = 1; DeletionFile deletion_file = 2; // Data-change marker; see AddFragment.data_change. @@ -446,8 +448,9 @@ message AddIndexSegment { * rewrites the segment). Empty when unavailable. */ repeated IndexFile files = 7; - // Data-change marker; see AddFragment.data_change. false marks a segment - // rebuild/compaction that does not reflect any change to the indexed data. + /* Data-change marker; see AddFragment.data_change. false marks a segment + * rebuild/compaction that does not reflect any change to the indexed data. + */ optional bool data_change = 8; /* * The base path this segment's files live under, for a segment imported from @@ -487,8 +490,9 @@ message FragmentCoverage { */ message RemoveIndexSegment { UUID uuid = 1; - // Data-change marker; see AddFragment.data_change. false marks removal of a - // segment superseded by compaction, not a change to the indexed data. + /* Data-change marker; see AddFragment.data_change. false marks removal of a + * segment superseded by compaction, not a change to the indexed data. + */ optional bool data_change = 2; /* * The logical index the segment belongs to. Carried so apply can check the diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto index 4078027f464..6aeefcaf242 100644 --- a/protos/transaction/transaction.proto +++ b/protos/transaction/transaction.proto @@ -366,8 +366,9 @@ message Transaction { Clone clone = 113; UpdateBases update_bases = 114; DataOverlay data_overlay = 115; - // Action-based transaction (Transaction V2). See actions.proto. - // DRAFT: currently rejected on load; no write path. + /* Action-based transaction (Transaction V2). See actions.proto. + * DRAFT: currently rejected on load; no write path. + */ CompositeOperation composite_operation = 116; } From ddafb459647d7760d7020384098e8b8f6ae3d400 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 8 Sep 2026 12:17:13 -0700 Subject: [PATCH 08/15] ci: point the format gate at the moved transaction protos `protos/transaction.proto` became `protos/transaction/`, and two globs in the area labeler still name the old path. One of them drives the format-spec vote gate, so a glob that matches nothing would let a change to these protos land without the PMC vote the gate exists to require. Both now match the directory, which covers the split files and any later sibling. The spec doc's link to the same path is repointed at the directory for the same reason: it names three files now. Co-Authored-By: Claude Opus 5 (1M context) --- .github/labeler-area.yml | 4 ++-- docs/src/format/table/transaction.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/labeler-area.yml b/.github/labeler-area.yml index 998e752fba6..2045aab45c5 100644 --- a/.github/labeler-area.yml +++ b/.github/labeler-area.yml @@ -44,7 +44,7 @@ A-format: - "protos/index_old.proto" - "protos/rowids.proto" - "protos/table.proto" - - "protos/transaction.proto" + - "protos/transaction/*.proto" - "docs/src/format/**" # Drives the format-spec vote gate (.github/workflows/format-vote-gate.yml): @@ -62,7 +62,7 @@ format-change: - "protos/index_old.proto" - "protos/rowids.proto" - "protos/table.proto" - - "protos/transaction.proto" + - "protos/transaction/*.proto" - "docs/src/format/**" # Lockfiles are intentionally not excluded: a pure dependency bump gets both diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index c88c3170988..8e3de3e6a7e 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -47,7 +47,7 @@ For detailed conflict detection and resolution mechanisms, see the [Conflict Res ## Transaction Types -The authoritative specification for transaction types is defined in [`protos/transaction.proto`](https://github.com/lancedb/lance/blob/main/protos/transaction.proto). +The authoritative specification for transaction types is defined in [`protos/transaction/`](https://github.com/lancedb/lance/tree/main/protos/transaction). Each transaction contains a `read_version` field indicating the table version from which the transaction was built, a `uuid` field uniquely identifying the transaction, and an `operation` field specifying one of the following transaction types: From bf918b592c6e5f329ab042a58c97076f6b2caaf2 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sun, 20 Sep 2026 10:25:51 -0700 Subject: [PATCH 09/15] ci: compare format-gate globs against the files they match `test_only_persisted_protos_are_format_changes` compared the raw config entries against a flat `protos/*.proto` glob, so moving the transaction protos into `protos/transaction/` broke it two ways at once: the gated entry became a glob rather than a path, and the files it covers stopped being visible to the non-recursive listing. Resolve both sides to concrete files before comparing. The test now states what it always meant -- every persisted proto is gated, and nothing else is -- and keeps holding as protos move between directories. Co-Authored-By: Claude Opus 5 (1M context) --- ci/test_labeler_area.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/test_labeler_area.py b/ci/test_labeler_area.py index ae4b74bd806..48b5aef1664 100644 --- a/ci/test_labeler_area.py +++ b/ci/test_labeler_area.py @@ -24,11 +24,19 @@ def test_format_labels_use_the_same_paths(): def test_only_persisted_protos_are_format_changes(): + """Every persisted proto is gated, and nothing else is. + + The config entries are globs and protos live in subdirectories, so both + sides are resolved to concrete files before being compared. + """ detected_proto_paths = { - path for path in paths_for("format-change") if path.startswith("protos/") + matched.relative_to(ROOT).as_posix() + for glob in paths_for("format-change") + if glob.startswith("protos/") + for matched in ROOT.glob(glob) } all_proto_paths = { - path.relative_to(ROOT).as_posix() for path in (ROOT / "protos").glob("*.proto") + path.relative_to(ROOT).as_posix() for path in (ROOT / "protos").rglob("*.proto") } assert detected_proto_paths == all_proto_paths - EXECUTION_PROTO_PATHS From bb419896d00a15e660d034812a7a065ad0e00193 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sun, 20 Sep 2026 10:25:57 -0700 Subject: [PATCH 10/15] feat(transaction): give CompositeOperation a typed uuid The operation's uuid was a string while every other uuid in the action protos -- and in `table.proto` generally -- is the `UUID` message. A string leaves the encoding unstated and costs 36 bytes where 16 will do. Addresses review feedback on the draft wire format. Transaction V2 is a pre-vote draft with no compatibility contract, so the field changes type in place rather than being deprecated alongside a replacement. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 2 +- rust/lance-table/src/transaction/proto.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index fd87fdcb021..241525962d6 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -135,7 +135,7 @@ message Ref { */ message CompositeOperation { // Unique identifier for this operation (matches Transaction.uuid semantics). - string uuid = 1; + UUID uuid = 1; // The dataset version this operation was planned against. uint64 read_version = 2; // The ordered list of user actions applied by this operation. diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 7d42e26eda4..49a364dbf14 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -871,7 +871,7 @@ mod tests { uuid: Uuid::new_v4().to_string(), operation: Some(pb::transaction::Operation::CompositeOperation( pb::CompositeOperation { - uuid: Uuid::new_v4().to_string(), + uuid: Some(pb::Uuid::from(&Uuid::new_v4())), read_version: 1, actions: vec![pb::UserAction { description: "append batch".to_string(), From c201a6774ba30ec921ed6b77f7c2f48951265826 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 21 Sep 2026 10:11:33 -0700 Subject: [PATCH 11/15] docs(protos): condense the actions.proto header The file-level comment ran to 810 words, which is more than a reader needs before the first message. Cut to under 300 by dropping the section headings, the points that only restated the message comments below them (relocation by watermark, index-segment duplication), and the STATUS/ADDITIVE-SAFETY prose, keeping the five reasons the wire shape is not obvious from the messages themselves. Also drops the TODO over `message Action` asking whether `data_change` belongs per-action or per-UserAction: per-action is the answer. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 122 ++++++++----------------------- 1 file changed, 29 insertions(+), 93 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 241525962d6..01c79b67562 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -13,102 +13,42 @@ package lance.table; /* * Action-based transactions (Transaction V2) — DRAFT. * - * A `CompositeOperation` replaces the single legacy `Operation` (see - * transaction.proto) with an ordered list of granular `Action`s that commit - * atomically as one manifest change. This file is the wire draft only. + * A `CompositeOperation` replaces the legacy `Operation` (see + * transaction.proto) with an ordered list of `Action`s committing + * atomically as one manifest change. This file is the wire draft only: the + * field numbers are not yet a stable contract, and support here is read-side + * fail-closed — rejected on load, with no write path. * - * STATUS - * ------ - * The messages and field numbers here are NOT yet a stable contract. Library - * support is read-side fail-closed only: a transaction carrying a - * `CompositeOperation` is rejected on load, and there is no write path. Apply, - * id translation, and conflict resolution are intentionally absent. + * Why it is shaped this way: * - * DESIGN RATIONALE - * ---------------- - * The reasoning below is the durable record of why the wire format is shaped - * this way; it is deliberately kept in-tree rather than in an external tracker. + * 1. Actions are deltas, not post-images: each records *the change* to the + * manifest, not the resulting state. That is what lets actions compose + * into one commit, and replay onto another branch, with no special cases. * - * 1. Actions are deltas, not post-images. Each action records *the change* to - * the manifest (add this file, tombstone this field), not the resulting - * state. Deltas are what make two things fall out uniformly with no special - * cases: composing several actions into one atomic commit, and merging or - * rebasing a branch (replay the deltas against the target). A few actions - * that carry non-derivable preconditions are the exception (see 8). + * 2. `Ref = Committed | Local`. A minting action cannot know its + * counter-allocated id (field, fragment, base) until commit, and gets a + * different one when replayed onto another target, so it names the id with + * a `Local` token resolved against the target's counters at apply. A + * writer that must bake an id into a file it writes in the same commit + * instead takes `Committed` ids from a range an earlier commit reserved + * (`ReserveFragmentIds`, `ReserveRowIds`). * - * 2. Minting vs. reference-stable actions. Minting actions allocate a new - * counter-based id (a field id, fragment id, or base id). Because the final - * id is not known until commit — and differs when the same action is - * replayed onto a different target during merge/rebase — a minting action - * names its new id with a `Local` token (see `Ref`) rather than a concrete - * value. Reference-stable actions touch already-committed ids at stable - * coordinates (a fragment's deletion vector, a config key); their target is - * unambiguous, so they may safely carry a post-image at rest. + * 3. Schema changes are per field (`AddField` / `DropField` / `AlterField`), + * never a wholesale replacement, so disjoint edits commute. `AlterField` + * preserves the field id and makes each mutable facet an independent + * optional, so concurrent alters of different facets commute too. * - * The exception is a writer that has to know an id *before* it commits, - * because it bakes that id into a file it writes in the same commit — an - * index whose payload holds row addresses or row ids cannot be written - * against a `Local` token. Such a writer takes ids from a range an earlier - * commit reserved (`ReserveFragmentIds`, `ReserveRowIds`) and names them as - * `Committed`. The reservation is what makes that safe: the ids are the - * writer's alone, so they stay valid at every later version and nothing has - * to be re-resolved when the operation is replayed. It costs an extra - * commit, which is why it is the exception and not the rule. + * 4. Index actions target a segment by UUID: the format has no index apart + * from its segments, and a logical index is the set of segments sharing a + * `name`. * - * 3. One reference type, `Ref = Committed | Local`. `Committed` is a concrete, - * already-assigned id. `Local` is a placeholder minted by an earlier `Add*` - * action in the same operation. `Local` resolves to a freshly-allocated - * committed id at apply time, and re-resolves against the *target's* - * counters on merge/rebase — which is exactly what lets two independent - * `AddField`s on divergent branches become two distinct fields on merge. + * 5. Assertions carry only preconditions no post-image can reconstruct, e.g. + * the keys a merge-insert inserted (`AssertUniqueKeys`). * - * 4. Relocation by counter watermark. When an operation is replayed onto a - * newer version, minting actions are re-applied against the target's - * counters and their `Local` refs re-resolve; ids assigned above the - * read-version watermark are known to be freshly minted and are rewritten - * consistently across every action that referenced them. - * - * 5. Field-level schema. Schema changes are expressed per field - * (`AddField` / `DropField` / `AlterField`), never as a wholesale - * replacement, so concurrent schema edits to disjoint fields commute. - * - * 6. Identity-preserving `AlterField`. Altering a field is distinct from - * dropping it and adding a new one: the field keeps its id. Each mutable - * facet (name, type, nullability) is an independent optional, so two - * concurrent alters of *different* facets of the same field commute - * (e.g. a widening cast alongside a non-null -> nullable relaxation). - * - * 7. Index segments, not indices. The format has no first-class "index" - * separate from its segments; a new index is written as its first segment, - * and a logical index is the set of segments sharing a `name`. The actions - * mirror that: add/remove/adjust operate on a segment (by UUID). The - * resulting per-segment config duplication is a pre-existing limitation this - * draft does not attempt to fix. - * - * 8. Assertions carry only non-derivable preconditions. Most conflict checks - * are computed from the actions themselves. The exception is a fact that - * cannot be reconstructed from any post-image — e.g. the set of keys a - * merge-insert inserted — which is carried explicitly as an assertion - * (`AssertUniqueKeys`) and checked against concurrent commits. - * - * WHAT STAYS OFF THE WIRE - * ----------------------- - * Two classes of information are deliberately not serialized, and are - * recomputed at conflict time instead: - * - Conflict *footprints* (which fields/fragments/keys an action touches). - * These are a pure function of each action, so storing them would only risk - * drift. - * - Large derivable row-level deltas, e.g. a deletion's affected rows or an - * update's matched row offsets. These can be recomputed by diffing against - * the read-version state, and serializing them would bloat transactions - * with potentially huge integer lists. - * - * ADDITIVE-SAFETY - * --------------- - * The schema is meant to evolve additively: message identities and the core - * mutation fields are pinned, and anything discovered during implementation - * should arrive as an *added* optional field rather than a reshaping of what is - * here. + * Deliberately off the wire and recomputed instead: conflict footprints, a + * pure function of each action, and large derivable row-level deltas, + * recoverable by diffing against the read-version state. The schema evolves + * additively — new optional fields, not a reshaping of what is here. */ /* @@ -161,10 +101,6 @@ message UserAction { /* * A single granular change (a delta) or assertion. Legacy operations decompose * into an ordered list of these. - * - * TODO: the `data_change` markers on the actions below are drafted per-action - * for maximum flexibility; revisit whether per-action or per-UserAction - * granularity is the right unit before the format stabilizes. */ message Action { oneof action { From f0a2e5caf5bfdabe356a74193d347f4d2f650422 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 21 Sep 2026 10:33:06 -0700 Subject: [PATCH 12/15] docs(format): mark Transaction V2 experimental and specify it Transaction V2 was labelled a DRAFT, which is not a status the project recognizes. The governance vote on experimental specification features passed on 2026-08-27 (discussion #8305), so use that status instead: EXPERIMENTAL, with the two commitments a reader needs stated where the label appears -- breaking changes without a separate vote, and removal if the stabilization vote does not pass. The process also requires the feature to be marked in the documentation, and `CompositeOperation` was not described in docs/src/format at all. Adds a section to the transaction specification covering the two-level CompositeOperation/UserAction shape, why actions are deltas with local id placeholders, and what the feature asks of a reader that does not implement it: reject a transaction it cannot decode rather than skip it, and tolerate an undecodable inline transaction when opening a table. Co-Authored-By: Claude Opus 5 (1M context) --- docs/src/format/table/transaction.md | 63 +++++++++++++++++++++++ protos/transaction/actions.proto | 19 +++---- protos/transaction/transaction.proto | 2 +- rust/lance-table/src/transaction/proto.rs | 8 +-- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index 8e3de3e6a7e..98f3db17d0f 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -556,6 +556,69 @@ An UpdateBases operation only modifies the base paths. As a result, it only conf UpdateBases operations and even then only conflicts if the two operations have base paths with the same id, name, or path. +## Action-Based Transactions (Transaction V2) + +!!! warning "Experimental" + + Transaction V2 is an experimental format feature under the process in + [Voting](../../community/voting.md). Its protobuf field numbers are not a + stable contract, breaking changes may be made without a separate vote, and + the feature is removed from the specification and the codebase if its + stabilization vote does not pass. Discussion: + [#5960](https://github.com/lance-format/lance/discussions/5960). + +Every operation above is a single named verb. A transaction may instead carry a +`CompositeOperation`: an ordered list of granular *actions* that apply +atomically as one manifest change. This lets one commit express a change no +single named operation covers, such as appending a fragment and updating an +index together, and makes the transaction a true diff of the manifest rather +than a description of intent. + +
+CompositeOperation protobuf message + +```protobuf +%%% proto.message.CompositeOperation %%% +``` + +
+ +A `CompositeOperation` holds `UserAction`s, each a human-recognizable step with +a description, which in turn hold the granular `Action`s applied to the +manifest. The two levels exist so the transaction history stays readable when +several operations are squashed into one: the descriptions survive, while the +action lists are flattened on apply. + +Actions are deltas rather than post-images, and an action that allocates a new +identifier (a field, fragment, or base id) names it with a local placeholder +that resolves against the target's counters at apply time. Both properties are +what let a composite operation be replayed against a newer version, or onto a +different branch, without rewriting it. + +The full action vocabulary and the reasoning behind its shape are defined in +`protos/transaction/actions.proto`. + +### Compatibility + +A `CompositeOperation` produces an ordinary manifest, so a reader that scans a +table whose latest version was written this way needs no knowledge of the +feature. Readers are affected in two places: + +- A reader that decodes the transaction itself — reading the transaction + history, or checking a concurrent commit for conflicts — sees an operation it + does not recognize. Implementations must reject it rather than treat it as a + no-op, so that a concurrent V2 commit aborts an in-flight commit instead of + being silently skipped. +- Transactions are usually inlined into the manifest. A reader that fails when + an inline transaction does not decode cannot open such a table at all. + Implementations must tolerate an undecodable inline transaction and continue + opening the table. In the Rust implementation this has been true since + v12.0.0. + +Conflict resolution between two `CompositeOperation`s is computed from the +actions themselves. Between a `CompositeOperation` and any named operation it +fails closed: the commit is rejected as a conflict rather than compared. + ## Conflict Resolution ### Terminology diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 01c79b67562..1e7acb2e7bf 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -11,15 +11,17 @@ import "google/protobuf/any.proto"; package lance.table; /* - * Action-based transactions (Transaction V2) — DRAFT. + * Action-based transactions (Transaction V2) — EXPERIMENTAL. * * A `CompositeOperation` replaces the legacy `Operation` (see * transaction.proto) with an ordered list of `Action`s committing - * atomically as one manifest change. This file is the wire draft only: the - * field numbers are not yet a stable contract, and support here is read-side + * atomically as one manifest change. This is an experimental feature under + * the process in docs/src/community/voting.md: the field numbers are not a + * stable contract, may change without a separate vote, and are removed if + * the stabilization vote does not pass. Support here is read-side * fail-closed — rejected on load, with no write path. * - * Why it is shaped this way: + * Why this shape: * * 1. Actions are deltas, not post-images: each records *the change* to the * manifest, not the resulting state. That is what lets actions compose @@ -27,7 +29,7 @@ package lance.table; * * 2. `Ref = Committed | Local`. A minting action cannot know its * counter-allocated id (field, fragment, base) until commit, and gets a - * different one when replayed onto another target, so it names the id with + * different one on replay onto another target, so it names the id with * a `Local` token resolved against the target's counters at apply. A * writer that must bake an id into a file it writes in the same commit * instead takes `Committed` ids from a range an earlier commit reserved @@ -45,10 +47,9 @@ package lance.table; * 5. Assertions carry only preconditions no post-image can reconstruct, e.g. * the keys a merge-insert inserted (`AssertUniqueKeys`). * - * Deliberately off the wire and recomputed instead: conflict footprints, a - * pure function of each action, and large derivable row-level deltas, - * recoverable by diffing against the read-version state. The schema evolves - * additively — new optional fields, not a reshaping of what is here. + * Off the wire and recomputed instead: conflict footprints, a pure function + * of each action, and large derivable row-level deltas, recoverable by + * diffing against the read-version state. */ /* diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto index 6aeefcaf242..71d7254df3d 100644 --- a/protos/transaction/transaction.proto +++ b/protos/transaction/transaction.proto @@ -367,7 +367,7 @@ message Transaction { UpdateBases update_bases = 114; DataOverlay data_overlay = 115; /* Action-based transaction (Transaction V2). See actions.proto. - * DRAFT: currently rejected on load; no write path. + * EXPERIMENTAL: currently rejected on load; no write path. */ CompositeOperation composite_operation = 116; } diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 49a364dbf14..034a2da9fe7 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -402,8 +402,8 @@ impl TryFrom for Transaction { .collect::>>()?, }, Some(pb::transaction::Operation::CompositeOperation(_)) => { - // Action-based transactions (Transaction V2) are a draft wire - // format (OSS-1530). This version of Lance recognizes the message + // Action-based transactions (Transaction V2) are an experimental + // wire format (OSS-1530). This version of Lance recognizes the message // but has no support for it: reject on load, fail-closed. Because // load_and_sort_new_transactions collects transactions with // try_collect, a concurrent V2 commit in the conflict window @@ -862,8 +862,8 @@ mod tests { #[test] fn test_composite_operation_rejected_on_load() { - // Action-based transactions (Transaction V2) are a draft wire format that - // this version of Lance does not support. Loading one must fail closed + // Action-based transactions (Transaction V2) are an experimental wire + // format that this version of Lance does not support. Loading one must fail closed // (never be silently skipped or leniently parsed), so that a concurrent // V2 commit in the conflict window aborts an in-flight commit. let message = pb::Transaction { From ea4b6f3fd72d24becb452f16c8a41c0e3450dcbc Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 21 Sep 2026 12:03:59 -0700 Subject: [PATCH 13/15] docs(format): warn that Transaction V2 breaks pre-v12 readers Review raised that the compatibility cost was understated, and the backports that would remove it are unlikely: several downstream packages bake Lance into size-constrained binaries, so older release lines will not pick up #7740. That makes this durable rather than transitional, so say so where a writer will see it. Committing a CompositeOperation does not merely make the transaction unreadable as history -- it makes that table version unopenable by Lance before v12.0.0, because those releases decode the manifest's inline transaction while opening the table and propagate the failure (#9454). Room in the actions.proto header, which is budgeted at under 300 words, comes from dropping the point about index actions targeting a segment; AddIndexSegment's own comment already says it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/src/format/table/transaction.md | 21 +++++++++++++++++++-- protos/transaction/actions.proto | 16 ++++++++-------- protos/transaction/transaction.proto | 8 ++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index 98f3db17d0f..64f6ccee9be 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -567,6 +567,21 @@ same id, name, or path. stabilization vote does not pass. Discussion: [#5960](https://github.com/lance-format/lance/discussions/5960). +!!! danger "Writing Transaction V2 breaks compatibility with readers before v12.0.0" + + A table version committed as a `CompositeOperation` **cannot be opened** by + Lance before v12.0.0 — not merely read as history. Those releases decode the + manifest's inline transaction section while opening the table, and an + operation they do not recognize decodes to no operation at all, which fails + the open. The fix + ([#7740](https://github.com/lance-format/lance/pull/7740)) shipped in + v12.0.0 and is not expected to be backported to earlier release lines; see + [#9454](https://github.com/lance-format/lance/issues/9454). + + Treat this as durable, not transitional. Writing a `CompositeOperation` is + opt-in for exactly this reason, and a table that has one anywhere in its + history raises the minimum reader version for that table to v12.0.0. + Every operation above is a single named verb. A transaction may instead carry a `CompositeOperation`: an ordered list of granular *actions* that apply atomically as one manifest change. This lets one commit express a change no @@ -612,8 +627,10 @@ feature. Readers are affected in two places: - Transactions are usually inlined into the manifest. A reader that fails when an inline transaction does not decode cannot open such a table at all. Implementations must tolerate an undecodable inline transaction and continue - opening the table. In the Rust implementation this has been true since - v12.0.0. + opening the table, because the transaction contents are not needed to read + data. In the Rust implementation this has been true only since v12.0.0, which + is what makes writing Transaction V2 a compatibility break against earlier + releases rather than a graceful degradation. Conflict resolution between two `CompositeOperation`s is computed from the actions themselves. Between a `CompositeOperation` and any named operation it diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 1e7acb2e7bf..45c6ef18cca 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -21,11 +21,15 @@ package lance.table; * the stabilization vote does not pass. Support here is read-side * fail-closed — rejected on load, with no write path. * + * COMPATIBILITY WARNING. A table version carrying one of these cannot be + * opened at all by Lance before v12.0.0, and the fix is not backported + * (issue #9454). Writing V2 is opt-in for that reason. + * * Why this shape: * * 1. Actions are deltas, not post-images: each records *the change* to the * manifest, not the resulting state. That is what lets actions compose - * into one commit, and replay onto another branch, with no special cases. + * into one commit and replay onto another branch with no special cases. * * 2. `Ref = Committed | Local`. A minting action cannot know its * counter-allocated id (field, fragment, base) until commit, and gets a @@ -40,16 +44,12 @@ package lance.table; * preserves the field id and makes each mutable facet an independent * optional, so concurrent alters of different facets commute too. * - * 4. Index actions target a segment by UUID: the format has no index apart - * from its segments, and a logical index is the set of segments sharing a - * `name`. - * - * 5. Assertions carry only preconditions no post-image can reconstruct, e.g. + * 4. Assertions carry only preconditions no post-image can reconstruct, e.g. * the keys a merge-insert inserted (`AssertUniqueKeys`). * * Off the wire and recomputed instead: conflict footprints, a pure function - * of each action, and large derivable row-level deltas, recoverable by - * diffing against the read-version state. + * of each action, and large row-level deltas recoverable from the + * read-version state. */ /* diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto index 71d7254df3d..7f020a827b6 100644 --- a/protos/transaction/transaction.proto +++ b/protos/transaction/transaction.proto @@ -367,7 +367,15 @@ message Transaction { UpdateBases update_bases = 114; DataOverlay data_overlay = 115; /* Action-based transaction (Transaction V2). See actions.proto. + * * EXPERIMENTAL: currently rejected on load; no write path. + * + * A table version carrying this arm cannot be opened at all by Lance + * before v12.0.0. Those releases decode the inline transaction while + * opening the table, and an arm they do not know decodes to no operation, + * which fails the open (issue #9454). The fix landed in v12.0.0 and is not + * backported, so this is a durable compatibility break rather than a + * transitional one. */ CompositeOperation composite_operation = 116; } From be2dd47ee4db30ca924592b79c90d87f574fe1c6 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 21 Sep 2026 12:57:45 -0700 Subject: [PATCH 14/15] feat(protos): let AddIndexSegment declare covering fields Review found the message could not represent IndexMetadata's covering_fields at all, so a covering index was not expressible as an action. Adds the field where the message is defined. Targets the contract in #9159 rather than the one on main: `fields` is the keyed columns only, `covering_fields` is an independent declaration of the carried ones, and the dependency set is the union. A new action has no historical encoding to stay compatible with, so it can take the contract the format is moving to and avoid a field-number change later. #9159 leaves FLAG_INDEPENDENT_COVERING_FIELDS unimplemented, so a manifest cannot yet express a column that is both keyed and carried. The overlapping form is therefore rejected at apply, and a disjoint one lowers to the legacy subset representation; both come out when the flag is implemented, with no change to this message. Co-Authored-By: Claude Opus 5 (1M context) --- docs/src/format/table/transaction.md | 11 +++++++++++ protos/transaction/actions.proto | 24 +++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index 64f6ccee9be..f7de5e2d54a 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -613,6 +613,17 @@ different branch, without rewriting it. The full action vocabulary and the reasoning behind its shape are defined in `protos/transaction/actions.proto`. +One vocabulary detail is a format contract rather than an implementation +choice. `AddIndexSegment.fields` names only the columns the segment is keyed +on, and `covering_fields` is an independent declaration of the columns whose +values it carries, so the segment'''s dependency set is the union of the two. +This is the contract an index action targets, not the legacy one where +`IndexMetadata.fields` means keyed columns followed by carried columns. A +declaration whose two lists overlap can only be represented by a manifest +declaring `FLAG_INDEPENDENT_COVERING_FIELDS`; until a release implements that +flag, apply rejects the overlapping form and lowers a disjoint one to the +legacy subset representation. + ### Compatibility A `CompositeOperation` produces an ordinary manifest, so a reader that scans a diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 45c6ef18cca..90a875ab38a 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -365,8 +365,30 @@ message AlterField { message AddIndexSegment { UUID uuid = 1; string name = 2; - // Indexed field ids (Committed, or Local for same-op minted fields). + /* + * The field ids this segment is keyed on -- the columns it can be searched + * by (Committed, or Local for same-op minted fields). Empty only for a + * system index keyed on no column at all, such as mem_wal or frag_reuse. + * + * Keys only. `IndexMetadata.fields` in a manifest written under the legacy + * contract instead means keyed columns followed by carried ones; this + * action does not inherit that, because it has no historical encoding to + * stay compatible with. Apply derives the manifest's representation. + */ repeated Ref fields = 3; + /* + * Field ids whose values the segment carries but is not necessarily keyed + * on. Independent of `fields` rather than a subset of it, and the segment's + * dependency set is the union of the two. Empty for a segment that carries + * no extra column. + * + * An id may appear in both lists, meaning a column the segment is keyed on + * *and* carries values for. That form requires the manifest to declare + * FLAG_INDEPENDENT_COVERING_FIELDS, which no release implements yet, so + * apply rejects an overlapping declaration until it does. A disjoint + * declaration lowers to the legacy subset form and commits today. + */ + repeated Ref covering_fields = 12; google.protobuf.Any index_details = 4; optional int32 index_version = 5; /* From d1bc81a8230448684c2b190324e50690d18dd2e7 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 21 Sep 2026 15:26:06 -0700 Subject: [PATCH 15/15] feat(protos): drop the duplicated identity from CompositeOperation CompositeOperation carried uuid and read_version, both copies of fields the enclosing Transaction already has. Nothing read them back: the to-wire path stamped them from the envelope and the read side dropped them. The stated purpose -- letting a squashed operation keep its provenance -- is one the fields cannot serve where they sat. Squashing collapses each original CompositeOperation into one UserAction, so the message holding the identity is exactly the one squashing destroys. If per-step provenance is wanted, it belongs on UserAction, and it can be added as an optional field when squashing exists to need it. Renumber actions to 1 rather than reserving 1 and 2: this message has not shipped, and the file states that its field numbers are not a stable contract. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 6 +----- rust/lance-table/src/transaction/proto.rs | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 90a875ab38a..bb7b5ff4b87 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -75,12 +75,8 @@ message Ref { * commit atomically as a single manifest change. */ message CompositeOperation { - // Unique identifier for this operation (matches Transaction.uuid semantics). - UUID uuid = 1; - // The dataset version this operation was planned against. - uint64 read_version = 2; // The ordered list of user actions applied by this operation. - repeated UserAction actions = 3; + repeated UserAction actions = 1; } /* diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 034a2da9fe7..ef6ae7bc5e9 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -871,8 +871,6 @@ mod tests { uuid: Uuid::new_v4().to_string(), operation: Some(pb::transaction::Operation::CompositeOperation( pb::CompositeOperation { - uuid: Some(pb::Uuid::from(&Uuid::new_v4())), - read_version: 1, actions: vec![pb::UserAction { description: "append batch".to_string(), actions: vec![pb::Action {