From 83aface67f5d161ea91c842c556d39b6c0d0382e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Sun, 18 Oct 2026 13:31:15 +0200 Subject: [PATCH 1/2] fix(core): the schema-less API reads documents a typed collection stored in overflow pages (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A typed DocumentCollection stores a document larger than one page across overflow pages: the slot holds [length][first overflow page][first chunk]. DynamicCollection never checked SlotFlags.HasOverflow and handed the slot bytes to BsonSpanReader, which threw NotSupportedException "Skipping type … not supported at position 7". FindAllAsync, FindByIdAsync, ScanAsync, BLQL, BLite Studio and BLiteMigration all failed on the first large document. New internal OverflowChainReader (Storage/) reassembles such a document, sync and async, from the layout DocumentCollection writes, and lists the chain's pages. DynamicCollection uses it in ReadDocumentAt, ReadDocumentAtAsync and ScanAsync, and DeleteSlot frees the chain on commit (FreePageOnCommit, as DocumentCollection.FreeOverflowChain does) so a schema-less delete or relocating update of such a document leaks no pages. DynamicCollection still does not write overflow pages; the limit is now public as MaxDocumentSize and the rejection message names it. BLiteMigration copies into a DynamicCollection, so it refuses a document larger than MaxDocumentSize with a NotSupportedException naming the collection, _id and size, before writing it; on failure ToMultiFileAsync also removes the companion files (.idx, WAL directory, collection directory) it created, and ToSingleFileAsync removes a target it created. README notes the schema-less size limit. Tests: DynamicOverflowReadTests, 9 tests; 6 of the 8 that compile on upstream/main fail there (3 with the issue's exception). Co-Authored-By: Claude Fable 5.1 --- README.md | 2 + src/BLite.Core/BLiteMigration.cs | 49 +++- src/BLite.Core/DynamicCollection.cs | 54 ++++- src/BLite.Core/Storage/OverflowChainReader.cs | 149 ++++++++++++ tests/BLite.Tests/DynamicOverflowReadTests.cs | 223 ++++++++++++++++++ 5 files changed, 472 insertions(+), 5 deletions(-) create mode 100644 src/BLite.Core/Storage/OverflowChainReader.cs create mode 100644 tests/BLite.Tests/DynamicOverflowReadTests.cs diff --git a/README.md b/README.md index 1a868fde..4056993d 100644 --- a/README.md +++ b/README.md @@ -776,6 +776,8 @@ BsonId id = await orders.InsertAsync(doc, ct); List ids = await orders.InsertBulkAsync([doc1, doc2, doc3], ct); ``` +> A schema-less document must fit in one page: `DynamicCollection.MaxDocumentSize` bytes of BSON (about 16 KB with the default page size); a larger one is rejected with `BLiteValidationException`. A typed `DocumentCollection` stores larger documents across overflow pages, and the schema-less API reads those, so a typed collection with large documents can be read, queried and shown in BLite Studio, but not copied by `BLiteMigration`. + ### Read ```csharp diff --git a/src/BLite.Core/BLiteMigration.cs b/src/BLite.Core/BLiteMigration.cs index cbec33d5..443476ab 100644 --- a/src/BLite.Core/BLiteMigration.cs +++ b/src/BLite.Core/BLiteMigration.cs @@ -92,6 +92,9 @@ public static async Task ToMultiFileAsync(string sourcePath, PageFileConfig targ // Target main .db is written to a temp path first so that we never // leave the source in a partially-overwritten state. var tempDbPath = sourcePath + ".migrating"; + // The target's companion files (.idx, WAL, collection directory) are created at their + // final paths; remember which of them this call creates, so a failure removes only those. + var createdCompanions = CompanionPathsThatDoNotExist(targetConfig, tempDbPath); try { using (var source = new BLiteEngine(sourcePath, sourceConfig)) @@ -110,10 +113,12 @@ public static async Task ToMultiFileAsync(string sourcePath, PageFileConfig targ } catch { - // Best-effort cleanup of the half-written temp file. + // Best-effort cleanup of the half-written temp file and the companion files this + // call created; the source is untouched. SafeDelete(tempDbPath); var tempWal = Path.ChangeExtension(tempDbPath, ".wal"); SafeDelete(tempWal); + DeletePaths(createdCompanions); throw; } } @@ -169,6 +174,7 @@ public static async Task ToSingleFileAsync( StringComparison.OrdinalIgnoreCase); var actualTargetPath = isInPlace ? targetPath + ".migrating" : targetPath; + bool targetExistedBefore = !isInPlace && File.Exists(actualTargetPath); try { @@ -190,7 +196,8 @@ public static async Task ToSingleFileAsync( } catch { - if (isInPlace) + // Remove the half-written target unless it is a file the caller already had. + if (isInPlace || !targetExistedBefore) { SafeDelete(actualTargetPath); SafeDelete(Path.ChangeExtension(actualTargetPath, ".wal")); @@ -250,6 +257,18 @@ private static async Task CopyCollectionsASync(BLiteEngine source, BLiteEngine t var batch = new List(BatchSize); await foreach (var doc in sourceColl.FindAllAsync()) { + // A typed collection may hold a document larger than a page (overflow pages); + // the target is a DynamicCollection, which does not write those (#301). Say + // which document it is instead of failing inside InsertBulkAsync. + if (doc.RawData.Length > targetColl.MaxDocumentSize) + { + doc.TryGetId(out var tooLargeId); + throw new NotSupportedException( + $"Collection '{collName}' holds a document of {doc.RawData.Length} bytes (_id {tooLargeId}), " + + $"larger than the {targetColl.MaxDocumentSize} bytes a schema-less collection can store in one page. " + + "BLiteMigration copies through the schema-less API and cannot migrate this database; the source is unchanged."); + } + batch.Add(doc); if (batch.Count >= BatchSize) { @@ -348,4 +367,30 @@ private static void SafeDelete(string path) try { if (File.Exists(path)) File.Delete(path); } catch { /* best-effort */ } } + + /// The companion paths of that do not exist yet, i.e. the ones a migration to it will create. + private static List CompanionPathsThatDoNotExist(PageFileConfig config, string mainDbPath) + { + var paths = new List(); + var walPath = config.WalPath ?? Path.ChangeExtension(mainDbPath, ".wal"); + var walDir = Path.GetDirectoryName(walPath); + if (!string.IsNullOrEmpty(walDir) && !Directory.Exists(walDir)) paths.Add(walDir); // e.g. the "wal/" directory of PageFileConfig.Server + else if (!File.Exists(walPath)) paths.Add(walPath); + if (config.IndexFilePath != null && !File.Exists(config.IndexFilePath)) paths.Add(config.IndexFilePath); + if (config.CollectionDataDirectory != null && !Directory.Exists(config.CollectionDataDirectory)) paths.Add(config.CollectionDataDirectory); + return paths; + } + + private static void DeletePaths(IEnumerable paths) + { + foreach (var path in paths) + { + try + { + if (Directory.Exists(path)) Directory.Delete(path, recursive: true); + else if (File.Exists(path)) File.Delete(path); + } + catch { /* best-effort */ } + } + } } diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index 4e144373..32220b65 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -52,6 +52,15 @@ public sealed class DynamicCollection : IDisposable // Cached delegate — avoids per-call closure allocation when passed to _fsi.FindPage. private readonly Func _isPageLocked; private readonly int _maxDocumentSizeForSinglePage; + + /// + /// The largest document, in BSON bytes, this collection writes: one page minus its header, + /// slot and reserve (about 16 KB with the default page size). A larger document is rejected + /// with . A typed DocumentCollection stores larger + /// documents across overflow pages, and this collection reads those (#301); it does not + /// write them, so such a document cannot be copied into a DynamicCollection. + /// + public int MaxDocumentSize { get; } private uint _currentDataPage; private bool _isTimeSeries; @@ -118,6 +127,7 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction _collectionName = collectionName ?? throw new ArgumentNullException(nameof(collectionName)); _idType = idType; _maxDocumentSizeForSinglePage = _storage.PageSize - 128; + MaxDocumentSize = _maxDocumentSizeForSinglePage - SlotEntry.Size; _fsi = freeSpaceIndex ?? new FreeSpaceIndex(_storage.PageSize); _isPageLocked = _storage.IsPageLocked; @@ -777,7 +787,7 @@ private async Task InsertCore(BsonDocument document, ITransaction transa else { // Unreachable: ValidateWrite rejects documents that do not fit in a page. - throw new BLiteValidationException($"Document size {docData.Length} exceeds maximum single page size. Overflow not yet supported in DynamicCollection."); + throw new BLiteValidationException($"Document size {docData.Length} exceeds the {_maxDocumentSizeForSinglePage - SlotEntry.Size} bytes a DynamicCollection document may occupy (one page). DynamicCollection does not write overflow pages; a typed DocumentCollection does."); } // Index the _id @@ -1019,7 +1029,19 @@ public async IAsyncEnumerable ScanAsync(BsonReaderPredicate predic var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset)); if (slot.Flags.HasFlag(SlotFlags.Deleted)) continue; - var data = buffer.AsSpan(slot.Offset, slot.Length); + // A document a typed collection stored across overflow pages (#301) is + // reassembled before the predicate sees it; its slot holds only the first chunk. + ReadOnlySpan data; + if (slot.Flags.HasFlag(SlotFlags.HasOverflow)) + { + var full = OverflowChainReader.Read(_storage, buffer, slot, txnId); + if (full == null) continue; + data = full; + } + else + { + data = buffer.AsSpan(slot.Offset, slot.Length); + } var reader = new BsonSpanReader(data, _storage.GetKeyReverseMap()); if (predicate(reader)) @@ -2232,7 +2254,7 @@ private void ValidateWrite(BsonId id, IndexKey key, BsonDocument document, BsonD var size = document.RawData.Length; if (!_isTimeSeries && size + SlotEntry.Size > _maxDocumentSizeForSinglePage) throw new BLiteValidationException( - $"Document size {size} exceeds maximum single page size. Overflow not yet supported in DynamicCollection."); + $"Document size {size} exceeds the {_maxDocumentSizeForSinglePage - SlotEntry.Size} bytes a DynamicCollection document may occupy (one page). DynamicCollection does not write overflow pages; a typed DocumentCollection does."); // An update keeps the id it was found by, so only a new id is checked: one stored before // MaxKeySize was lowered (#217) must not make its document impossible to update. @@ -2462,6 +2484,14 @@ private bool TryUpdateInPlace(DocumentLocation location, ReadOnlyMemory da if (slot.Flags.HasFlag(SlotFlags.Deleted)) return null; + // A typed collection stores a document larger than a page across overflow pages; + // the slot then holds [length][first overflow page][first chunk] (#301). + if (slot.Flags.HasFlag(SlotFlags.HasOverflow)) + { + var full = OverflowChainReader.Read(_storage, buffer, slot, txnId); + return full == null ? null : new BsonDocument(full, _storage.GetKeyReverseMap(), _storage.GetKeyMap()); + } + // Copy document data (buffer is pooled, data must outlive it) var docData = buffer.AsSpan(slot.Offset, slot.Length).ToArray(); return new BsonDocument(docData, _storage.GetKeyReverseMap(), _storage.GetKeyMap()); @@ -2507,6 +2537,14 @@ private bool TryUpdateInPlace(DocumentLocation location, ReadOnlyMemory da var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset)); if (slot.Flags.HasFlag(SlotFlags.Deleted)) return null; + if (slot.Flags.HasFlag(SlotFlags.HasOverflow)) + { + // Overflow document of a typed collection (#301); the reader copies what it + // needs from the page buffer before it awaits. + var full = await OverflowChainReader.ReadAsync(_storage, buffer, slot, txnId, ct).ConfigureAwait(false); + return full == null ? null : new BsonDocument(full, _storage.GetKeyReverseMap(), _storage.GetKeyMap()); + } + // Span created after the only await — safe; ToArray() copies before buffer is returned var docData = buffer.AsSpan(slot.Offset, slot.Length).ToArray(); return new BsonDocument(docData, _storage.GetKeyReverseMap(), _storage.GetKeyMap()); @@ -2682,6 +2720,16 @@ private void DeleteSlot(DocumentLocation location, ITransaction transaction) { var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size); var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset)); + + // A document a typed collection stored across overflow pages (#301): recycle + // its chain once the transaction commits, as DocumentCollection.FreeOverflowChain + // does; until then a rollback returns to the committed document, which uses them. + if (slot.Flags.HasFlag(SlotFlags.HasOverflow)) + { + foreach (var overflowPageId in OverflowChainReader.ChainPages(_storage, buffer, slot, transaction.TransactionId)) + _storage.FreePageOnCommit(overflowPageId, transaction.TransactionId); + } + slot.Flags |= SlotFlags.Deleted; slot.WriteTo(buffer.AsSpan(slotOffset)); diff --git a/src/BLite.Core/Storage/OverflowChainReader.cs b/src/BLite.Core/Storage/OverflowChainReader.cs new file mode 100644 index 00000000..18279e19 --- /dev/null +++ b/src/BLite.Core/Storage/OverflowChainReader.cs @@ -0,0 +1,149 @@ +using System.Buffers; +using System.Buffers.Binary; + +namespace BLite.Core.Storage; + +/// +/// Reassembles a document that a typed DocumentCollection stored across overflow pages, +/// so that the schema-less read paths (DynamicCollection, and through it BLQL, Studio and +/// BLiteMigration) can read every document the typed API wrote (#301). +/// +/// A slot flagged holds +/// [TotalLength (4)] [FirstOverflowPage (4)] [first chunk of the document]; every overflow +/// page holds PageSize - SlottedPageHeader.Size bytes of payload after its header and +/// names the next page in (0 ends the chain). +/// This is the layout DocumentCollection.InsertWithOverflow writes and +/// DocumentCollection.ReadRawBytesAt reads. +/// +/// +internal static class OverflowChainReader +{ + /// Bytes of slot payload that precede the first chunk: total length and first overflow page id. + public const int SlotMetadataSize = 8; + + /// + /// Reads the whole document whose first chunk is in of + /// . Returns null when the slot does not carry an + /// overflow document or the chain is inconsistent (truncated, or shorter than its declared + /// length), never a partial document. + /// + public static byte[]? Read(StorageEngine storage, ReadOnlySpan pageBuffer, in SlotEntry slot, ulong txnId) + { + if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk)) + return null; + + var document = new byte[totalLength]; + firstChunk.CopyTo(document); + int offset = firstChunk.Length; + + var overflowBuffer = ArrayPool.Shared.Rent(storage.PageSize); + try + { + while (nextPageId != 0 && offset < totalLength) + { + storage.ReadPage(nextPageId, txnId, overflowBuffer); + nextPageId = AppendChunk(storage.PageSize, overflowBuffer, document, ref offset); + } + } + finally + { + ArrayPool.Shared.Return(overflowBuffer); + } + + return offset == totalLength ? document : null; + } + + /// Async counterpart of , reading the chain with . + public static async ValueTask ReadAsync(StorageEngine storage, byte[] pageBuffer, SlotEntry slot, ulong txnId, CancellationToken ct) + { + // Everything taken from the page buffer is copied before the first await: the caller's + // buffer may be pooled, and a span cannot live across an await anyway. + if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk)) + return null; + + var document = new byte[totalLength]; + firstChunk.CopyTo(document); + int offset = firstChunk.Length; + + var overflowBuffer = ArrayPool.Shared.Rent(storage.PageSize); + try + { + while (nextPageId != 0 && offset < totalLength) + { + ct.ThrowIfCancellationRequested(); + await storage.ReadPageAsync(nextPageId, txnId, overflowBuffer.AsMemory(0, storage.PageSize), ct).ConfigureAwait(false); + nextPageId = AppendChunk(storage.PageSize, overflowBuffer, document, ref offset); + } + } + finally + { + ArrayPool.Shared.Return(overflowBuffer); + } + + return offset == totalLength ? document : null; + } + + /// + /// The page ids of the overflow chain that starts in , in chain + /// order, so a caller that deletes the document can free them. Empty when the slot carries + /// no overflow document. Stops at a page that does not look like an overflow page, so a + /// chain corrupted by an earlier failure cannot free a page another document uses. + /// + public static List ChainPages(StorageEngine storage, ReadOnlySpan pageBuffer, in SlotEntry slot, ulong txnId) + { + var pages = new List(); + if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk)) + return pages; + + int offset = firstChunk.Length; + var overflowBuffer = ArrayPool.Shared.Rent(storage.PageSize); + try + { + while (nextPageId != 0 && offset < totalLength) + { + storage.ReadPage(nextPageId, txnId, overflowBuffer); + var header = SlottedPageHeader.ReadFrom(overflowBuffer); + if (header.PageType != PageType.Overflow || pages.Contains(nextPageId)) + break; + pages.Add(nextPageId); + offset += Math.Min(storage.PageSize - SlottedPageHeader.Size, totalLength - offset); + nextPageId = header.NextOverflowPage; + } + } + finally + { + ArrayPool.Shared.Return(overflowBuffer); + } + return pages; + } + + private static bool TryReadHead(ReadOnlySpan pageBuffer, in SlotEntry slot, out int totalLength, out uint nextPageId, out ReadOnlySpan firstChunk) + { + totalLength = 0; + nextPageId = 0; + firstChunk = default; + + if ((slot.Flags & SlotFlags.HasOverflow) == 0) return false; + if (slot.Length < SlotMetadataSize || slot.Offset + slot.Length > pageBuffer.Length) return false; + + var payload = pageBuffer.Slice(slot.Offset, slot.Length); + totalLength = BinaryPrimitives.ReadInt32LittleEndian(payload); + nextPageId = BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(4)); + + int firstChunkSize = slot.Length - SlotMetadataSize; + if (totalLength <= 0 || firstChunkSize > totalLength) return false; + + firstChunk = payload.Slice(SlotMetadataSize, firstChunkSize); + return true; + } + + /// Copies one overflow page's payload into and returns the next page id. + private static uint AppendChunk(int pageSize, byte[] overflowBuffer, byte[] document, ref int offset) + { + var header = SlottedPageHeader.ReadFrom(overflowBuffer); + int chunkSize = Math.Min(pageSize - SlottedPageHeader.Size, document.Length - offset); + overflowBuffer.AsSpan(SlottedPageHeader.Size, chunkSize).CopyTo(document.AsSpan(offset)); + offset += chunkSize; + return header.NextOverflowPage; + } +} diff --git a/tests/BLite.Tests/DynamicOverflowReadTests.cs b/tests/BLite.Tests/DynamicOverflowReadTests.cs new file mode 100644 index 00000000..c2f48c99 --- /dev/null +++ b/tests/BLite.Tests/DynamicOverflowReadTests.cs @@ -0,0 +1,223 @@ +using BLite.Bson; +using BLite.Core; +using BLite.Core.Query.Blql; +using BLite.Core.Storage; +using BLite.Shared; + +namespace BLite.Tests; + +/// +/// A typed DocumentCollection stores a document larger than one page across overflow +/// pages. The schema-less API (, and through it BLQL, Studio and +/// ) used to hand the slot bytes to the BSON reader as if they were +/// a document and threw "Skipping type 69 not supported" (EntglDb/BLite#301). These tests write +/// through and read, query, delete and migrate through . +/// +public class DynamicOverflowReadTests : IDisposable +{ + private const int LargeNameLength = 40_000; // ~2.5 default pages + private readonly string _dir = Path.Combine(Path.GetTempPath(), $"blite_overflow_{Guid.NewGuid():N}"); + private readonly string _dbPath; + + public DynamicOverflowReadTests() + { + Directory.CreateDirectory(_dir); + _dbPath = Path.Combine(_dir, "app.db"); + } + + public void Dispose() + { + try { Directory.Delete(_dir, recursive: true); } catch { /* best-effort */ } + } + + private static string LargeName(char c = 'x') => new string(c, LargeNameLength); + + /// Writes one small and one large (overflow) user through the typed API; returns their ids. + private async Task<(ObjectId small, ObjectId large)> WriteTypedAsync() + { + using var db = new TestDbContext(_dbPath); + var small = new User { Name = "small", Age = 1 }; + var large = new User { Name = LargeName(), Age = 99 }; + await db.Users.InsertAsync(small); + await db.Users.InsertAsync(large); + await db.SaveChangesAsync(); + return (small.Id, large.Id); + } + + private static string NameOf(BsonDocument d) { d.TryGetString("name", out var n); return n!; } + + [Fact] + public async Task FindAllAsync_ReturnsTheOverflowDocument_Whole() + { + await WriteTypedAsync(); + + using var engine = new BLiteEngine(_dbPath); + var docs = await engine.GetOrCreateCollection("users").FindAllAsync().ToListAsync(); + + Assert.Equal(2, docs.Count); + var large = Assert.Single(docs, d => NameOf(d).Length > 10); + Assert.Equal(LargeName(), NameOf(large)); + Assert.True(large.TryGetInt32("age", out var age)); + Assert.Equal(99, age); + Assert.Equal(2, await engine.GetOrCreateCollection("users").CountAsync()); + } + + [Fact] + public async Task FindByIdAsync_ReturnsTheOverflowDocument_Whole() + { + var (_, largeId) = await WriteTypedAsync(); + + using var engine = new BLiteEngine(_dbPath); + var doc = await engine.GetOrCreateCollection("users").FindByIdAsync(largeId); + + Assert.NotNull(doc); + Assert.Equal(LargeName(), NameOf(doc!)); + } + + [Fact] + public async Task Blql_FiltersAndSorts_OverflowDocuments() + { + await WriteTypedAsync(); + + using var engine = new BLiteEngine(_dbPath); + var col = engine.GetOrCreateCollection("users"); + + var byAge = col.Query(BlqlFilter.Eq("age", 99)).ToList(); + Assert.Equal(LargeName(), NameOf(Assert.Single(byAge))); + + var sorted = col.Query().OrderByDescending("age").ToList(); + Assert.Equal(2, sorted.Count); + Assert.Equal(LargeName(), NameOf(sorted[0])); + Assert.Equal("small", NameOf(sorted[1])); + + // An index on the field: the index-served path reads the same slots. + await col.CreateIndexAsync("age"); + await engine.CommitAsync(); + var viaIndex = col.Query(BlqlFilter.Gte("age", 50)).ToList(); + Assert.Equal(LargeName(), NameOf(Assert.Single(viaIndex))); + } + + [Fact] + public async Task ScanAsync_PredicateSeesTheWholeDocument() + { + await WriteTypedAsync(); + + using var engine = new BLiteEngine(_dbPath); + var col = engine.GetOrCreateCollection("users"); + + // The predicate gets a reader over the reassembled document, so all of its bytes are + // there; a slot alone holds at most one page of them. + var matched = await col.ScanAsync(reader => reader.Remaining > LargeNameLength).ToListAsync(); + + Assert.Equal(LargeName(), NameOf(Assert.Single(matched))); + } + + [Fact] + public async Task TypedDelete_OfTheOverflowDocument_IsSeenBySchemaLessApi() + { + var (_, largeId) = await WriteTypedAsync(); + using (var db = new TestDbContext(_dbPath)) + { + Assert.True(await db.Users.DeleteAsync(largeId)); + await db.SaveChangesAsync(); + } + + using var engine = new BLiteEngine(_dbPath); + var docs = await engine.GetOrCreateCollection("users").FindAllAsync().ToListAsync(); + Assert.Equal("small", NameOf(Assert.Single(docs))); + } + + [Fact] + public async Task SchemaLessDelete_OfTheOverflowDocument_IsSeenByTypedApi_AndTheSpaceIsReused() + { + var (_, largeId) = await WriteTypedAsync(); + long sizeBefore; + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("users"); + Assert.True(await col.DeleteAsync(largeId)); + await engine.CommitAsync(); + await engine.CheckpointAsync(); + Assert.Null(await col.FindByIdAsync(largeId)); + sizeBefore = new FileInfo(_dbPath).Length; + } + + using (var db = new TestDbContext(_dbPath)) + { + Assert.Null(await db.Users.FindByIdAsync(largeId)); + Assert.Equal(1, await db.Users.CountAsync()); + + // The overflow pages were freed on commit: another document of the same size fits + // into the recycled pages and the file does not grow by its size again. + await db.Users.InsertAsync(new User { Name = LargeName('y'), Age = 7 }); + await db.SaveChangesAsync(); + db.ForceCheckpoint(); + } + var sizeAfter = new FileInfo(_dbPath).Length; + Assert.True(sizeAfter - sizeBefore < LargeNameLength, + $"file grew by {sizeAfter - sizeBefore} bytes; the freed overflow pages were not reused"); + } + + [Fact] + public async Task SchemaLessUpdate_OfTheOverflowDocument_ReplacesIt_ForBothApis() + { + var (_, largeId) = await WriteTypedAsync(); + using (var engine = new BLiteEngine(_dbPath)) + { + var col = engine.GetOrCreateCollection("users"); + var replacement = col.CreateDocument(["_id", "name", "age"], b => b + .AddId((BsonId)largeId) + .AddString("name", "small now") + .AddInt32("age", 42)); + Assert.True(await col.UpdateAsync(largeId, replacement)); + await engine.CommitAsync(); + + var read = await col.FindByIdAsync(largeId); + Assert.Equal("small now", NameOf(read!)); + } + + using var db = new TestDbContext(_dbPath); + var typed = await db.Users.FindByIdAsync(largeId); + Assert.NotNull(typed); + Assert.Equal("small now", typed!.Name); + Assert.Equal(42, typed.Age); + } + + [Fact] + public async Task SchemaLessInsert_LargerThanAPage_IsRejected_NamingTheLimit() + { + using var engine = new BLiteEngine(_dbPath); + var col = engine.GetOrCreateCollection("users"); + var doc = col.CreateDocument(["name"], b => b.AddString("name", LargeName())); + + var ex = await Assert.ThrowsAsync(() => col.InsertAsync(doc).AsTask()); + Assert.Contains(col.MaxDocumentSize.ToString(), ex.Message); + Assert.True(col.MaxDocumentSize > 0 && col.MaxDocumentSize < engine.Storage.PageSize); + } + + [Fact] + public async Task Migration_RefusesTheOverflowDocument_ByName_AndLeavesNothingBehind() + { + var (_, largeId) = await WriteTypedAsync(); + var config = PageFileConfig.Server(_dbPath); + + var ex = await Assert.ThrowsAsync(() => BLiteMigration.ToMultiFileAsync(_dbPath, config)); + + Assert.Contains("'users'", ex.Message); + Assert.Contains(largeId.ToString(), ex.Message); + Assert.Contains("one page", ex.Message); + + // The source is intact and still readable by both APIs. + using (var db = new TestDbContext(_dbPath)) + { + Assert.Equal(2, await db.Users.CountAsync()); + Assert.Equal(LargeName(), (await db.Users.FindByIdAsync(largeId))!.Name); + } + + // Nothing of the target is left: no temp file, no .idx, no WAL directory, no collection directory. + Assert.False(File.Exists(_dbPath + ".migrating")); + Assert.False(File.Exists(config.IndexFilePath!), config.IndexFilePath); + Assert.False(File.Exists(config.WalPath!), config.WalPath); + Assert.False(Directory.Exists(config.CollectionDataDirectory!), config.CollectionDataDirectory); + } +} From df2e66cc7989bb6fc3ecd5eb30c6f457899faf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Mon, 19 Oct 2026 06:23:30 +0200 Subject: [PATCH 2/2] fix(core): keep DynamicCollection.MaxDocumentSize internal (#301) Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/BLite.Core/DynamicCollection.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4056993d..f922df89 100644 --- a/README.md +++ b/README.md @@ -776,7 +776,7 @@ BsonId id = await orders.InsertAsync(doc, ct); List ids = await orders.InsertBulkAsync([doc1, doc2, doc3], ct); ``` -> A schema-less document must fit in one page: `DynamicCollection.MaxDocumentSize` bytes of BSON (about 16 KB with the default page size); a larger one is rejected with `BLiteValidationException`. A typed `DocumentCollection` stores larger documents across overflow pages, and the schema-less API reads those, so a typed collection with large documents can be read, queried and shown in BLite Studio, but not copied by `BLiteMigration`. +> A schema-less document must fit in one page: about 16 KB of BSON with the default page size; a larger one is rejected with `BLiteValidationException`. A typed `DocumentCollection` stores larger documents across overflow pages, and the schema-less API reads those, so a typed collection with large documents can be read, queried and shown in BLite Studio, but not copied by `BLiteMigration`. ### Read diff --git a/src/BLite.Core/DynamicCollection.cs b/src/BLite.Core/DynamicCollection.cs index 32220b65..a087518c 100644 --- a/src/BLite.Core/DynamicCollection.cs +++ b/src/BLite.Core/DynamicCollection.cs @@ -60,7 +60,7 @@ public sealed class DynamicCollection : IDisposable /// documents across overflow pages, and this collection reads those (#301); it does not /// write them, so such a document cannot be copied into a DynamicCollection. /// - public int MaxDocumentSize { get; } + internal int MaxDocumentSize { get; } private uint _currentDataPage; private bool _isTimeSeries;