fix(core): the schema-less API reads documents a typed collection stored in overflow pages (#301)

README.md

@@ -776,6 +776,8 @@ BsonId id = await orders.InsertAsync(doc, ct);
 List<BsonId> ids = await orders.InsertBulkAsync([doc1, doc2, doc3], ct);
 ```
 
+> A schema-less document must fit in one page: about 16 KB of BSON with the default page size; a larger one is rejected with `BLiteValidationException`. A typed `DocumentCollection` stores larger documents across overflow pages, and the schema-less API reads those, so a typed collection with large documents can be read, queried and shown in BLite Studio, but not copied by `BLiteMigration`.
+
 ### Read
 
 ```csharp

src/BLite.Core/BLiteMigration.cs

@@ -92,6 +92,9 @@ public static async Task ToMultiFileAsync(string sourcePath, PageFileConfig targ
         // Target main .db is written to a temp path first so that we never
         // leave the source in a partially-overwritten state.
         var tempDbPath = sourcePath + ".migrating";
+        // The target's companion files (.idx, WAL, collection directory) are created at their
+        // final paths; remember which of them this call creates, so a failure removes only those.
+        var createdCompanions = CompanionPathsThatDoNotExist(targetConfig, tempDbPath);
         try
         {
             using (var source = new BLiteEngine(sourcePath, sourceConfig))
@@ -110,10 +113,12 @@ public static async Task ToMultiFileAsync(string sourcePath, PageFileConfig targ
         }
         catch
         {
-            // Best-effort cleanup of the half-written temp file.
+            // Best-effort cleanup of the half-written temp file and the companion files this
+            // call created; the source is untouched.
             SafeDelete(tempDbPath);
             var tempWal = Path.ChangeExtension(tempDbPath, ".wal");
             SafeDelete(tempWal);
+            DeletePaths(createdCompanions);
             throw;
         }
     }
@@ -169,6 +174,7 @@ public static async Task ToSingleFileAsync(
             StringComparison.OrdinalIgnoreCase);
 
         var actualTargetPath = isInPlace ? targetPath + ".migrating" : targetPath;
+        bool targetExistedBefore = !isInPlace && File.Exists(actualTargetPath);
 
         try
         {
@@ -190,7 +196,8 @@ public static async Task ToSingleFileAsync(
         }
         catch
         {
-            if (isInPlace)
+            // Remove the half-written target unless it is a file the caller already had.
+            if (isInPlace || !targetExistedBefore)
             {
                 SafeDelete(actualTargetPath);
                 SafeDelete(Path.ChangeExtension(actualTargetPath, ".wal"));
@@ -250,6 +257,18 @@ private static async Task CopyCollectionsASync(BLiteEngine source, BLiteEngine t
             var batch = new List<BsonDocument>(BatchSize);
             await foreach (var doc in sourceColl.FindAllAsync())
             {
+                // A typed collection may hold a document larger than a page (overflow pages);
+                // the target is a DynamicCollection, which does not write those (#301). Say
+                // which document it is instead of failing inside InsertBulkAsync.
+                if (doc.RawData.Length > targetColl.MaxDocumentSize)
+                {
+                    doc.TryGetId(out var tooLargeId);
+                    throw new NotSupportedException(
+                        $"Collection '{collName}' holds a document of {doc.RawData.Length} bytes (_id {tooLargeId}), " +
+                        $"larger than the {targetColl.MaxDocumentSize} bytes a schema-less collection can store in one page. " +
+                        "BLiteMigration copies through the schema-less API and cannot migrate this database; the source is unchanged.");
+                }
+
                 batch.Add(doc);
                 if (batch.Count >= BatchSize)
                 {
@@ -348,4 +367,30 @@ private static void SafeDelete(string path)
         try { if (File.Exists(path)) File.Delete(path); }
         catch { /* best-effort */ }
     }
+
+    /// <summary>The companion paths of <paramref name="config"/> that do not exist yet, i.e. the ones a migration to it will create.</summary>
+    private static List<string> CompanionPathsThatDoNotExist(PageFileConfig config, string mainDbPath)
+    {
+        var paths = new List<string>();
+        var walPath = config.WalPath ?? Path.ChangeExtension(mainDbPath, ".wal");
+        var walDir = Path.GetDirectoryName(walPath);
+        if (!string.IsNullOrEmpty(walDir) && !Directory.Exists(walDir)) paths.Add(walDir);   // e.g. the "wal/" directory of PageFileConfig.Server
+        else if (!File.Exists(walPath)) paths.Add(walPath);
+        if (config.IndexFilePath != null && !File.Exists(config.IndexFilePath)) paths.Add(config.IndexFilePath);
+        if (config.CollectionDataDirectory != null && !Directory.Exists(config.CollectionDataDirectory)) paths.Add(config.CollectionDataDirectory);
+        return paths;
+    }
+
+    private static void DeletePaths(IEnumerable<string> paths)
+    {
+        foreach (var path in paths)
+        {
+            try
+            {
+                if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
+                else if (File.Exists(path)) File.Delete(path);
+            }
+            catch { /* best-effort */ }
+        }
+    }
 }

src/BLite.Core/DynamicCollection.cs

@@ -52,6 +52,15 @@ public sealed class DynamicCollection : IDisposable
     // Cached delegate — avoids per-call closure allocation when passed to _fsi.FindPage.
     private readonly Func<uint, ulong, bool> _isPageLocked;
     private readonly int _maxDocumentSizeForSinglePage;
+
+    /// <summary>
+    /// The largest document, in BSON bytes, this collection writes: one page minus its header,
+    /// slot and reserve (about 16 KB with the default page size). A larger document is rejected
+    /// with <see cref="BLiteValidationException"/>. A typed <c>DocumentCollection</c> stores larger
+    /// documents across overflow pages, and this collection reads those (#301); it does not
+    /// write them, so such a document cannot be copied into a <c>DynamicCollection</c>.
+    /// </summary>
+    internal int MaxDocumentSize { get; }
     private uint _currentDataPage;
     private bool _isTimeSeries;
 
@@ -118,6 +127,7 @@ internal DynamicCollection(StorageEngine storage, ITransactionHolder transaction
         _collectionName = collectionName ?? throw new ArgumentNullException(nameof(collectionName));
         _idType = idType;
         _maxDocumentSizeForSinglePage = _storage.PageSize - 128;
+        MaxDocumentSize = _maxDocumentSizeForSinglePage - SlotEntry.Size;
         _fsi = freeSpaceIndex ?? new FreeSpaceIndex(_storage.PageSize);
         _isPageLocked = _storage.IsPageLocked;
 
@@ -777,7 +787,7 @@ private async Task<BsonId> InsertCore(BsonDocument document, ITransaction transa
         else
         {
             // Unreachable: ValidateWrite rejects documents that do not fit in a page.
-            throw new BLiteValidationException($"Document size {docData.Length} exceeds maximum single page size. Overflow not yet supported in DynamicCollection.");
+            throw new BLiteValidationException($"Document size {docData.Length} exceeds the {_maxDocumentSizeForSinglePage - SlotEntry.Size} bytes a DynamicCollection document may occupy (one page). DynamicCollection does not write overflow pages; a typed DocumentCollection does.");
         }
 
         // Index the _id
@@ -1019,7 +1029,19 @@ public async IAsyncEnumerable<BsonDocument> ScanAsync(BsonReaderPredicate predic
                 var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
                 if (slot.Flags.HasFlag(SlotFlags.Deleted)) continue;
 
-                var data = buffer.AsSpan(slot.Offset, slot.Length);
+                // A document a typed collection stored across overflow pages (#301) is
+                // reassembled before the predicate sees it; its slot holds only the first chunk.
+                ReadOnlySpan<byte> data;
+                if (slot.Flags.HasFlag(SlotFlags.HasOverflow))
+                {
+                    var full = OverflowChainReader.Read(_storage, buffer, slot, txnId);
+                    if (full == null) continue;
+                    data = full;
+                }
+                else
+                {
+                    data = buffer.AsSpan(slot.Offset, slot.Length);
+                }
                 var reader = new BsonSpanReader(data, _storage.GetKeyReverseMap());
 
                 if (predicate(reader))
@@ -2232,7 +2254,7 @@ private void ValidateWrite(BsonId id, IndexKey key, BsonDocument document, BsonD
         var size = document.RawData.Length;
         if (!_isTimeSeries && size + SlotEntry.Size > _maxDocumentSizeForSinglePage)
             throw new BLiteValidationException(
-                $"Document size {size} exceeds maximum single page size. Overflow not yet supported in DynamicCollection.");
+                $"Document size {size} exceeds the {_maxDocumentSizeForSinglePage - SlotEntry.Size} bytes a DynamicCollection document may occupy (one page). DynamicCollection does not write overflow pages; a typed DocumentCollection does.");
 
         // An update keeps the id it was found by, so only a new id is checked: one stored before
         // MaxKeySize was lowered (#217) must not make its document impossible to update.
@@ -2462,6 +2484,14 @@ private bool TryUpdateInPlace(DocumentLocation location, ReadOnlyMemory<byte> da
             if (slot.Flags.HasFlag(SlotFlags.Deleted))
                 return null;
 
+            // A typed collection stores a document larger than a page across overflow pages;
+            // the slot then holds [length][first overflow page][first chunk] (#301).
+            if (slot.Flags.HasFlag(SlotFlags.HasOverflow))
+            {
+                var full = OverflowChainReader.Read(_storage, buffer, slot, txnId);
+                return full == null ? null : new BsonDocument(full, _storage.GetKeyReverseMap(), _storage.GetKeyMap());
+            }
+
             // Copy document data (buffer is pooled, data must outlive it)
             var docData = buffer.AsSpan(slot.Offset, slot.Length).ToArray();
             return new BsonDocument(docData, _storage.GetKeyReverseMap(), _storage.GetKeyMap());
@@ -2507,6 +2537,14 @@ private bool TryUpdateInPlace(DocumentLocation location, ReadOnlyMemory<byte> da
             var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
             if (slot.Flags.HasFlag(SlotFlags.Deleted)) return null;
 
+            if (slot.Flags.HasFlag(SlotFlags.HasOverflow))
+            {
+                // Overflow document of a typed collection (#301); the reader copies what it
+                // needs from the page buffer before it awaits.
+                var full = await OverflowChainReader.ReadAsync(_storage, buffer, slot, txnId, ct).ConfigureAwait(false);
+                return full == null ? null : new BsonDocument(full, _storage.GetKeyReverseMap(), _storage.GetKeyMap());
+            }
+
             // Span created after the only await — safe; ToArray() copies before buffer is returned
             var docData = buffer.AsSpan(slot.Offset, slot.Length).ToArray();
             return new BsonDocument(docData, _storage.GetKeyReverseMap(), _storage.GetKeyMap());
@@ -2682,6 +2720,16 @@ private void DeleteSlot(DocumentLocation location, ITransaction transaction)
             {
                 var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
                 var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
+
+                // A document a typed collection stored across overflow pages (#301): recycle
+                // its chain once the transaction commits, as DocumentCollection.FreeOverflowChain
+                // does; until then a rollback returns to the committed document, which uses them.
+                if (slot.Flags.HasFlag(SlotFlags.HasOverflow))
+                {
+                    foreach (var overflowPageId in OverflowChainReader.ChainPages(_storage, buffer, slot, transaction.TransactionId))
+                        _storage.FreePageOnCommit(overflowPageId, transaction.TransactionId);
+                }
+
                 slot.Flags |= SlotFlags.Deleted;
                 slot.WriteTo(buffer.AsSpan(slotOffset));
 

src/BLite.Core/Storage/OverflowChainReader.cs

@@ -0,0 +1,149 @@
+using System.Buffers;
+using System.Buffers.Binary;
+
+namespace BLite.Core.Storage;
+
+/// <summary>
+/// Reassembles a document that a typed <c>DocumentCollection</c> stored across overflow pages,
+/// so that the schema-less read paths (<c>DynamicCollection</c>, and through it BLQL, Studio and
+/// <c>BLiteMigration</c>) can read every document the typed API wrote (#301).
+/// <para>
+/// A slot flagged <see cref="SlotFlags.HasOverflow"/> holds
+/// <c>[TotalLength (4)] [FirstOverflowPage (4)] [first chunk of the document]</c>; every overflow
+/// page holds <c>PageSize - SlottedPageHeader.Size</c> bytes of payload after its header and
+/// names the next page in <see cref="SlottedPageHeader.NextOverflowPage"/> (0 ends the chain).
+/// This is the layout <c>DocumentCollection.InsertWithOverflow</c> writes and
+/// <c>DocumentCollection.ReadRawBytesAt</c> reads.
+/// </para>
+/// </summary>
+internal static class OverflowChainReader
+{
+    /// <summary>Bytes of slot payload that precede the first chunk: total length and first overflow page id.</summary>
+    public const int SlotMetadataSize = 8;
+
+    /// <summary>
+    /// Reads the whole document whose first chunk is in <paramref name="slot"/> of
+    /// <paramref name="pageBuffer"/>. Returns <c>null</c> when the slot does not carry an
+    /// overflow document or the chain is inconsistent (truncated, or shorter than its declared
+    /// length), never a partial document.
+    /// </summary>
+    public static byte[]? Read(StorageEngine storage, ReadOnlySpan<byte> pageBuffer, in SlotEntry slot, ulong txnId)
+    {
+        if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk))
+            return null;
+
+        var document = new byte[totalLength];
+        firstChunk.CopyTo(document);
+        int offset = firstChunk.Length;
+
+        var overflowBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
+        try
+        {
+            while (nextPageId != 0 && offset < totalLength)
+            {
+                storage.ReadPage(nextPageId, txnId, overflowBuffer);
+                nextPageId = AppendChunk(storage.PageSize, overflowBuffer, document, ref offset);
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(overflowBuffer);
+        }
+
+        return offset == totalLength ? document : null;
+    }
+
+    /// <summary>Async counterpart of <see cref="Read"/>, reading the chain with <see cref="StorageEngine.ReadPageAsync"/>.</summary>
+    public static async ValueTask<byte[]?> ReadAsync(StorageEngine storage, byte[] pageBuffer, SlotEntry slot, ulong txnId, CancellationToken ct)
+    {
+        // Everything taken from the page buffer is copied before the first await: the caller's
+        // buffer may be pooled, and a span cannot live across an await anyway.
+        if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk))
+            return null;
+
+        var document = new byte[totalLength];
+        firstChunk.CopyTo(document);
+        int offset = firstChunk.Length;
+
+        var overflowBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
+        try
+        {
+            while (nextPageId != 0 && offset < totalLength)
+            {
+                ct.ThrowIfCancellationRequested();
+                await storage.ReadPageAsync(nextPageId, txnId, overflowBuffer.AsMemory(0, storage.PageSize), ct).ConfigureAwait(false);
+                nextPageId = AppendChunk(storage.PageSize, overflowBuffer, document, ref offset);
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(overflowBuffer);
+        }
+
+        return offset == totalLength ? document : null;
+    }
+
+    /// <summary>
+    /// The page ids of the overflow chain that starts in <paramref name="slot"/>, in chain
+    /// order, so a caller that deletes the document can free them. Empty when the slot carries
+    /// no overflow document. Stops at a page that does not look like an overflow page, so a
+    /// chain corrupted by an earlier failure cannot free a page another document uses.
+    /// </summary>
+    public static List<uint> ChainPages(StorageEngine storage, ReadOnlySpan<byte> pageBuffer, in SlotEntry slot, ulong txnId)
+    {
+        var pages = new List<uint>();
+        if (!TryReadHead(pageBuffer, slot, out var totalLength, out var nextPageId, out var firstChunk))
+            return pages;
+
+        int offset = firstChunk.Length;
+        var overflowBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
+        try
+        {
+            while (nextPageId != 0 && offset < totalLength)
+            {
+                storage.ReadPage(nextPageId, txnId, overflowBuffer);
+                var header = SlottedPageHeader.ReadFrom(overflowBuffer);
+                if (header.PageType != PageType.Overflow || pages.Contains(nextPageId))
+                    break;
+                pages.Add(nextPageId);
+                offset += Math.Min(storage.PageSize - SlottedPageHeader.Size, totalLength - offset);
+                nextPageId = header.NextOverflowPage;
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(overflowBuffer);
+        }
+        return pages;
+    }
+
+    private static bool TryReadHead(ReadOnlySpan<byte> pageBuffer, in SlotEntry slot, out int totalLength, out uint nextPageId, out ReadOnlySpan<byte> firstChunk)
+    {
+        totalLength = 0;
+        nextPageId = 0;
+        firstChunk = default;
+
+        if ((slot.Flags & SlotFlags.HasOverflow) == 0) return false;
+        if (slot.Length < SlotMetadataSize || slot.Offset + slot.Length > pageBuffer.Length) return false;
+
+        var payload = pageBuffer.Slice(slot.Offset, slot.Length);
+        totalLength = BinaryPrimitives.ReadInt32LittleEndian(payload);
+        nextPageId = BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(4));
+
+        int firstChunkSize = slot.Length - SlotMetadataSize;
+        if (totalLength <= 0 || firstChunkSize > totalLength) return false;
+
+        firstChunk = payload.Slice(SlotMetadataSize, firstChunkSize);
+        return true;
+    }
+
+    /// <summary>Copies one overflow page's payload into <paramref name="document"/> and returns the next page id.</summary>
+    private static uint AppendChunk(int pageSize, byte[] overflowBuffer, byte[] document, ref int offset)
+    {
+        var header = SlottedPageHeader.ReadFrom(overflowBuffer);
+        int chunkSize = Math.Min(pageSize - SlottedPageHeader.Size, document.Length - offset);
+        overflowBuffer.AsSpan(SlottedPageHeader.Size, chunkSize).CopyTo(document.AsSpan(offset));
+        offset += chunkSize;
+        return header.NextOverflowPage;
+    }
+}