From f5a6399bf5235147a4fb4b88bc434e6f6a78bacd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:15:34 +0000 Subject: [PATCH 1/2] Stream ISO 8211 records and slice field data instead of copying it Breaking: Iso8211Field.Data is now ReadOnlyMemory (was byte[]). - Iso8211DocumentReader.EnumerateRecords(ReadOnlyMemory) returns one record at a time, so reading a file no longer needs memory that grows with its size. The S-57 reader and, with a one-line change, EncDotNet.S100's S-101 reader can share it. - Field data is a slice of the data read rather than a copy per field. Read(byte[]), Read(ReadOnlySpan), files and streams read from one private copy, so the document never shares the caller's buffer; the new Read(ReadOnlyMemory) and EnumerateRecords slice the caller's memory directly and say so. - One string per distinct tag instead of one per directory entry. - Iso8211FieldReader takes ReadOnlyMemory, so existing calls such as new Iso8211FieldReader(definition, field.Data) compile unchanged. - The break is recorded in CompatibilitySuppressions.xml for the next minor release, per docs/versioning.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YagvELyJKBANk61DyFGUNJ --- docs/reading-iso8211.md | 31 +++ scripts/DumpCatalog.cs | 4 +- scripts/DumpChart.cs | 4 +- .../CompatibilitySuppressions.xml | 18 ++ .../Iso8211DocumentReader.cs | 215 ++++++++++++++++-- src/EncDotNet.Iso8211/Iso8211Field.cs | 12 +- src/EncDotNet.Iso8211/Iso8211FieldReader.cs | 20 ++ src/EncDotNet.Iso8211/Iso8211RecordWriter.cs | 2 +- .../Iso8211DocumentReaderStreamingTests.cs | 107 +++++++++ .../Iso8211FieldReaderEquivalenceTests.cs | 9 +- 10 files changed, 389 insertions(+), 33 deletions(-) create mode 100644 src/EncDotNet.Iso8211/CompatibilitySuppressions.xml create mode 100644 tests/EncDotNet.UnitTests/Iso8211DocumentReaderStreamingTests.cs diff --git a/docs/reading-iso8211.md b/docs/reading-iso8211.md index ba49334..16bf749 100644 --- a/docs/reading-iso8211.md +++ b/docs/reading-iso8211.md @@ -104,6 +104,37 @@ subfields it lacks. Character data is read as UTF-8 by default. Pass `lexicalLevel: 2` to the `Iso8211FieldReader` constructor for fields encoded as UCS-2 (UTF-16LE). +## Read a large file record by record + +`Iso8211DocumentReader.EnumerateRecords` returns one record at a time instead +of building the whole document, so memory use doesn't grow with the size of the +file. Each record can be released once you have processed it: + +```csharp +using EncDotNet.Iso8211; + +byte[] data = File.ReadAllBytes("path/to/file.000"); +Iso8211DataDescriptiveRecord? ddr = null; +int fieldCount = 0; + +foreach (Iso8211Record record in Iso8211DocumentReader.EnumerateRecords(data)) +{ + if (record.IsDataDescriptiveRecord) + { + ddr = Iso8211DataDescriptiveRecordReader.Read(record); + continue; + } + + fieldCount += record.Fields.Count; +} +``` + +Each field's `Data` is a `ReadOnlyMemory` slice of the data read, not a +copy. `EnumerateRecords` and `Read(ReadOnlyMemory)` slice the memory you +pass, so don't modify it while you use the records. The overloads that take a +`byte[]`, a span, a file or a stream read from a private copy, so you can reuse +your buffer straight away. + ## Scan a file token by token `Iso8211Reader` works like `System.Text.Json.Utf8JsonReader`: call `Read()` diff --git a/scripts/DumpCatalog.cs b/scripts/DumpCatalog.cs index 2df65c9..80d2d92 100644 --- a/scripts/DumpCatalog.cs +++ b/scripts/DumpCatalog.cs @@ -105,7 +105,7 @@ } // Hex dump - DumpHex(field.Data, "│ │ "); + DumpHex(field.Data.Span, "│ │ "); Console.WriteLine("│ └────────────────────────────────────────────────────"); } @@ -205,7 +205,7 @@ // Helper: Hex dump // ============================================================================ -static void DumpHex(byte[] data, string linePrefix) +static void DumpHex(ReadOnlySpan data, string linePrefix) { const int bytesPerLine = 16; int maxBytes = Math.Min(data.Length, 256); diff --git a/scripts/DumpChart.cs b/scripts/DumpChart.cs index 685687a..df1c9b3 100644 --- a/scripts/DumpChart.cs +++ b/scripts/DumpChart.cs @@ -106,7 +106,7 @@ } // Hex dump - DumpHex(field.Data, "│ │ "); + DumpHex(field.Data.Span, "│ │ "); Console.WriteLine("│ └────────────────────────────────────────────────────"); } @@ -359,7 +359,7 @@ // Helper: Hex dump // ============================================================================ -static void DumpHex(byte[] data, string linePrefix) +static void DumpHex(ReadOnlySpan data, string linePrefix) { const int bytesPerLine = 16; int maxBytes = Math.Min(data.Length, 256); diff --git a/src/EncDotNet.Iso8211/CompatibilitySuppressions.xml b/src/EncDotNet.Iso8211/CompatibilitySuppressions.xml new file mode 100644 index 0000000..1b1008c --- /dev/null +++ b/src/EncDotNet.Iso8211/CompatibilitySuppressions.xml @@ -0,0 +1,18 @@ + + + + + CP0002 + M:EncDotNet.Iso8211.Iso8211Field.get_Data + lib/net10.0/EncDotNet.Iso8211.dll + lib/net10.0/EncDotNet.Iso8211.dll + true + + + CP0002 + M:EncDotNet.Iso8211.Iso8211Field.get_Data + lib/net8.0/EncDotNet.Iso8211.dll + lib/net8.0/EncDotNet.Iso8211.dll + true + + \ No newline at end of file diff --git a/src/EncDotNet.Iso8211/Iso8211DocumentReader.cs b/src/EncDotNet.Iso8211/Iso8211DocumentReader.cs index e7b4e16..e059cfe 100644 --- a/src/EncDotNet.Iso8211/Iso8211DocumentReader.cs +++ b/src/EncDotNet.Iso8211/Iso8211DocumentReader.cs @@ -1,33 +1,67 @@ +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + namespace EncDotNet.Iso8211; /// /// Provides methods to read ISO 8211 formatted data and return structured objects. /// /// -/// This reader uses internally for parsing -/// and builds a complete object model of the ISO 8211 data. +/// +/// This reader uses internally for parsing. The Read methods +/// build a complete ; +/// returns one record at a time, so memory use doesn't grow with the size of the file. +/// +/// +/// Each field's is a slice of the data read rather than a copy +/// of its own. The methods that take a array, a span, a file or a stream read +/// from a private copy, so the document never shares memory with the caller. The methods that +/// take a slice the caller's memory directly. +/// /// public static class Iso8211DocumentReader { /// /// Reads an ISO 8211 document from a byte array. /// - /// The ISO 8211 data to read. + /// The ISO 8211 data to read. The document is read from a copy, so the + /// array can be reused afterwards. /// The parsed ISO 8211 document. public static Iso8211Document Read(byte[] data) { - return Read(data.AsSpan()); + ArgumentNullException.ThrowIfNull(data); + return Read(data.AsMemory().ToArray().AsMemory()); } /// /// Reads an ISO 8211 document from a span of bytes. /// - /// The ISO 8211 data to read. + /// The ISO 8211 data to read. The document is read from a copy. /// The parsed ISO 8211 document. public static Iso8211Document Read(ReadOnlySpan data) { - var reader = new Iso8211Reader(data); - return Read(ref reader); + return Read(data.ToArray().AsMemory()); + } + + /// + /// Reads an ISO 8211 document from memory, without copying it. + /// + /// The ISO 8211 data to read. + /// The parsed ISO 8211 document. + /// + /// Every field's is a slice of , so + /// the document holds a reference to it. Don't modify or reuse the memory while the document + /// is in use. To read from a buffer you will reuse, call , which + /// reads from a copy. + /// + public static Iso8211Document Read(ReadOnlyMemory data) + { + return new Iso8211Document + { + Records = [.. EnumerateRecords(data)] + }; } /// @@ -37,8 +71,7 @@ public static Iso8211Document Read(ReadOnlySpan data) /// The parsed ISO 8211 document. public static Iso8211Document ReadFromFile(string path) { - var data = File.ReadAllBytes(path); - return Read(data); + return Read(File.ReadAllBytes(path).AsMemory()); } /// @@ -50,7 +83,7 @@ public static Iso8211Document ReadFromFile(string path) public static async Task ReadFromFileAsync(string path, CancellationToken cancellationToken = default) { var data = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); - return Read(data); + return Read(data.AsMemory()); } /// @@ -62,7 +95,7 @@ public static Iso8211Document Read(Stream stream) { using var memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); - return Read(memoryStream.ToArray()); + return Read(WrittenBytes(memoryStream)); } /// @@ -75,24 +108,28 @@ public static async Task ReadAsync(Stream stream, CancellationT { using var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); - return Read(memoryStream.ToArray()); + return Read(WrittenBytes(memoryStream)); } /// - /// Reads an ISO 8211 document using a ForwardOnlyIso8211Reader. + /// Reads an ISO 8211 document using an . /// /// The forward-only reader to use for parsing. /// The parsed ISO 8211 document. + /// + /// The reader only exposes spans, so each field's data is copied. When you have the data as + /// memory, avoids the copies. + /// public static Iso8211Document Read(ref Iso8211Reader parser) { var records = new List(); + var tags = new TagTable(); while (parser.Read()) { if (parser.TokenType == Iso8211TokenType.StartRecord) { - var record = ReadRecord(ref parser); - records.Add(record); + records.Add(ReadRecord(ref parser, tags, ReadOnlyMemory.Empty, default)); } else if (parser.TokenType == Iso8211TokenType.EndOfData) { @@ -106,12 +143,89 @@ public static Iso8211Document Read(ref Iso8211Reader parser) }; } + /// + /// Reads ISO 8211 records one at a time, without building a whole document. + /// + /// The ISO 8211 data to read. + /// + /// The records in file order, the Data Descriptive Record (DDR) first. Records are read as + /// the sequence is enumerated, so a record that has been processed can be released. + /// + /// + /// + /// Every field's is a slice of , so + /// don't modify the memory while the records are in use. + /// + /// + /// Enumeration stops at the end of the data, or at a record that can't be read, such as one + /// cut short by the end of the data. + /// + /// + /// + /// + /// byte[] data = File.ReadAllBytes("path/to/file.000"); + /// Iso8211DataDescriptiveRecord? ddr = null; + /// foreach (Iso8211Record record in Iso8211DocumentReader.EnumerateRecords(data)) + /// { + /// if (record.IsDataDescriptiveRecord) + /// { + /// ddr = Iso8211DataDescriptiveRecordReader.Read(record); + /// continue; + /// } + /// + /// // Decode the record's fields with ddr, then let it go. + /// } + /// + /// + public static IEnumerable EnumerateRecords(ReadOnlyMemory data) + { + var tags = new TagTable(); + var offset = 0; + + while (TryReadRecordAt(data, offset, tags, out var record, out var length)) + { + yield return record; + offset += length; + } + } + + /// + /// Reads the record that starts at . + /// + /// + /// A separate method because is a ref struct, which an + /// iterator can't hold across a yield. + /// + private static bool TryReadRecordAt(ReadOnlyMemory data, int offset, TagTable tags, out Iso8211Record record, out int length) + { + var remaining = data.Span[offset..]; + var parser = new Iso8211Reader(remaining); + if (!parser.Read() || parser.TokenType != Iso8211TokenType.StartRecord) + { + record = null!; + length = 0; + return false; + } + + // The leader's length, or the length derived from the directory for a record too long + // for the leader's five digits. + length = parser.CurrentLeader.RecordLength; + record = ReadRecord(ref parser, tags, data[offset..], remaining); + return true; + } + /// /// Reads a single ISO 8211 record from the reader. /// /// The forward-only reader positioned at a StartRecord token. + /// Tag strings already seen, so each distinct tag is allocated once. + /// + /// The memory reads, which fields are sliced from; empty to copy + /// each field's data instead. + /// + /// The span was created over. /// The parsed record. - private static Iso8211Record ReadRecord(ref Iso8211Reader parser) + private static Iso8211Record ReadRecord(ref Iso8211Reader parser, TagTable tags, ReadOnlyMemory source, ReadOnlySpan sourceSpan) { if (parser.TokenType != Iso8211TokenType.StartRecord) { @@ -119,8 +233,8 @@ private static Iso8211Record ReadRecord(ref Iso8211Reader parser) } var leader = Iso8211RecordLeader.FromLeader(parser.CurrentLeader); - var directoryEntries = new List(); - var fields = new List(); + var directoryEntries = new List(parser.DirectoryEntryCount); + var fields = new List(parser.DirectoryEntryCount); // Read directory entries while (parser.Read()) @@ -129,7 +243,7 @@ private static Iso8211Record ReadRecord(ref Iso8211Reader parser) { var entry = new Iso8211DirectoryEntry { - Tag = parser.GetTagString(), + Tag = tags.Get(parser.CurrentTag), Length = parser.CurrentLength, Position = parser.CurrentPosition }; @@ -138,7 +252,7 @@ private static Iso8211Record ReadRecord(ref Iso8211Reader parser) else if (parser.TokenType == Iso8211TokenType.Field) { // First field read via auto-transition from directory to field area - fields.Add(new Iso8211Field { Tag = directoryEntries[fields.Count].Tag, Data = parser.ValueSpan.ToArray() }); + fields.Add(new Iso8211Field { Tag = directoryEntries[fields.Count].Tag, Data = FieldData(ref parser, source, sourceSpan) }); break; } else if (parser.TokenType == Iso8211TokenType.EndRecord) @@ -154,7 +268,7 @@ private static Iso8211Record ReadRecord(ref Iso8211Reader parser) { var entry = directoryEntries[i]; parser.TryReadFieldDirect(entry.Position, entry.Length); - fields.Add(new Iso8211Field { Tag = entry.Tag, Data = parser.ValueSpan.ToArray() }); + fields.Add(new Iso8211Field { Tag = entry.Tag, Data = FieldData(ref parser, source, sourceSpan) }); } // Advance past end of record @@ -170,4 +284,63 @@ private static Iso8211Record ReadRecord(ref Iso8211Reader parser) Fields = fields }; } + + /// + /// Gets the current field's data: a slice of , or a copy when it's empty. + /// + private static ReadOnlyMemory FieldData(ref Iso8211Reader parser, ReadOnlyMemory source, ReadOnlySpan sourceSpan) + { + var value = parser.ValueSpan; + if (source.IsEmpty) + { + return value.ToArray(); + } + + // The reader's value is a slice of the span it was given; find where it starts. + var start = (int)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(sourceSpan), ref MemoryMarshal.GetReference(value)); + return source.Slice(start, value.Length); + } + + /// + /// Gets the bytes written to a memory stream, without copying them. + /// + private static ReadOnlyMemory WrittenBytes(MemoryStream stream) => + stream.TryGetBuffer(out var buffer) ? buffer.AsMemory() : stream.ToArray(); + + /// + /// Shares one string per distinct field tag, so a document with tens of thousands of fields + /// holds a handful of tag strings rather than one per directory entry. + /// + private sealed class TagTable + { + private readonly Dictionary _shortTags = []; + + public string Get(ReadOnlySpan tag) + { + if (tag.IsEmpty) + { + // As Iso8211Reader.GetTagString reports it. + throw new InvalidOperationException("No tag available for the current token."); + } + + // Tags are at most a few ASCII characters, usually four. Pack those bytes and the + // length into one key; the length keeps "ABC" and "ABC\0" apart. + if (tag.Length <= 4) + { + Span packed = stackalloc byte[4]; + packed.Clear(); + tag.CopyTo(packed); + var key = BinaryPrimitives.ReadUInt32LittleEndian(packed) | ((long)tag.Length << 32); + if (!_shortTags.TryGetValue(key, out var value)) + { + value = Encoding.ASCII.GetString(tag); + _shortTags[key] = value; + } + + return value; + } + + return Encoding.ASCII.GetString(tag); + } + } } diff --git a/src/EncDotNet.Iso8211/Iso8211Field.cs b/src/EncDotNet.Iso8211/Iso8211Field.cs index befb281..5e6c9f8 100644 --- a/src/EncDotNet.Iso8211/Iso8211Field.cs +++ b/src/EncDotNet.Iso8211/Iso8211Field.cs @@ -13,9 +13,15 @@ public sealed record Iso8211Field public string Tag { get; init; } = string.Empty; /// - /// Gets the raw field data. + /// Gets the raw field data, without the field terminator. /// - public byte[] Data { get; init; } = Array.Empty(); + /// + /// Fields read by are slices of the data it read, not + /// copies. Whether that data is shared with the caller depends on the read method: see + /// . A array + /// converts implicitly when you build a field. + /// + public ReadOnlyMemory Data { get; init; } /// /// Gets the field data as a string using the specified encoding. @@ -25,6 +31,6 @@ public sealed record Iso8211Field public string GetDataString(Encoding? encoding = null) { encoding ??= Encoding.ASCII; - return encoding.GetString(Data).TrimEnd('\x1f', '\x1e'); + return encoding.GetString(Data.Span).TrimEnd('\x1f', '\x1e'); } } diff --git a/src/EncDotNet.Iso8211/Iso8211FieldReader.cs b/src/EncDotNet.Iso8211/Iso8211FieldReader.cs index 92a4f2d..ec05850 100644 --- a/src/EncDotNet.Iso8211/Iso8211FieldReader.cs +++ b/src/EncDotNet.Iso8211/Iso8211FieldReader.cs @@ -89,6 +89,26 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, i { } + /// + /// Initializes a new instance of the class. + /// + /// The field definition from the DDR describing this field's structure. + /// + /// The raw field data to read, such as an . The reader reads it in + /// place, without copying it. + /// + /// + /// The lexical level for character data (0 or 1 = ASCII/ISO 8859, 2 = UCS-2/UTF-16LE). + /// When set to 2, variable-length character subfields use 2-byte terminators (0x1F 0x00 / 0x1E 0x00). + /// + /// + /// Thrown when is null. + /// + public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlyMemory data, int lexicalLevel = 0) + : this(fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)), data, lexicalLevel, validated: true) + { + } + private Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlyMemory data, int lexicalLevel, bool validated) { _ = validated; diff --git a/src/EncDotNet.Iso8211/Iso8211RecordWriter.cs b/src/EncDotNet.Iso8211/Iso8211RecordWriter.cs index dd1ffe2..4057ef5 100644 --- a/src/EncDotNet.Iso8211/Iso8211RecordWriter.cs +++ b/src/EncDotNet.Iso8211/Iso8211RecordWriter.cs @@ -65,7 +65,7 @@ public static void Write(Iso8211Record record, Iso8211WriterOptions options, Lis // Field area. for (int i = 0; i < fields.Count; i++) { - output.AddRange(fields[i].Data); + output.AddRange(fields[i].Data.Span); output.Add(FieldTerminator); } } diff --git a/tests/EncDotNet.UnitTests/Iso8211DocumentReaderStreamingTests.cs b/tests/EncDotNet.UnitTests/Iso8211DocumentReaderStreamingTests.cs new file mode 100644 index 0000000..1cbc293 --- /dev/null +++ b/tests/EncDotNet.UnitTests/Iso8211DocumentReaderStreamingTests.cs @@ -0,0 +1,107 @@ +using System.Runtime.InteropServices; +using EncDotNet.Iso8211; +using EncDotNet.Testing; + +namespace EncDotNet.UnitTests; + +/// +/// Tests for and for how +/// each read method shares, or doesn't share, the caller's memory. +/// +public class Iso8211DocumentReaderStreamingTests +{ + [Fact] + public void EnumerateRecords_ReturnsTheSameRecordsAsRead() + { + // The Large profile includes a record longer than 99 999 bytes, whose leader length is 00000. + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Large); + + var document = Iso8211DocumentReader.Read(bytes); + var streamed = Iso8211DocumentReader.EnumerateRecords(bytes).ToList(); + + Assert.Equal(document.Records.Count, streamed.Count); + for (var r = 0; r < streamed.Count; r++) + { + var expected = document.Records[r]; + var actual = streamed[r]; + Assert.Equal(expected.Leader, actual.Leader); + Assert.Equal(expected.Directory, actual.Directory); + Assert.Equal(expected.Fields.Count, actual.Fields.Count); + for (var f = 0; f < actual.Fields.Count; f++) + { + Assert.Equal(expected.Fields[f].Tag, actual.Fields[f].Tag); + Assert.True(expected.Fields[f].Data.Span.SequenceEqual(actual.Fields[f].Data.Span), $"record {r} field {f}"); + } + } + } + + [Fact] + public void EnumerateRecords_IsLazy() + { + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Harbour); + + // Taking the first two records reads only those two. + var first = Iso8211DocumentReader.EnumerateRecords(bytes).Take(2).ToList(); + + Assert.True(first[0].IsDataDescriptiveRecord); + Assert.Equal("0001", first[1].Fields[0].Tag); + } + + [Fact] + public void EnumerateRecords_StopsAtATruncatedRecord() + { + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Harbour); + var all = Iso8211DocumentReader.Read(bytes).Records.Count; + + var truncated = Iso8211DocumentReader.EnumerateRecords(bytes.AsMemory(0, bytes.Length - 10)).Count(); + + Assert.Equal(all - 1, truncated); + } + + [Fact] + public void Read_ByteArray_ReadsFromACopy() + { + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Harbour); + var document = Iso8211DocumentReader.Read(bytes); + var field = document.Records[1].Fields[1]; + var before = field.Data.ToArray(); + + // Reusing the caller's buffer must not change the document. + Array.Clear(bytes); + + Assert.Equal(before, field.Data.ToArray()); + } + + [Fact] + public void Read_Memory_SlicesTheCallersMemory() + { + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Harbour); + var document = Iso8211DocumentReader.Read(bytes.AsMemory()); + + Assert.True(MemoryMarshal.TryGetArray(document.Records[1].Fields[1].Data, out var segment)); + Assert.Same(bytes, segment.Array); + } + + [Fact] + public void Read_Stream_ReadsTheSameRecords() + { + var bytes = SyntheticS57Cell.Create(SyntheticCellProfile.Harbour); + + var document = Iso8211DocumentReader.Read(new MemoryStream(bytes)); + + Assert.Equal(Iso8211DocumentReader.Read(bytes).Records.Count, document.Records.Count); + Assert.True(document.Records[^1].Fields[^1].Data.Span.SequenceEqual( + Iso8211DocumentReader.Read(bytes).Records[^1].Fields[^1].Data.Span)); + } + + [Fact] + public void Read_SharesOneStringPerTag() + { + var document = Iso8211DocumentReader.Read(SyntheticS57Cell.Create(SyntheticCellProfile.Harbour)); + + var vrids = document.DataRecords.SelectMany(r => r.Fields).Where(f => f.Tag == "VRID").Select(f => f.Tag).ToList(); + + Assert.True(vrids.Count > 1); + Assert.All(vrids, tag => Assert.Same(vrids[0], tag)); + } +} diff --git a/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs b/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs index 2f808a8..dcc0a48 100644 --- a/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs +++ b/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs @@ -106,15 +106,16 @@ private static void AssertCellReadsTheSame(byte[] cell, bool truncate) continue; } - AssertReadsTheSame(definition, field.Data, lexicalLevel: 0); + var data = field.Data.ToArray(); + AssertReadsTheSame(definition, data, lexicalLevel: 0); // Truncating every field is slow; a few hundred cover each field type many times. if (truncate && truncated++ < 150) { - for (var length = 0; length < field.Data.Length; length++) + for (var length = 0; length < data.Length; length++) { - AssertReadsTheSame(definition, field.Data[..length], lexicalLevel: 0); - AssertReadsTheSame(definition, [.. field.Data[..length], FieldTerminator], lexicalLevel: 0); + AssertReadsTheSame(definition, data[..length], lexicalLevel: 0); + AssertReadsTheSame(definition, [.. data[..length], FieldTerminator], lexicalLevel: 0); } } } From 48546457cf06cac995ec0c811ad2a58297bdd9d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 23:15:34 +0000 Subject: [PATCH 2/2] Read S-57 cells record by record with right-sized collections S57DocumentReader now streams records with Iso8211DocumentReader.EnumerateRecords instead of building the whole ISO 8211 document, and reads a byte array in place, since the S-57 model keeps decoded values rather than slices of the input. It also sizes each record's collection builders from the fields' group counts, skips them when a field is absent, and finds fields with a loop instead of GetFieldsByTag's iterator. Previously every record allocated several eight-slot builder arrays and copied each again on ToImmutable. On the synthetic harbour cell, S57DocumentReader.Read allocates 3.7 MB, down from 8.0 MB. Its output is unchanged: the fingerprint tests pass, and locally so do fingerprints of real NOAA cells. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YagvELyJKBANk61DyFGUNJ --- src/EncDotNet.S57/S57DocumentReader.cs | 154 +++++++++++++----- .../AllocationBudgetTests.cs | 18 ++ .../PerformanceBudgets.cs | 7 +- 3 files changed, 139 insertions(+), 40 deletions(-) diff --git a/src/EncDotNet.S57/S57DocumentReader.cs b/src/EncDotNet.S57/S57DocumentReader.cs index b482137..3cf41fe 100644 --- a/src/EncDotNet.S57/S57DocumentReader.cs +++ b/src/EncDotNet.S57/S57DocumentReader.cs @@ -31,8 +31,11 @@ public static class S57DocumentReader /// The parsed S-57 document. public static S57Document Read(byte[] data, ILogger? logger = null) { - var iso8211Document = Iso8211DocumentReader.Read(data); - return ParseDocument(iso8211Document, logger); + ArgumentNullException.ThrowIfNull(data); + + // The S-57 model holds decoded values, not slices of the input, so the records can be + // read from the caller's array without a copy. + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(data), logger); } /// @@ -43,8 +46,7 @@ public static S57Document Read(byte[] data, ILogger? logger = null) /// The parsed S-57 document. public static S57Document Read(ReadOnlySpan data, ILogger? logger = null) { - var iso8211Document = Iso8211DocumentReader.Read(data); - return ParseDocument(iso8211Document, logger); + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(data.ToArray()), logger); } /// @@ -55,8 +57,7 @@ public static S57Document Read(ReadOnlySpan data, ILogger? logger = null) /// The parsed S-57 document. public static S57Document ReadFromFile(string path, ILogger? logger = null) { - var iso8211Document = Iso8211DocumentReader.ReadFromFile(path); - return ParseDocument(iso8211Document, logger); + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(File.ReadAllBytes(path)), logger); } /// @@ -68,8 +69,8 @@ public static S57Document ReadFromFile(string path, ILogger? logger = null) /// A task that represents the asynchronous read operation. public static async Task ReadFromFileAsync(string path, ILogger? logger = null, CancellationToken cancellationToken = default) { - var iso8211Document = await Iso8211DocumentReader.ReadFromFileAsync(path, cancellationToken).ConfigureAwait(false); - return ParseDocument(iso8211Document, logger); + var data = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(data), logger); } /// @@ -80,8 +81,9 @@ public static async Task ReadFromFileAsync(string path, ILogger? lo /// The parsed S-57 document. public static S57Document Read(Stream stream, ILogger? logger = null) { - var iso8211Document = Iso8211DocumentReader.Read(stream); - return ParseDocument(iso8211Document, logger); + using var memoryStream = new MemoryStream(); + stream.CopyTo(memoryStream); + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(WrittenBytes(memoryStream)), logger); } /// @@ -93,19 +95,25 @@ public static S57Document Read(Stream stream, ILogger? logger = null) /// A task that represents the asynchronous read operation. public static async Task ReadAsync(Stream stream, ILogger? logger = null, CancellationToken cancellationToken = default) { - var iso8211Document = await Iso8211DocumentReader.ReadAsync(stream, cancellationToken).ConfigureAwait(false); - return ParseDocument(iso8211Document, logger); + using var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); + return ParseRecords(Iso8211DocumentReader.EnumerateRecords(WrittenBytes(memoryStream)), logger); } /// - /// Parses an ISO 8211 document into an S-57 document. + /// Gets the bytes written to a memory stream, without copying them. /// - private static S57Document ParseDocument(Iso8211Document iso8211Document, ILogger? logger) + private static ReadOnlyMemory WrittenBytes(MemoryStream stream) => + stream.TryGetBuffer(out var buffer) ? buffer.AsMemory() : stream.ToArray(); + + /// + /// Parses ISO 8211 records, read one at a time, into an S-57 document. + /// + private static S57Document ParseRecords(IEnumerable records, ILogger? logger) { - // Parse the Data Descriptive Record (DDR) to get field definitions - var ddr = iso8211Document.DataDescriptiveRecord is not null - ? Iso8211DataDescriptiveRecordReader.Read(iso8211Document.DataDescriptiveRecord) - : null; + // The Data Descriptive Record (DDR), when it is the first record, gives the field definitions + Iso8211DataDescriptiveRecord? ddr = null; + var first = true; S57DataSetIdentification? dsid = null; S57DataSetParameters? dspm = null; @@ -113,8 +121,23 @@ private static S57Document ParseDocument(Iso8211Document iso8211Document, ILogge var vectorRecords = ImmutableArray.CreateBuilder(); int nall = 0; // NATF lexical level (0=ASCII, 1=ISO 8859, 2=UCS-2) - foreach (var record in iso8211Document.DataRecords) + foreach (var record in records) { + if (first) + { + first = false; + if (record.IsDataDescriptiveRecord) + { + ddr = Iso8211DataDescriptiveRecordReader.Read(record); + continue; + } + } + + if (record.IsDataDescriptiveRecord) + { + continue; // Only data records carry S-57 content + } + // Check for DSID field var dsidField = record.GetFieldByTag(S57FieldTags.DSID); if (dsidField != null) @@ -500,11 +523,18 @@ private static S57DataSetParameters ParseDataSetParameters(Iso8211Record record, /// private static ImmutableArray ParseAttributes(Iso8211Record record, string fieldTag, Iso8211DataDescriptiveRecord? ddr, ILogger? logger, int lexicalLevel = 0) { - var attributes = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? attributes = null; var fieldDef = ddr?.GetFieldDefinition(fieldTag); - foreach (var field in record.GetFieldsByTag(fieldTag)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != fieldTag) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -512,6 +542,7 @@ private static ImmutableArray ParseAttributes(Iso8211Record r // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data, lexicalLevel); + attributes ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -532,7 +563,7 @@ private static ImmutableArray ParseAttributes(Iso8211Record r } } - return attributes.ToImmutable(); + return Complete(attributes); } @@ -542,11 +573,18 @@ private static ImmutableArray ParseAttributes(Iso8211Record r /// private static ImmutableArray ParseSpatialPointers(Iso8211Record record, Iso8211DataDescriptiveRecord? ddr, ILogger? logger) { - var pointers = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? pointers = null; var fieldDef = ddr?.GetFieldDefinition(S57FieldTags.FSPT); - foreach (var field in record.GetFieldsByTag(S57FieldTags.FSPT)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != S57FieldTags.FSPT) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -554,6 +592,7 @@ private static ImmutableArray ParseSpatialPointers(Iso8211Rec // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); + pointers ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -581,7 +620,7 @@ private static ImmutableArray ParseSpatialPointers(Iso8211Rec } } - return pointers.ToImmutable(); + return Complete(pointers); } /// @@ -589,11 +628,18 @@ private static ImmutableArray ParseSpatialPointers(Iso8211Rec /// private static ImmutableArray ParseFeaturePointers(Iso8211Record record, Iso8211DataDescriptiveRecord? ddr, ILogger? logger) { - var pointers = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? pointers = null; var fieldDef = ddr?.GetFieldDefinition(S57FieldTags.FFPT); - foreach (var field in record.GetFieldsByTag(S57FieldTags.FFPT)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != S57FieldTags.FFPT) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -601,6 +647,7 @@ private static ImmutableArray ParseFeaturePointers(Iso8211Rec // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); + pointers ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -626,7 +673,7 @@ private static ImmutableArray ParseFeaturePointers(Iso8211Rec } } - return pointers.ToImmutable(); + return Complete(pointers); } /// @@ -634,11 +681,18 @@ private static ImmutableArray ParseFeaturePointers(Iso8211Rec /// private static ImmutableArray ParseVectorPointers(Iso8211Record record, Iso8211DataDescriptiveRecord? ddr, ILogger? logger) { - var pointers = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? pointers = null; var fieldDef = ddr?.GetFieldDefinition(S57FieldTags.VRPT); - foreach (var field in record.GetFieldsByTag(S57FieldTags.VRPT)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != S57FieldTags.VRPT) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -646,6 +700,7 @@ private static ImmutableArray ParseVectorPointers(Iso8211Recor // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); + pointers ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -675,7 +730,7 @@ private static ImmutableArray ParseVectorPointers(Iso8211Recor } } - return pointers.ToImmutable(); + return Complete(pointers); } /// @@ -683,11 +738,18 @@ private static ImmutableArray ParseVectorPointers(Iso8211Recor /// private static ImmutableArray ParseCoordinates2D(Iso8211Record record, Iso8211DataDescriptiveRecord? ddr, ILogger? logger) { - var coordinates = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? coordinates = null; var fieldDef = ddr?.GetFieldDefinition(S57FieldTags.SG2D); - foreach (var field in record.GetFieldsByTag(S57FieldTags.SG2D)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != S57FieldTags.SG2D) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -695,6 +757,7 @@ private static ImmutableArray ParseCoordinates2D(Iso8211Record // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); + coordinates ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -717,7 +780,7 @@ private static ImmutableArray ParseCoordinates2D(Iso8211Record } } - return coordinates.ToImmutable(); + return Complete(coordinates); } /// @@ -725,11 +788,18 @@ private static ImmutableArray ParseCoordinates2D(Iso8211Record /// private static ImmutableArray ParseSoundings(Iso8211Record record, Iso8211DataDescriptiveRecord? ddr, ILogger? logger) { - var soundings = ImmutableArray.CreateBuilder(); + ImmutableArray.Builder? soundings = null; var fieldDef = ddr?.GetFieldDefinition(S57FieldTags.SG3D); - foreach (var field in record.GetFieldsByTag(S57FieldTags.SG3D)) + // A loop rather than GetFieldsByTag, which allocates an iterator per call. + for (var i = 0; i < record.Fields.Count; i++) { + var field = record.Fields[i]; + if (field.Tag != S57FieldTags.SG3D) + { + continue; + } + if (fieldDef == null || !fieldDef.HasRepeatingGroup) { continue; // No field definition or not a repeating field - skip @@ -737,6 +807,7 @@ private static ImmutableArray ParseSoundings(Iso8211Record record, // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); + soundings ??= ImmutableArray.CreateBuilder(reader.GroupCount); try { @@ -761,9 +832,18 @@ private static ImmutableArray ParseSoundings(Iso8211Record record, } } - return soundings.ToImmutable(); + return Complete(soundings); } + /// + /// Completes a builder sized from the fields' group counts, without copying when it is full, + /// and without allocating at all when no field was present. + /// + private static ImmutableArray Complete(ImmutableArray.Builder? builder) => + builder is null ? [] + : builder.Count == builder.Capacity ? builder.MoveToImmutable() + : builder.ToImmutable(); + /// /// Logs a warning when a field's data ends partway through a repetition of its group. The /// complete repetitions before it are kept, and the partial one is dropped. diff --git a/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs b/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs index 5762dcc..47fc6da 100644 --- a/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs +++ b/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs @@ -42,6 +42,24 @@ public void Iso8211DocumentReader_Read_StaysWithinBudget() PerformanceBudget.AssertAllocated("Iso8211DocumentReader.Read", allocated, PerformanceBudgets.Iso8211DocumentReaderRead); } + [Fact] + public void Iso8211DocumentReader_EnumerateRecords_StaysWithinBudget() + { + // Streaming reads the caller's memory in place and keeps no document. + ReadOnlyMemory bytes = Cell.Bytes; + + var allocated = MemoryMeter.AllocatedBytes(() => + { + foreach (var record in Iso8211DocumentReader.EnumerateRecords(bytes)) + { + _ = record.Fields.Count; + } + }); + + PerformanceBudget.AssertAllocated( + "Iso8211DocumentReader.EnumerateRecords", allocated, PerformanceBudgets.Iso8211DocumentReaderEnumerateRecords); + } + [Fact] public void Iso8211DataDescriptiveRecordReader_Read_StaysWithinBudget() { diff --git a/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs b/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs index 4396c2f..a11d084 100644 --- a/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs +++ b/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs @@ -18,17 +18,18 @@ internal static class PerformanceBudgets { // Allocated bytes for one operation. public const long Iso8211ReaderAllTokens = 0; - public const long Iso8211DocumentReaderRead = 2_631_240; + public const long Iso8211DocumentReaderRead = 2_088_976; + public const long Iso8211DocumentReaderEnumerateRecords = 1_599_720; public const long Iso8211DataDescriptiveRecordReaderRead = 32_808; public const long Iso8211FieldReaderCoordinates = 846_376; public const long Iso8211FieldReaderCoordinatesByOrdinal = 141_264; public const long Iso8211DocumentWriterWrite = 2_318_456; - public const long S57DocumentReaderRead = 7_991_888; + public const long S57DocumentReaderRead = 3_713_184; public const long S57DocumentApplyChanges = 685_440; public const long S57ChartFromDocument = 1_030_312; // Retained bytes held by the resulting model. - public const long Iso8211DocumentRetained = 2_475_768; + public const long Iso8211DocumentRetained = 2_048_712; public const long S57DocumentRetained = 1_194_928; public const long S57ChartRetained = 1_145_696; }