From 8b25cb06f74de05b98ce18d0b7b484f5631319b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 22:20:35 +0000 Subject: [PATCH 1/2] Read ISO 8211 subfields without allocating per value Iso8211FieldReader allocated about 430 bytes per coordinate: a LINQ closure, delegate and enumerator for every name lookup, a boxed value for every read, a group object per repetition, and a subfield offset table built in a List and copied by ToArray. - Work out each field definition's layout once (Iso8211FieldLayout, cached per definition in a ConditionalWeakTable, not on the record): subfield names, formats and, when every subfield is fixed width, the byte offsets of a repetition. Fixed-width fields such as SG2D, SG3D, FSPT and VRPT then need no per-instance offset table. - Convert values to T without boxing, and decode character data once. - Add GetOrdinal plus GetSubfield/TryGetSubfield/GetSubfieldBytes by group index and ordinal, for reading long repeating groups without a name lookup or group object per repetition. Additive only. - Replace LINQ lookups in GetSubfieldDefinition, GetFieldDefinition, GetFieldByTag and GetFieldsByTag with loops. - Use the ordinal accessors in S57DocumentReader's group loops, keeping its warn-and-skip handling of malformed fields. S57DocumentReader on the synthetic harbour cell allocates 8.0 MB, down from 29.0 MB. Behavior is pinned by a frozen copy of the previous field reader that a differential test compares against (synthetic, truncated, hand-built and, locally, real cells) and by fingerprints of the S-57 documents the reader produced before the change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YagvELyJKBANk61DyFGUNJ --- docs/reading-iso8211.md | 22 + .../Iso8211DataDescriptiveRecord.cs | 15 +- .../Iso8211FieldDefinition.cs | 15 +- src/EncDotNet.Iso8211/Iso8211FieldLayout.cs | 137 ++++ src/EncDotNet.Iso8211/Iso8211FieldReader.cs | 637 ++++++++++------ src/EncDotNet.Iso8211/Iso8211Record.cs | 26 +- src/EncDotNet.Iso8211/Iso8211SubfieldGroup.cs | 40 +- src/EncDotNet.S57/S57DocumentReader.cs | 199 ++--- .../AllocationBudgetTests.cs | 31 + .../PerformanceBudget.cs | 7 +- .../PerformanceBudgets.cs | 7 +- .../Iso8211FieldReaderEquivalenceTests.cs | 323 +++++++++ .../Reference/LegacyIso8211FieldReader.cs | 680 ++++++++++++++++++ .../S57DocumentFingerprint.cs | 98 +++ .../S57DocumentReaderFingerprintTests.cs | 60 ++ 15 files changed, 1940 insertions(+), 357 deletions(-) create mode 100644 src/EncDotNet.Iso8211/Iso8211FieldLayout.cs create mode 100644 tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs create mode 100644 tests/EncDotNet.UnitTests/Reference/LegacyIso8211FieldReader.cs create mode 100644 tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs create mode 100644 tests/EncDotNet.UnitTests/S57DocumentReaderFingerprintTests.cs diff --git a/docs/reading-iso8211.md b/docs/reading-iso8211.md index cb115f1..ba49334 100644 --- a/docs/reading-iso8211.md +++ b/docs/reading-iso8211.md @@ -79,6 +79,28 @@ foreach (Iso8211Record record in document.DataRecords.Take(3)) of coordinates, `GetSubfieldValues` returns every value of one subfield and `GetSubfieldGroups` returns each group. +To read a long repeating group, such as the thousands of coordinates in an +edge, look up each subfield's ordinal once with `GetOrdinal`, then read each +repetition by group index and ordinal. This avoids a name lookup and a group +object for every repetition: + +```csharp +var reader = new Iso8211FieldReader(ddr.GetFieldDefinition("SG2D")!, field.Data); +int y = reader.GetOrdinal("YCOO"); +int x = reader.GetOrdinal("XCOO"); + +var points = new List<(int Y, int X)>(reader.GroupCount); +for (int group = 0; group < reader.GroupCount; group++) +{ + points.Add((reader.GetSubfield(group, y), reader.GetSubfield(group, x))); +} +``` + +`GroupCount` counts complete repetitions only. If the field data ends partway +through one, `GetSubfieldGroups` still returns that partial group, and +`TryGetSubfield(group, ordinal, out value)` returns `false` for the +subfields it lacks. + Character data is read as UTF-8 by default. Pass `lexicalLevel: 2` to the `Iso8211FieldReader` constructor for fields encoded as UCS-2 (UTF-16LE). diff --git a/src/EncDotNet.Iso8211/Iso8211DataDescriptiveRecord.cs b/src/EncDotNet.Iso8211/Iso8211DataDescriptiveRecord.cs index 8f4666b..d4cd52e 100644 --- a/src/EncDotNet.Iso8211/Iso8211DataDescriptiveRecord.cs +++ b/src/EncDotNet.Iso8211/Iso8211DataDescriptiveRecord.cs @@ -31,6 +31,17 @@ public sealed record Iso8211DataDescriptiveRecord /// /// The field tag to search for. /// The field definition with the specified tag, or null if not found. - public Iso8211FieldDefinition? GetFieldDefinition(string tag) => - FieldDefinitions.FirstOrDefault(f => f.Tag == tag); + public Iso8211FieldDefinition? GetFieldDefinition(string tag) + { + var definitions = FieldDefinitions; + for (var i = 0; i < definitions.Count; i++) + { + if (definitions[i].Tag == tag) + { + return definitions[i]; + } + } + + return null; + } } diff --git a/src/EncDotNet.Iso8211/Iso8211FieldDefinition.cs b/src/EncDotNet.Iso8211/Iso8211FieldDefinition.cs index 8d04968..0755252 100644 --- a/src/EncDotNet.Iso8211/Iso8211FieldDefinition.cs +++ b/src/EncDotNet.Iso8211/Iso8211FieldDefinition.cs @@ -84,6 +84,17 @@ public sealed record Iso8211FieldDefinition /// /// The subfield name to search for. /// The subfield definition with the specified name, or null if not found. - public Iso8211SubfieldDefinition? GetSubfieldDefinition(string name) => - SubfieldDefinitions.FirstOrDefault(s => s.Name == name); + public Iso8211SubfieldDefinition? GetSubfieldDefinition(string name) + { + var definitions = SubfieldDefinitions; + for (var i = 0; i < definitions.Count; i++) + { + if (definitions[i].Name == name) + { + return definitions[i]; + } + } + + return null; + } } diff --git a/src/EncDotNet.Iso8211/Iso8211FieldLayout.cs b/src/EncDotNet.Iso8211/Iso8211FieldLayout.cs new file mode 100644 index 0000000..c3c5749 --- /dev/null +++ b/src/EncDotNet.Iso8211/Iso8211FieldLayout.cs @@ -0,0 +1,137 @@ +using System.Runtime.CompilerServices; + +namespace EncDotNet.Iso8211; + +/// +/// The structure of an , worked out once and shared by +/// every that reads a field with that definition. +/// +/// +/// +/// Layouts are cached per definition instance in a +/// rather than stored on the definition, because is a record: +/// a cached member would take part in its generated equality, and a with expression would +/// copy a stale layout onto a definition whose subfields had changed. Definitions are treated as +/// immutable once a reader has used them. +/// +/// +/// When every subfield has a fixed width, as in coordinate and pointer fields such as SG2D, SG3D, +/// FSPT and VRPT, the byte offset of any subfield instance follows from its position alone, so +/// readers don't need a per-instance offset table. +/// +/// +internal sealed class Iso8211FieldLayout +{ + private static readonly ConditionalWeakTable Cache = new(); + + private readonly string[] _names; + + private Iso8211FieldLayout(Iso8211FieldDefinition definition) + { + var subfields = definition.SubfieldDefinitions; + Count = subfields.Count; + _names = new string[Count]; + Formats = new Iso8211SubfieldFormat[Count]; + Offsets = new int[Count]; + + for (var i = 0; i < Count; i++) + { + _names[i] = subfields[i].Name; + Formats[i] = subfields[i].Format; + } + + RepeatStart = definition.RepeatingSubfieldStartIndex; + HasRepeatingGroup = RepeatStart >= 0; + PrefixCount = HasRepeatingGroup ? RepeatStart : Count; + RepeatCount = HasRepeatingGroup ? Count - RepeatStart : 0; + + // A repeating group that starts past the last subfield is malformed; leave it to the + // general path, which reports it as it always has. + IsFixedWidth = Count > 0 && (!HasRepeatingGroup || RepeatCount > 0); + for (var i = 0; i < Count && IsFixedWidth; i++) + { + IsFixedWidth = Formats[i].IsFixedWidth; + } + + if (IsFixedWidth) + { + // Offsets of the leading subfields from the start of the field, and of the + // repeating subfields from the start of their group. + var offset = 0; + for (var i = 0; i < PrefixCount; i++) + { + Offsets[i] = offset; + offset += Formats[i].Width; + } + + PrefixBytes = offset; + offset = 0; + for (var i = PrefixCount; i < Count; i++) + { + Offsets[i] = offset; + offset += Formats[i].Width; + } + + Stride = offset; + } + } + + /// Gets the number of subfield definitions. + public int Count { get; } + + /// Gets each subfield's format, by definition position. + public Iso8211SubfieldFormat[] Formats { get; } + + /// Gets whether the field has a repeating subfield group. + public bool HasRepeatingGroup { get; } + + /// Gets the position of the first repeating subfield, or -1. + public int RepeatStart { get; } + + /// Gets the number of subfields before the repeating group (all of them if none repeat). + public int PrefixCount { get; } + + /// Gets the number of subfields in each repetition of the group. + public int RepeatCount { get; } + + /// Gets whether every subfield has a fixed width. + public bool IsFixedWidth { get; } + + /// + /// Gets each subfield's byte offset when : from the start of the + /// field for leading subfields, and from the start of the group for repeating ones. + /// + public int[] Offsets { get; } + + /// Gets the total width of the leading subfields when . + public int PrefixBytes { get; } + + /// Gets the width of one repetition of the group when . + public int Stride { get; } + + /// + /// Gets the layout for a field definition, building it on first use. + /// + public static Iso8211FieldLayout For(Iso8211FieldDefinition definition) => + Cache.GetValue(definition, static d => new Iso8211FieldLayout(d)); + + /// + /// Gets the position of the first subfield definition with the given name, or -1. + /// + /// + /// A field has a handful of four-character names, so a loop is as fast as a dictionary and + /// allocates nothing. + /// + public int IndexOf(string name) + { + for (var i = 0; i < _names.Length; i++) + { + if (string.Equals(_names[i], name, StringComparison.Ordinal)) + { + return i; + } + } + + return -1; + } +} diff --git a/src/EncDotNet.Iso8211/Iso8211FieldReader.cs b/src/EncDotNet.Iso8211/Iso8211FieldReader.cs index ac1a3bd..92a4f2d 100644 --- a/src/EncDotNet.Iso8211/Iso8211FieldReader.cs +++ b/src/EncDotNet.Iso8211/Iso8211FieldReader.cs @@ -41,16 +41,21 @@ public sealed class Iso8211FieldReader private const byte FieldTerminator = 0x1E; private readonly Iso8211FieldDefinition _fieldDefinition; - private readonly byte[] _data; - private readonly ParsedSubfield[] _parsedSubfields; - private readonly int _groupCount; + private readonly Iso8211FieldLayout _layout; + private readonly ReadOnlyMemory _data; private readonly int _lexicalLevel; + // Offsets of each parsed subfield instance. Null when every subfield is fixed-width: the + // layout then gives any instance's offset from its position, so no table is needed. + private readonly ParsedSubfield[]? _parsedSubfields; + private readonly int _subfieldCount; + private readonly int _groupCount; + /// /// Initializes a new instance of the class. /// /// The field definition from the DDR describing this field's structure. - /// The raw field data to read. + /// The raw field data to read. The reader keeps its own copy. /// /// The lexical level for character data (0 or 1 = ASCII/ISO 8859, 2 = UCS-2/UTF-16LE). /// When set to 2, variable-length character subfields use 2-byte terminators (0x1F 0x00 / 0x1E 0x00). @@ -59,18 +64,15 @@ public sealed class Iso8211FieldReader /// Thrown when is null. /// public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlySpan data, int lexicalLevel = 0) + : this(fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)), data.ToArray().AsMemory(), lexicalLevel, validated: true) { - _fieldDefinition = fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)); - _data = data.ToArray(); - _lexicalLevel = lexicalLevel; - (_parsedSubfields, _groupCount) = ParseSubfields(); } /// /// Initializes a new instance of the class. /// /// The field definition from the DDR describing this field's structure. - /// The raw field data to read. + /// The raw field data to read. The reader reads the array in place, without copying it. /// /// The lexical level for character data (0 or 1 = ASCII/ISO 8859, 2 = UCS-2/UTF-16LE). /// When set to 2, variable-length character subfields use 2-byte terminators (0x1F 0x00 / 0x1E 0x00). @@ -79,11 +81,34 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlySpan or is null. /// public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, int lexicalLevel = 0) + : this( + fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)), + (data ?? throw new ArgumentNullException(nameof(data))).AsMemory(), + lexicalLevel, + validated: true) { - _fieldDefinition = fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)); - _data = data ?? throw new ArgumentNullException(nameof(data)); + } + + private Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlyMemory data, int lexicalLevel, bool validated) + { + _ = validated; + _fieldDefinition = fieldDefinition; + _layout = Iso8211FieldLayout.For(fieldDefinition); + _data = data; _lexicalLevel = lexicalLevel; - (_parsedSubfields, _groupCount) = ParseSubfields(); + + var span = data.Span; + if (_layout.IsFixedWidth) + { + _subfieldCount = ParseSubfields(span, [], out _groupCount); + } + else + { + // Count first, then fill an exactly sized table. + var count = ParseSubfields(span, [], out _); + _parsedSubfields = count == 0 ? [] : new ParsedSubfield[count]; + _subfieldCount = ParseSubfields(span, _parsedSubfields, out _groupCount); + } } /// @@ -94,7 +119,7 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, i /// /// Gets the raw field data. /// - public ReadOnlySpan Data => _data; + public ReadOnlySpan Data => _data.Span; /// /// Gets the number of parsed subfield instances. @@ -102,7 +127,7 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, i /// /// For fields with repeating groups, this count includes all instances across all repetitions. /// - public int SubfieldCount => _parsedSubfields.Length; + public int SubfieldCount => _subfieldCount; /// /// Gets the number of subfield groups (repetitions) in this field. @@ -110,6 +135,7 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, i /// /// For fields without repeating groups, this will be 1. /// For fields with repeating groups, this is the number of times the group repeats. + /// A final repetition cut short by the end of the field data is not counted. /// public int GroupCount => _groupCount; @@ -118,6 +144,41 @@ public Iso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, i /// public bool HasRepeatingGroups => _fieldDefinition.HasRepeatingGroup && _groupCount > 1; + /// + /// Gets the ordinal of a subfield: its position in the field definition's + /// . + /// + /// The subfield name, such as "YCOO". + /// The zero-based ordinal of the first subfield with that name. + /// + /// Thrown when the field definition has no subfield with the specified name. + /// + /// + /// Look the ordinal up once and pass it to to read many + /// repetitions of a group without repeating the name lookup for each one. + /// + /// + /// + /// var reader = new Iso8211FieldReader(ddr.GetFieldDefinition("SG2D")!, field.Data); + /// var y = reader.GetOrdinal("YCOO"); + /// var x = reader.GetOrdinal("XCOO"); + /// for (var group = 0; group < reader.GroupCount; group++) + /// { + /// points.Add((reader.GetSubfield<int>(group, y), reader.GetSubfield<int>(group, x))); + /// } + /// + /// + public int GetOrdinal(string name) + { + var ordinal = _layout.IndexOf(name); + if (ordinal < 0) + { + throw new KeyNotFoundException($"Subfield '{name}' not found in field definition '{_fieldDefinition.Tag}'."); + } + + return ordinal; + } + /// /// Gets the value of a subfield by name. /// @@ -159,25 +220,70 @@ public T GetSubfield(string name) /// true if the subfield was found; otherwise, false. public bool TryGetSubfield(string name, out T value) { - var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); - if (subfieldDef is null) + // Instances follow definition order, so a subfield's first instance is at its ordinal. + var ordinal = _layout.IndexOf(name); + if (ordinal < 0 || ordinal >= _subfieldCount) { value = default!; return false; } - // Find the first parsed subfield matching this definition - foreach (var parsed in _parsedSubfields) + value = ConvertAt(ordinal); + return true; + } + + /// + /// Gets the value of a subfield in one repetition of the field's repeating group. + /// + /// The type to convert the subfield value to. + /// + /// The zero-based repetition, from 0 to - 1. For a field without a + /// repeating group, the only repetition is 0. + /// + /// The subfield's ordinal, from . + /// The subfield value converted to type . + /// + /// Thrown when is not a subfield of the repeating group, or the + /// field data has no such repetition. + /// + /// + /// Thrown when the subfield value cannot be converted to the requested type. + /// + public T GetSubfield(int groupIndex, int ordinal) + { + if (!TryGetInstanceIndex(groupIndex, ordinal, out var index)) { - if (parsed.DefinitionIndex == subfieldDef.Index) - { - value = ConvertValue(parsed, subfieldDef); - return true; - } + throw new ArgumentOutOfRangeException( + nameof(groupIndex), + $"Field '{_fieldDefinition.Tag}' has no subfield {ordinal} in repetition {groupIndex}."); + } + + return ConvertAt(index); + } + + /// + /// Tries to get the value of a subfield in one repetition of the field's repeating group. + /// + /// The type to convert the subfield value to. + /// The zero-based repetition. + /// The subfield's ordinal, from . + /// + /// When this method returns, contains the subfield value if present; otherwise, the default value. + /// + /// + /// true if the repetition contains the subfield; false if + /// is not in the repeating group or the field data ends before it. + /// + public bool TryGetSubfield(int groupIndex, int ordinal, out T value) + { + if (!TryGetInstanceIndex(groupIndex, ordinal, out var index)) + { + value = default!; + return false; } - value = default!; - return false; + value = ConvertAt(index); + return true; } /// @@ -191,23 +297,30 @@ public bool TryGetSubfield(string name, out T value) /// public ImmutableArray GetSubfieldValues(string name) { - var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); - if (subfieldDef is null) + var ordinal = _layout.IndexOf(name); + if (ordinal < 0) { throw new KeyNotFoundException($"Subfield '{name}' not found in field definition '{_fieldDefinition.Tag}'."); } - var builder = ImmutableArray.CreateBuilder(); + if (ordinal >= _subfieldCount) + { + return []; + } - foreach (var parsed in _parsedSubfields) + if (!_layout.HasRepeatingGroup || ordinal < _layout.RepeatStart) { - if (parsed.DefinitionIndex == subfieldDef.Index) - { - builder.Add(ConvertValue(parsed, subfieldDef)); - } + return [ConvertAt(ordinal)]; } - return builder.ToImmutable(); + // Instances of a repeating subfield sit one group apart. + var builder = ImmutableArray.CreateBuilder(((_subfieldCount - 1 - ordinal) / _layout.RepeatCount) + 1); + for (var index = ordinal; index < _subfieldCount; index += _layout.RepeatCount) + { + builder.Add(ConvertAt(index)); + } + + return builder.MoveToImmutable(); } /// @@ -221,14 +334,12 @@ public ImmutableArray GetSubfieldValues(string name) /// public T GetSubfieldAt(int index) { - if (index < 0 || index >= _parsedSubfields.Length) + if (index < 0 || index >= _subfieldCount) { throw new ArgumentOutOfRangeException(nameof(index)); } - var parsed = _parsedSubfields[index]; - var subfieldDef = _fieldDefinition.SubfieldDefinitions[parsed.DefinitionIndex]; - return ConvertValue(parsed, subfieldDef); + return ConvertAt(index); } /// @@ -241,26 +352,46 @@ public T GetSubfieldAt(int index) /// /// /// For repeating subfields, this returns the bytes from the first occurrence. - /// Use to retrieve + /// Use or + /// to retrieve /// bytes for a specific group. /// public ReadOnlySpan GetSubfieldBytes(string name) { - var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); - if (subfieldDef is null) + var ordinal = _layout.IndexOf(name); + if (ordinal < 0) { throw new KeyNotFoundException($"Subfield '{name}' not found in field '{_fieldDefinition.Tag}'."); } - foreach (var parsed in _parsedSubfields) + if (ordinal >= _subfieldCount) { - if (parsed.DefinitionIndex == subfieldDef.Index) - { - return _data.AsSpan(parsed.Offset, parsed.Length); - } + throw new KeyNotFoundException($"Subfield '{name}' not found in parsed field data."); } - throw new KeyNotFoundException($"Subfield '{name}' not found in parsed field data."); + return BytesAt(ordinal); + } + + /// + /// Gets the raw bytes of a subfield in one repetition of the field's repeating group. + /// + /// The zero-based repetition. + /// The subfield's ordinal, from . + /// A span containing the raw subfield data. + /// + /// Thrown when is not a subfield of the repeating group, or the + /// field data has no such repetition. + /// + public ReadOnlySpan GetSubfieldBytes(int groupIndex, int ordinal) + { + if (!TryGetInstanceIndex(groupIndex, ordinal, out var index)) + { + throw new ArgumentOutOfRangeException( + nameof(groupIndex), + $"Field '{_fieldDefinition.Tag}' has no subfield {ordinal} in repetition {groupIndex}."); + } + + return BytesAt(index); } /// @@ -273,13 +404,12 @@ public ReadOnlySpan GetSubfieldBytes(string name) /// public ReadOnlySpan GetSubfieldBytesAt(int index) { - if (index < 0 || index >= _parsedSubfields.Length) + if (index < 0 || index >= _subfieldCount) { throw new ArgumentOutOfRangeException(nameof(index)); } - var parsed = _parsedSubfields[index]; - return _data.AsSpan(parsed.Offset, parsed.Length); + return BytesAt(index); } /// @@ -291,13 +421,16 @@ public ReadOnlySpan GetSubfieldBytesAt(int index) /// /// /// For fields without repeating groups, this returns a single group containing all subfields. + /// Unlike , the enumeration includes a final repetition cut short by + /// the end of the field data. To read many repetitions without allocating a group object for + /// each, use with . /// public IEnumerable GetSubfieldGroups() { if (!_fieldDefinition.HasRepeatingGroup) { // Single group containing all subfields - yield return new Iso8211SubfieldGroup(this, 0, _parsedSubfields.Length, 0); + yield return new Iso8211SubfieldGroup(this, 0, _subfieldCount, 0); yield break; } @@ -309,9 +442,9 @@ public IEnumerable GetSubfieldGroups() int currentIndex = fixedCount; int groupIndex = 0; - while (currentIndex < _parsedSubfields.Length) + while (currentIndex < _subfieldCount) { - var groupLength = Math.Min(repeatCount, _parsedSubfields.Length - currentIndex); + var groupLength = Math.Min(repeatCount, _subfieldCount - currentIndex); yield return new Iso8211SubfieldGroup(this, currentIndex, groupLength, groupIndex); currentIndex += groupLength; groupIndex++; @@ -331,37 +464,114 @@ public IEnumerable GetSubfieldGroups() public T GetFixedSubfield(string name) => GetSubfield(name); /// - /// Parses all subfields in the field data. + /// Gets the ordinal of a subfield by name, or -1, for . /// - /// - /// A tuple containing the array of parsed subfields and the number of groups. - /// - private (ParsedSubfield[] subfields, int groupCount) ParseSubfields() + internal int IndexOfSubfield(string name) => _layout.IndexOf(name); + + /// + /// Maps a repetition and subfield ordinal to the index of the parsed subfield instance. + /// + private bool TryGetInstanceIndex(int groupIndex, int ordinal, out int index) + { + index = -1; + if (groupIndex < 0 || ordinal < 0 || ordinal >= _layout.Count) + { + return false; + } + + if (!_layout.HasRepeatingGroup) + { + index = groupIndex == 0 ? ordinal : -1; + } + else if (ordinal >= _layout.RepeatStart) + { + // Instances follow definition order: the leading subfields, then each repetition. + index = _layout.RepeatStart + (groupIndex * _layout.RepeatCount) + (ordinal - _layout.RepeatStart); + } + + return index >= 0 && index < _subfieldCount; + } + + /// + /// Locates a parsed subfield instance: its definition ordinal, offset and length. + /// + private void Locate(int index, out int ordinal, out int offset, out int length) { - var subfields = _fieldDefinition.SubfieldDefinitions; - if (subfields.Count == 0) + if (_parsedSubfields is not null) + { + var parsed = _parsedSubfields[index]; + ordinal = parsed.DefinitionIndex; + offset = parsed.Offset; + length = parsed.Length; + return; + } + + // Fixed-width layout: every instance before the last is full width, so offsets follow + // from position. Only the final instance can be cut short by the end of the data. + var layout = _layout; + if (index < layout.PrefixCount) { - return ([], 0); + ordinal = index; + offset = layout.Offsets[index]; } + else + { + var inGroups = index - layout.PrefixCount; + var group = inGroups / layout.RepeatCount; + ordinal = layout.RepeatStart + (inGroups - (group * layout.RepeatCount)); + offset = layout.PrefixBytes + (group * layout.Stride) + layout.Offsets[ordinal]; + } + + length = Math.Min(layout.Formats[ordinal].Width, _data.Length - offset); + } + + private ReadOnlySpan BytesAt(int index) + { + Locate(index, out _, out var offset, out var length); + return _data.Span.Slice(offset, length); + } + + private T ConvertAt(int index) + { + Locate(index, out var ordinal, out var offset, out var length); + var format = _layout.Formats[ordinal]; + var span = _data.Span.Slice(offset, length); + + return format.FormatType switch + { + Iso8211SubfieldFormatType.CharacterData => ConvertCharacterData(span), + Iso8211SubfieldFormatType.Integer => ConvertAsciiInteger(span), + Iso8211SubfieldFormatType.Real => ConvertAsciiReal(span), + Iso8211SubfieldFormatType.UnsignedInteger => ConvertUnsignedBinary(span, format.Width), + Iso8211SubfieldFormatType.SignedInteger => ConvertSignedBinary(span, format.Width), + Iso8211SubfieldFormatType.FloatingPoint => ConvertFloatingBinary(span, format.Width), + Iso8211SubfieldFormatType.BitString => ConvertBitString(span), + _ => throw new InvalidOperationException($"Unknown format type: {format.FormatType}") + }; + } - // Pre-size: for repeating groups, estimate ~4 bytes per subfield - int initialCapacity = subfields.Count; - if (_fieldDefinition.HasRepeatingGroup && _data.Length > 0) + /// + /// Walks the field data, recording each subfield instance in when + /// it isn't empty. + /// + /// The number of subfield instances. + private int ParseSubfields(ReadOnlySpan data, Span output, out int groupCount) + { + var subfields = _layout.Formats; + groupCount = 0; + if (subfields.Length == 0) { - initialCapacity = Math.Max(initialCapacity, _data.Length / 4); + return 0; } - var parsed = new List(initialCapacity); - var data = _data.AsSpan(); + var count = 0; int offset = 0; int subfieldIndex = 0; - int groupCount = 0; bool inRepeatingGroup = false; while (offset < data.Length) { - var subfieldDef = subfields[subfieldIndex]; - var format = subfieldDef.Format; + var format = subfields[subfieldIndex]; // For variable-length subfields, check if we've hit a field terminator. // Fixed-width binary subfields may contain 0x1E (30) as valid data, so we @@ -373,7 +583,7 @@ public IEnumerable GetSubfieldGroups() // For fixed-width subfields in repeating groups, check if we've hit the end: // If the remaining data is just the field terminator, we're done. - if (format.IsFixedWidth && _fieldDefinition.HasRepeatingGroup) + if (format.IsFixedWidth && _layout.HasRepeatingGroup) { // If remaining bytes < expected width AND the first remaining byte is FT, stop int remaining = data.Length - offset; @@ -386,41 +596,34 @@ public IEnumerable GetSubfieldGroups() // Calculate the length of this subfield int length = CalculateSubfieldLength(data, offset, format); - parsed.Add(new ParsedSubfield + if (!output.IsEmpty) { - DefinitionIndex = subfieldIndex, - Offset = offset, - Length = length, - GroupIndex = groupCount - }); + output[count] = new ParsedSubfield(subfieldIndex, offset, length); + } + count++; offset += length; // Skip unit terminator if present - only for variable-length subfields // Fixed-width binary subfields (b11, b12, b14, b21, b22, b24) don't have UTs - if (!format.IsFixedWidth) + if (!format.IsFixedWidth && IsUnitTerminator(data, offset)) { - if (IsUnitTerminator(data, offset)) - { - offset += TerminatorWidth; - } - else if (IsFieldTerminator(data, offset)) - { - // Field terminator acts as the final unit terminator; don't advance - // past it so the next iteration's FT check can break the loop. - } + offset += TerminatorWidth; } + // A field terminator acts as the final unit terminator; it isn't skipped, so the + // next iteration's check ends the loop. + // Advance to next subfield definition subfieldIndex++; // Handle repeating groups - if (subfieldIndex >= subfields.Count) + if (subfieldIndex >= subfields.Length) { - if (_fieldDefinition.HasRepeatingGroup) + if (_layout.HasRepeatingGroup) { // Reset to the start of the repeating group - subfieldIndex = _fieldDefinition.RepeatingSubfieldStartIndex; + subfieldIndex = _layout.RepeatStart; groupCount++; inRepeatingGroup = true; } @@ -430,28 +633,26 @@ public IEnumerable GetSubfieldGroups() break; } } - else if (!inRepeatingGroup && subfieldIndex == _fieldDefinition.RepeatingSubfieldStartIndex && _fieldDefinition.HasRepeatingGroup) + else if (!inRepeatingGroup && subfieldIndex == _layout.RepeatStart && _layout.HasRepeatingGroup) { - // Just entered the repeating group for the first time. - // Set to 0; the wrap-around at line ~424 will increment to 1 - // after the first complete group is parsed. + // Just entered the repeating group for the first time; the wrap-around above + // counts each group once it is complete. groupCount = 0; inRepeatingGroup = true; } } // If we never entered a repeating group but have subfields, we have 1 group - if (!inRepeatingGroup && parsed.Count > 0) + if (!inRepeatingGroup && count > 0) { groupCount = 1; } - return (parsed.ToArray(), groupCount); + return count; } /// - /// Gets the width of terminators based on the lexical level. - /// For lexical level 2 (UCS-2), terminators are 2 bytes (e.g. 0x1F 0x00). + /// Gets the width of a terminator for the current lexical level. /// private int TerminatorWidth => _lexicalLevel >= 2 ? 2 : 1; @@ -506,189 +707,193 @@ private int CalculateSubfieldLength(ReadOnlySpan data, int offset, Iso8211 return length; } - /// - /// Converts a parsed subfield value to the requested type. - /// - private T ConvertValue(ParsedSubfield parsed, Iso8211SubfieldDefinition subfieldDef) - { - var span = _data.AsSpan(parsed.Offset, parsed.Length); - var format = subfieldDef.Format; - - object result = format.FormatType switch - { - Iso8211SubfieldFormatType.CharacterData => ConvertCharacterData(span), - Iso8211SubfieldFormatType.Integer => ConvertAsciiInteger(span), - Iso8211SubfieldFormatType.Real => ConvertAsciiReal(span), - Iso8211SubfieldFormatType.UnsignedInteger => ConvertUnsignedBinary(span, format.Width), - Iso8211SubfieldFormatType.SignedInteger => ConvertSignedBinary(span, format.Width), - Iso8211SubfieldFormatType.FloatingPoint => ConvertFloatingBinary(span, format.Width), - Iso8211SubfieldFormatType.BitString => ConvertBitString(span), - _ => throw new InvalidOperationException($"Unknown format type: {format.FormatType}") - }; - - return (T)result; - } + // The conversions below return T directly. Each `(T)(object)value` sits behind a + // `typeof(T) == typeof(...)` check, which the JIT folds for value types, so optimized code + // neither boxes nor allocates. /// /// Converts character data to the requested type. /// - private object ConvertCharacterData(ReadOnlySpan data) + private T ConvertCharacterData(ReadOnlySpan data) { - var encoding = _lexicalLevel >= 2 ? Encoding.Unicode : Encoding.UTF8; - var str = encoding.GetString(data).TrimEnd('\x1F', '\x1E', '\0', ' '); + if (typeof(T) == typeof(byte[])) + { + return (T)(object)data.ToArray(); + } + + var str = DecodeCharacterData(data); if (typeof(T) == typeof(string)) { - return str; + return (T)(object)str; } if (typeof(T) == typeof(int)) { - return int.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)int.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(long)) { - return long.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)long.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(double)) { - return double.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)double.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(float)) { - return float.Parse(str, CultureInfo.InvariantCulture); - } - if (typeof(T) == typeof(byte[])) - { - return data.ToArray(); + return (T)(object)float.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(byte)) { - return byte.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)byte.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(sbyte)) { - return sbyte.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)sbyte.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(short)) { - return short.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)short.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(ushort)) { - return ushort.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)ushort.Parse(str, CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert character data to type {typeof(T).Name}."); } /// - /// Converts ASCII integer data to the requested type. + /// Decodes character data, dropping trailing terminators, NULs and spaces. + /// + private string DecodeCharacterData(ReadOnlySpan data) + { + if (_lexicalLevel >= 2) + { + return Encoding.Unicode.GetString(data).TrimEnd(CharacterPadding); + } + + // In UTF-8 these four characters are single bytes that never occur inside a multi-byte + // sequence, so trimming the bytes before decoding matches trimming the decoded string, + // and decodes once instead of allocating a second, trimmed string. + var end = data.Length; + while (end > 0 && data[end - 1] is UnitTerminator or FieldTerminator or 0x00 or (byte)' ') + { + end--; + } + + return Encoding.UTF8.GetString(data[..end]); + } + + private static readonly char[] CharacterPadding = ['\x1F', '\x1E', '\0', ' ']; + + /// + /// Converts ASCII integer data (I format) to the requested type. /// - private static object ConvertAsciiInteger(ReadOnlySpan data) + private static T ConvertAsciiInteger(ReadOnlySpan data) { var str = Encoding.ASCII.GetString(data).Trim(); if (string.IsNullOrEmpty(str)) { - return default(T)!; + return default!; } if (typeof(T) == typeof(string)) { - return str; + return (T)(object)str; } if (typeof(T) == typeof(int)) { - return int.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)int.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(uint)) { - return uint.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)uint.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(long)) { - return long.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)long.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(ulong)) { - return ulong.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)ulong.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(short)) { - return short.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)short.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(ushort)) { - return ushort.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)ushort.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(byte)) { - return byte.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)byte.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(sbyte)) { - return sbyte.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)sbyte.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(double)) { - return double.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)double.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(float)) { - return float.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)float.Parse(str, CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert ASCII integer to type {typeof(T).Name}."); } /// - /// Converts ASCII real number data to the requested type. + /// Converts ASCII real data (R format) to the requested type. /// - private static object ConvertAsciiReal(ReadOnlySpan data) + private static T ConvertAsciiReal(ReadOnlySpan data) { var str = Encoding.ASCII.GetString(data).Trim(); if (string.IsNullOrEmpty(str)) { - return default(T)!; + return default!; } if (typeof(T) == typeof(string)) { - return str; + return (T)(object)str; } if (typeof(T) == typeof(double)) { - return double.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)double.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(float)) { - return float.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)float.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(decimal)) { - return decimal.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)decimal.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(int)) { - return (int)double.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)(int)double.Parse(str, CultureInfo.InvariantCulture); } if (typeof(T) == typeof(long)) { - return (long)double.Parse(str, CultureInfo.InvariantCulture); + return (T)(object)(long)double.Parse(str, CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert ASCII real to type {typeof(T).Name}."); } /// - /// Converts unsigned binary integer data to the requested type. + /// Converts unsigned binary data (b1w format) to the requested type. /// - private static object ConvertUnsignedBinary(ReadOnlySpan data, int width) + private static T ConvertUnsignedBinary(ReadOnlySpan data, int width) { // Use actual data length if shorter than expected width int actualWidth = Math.Min(width, data.Length); - ulong value = actualWidth switch { 1 => data[0], @@ -700,60 +905,59 @@ private static object ConvertUnsignedBinary(ReadOnlySpan data, int widt if (typeof(T) == typeof(byte)) { - return (byte)value; + return (T)(object)(byte)value; } if (typeof(T) == typeof(ushort)) { - return (ushort)value; + return (T)(object)(ushort)value; } if (typeof(T) == typeof(uint)) { - return (uint)value; + return (T)(object)(uint)value; } if (typeof(T) == typeof(ulong)) { - return value; + return (T)(object)value; } if (typeof(T) == typeof(int)) { - return (int)value; + return (T)(object)(int)value; } if (typeof(T) == typeof(long)) { - return (long)value; + return (T)(object)(long)value; } if (typeof(T) == typeof(short)) { - return (short)value; + return (T)(object)(short)value; } if (typeof(T) == typeof(sbyte)) { - return (sbyte)value; + return (T)(object)(sbyte)value; } if (typeof(T) == typeof(double)) { - return (double)value; + return (T)(object)(double)value; } if (typeof(T) == typeof(float)) { - return (float)value; + return (T)(object)(float)value; } if (typeof(T) == typeof(string)) { - return value.ToString(CultureInfo.InvariantCulture); + return (T)(object)value.ToString(CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert unsigned binary to type {typeof(T).Name}."); } /// - /// Converts signed binary integer data to the requested type. + /// Converts signed binary data (b2w format) to the requested type. /// - private static object ConvertSignedBinary(ReadOnlySpan data, int width) + private static T ConvertSignedBinary(ReadOnlySpan data, int width) { // Use actual data length if shorter than expected width int actualWidth = Math.Min(width, data.Length); - long value = actualWidth switch { 1 => (sbyte)data[0], @@ -765,59 +969,58 @@ private static object ConvertSignedBinary(ReadOnlySpan data, int width) if (typeof(T) == typeof(sbyte)) { - return (sbyte)value; + return (T)(object)(sbyte)value; } if (typeof(T) == typeof(short)) { - return (short)value; + return (T)(object)(short)value; } if (typeof(T) == typeof(int)) { - return (int)value; + return (T)(object)(int)value; } if (typeof(T) == typeof(long)) { - return value; + return (T)(object)value; } if (typeof(T) == typeof(byte)) { - return (byte)value; + return (T)(object)(byte)value; } if (typeof(T) == typeof(ushort)) { - return (ushort)value; + return (T)(object)(ushort)value; } if (typeof(T) == typeof(uint)) { - return (uint)value; + return (T)(object)(uint)value; } if (typeof(T) == typeof(ulong)) { - return (ulong)value; + return (T)(object)(ulong)value; } if (typeof(T) == typeof(double)) { - return (double)value; + return (T)(object)(double)value; } if (typeof(T) == typeof(float)) { - return (float)value; + return (T)(object)(float)value; } if (typeof(T) == typeof(string)) { - return value.ToString(CultureInfo.InvariantCulture); + return (T)(object)value.ToString(CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert signed binary to type {typeof(T).Name}."); } /// - /// Converts floating-point binary data (ISO 8211 b4x/b5x, IEEE 754) to the requested type. + /// Converts floating-point binary data (b4w format) to the requested type. /// - private static object ConvertFloatingBinary(ReadOnlySpan data, int width) + private static T ConvertFloatingBinary(ReadOnlySpan data, int width) { int actualWidth = Math.Min(width, data.Length); - double value = actualWidth switch { 4 => BinaryPrimitives.ReadSingleLittleEndian(data), @@ -828,49 +1031,46 @@ private static object ConvertFloatingBinary(ReadOnlySpan data, int widt if (typeof(T) == typeof(double)) { - return value; + return (T)(object)value; } if (typeof(T) == typeof(float)) { - return (float)value; + return (T)(object)(float)value; } if (typeof(T) == typeof(decimal)) { - return (decimal)value; + return (T)(object)(decimal)value; } if (typeof(T) == typeof(int)) { - return (int)value; + return (T)(object)(int)value; } if (typeof(T) == typeof(long)) { - return (long)value; + return (T)(object)(long)value; } if (typeof(T) == typeof(string)) { - return value.ToString(CultureInfo.InvariantCulture); + return (T)(object)value.ToString(CultureInfo.InvariantCulture); } throw new InvalidOperationException($"Cannot convert floating-point binary to type {typeof(T).Name}."); } /// - /// Converts bit string data (ISO 8211 B(n) format) to the requested type. + /// Converts bit string data (B(n) format) to the requested type. /// - /// - /// Bit strings are fixed-width binary data. They can be returned as byte[] - /// for raw access, or as integer types when the width matches. - /// - private static object ConvertBitString(ReadOnlySpan data) + private static T ConvertBitString(ReadOnlySpan data) { if (typeof(T) == typeof(byte[])) { - return data.ToArray(); + return (T)(object)data.ToArray(); } + if (typeof(T) == typeof(string)) { // Return hex representation - return Convert.ToHexString(data); + return (T)(object)Convert.ToHexString(data); } // For integer types, delegate to unsigned binary conversion @@ -878,7 +1078,7 @@ private static object ConvertBitString(ReadOnlySpan data) } /// - /// Reads an unsigned integer with a non-standard width. + /// Reads an unsigned little-endian integer of arbitrary width. /// private static ulong ReadUnsignedLittleEndian(ReadOnlySpan data, int width) { @@ -891,7 +1091,7 @@ private static ulong ReadUnsignedLittleEndian(ReadOnlySpan data, int width } /// - /// Reads a signed integer with a non-standard width. + /// Reads a signed little-endian integer of arbitrary width. /// private static long ReadSignedLittleEndian(ReadOnlySpan data, int width) { @@ -910,28 +1110,7 @@ private static long ReadSignedLittleEndian(ReadOnlySpan data, int width) } /// - /// Represents a parsed subfield with its location in the field data. + /// The position of one subfield instance within the field data. /// - private readonly struct ParsedSubfield - { - /// - /// The index of the subfield definition this parsed subfield corresponds to. - /// - public int DefinitionIndex { get; init; } - - /// - /// The byte offset of this subfield within the field data. - /// - public int Offset { get; init; } - - /// - /// The length of this subfield's data in bytes. - /// - public int Length { get; init; } - - /// - /// The group index for repeating subfields. - /// - public int GroupIndex { get; init; } - } + private readonly record struct ParsedSubfield(int DefinitionIndex, int Offset, int Length); } diff --git a/src/EncDotNet.Iso8211/Iso8211Record.cs b/src/EncDotNet.Iso8211/Iso8211Record.cs index b88bf4a..f78286a 100644 --- a/src/EncDotNet.Iso8211/Iso8211Record.cs +++ b/src/EncDotNet.Iso8211/Iso8211Record.cs @@ -30,12 +30,34 @@ public sealed record Iso8211Record /// /// The tag to search for. /// The field with the specified tag, or null if not found. - public Iso8211Field? GetFieldByTag(string tag) => Fields.FirstOrDefault(f => f.Tag == tag); + public Iso8211Field? GetFieldByTag(string tag) + { + var fields = Fields; + for (var i = 0; i < fields.Count; i++) + { + if (fields[i].Tag == tag) + { + return fields[i]; + } + } + + return null; + } /// /// Gets all fields with the specified tag. /// /// The tag to search for. /// All fields with the specified tag. - public IEnumerable GetFieldsByTag(string tag) => Fields.Where(f => f.Tag == tag); + public IEnumerable GetFieldsByTag(string tag) + { + var fields = Fields; + for (var i = 0; i < fields.Count; i++) + { + if (fields[i].Tag == tag) + { + yield return fields[i]; + } + } + } } diff --git a/src/EncDotNet.Iso8211/Iso8211SubfieldGroup.cs b/src/EncDotNet.Iso8211/Iso8211SubfieldGroup.cs index 6105e64..d8c408f 100644 --- a/src/EncDotNet.Iso8211/Iso8211SubfieldGroup.cs +++ b/src/EncDotNet.Iso8211/Iso8211SubfieldGroup.cs @@ -43,17 +43,7 @@ internal Iso8211SubfieldGroup(Iso8211FieldReader reader, int startIndex, int len /// public T GetSubfield(string name) { - var subfieldDef = _reader.FieldDefinition.GetSubfieldDefinition(name); - if (subfieldDef is null) - { - throw new KeyNotFoundException($"Subfield '{name}' not found in field definition."); - } - - var groupSubfieldIndex = subfieldDef.Index; - if (_reader.FieldDefinition.HasRepeatingGroup) - { - groupSubfieldIndex = subfieldDef.Index - _reader.FieldDefinition.RepeatingSubfieldStartIndex; - } + var groupSubfieldIndex = GetGroupSubfieldIndex(name); if (groupSubfieldIndex >= 0 && groupSubfieldIndex < _length) { @@ -73,17 +63,7 @@ public T GetSubfield(string name) /// public ReadOnlySpan GetSubfieldBytes(string name) { - var subfieldDef = _reader.FieldDefinition.GetSubfieldDefinition(name); - if (subfieldDef is null) - { - throw new KeyNotFoundException($"Subfield '{name}' not found in field definition."); - } - - var groupSubfieldIndex = subfieldDef.Index; - if (_reader.FieldDefinition.HasRepeatingGroup) - { - groupSubfieldIndex = subfieldDef.Index - _reader.FieldDefinition.RepeatingSubfieldStartIndex; - } + var groupSubfieldIndex = GetGroupSubfieldIndex(name); if (groupSubfieldIndex >= 0 && groupSubfieldIndex < _length) { @@ -93,6 +73,22 @@ public ReadOnlySpan GetSubfieldBytes(string name) throw new KeyNotFoundException($"Subfield '{name}' not found in group {_groupIndex}."); } + /// + /// Gets a named subfield's position within a repetition of the group. + /// + private int GetGroupSubfieldIndex(string name) + { + var ordinal = _reader.IndexOfSubfield(name); + if (ordinal < 0) + { + throw new KeyNotFoundException($"Subfield '{name}' not found in field definition."); + } + + return _reader.FieldDefinition.HasRepeatingGroup + ? ordinal - _reader.FieldDefinition.RepeatingSubfieldStartIndex + : ordinal; + } + /// /// Gets the value of a subfield within this group by its position. /// diff --git a/src/EncDotNet.S57/S57DocumentReader.cs b/src/EncDotNet.S57/S57DocumentReader.cs index 6cd8952..b482137 100644 --- a/src/EncDotNet.S57/S57DocumentReader.cs +++ b/src/EncDotNet.S57/S57DocumentReader.cs @@ -513,19 +513,22 @@ private static ImmutableArray ParseAttributes(Iso8211Record r // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data, lexicalLevel); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var attl = reader.GetOrdinal(S57SubfieldNames.ATTL); + var atvl = reader.GetOrdinal(S57SubfieldNames.ATVL); + for (var group = 0; group < reader.GroupCount; group++) { - var attl = group.GetSubfield(S57SubfieldNames.ATTL); - var atvl = group.GetSubfield(S57SubfieldNames.ATVL); - attributes.Add(new S57AttributeValue(attl, atvl)); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse attribute in {FieldTag} field.", fieldTag); - break; + attributes.Add(new S57AttributeValue( + reader.GetSubfield(group, attl), + reader.GetSubfield(group, atvl))); } + + WarnIfTruncated(reader, logger, "Failed to parse attribute in {FieldTag} field.", fieldTag); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse attribute in {FieldTag} field.", fieldTag); } } @@ -552,32 +555,29 @@ private static ImmutableArray ParseSpatialPointers(Iso8211Rec // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var name = reader.GetOrdinal(S57SubfieldNames.NAME); + var ornt = reader.GetOrdinal(S57SubfieldNames.ORNT); + var usag = reader.GetOrdinal(S57SubfieldNames.USAG); + var mask = reader.GetOrdinal(S57SubfieldNames.MASK); + for (var group = 0; group < reader.GroupCount; group++) { - // NAME is a composite subfield (5 bytes: RCNM(1) + RCID(4)) - // Read it as raw bytes and decompose - var nameBytes = group.GetSubfieldBytes(S57SubfieldNames.NAME); - var name = DecomposeNameField(nameBytes); - - var ornt = group.GetSubfield(S57SubfieldNames.ORNT); - var usag = group.GetSubfield(S57SubfieldNames.USAG); - var mask = group.GetSubfield(S57SubfieldNames.MASK); - pointers.Add(new S57SpatialPointer { - Name = name, - Orientation = (S57Orientation)ornt, - Usage = (S57UsageIndicator)usag, - Mask = (S57MaskingIndicator)mask + // NAME is a composite subfield (5 bytes: RCNM(1) + RCID(4)), read as raw bytes + Name = DecomposeNameField(reader.GetSubfieldBytes(group, name)), + Orientation = (S57Orientation)reader.GetSubfield(group, ornt), + Usage = (S57UsageIndicator)reader.GetSubfield(group, usag), + Mask = (S57MaskingIndicator)reader.GetSubfield(group, mask) }); } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse spatial pointer in FSPT field."); - break; - } + + WarnIfTruncated(reader, logger, "Failed to parse spatial pointer in {FieldTag} field.", S57FieldTags.FSPT); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse spatial pointer in FSPT field."); } } @@ -602,29 +602,27 @@ private static ImmutableArray ParseFeaturePointers(Iso8211Rec // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var lnam = reader.GetOrdinal(S57SubfieldNames.LNAM); + var rind = reader.GetOrdinal(S57SubfieldNames.RIND); + var comt = reader.GetOrdinal(S57SubfieldNames.COMT); + for (var group = 0; group < reader.GroupCount; group++) { - // LNAM is a composite subfield (8 bytes: AGEN(2) + FIDN(4) + FIDS(2)) - var lnamBytes = group.GetSubfieldBytes(S57SubfieldNames.LNAM); - var lnam = DecomposeLongNameField(lnamBytes); - - var rind = group.GetSubfield(S57SubfieldNames.RIND); - var comt = group.GetSubfield(S57SubfieldNames.COMT); - pointers.Add(new S57FeaturePointer { - Name = lnam, - Relationship = (S57RelationshipIndicator)rind, - Comment = comt + // LNAM is a composite subfield (8 bytes: AGEN(2) + FIDN(4) + FIDS(2)) + Name = DecomposeLongNameField(reader.GetSubfieldBytes(group, lnam)), + Relationship = (S57RelationshipIndicator)reader.GetSubfield(group, rind), + Comment = reader.GetSubfield(group, comt) }); } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse feature pointer in FFPT field."); - break; - } + + WarnIfTruncated(reader, logger, "Failed to parse feature pointer in {FieldTag} field.", S57FieldTags.FFPT); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse feature pointer in FFPT field."); } } @@ -649,33 +647,31 @@ private static ImmutableArray ParseVectorPointers(Iso8211Recor // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var name = reader.GetOrdinal(S57SubfieldNames.NAME); + var ornt = reader.GetOrdinal(S57SubfieldNames.ORNT); + var usag = reader.GetOrdinal(S57SubfieldNames.USAG); + var topi = reader.GetOrdinal(S57SubfieldNames.TOPI); + var mask = reader.GetOrdinal(S57SubfieldNames.MASK); + for (var group = 0; group < reader.GroupCount; group++) { - // NAME is a composite subfield (5 bytes: RCNM(1) + RCID(4)) - var nameBytes = group.GetSubfieldBytes(S57SubfieldNames.NAME); - var name = DecomposeNameField(nameBytes); - - var ornt = group.GetSubfield(S57SubfieldNames.ORNT); - var usag = group.GetSubfield(S57SubfieldNames.USAG); - var topi = group.GetSubfield(S57SubfieldNames.TOPI); - var mask = group.GetSubfield(S57SubfieldNames.MASK); - pointers.Add(new S57VectorPointer { - Name = name, - Orientation = (S57Orientation)ornt, - Usage = (S57UsageIndicator)usag, - Topology = (S57TopologyIndicator)topi, - Mask = (S57MaskingIndicator)mask + // NAME is a composite subfield (5 bytes: RCNM(1) + RCID(4)), read as raw bytes + Name = DecomposeNameField(reader.GetSubfieldBytes(group, name)), + Orientation = (S57Orientation)reader.GetSubfield(group, ornt), + Usage = (S57UsageIndicator)reader.GetSubfield(group, usag), + Topology = (S57TopologyIndicator)reader.GetSubfield(group, topi), + Mask = (S57MaskingIndicator)reader.GetSubfield(group, mask) }); } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse vector pointer in VRPT field."); - break; - } + + WarnIfTruncated(reader, logger, "Failed to parse vector pointer in {FieldTag} field.", S57FieldTags.VRPT); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse vector pointer in VRPT field."); } } @@ -700,24 +696,24 @@ private static ImmutableArray ParseCoordinates2D(Iso8211Record // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var ycoo = reader.GetOrdinal(S57SubfieldNames.YCOO); + var xcoo = reader.GetOrdinal(S57SubfieldNames.XCOO); + for (var group = 0; group < reader.GroupCount; group++) { - var ycoo = group.GetSubfield(S57SubfieldNames.YCOO); - var xcoo = group.GetSubfield(S57SubfieldNames.XCOO); - coordinates.Add(new S57Coordinate2D { - Y = ycoo, - X = xcoo + Y = reader.GetSubfield(group, ycoo), + X = reader.GetSubfield(group, xcoo) }); } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse 2D coordinate in SG2D field."); - break; - } + + WarnIfTruncated(reader, logger, "Failed to parse 2D coordinate in {FieldTag} field.", S57FieldTags.SG2D); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse 2D coordinate in SG2D field."); } } @@ -742,32 +738,47 @@ private static ImmutableArray ParseSoundings(Iso8211Record record, // Use DDR-based field reader with repeating groups var reader = new Iso8211FieldReader(fieldDef, field.Data); - foreach (var group in reader.GetSubfieldGroups()) + try { - try + var ycoo = reader.GetOrdinal(S57SubfieldNames.YCOO); + var xcoo = reader.GetOrdinal(S57SubfieldNames.XCOO); + var ve3d = reader.GetOrdinal(S57SubfieldNames.VE3D); + for (var group = 0; group < reader.GroupCount; group++) { - var ycoo = group.GetSubfield(S57SubfieldNames.YCOO); - var xcoo = group.GetSubfield(S57SubfieldNames.XCOO); - var ve3d = group.GetSubfield(S57SubfieldNames.VE3D); - soundings.Add(new S57Sounding { - Y = ycoo, - X = xcoo, - Depth = ve3d + Y = reader.GetSubfield(group, ycoo), + X = reader.GetSubfield(group, xcoo), + Depth = reader.GetSubfield(group, ve3d) }); } - catch (Exception ex) - { - logger?.LogWarning(ex, "Failed to parse sounding in SG3D field."); - break; - } + + WarnIfTruncated(reader, logger, "Failed to parse sounding in {FieldTag} field.", S57FieldTags.SG3D); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse sounding in SG3D field."); } } return soundings.ToImmutable(); } + /// + /// Logs a warning when a field's data ends partway through a repetition of its group. The + /// complete repetitions before it are kept, and the partial one is dropped. + /// + private static void WarnIfTruncated(Iso8211FieldReader reader, ILogger? logger, string message, string fieldTag) + { + var definition = reader.FieldDefinition; + var repeatCount = definition.SubfieldDefinitions.Count - definition.RepeatingSubfieldStartIndex; + var complete = definition.RepeatingSubfieldStartIndex + (reader.GroupCount * repeatCount); + if (reader.SubfieldCount > complete) + { + logger?.LogWarning(message, fieldTag); + } + } + /// /// Decomposes a NAME composite subfield (5 bytes: RCNM(1) + RCID(4)) into an S57RecordName. /// diff --git a/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs b/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs index b32172f..5762dcc 100644 --- a/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs +++ b/tests/EncDotNet.PerformanceTests/AllocationBudgetTests.cs @@ -80,6 +80,37 @@ public void Iso8211FieldReader_DecodingCoordinates_StaysWithinBudget() PerformanceBudget.AssertAllocated("Iso8211FieldReader SG2D", allocated, PerformanceBudgets.Iso8211FieldReaderCoordinates); } + [Fact] + public void Iso8211FieldReader_DecodingCoordinatesByOrdinal_StaysWithinBudget() + { + // The ordinal accessors are the allocation-free way to read a repeating group: the + // name lookups happen once per field, and no group objects are created. + var ddr = Iso8211DataDescriptiveRecordReader.Read(Cell.Iso8211Document.DataDescriptiveRecord!); + var definition = ddr.GetFieldDefinition("SG2D")!; + var fields = Cell.Iso8211Document.DataRecords + .SelectMany(r => r.Fields) + .Where(f => f.Tag == "SG2D") + .ToArray(); + + var allocated = MemoryMeter.AllocatedBytes(() => + { + foreach (var field in fields) + { + var reader = new Iso8211FieldReader(definition, field.Data); + var y = reader.GetOrdinal("YCOO"); + var x = reader.GetOrdinal("XCOO"); + for (var group = 0; group < reader.GroupCount; group++) + { + _ = reader.GetSubfield(group, y); + _ = reader.GetSubfield(group, x); + } + } + }); + + PerformanceBudget.AssertAllocated( + "Iso8211FieldReader SG2D by ordinal", allocated, PerformanceBudgets.Iso8211FieldReaderCoordinatesByOrdinal); + } + [Fact] public void Iso8211DocumentWriter_Write_StaysWithinBudget() { diff --git a/tests/EncDotNet.PerformanceTests/PerformanceBudget.cs b/tests/EncDotNet.PerformanceTests/PerformanceBudget.cs index c996746..9b76781 100644 --- a/tests/EncDotNet.PerformanceTests/PerformanceBudget.cs +++ b/tests/EncDotNet.PerformanceTests/PerformanceBudget.cs @@ -63,11 +63,12 @@ private static void AssertWithin(string scenario, string kind, long actual, long if (actual < lower) { Assert.Fail( - $"{scenario} {kind} {actual:N0} bytes, under its budget of {budget:N0} by {Percent(budget, actual)}. " + + $"{scenario} {kind} {actual:N0} bytes, under its budget of {budget:N0} by {Percent(actual, budget)}. " + $"Lower the budget in PerformanceBudgets.cs to {actual:N0} to lock in the improvement."); } } - private static string Percent(long larger, long smaller) => - smaller == 0 ? "∞" : $"{(larger - smaller) * 100.0 / smaller:F1}%"; + // The size of the difference as a share of the budget. + private static string Percent(long actual, long budget) => + budget == 0 ? "∞" : $"{Math.Abs(actual - budget) * 100.0 / budget:F1}%"; } diff --git a/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs b/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs index 168a67d..4396c2f 100644 --- a/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs +++ b/tests/EncDotNet.PerformanceTests/PerformanceBudgets.cs @@ -19,10 +19,11 @@ internal static class PerformanceBudgets // Allocated bytes for one operation. public const long Iso8211ReaderAllTokens = 0; public const long Iso8211DocumentReaderRead = 2_631_240; - public const long Iso8211DataDescriptiveRecordReaderRead = 50_136; - public const long Iso8211FieldReaderCoordinates = 6_194_360; + public const long Iso8211DataDescriptiveRecordReaderRead = 32_808; + public const long Iso8211FieldReaderCoordinates = 846_376; + public const long Iso8211FieldReaderCoordinatesByOrdinal = 141_264; public const long Iso8211DocumentWriterWrite = 2_318_456; - public const long S57DocumentReaderRead = 29_024_040; + public const long S57DocumentReaderRead = 7_991_888; public const long S57DocumentApplyChanges = 685_440; public const long S57ChartFromDocument = 1_030_312; diff --git a/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs b/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs new file mode 100644 index 0000000..2f808a8 --- /dev/null +++ b/tests/EncDotNet.UnitTests/Iso8211FieldReaderEquivalenceTests.cs @@ -0,0 +1,323 @@ +using System.Globalization; +using System.Text; +using EncDotNet.Iso8211; +using EncDotNet.Testing; +using EncDotNet.UnitTests.Reference; + +namespace EncDotNet.UnitTests; + +/// +/// Checks that returns exactly what it did before its +/// performance rewrite, by comparing it with , a frozen +/// copy of the earlier implementation, on real-shaped, truncated and hand-built fields. +/// +public class Iso8211FieldReaderEquivalenceTests +{ + private const byte UnitTerminator = 0x1F; + private const byte FieldTerminator = 0x1E; + + [Fact] + public void EveryFieldOfTheSyntheticCell_ReadsTheSame() + { + // The larger profiles have the same field shapes, only more of them. + var profile = SyntheticCellProfile.Harbour; + AssertCellReadsTheSame(SyntheticS57Cell.Create(profile), truncate: false); + AssertCellReadsTheSame(SyntheticS57Cell.CreateUpdate(profile, 1), truncate: false); + } + + [Fact] + public void TruncatedFieldsOfTheSyntheticCell_ReadTheSame() + { + // Every prefix of the data, with and without a trailing field terminator, exercises + // partial subfields, partial groups and the terminator checks. + AssertCellReadsTheSame(SyntheticS57Cell.Create(SyntheticCellProfile.Harbour), truncate: true); + } + + [Fact] + public void EveryFieldOfRealCells_ReadsTheSame() + { + // Real cells aren't committed. Point ENCDOTNET_ENC_ROOT at a directory of them, as for + // the benchmarks, to include them. + var root = Environment.GetEnvironmentVariable("ENCDOTNET_ENC_ROOT"); + if (string.IsNullOrEmpty(root) || !Directory.Exists(root)) + { + return; + } + + foreach (var path in Directory.EnumerateFiles(root, "*.0??", SearchOption.AllDirectories)) + { + if (Path.GetExtension(path)[1..].All(char.IsAsciiDigit)) + { + AssertCellReadsTheSame(File.ReadAllBytes(path), truncate: false); + } + } + } + + public static TheoryData HandBuiltLayouts => new() + { + // Name, descriptors ('*' starts the repeating group), repeating start, formats. + { "all fixed, no group", "RCNM!RCID!PRIM!OBJL", -1, "b11,b14,b11,b12" }, + { "fixed group", "YCOO!XCOO", 0, "b24,b24" }, + { "fixed prefix then fixed group (S-101 C3IL)", "VCID!YCOO!XCOO!ZCOO", 1, "b11,b24,b24,b24" }, + { "variable group", "ATTL!ATVL", 0, "b12,A" }, + { "fixed prefix then variable group", "RCNM!ATTL!ATVL", 1, "b11,b12,A" }, + { "variable, no group", "DSNM!EDTN!UPDN", -1, "A,A,A" }, + { "ascii numbers", "NUM!REAL!TEXT", -1, "I,R,A" }, + { "fixed ascii", "CODE!NUM", -1, "A(3),I(4)" }, + { "signed and float", "S1!S2!S4!F4!F8", -1, "b21,b22,b24,b44,b48" }, + { "odd widths", "U3!S3!U5", -1, "b13,b23,b15" }, + { "bit string group", "NAME!ORNT", 0, "B(40),b11" }, + }; + + [Theory] + [MemberData(nameof(HandBuiltLayouts))] + public void HandBuiltLayouts_ReadTheSame(string name, string labels, int repeatingStart, string formats) + { + _ = name; + var definition = Definition(labels.Split('!'), repeatingStart, formats.Split(',')); + var random = new Random(labels.Length * 31 + formats.Length); + + foreach (var lexicalLevel in new[] { 0, 2 }) + { + for (var sample = 0; sample < 8; sample++) + { + var data = RandomFieldData(definition, random, lexicalLevel); + for (var length = 0; length <= data.Length; length++) + { + AssertReadsTheSame(definition, data[..length], lexicalLevel); + } + } + } + } + + private static void AssertCellReadsTheSame(byte[] cell, bool truncate) + { + var document = Iso8211DocumentReader.Read(cell); + var ddr = Iso8211DataDescriptiveRecordReader.Read(document.DataDescriptiveRecord!); + var truncated = 0; + + foreach (var record in document.DataRecords) + { + foreach (var field in record.Fields) + { + var definition = ddr.GetFieldDefinition(field.Tag); + if (definition is null) + { + continue; + } + + AssertReadsTheSame(definition, field.Data, lexicalLevel: 0); + + // Truncating every field is slow; a few hundred cover each field type many times. + if (truncate && truncated++ < 150) + { + for (var length = 0; length < field.Data.Length; length++) + { + AssertReadsTheSame(definition, field.Data[..length], lexicalLevel: 0); + AssertReadsTheSame(definition, [.. field.Data[..length], FieldTerminator], lexicalLevel: 0); + } + } + } + } + } + + private static void AssertReadsTheSame(Iso8211FieldDefinition definition, byte[] data, int lexicalLevel) + { + var context = $"{definition.Tag} [{Convert.ToHexString(data)}] level {lexicalLevel}"; + var legacy = new LegacyIso8211FieldReader(definition, data, lexicalLevel); + var current = new Iso8211FieldReader(definition, data, lexicalLevel); + + Assert.True(legacy.SubfieldCount == current.SubfieldCount, $"SubfieldCount: {context}"); + Assert.True(legacy.GroupCount == current.GroupCount, $"GroupCount: {context}"); + Assert.True(legacy.HasRepeatingGroups == current.HasRepeatingGroups, $"HasRepeatingGroups: {context}"); + + for (var i = 0; i < legacy.SubfieldCount; i++) + { + Assert.True( + legacy.GetSubfieldBytesAt(i).SequenceEqual(current.GetSubfieldBytesAt(i)), + $"GetSubfieldBytesAt({i}): {context}"); + AssertSame(() => legacy.GetSubfieldAt(i), () => current.GetSubfieldAt(i), $"GetSubfieldAt({i}): {context}"); + AssertSame(() => legacy.GetSubfieldAt(i), () => current.GetSubfieldAt(i), $"GetSubfieldAt({i}): {context}"); + AssertSame(() => legacy.GetSubfieldAt(i), () => current.GetSubfieldAt(i), $"GetSubfieldAt({i}): {context}"); + AssertSame(() => legacy.GetSubfieldAt(i), () => current.GetSubfieldAt(i), $"GetSubfieldAt({i}): {context}"); + AssertSame(() => legacy.GetSubfieldAt(i), () => current.GetSubfieldAt(i), $"GetSubfieldAt({i}): {context}"); + AssertSame(() => Hex(legacy.GetSubfieldAt(i)), () => Hex(current.GetSubfieldAt(i)), $"GetSubfieldAt({i}): {context}"); + } + + foreach (var subfield in definition.SubfieldDefinitions) + { + var name = subfield.Name; + AssertSame( + () => legacy.TryGetSubfield(name, out var v) ? v : "", + () => current.TryGetSubfield(name, out var v) ? v : "", + $"TryGetSubfield({name}): {context}"); + AssertSame( + () => legacy.TryGetSubfield(name, out var v) ? v.ToString(CultureInfo.InvariantCulture) : "", + () => current.TryGetSubfield(name, out var v) ? v.ToString(CultureInfo.InvariantCulture) : "", + $"TryGetSubfield({name}): {context}"); + AssertSame( + () => string.Join("|", legacy.GetSubfieldValues(name)), + () => string.Join("|", current.GetSubfieldValues(name)), + $"GetSubfieldValues({name}): {context}"); + AssertSame( + () => Hex(legacy.GetSubfieldBytes(name).ToArray()), + () => Hex(current.GetSubfieldBytes(name).ToArray()), + $"GetSubfieldBytes({name}): {context}"); + } + + AssertGroupAccessorsAgree(current, context); + } + + /// + /// Checks the new ordinal accessors against the group objects and indexed accessors. + /// + private static void AssertGroupAccessorsAgree(Iso8211FieldReader reader, string context) + { + var definition = reader.FieldDefinition; + var groupStart = definition.HasRepeatingGroup ? definition.RepeatingSubfieldStartIndex : 0; + var groups = reader.GetSubfieldGroups().ToArray(); + + for (var g = 0; g < groups.Length; g++) + { + for (var k = 0; k < groups[g].Count; k++) + { + var ordinal = groupStart + k; + Assert.Equal(ordinal, reader.GetOrdinal(definition.SubfieldDefinitions[ordinal].Name)); + AssertSame( + () => groups[g].GetSubfieldAt(k), + () => reader.GetSubfield(g, ordinal), + $"GetSubfield({g}, {ordinal}): {context}"); + // Like TryGetSubfield(string), the Try form reports absence, not conversion failures. + AssertSame( + () => reader.TryGetSubfield(g, ordinal, out var value) ? value : "", + () => reader.GetSubfield(g, ordinal), + $"TryGetSubfield({g}, {ordinal}): {context}"); + } + + // Ordinals past a truncated final group are absent, not an error. + for (var k = groups[g].Count; k < definition.SubfieldDefinitions.Count - groupStart; k++) + { + Assert.False(reader.TryGetSubfield(g, groupStart + k, out _), $"TryGetSubfield past end: {context}"); + } + } + + Assert.False(reader.TryGetSubfield(groups.Length, groupStart, out _), $"TryGetSubfield past last group: {context}"); + } + + private static void AssertSame(Func legacy, Func current, string message) + { + Assert.True(Outcome(legacy) == Outcome(current), $"{message}: {Outcome(legacy)} vs {Outcome(current)}"); + } + + private static string Outcome(Func read) + { + try + { + return "value " + Convert.ToString(read(), CultureInfo.InvariantCulture); + } + catch (Exception ex) + { + return "throws " + ex.GetType().Name; + } + } + + private static string Hex(byte[] bytes) => Convert.ToHexString(bytes); + + private static Iso8211FieldDefinition Definition(string[] labels, int repeatingStart, string[] formats) + { + var subfields = new List(); + for (var i = 0; i < labels.Length; i++) + { + subfields.Add(new Iso8211SubfieldDefinition { Name = labels[i], Index = i, Format = ParseFormat(formats[i]) }); + } + + return new Iso8211FieldDefinition + { + Tag = "TEST", + DataStructureCode = repeatingStart >= 0 ? Iso8211DataStructureCode.Array : Iso8211DataStructureCode.Vector, + DataTypeCode = Iso8211DataTypeCode.MixedDataTypes, + SubfieldDefinitions = subfields, + RepeatingSubfieldStartIndex = repeatingStart, + }; + } + + private static Iso8211SubfieldFormat ParseFormat(string format) => format switch + { + "A" => new() { FormatType = Iso8211SubfieldFormatType.CharacterData }, + "I" => new() { FormatType = Iso8211SubfieldFormatType.Integer }, + "R" => new() { FormatType = Iso8211SubfieldFormatType.Real }, + _ when format.StartsWith("A(", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.CharacterData, Width = int.Parse(format[2..^1], CultureInfo.InvariantCulture) }, + _ when format.StartsWith("I(", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.Integer, Width = int.Parse(format[2..^1], CultureInfo.InvariantCulture) }, + _ when format.StartsWith("B(", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.BitString, Width = int.Parse(format[2..^1], CultureInfo.InvariantCulture) / 8 }, + _ when format.StartsWith("b1", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.UnsignedInteger, Width = format[2] - '0' }, + _ when format.StartsWith("b2", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.SignedInteger, Width = format[2] - '0' }, + _ when format.StartsWith("b4", StringComparison.Ordinal) => new() { FormatType = Iso8211SubfieldFormatType.FloatingPoint, Width = format[2] - '0' }, + _ => throw new ArgumentOutOfRangeException(nameof(format), format), + }; + + /// + /// Builds plausible field data: the leading subfields, then up to five repetitions, with + /// variable-length values that sometimes contain spaces or are empty. + /// + private static byte[] RandomFieldData(Iso8211FieldDefinition definition, Random random, int lexicalLevel) + { + var subfields = definition.SubfieldDefinitions; + var start = definition.HasRepeatingGroup ? definition.RepeatingSubfieldStartIndex : 0; + var bytes = new List(); + + void Append(Iso8211SubfieldFormat format) + { + if (format.IsFixedWidth) + { + if (format.FormatType is Iso8211SubfieldFormatType.CharacterData or Iso8211SubfieldFormatType.Integer) + { + bytes.AddRange(Encoding.ASCII.GetBytes(random.Next(0, 10_000).ToString(CultureInfo.InvariantCulture).PadLeft(format.Width, '0')[..format.Width])); + } + else + { + for (var b = 0; b < format.Width; b++) + { + // Include terminator values, which fixed-width binary data may hold. + bytes.Add(random.Next(4) == 0 ? FieldTerminator : (byte)random.Next(256)); + } + } + + return; + } + + var text = format.FormatType switch + { + Iso8211SubfieldFormatType.Integer => random.Next(-500, 500).ToString(CultureInfo.InvariantCulture), + Iso8211SubfieldFormatType.Real => (random.NextDouble() * 100).ToString("F2", CultureInfo.InvariantCulture), + _ => random.Next(5) == 0 ? string.Empty : new string('X', random.Next(1, 8)) + (random.Next(2) == 0 ? " " : string.Empty), + }; + if (lexicalLevel >= 2) + { + bytes.AddRange(Encoding.Unicode.GetBytes(text)); + bytes.Add(UnitTerminator); + bytes.Add(0); + } + else + { + bytes.AddRange(Encoding.ASCII.GetBytes(text)); + bytes.Add(UnitTerminator); + } + } + + for (var i = 0; i < start; i++) + { + Append(subfields[i].Format); + } + + var repetitions = definition.HasRepeatingGroup ? random.Next(0, 6) : 1; + for (var r = 0; r < repetitions; r++) + { + for (var i = start; i < subfields.Count; i++) + { + Append(subfields[i].Format); + } + } + + return [.. bytes]; + } +} diff --git a/tests/EncDotNet.UnitTests/Reference/LegacyIso8211FieldReader.cs b/tests/EncDotNet.UnitTests/Reference/LegacyIso8211FieldReader.cs new file mode 100644 index 0000000..60dbff5 --- /dev/null +++ b/tests/EncDotNet.UnitTests/Reference/LegacyIso8211FieldReader.cs @@ -0,0 +1,680 @@ +using System.Buffers.Binary; +using System.Collections.Immutable; +using System.Globalization; +using System.Text; + +using EncDotNet.Iso8211; + +namespace EncDotNet.UnitTests.Reference; + +/// +/// A frozen copy of as it was before its performance rewrite, +/// used only as the reference that the differential tests compare the current reader against. +/// +internal sealed class LegacyIso8211FieldReader +{ + private const byte UnitTerminator = 0x1F; + private const byte FieldTerminator = 0x1E; + + private readonly Iso8211FieldDefinition _fieldDefinition; + private readonly byte[] _data; + private readonly ParsedSubfield[] _parsedSubfields; + private readonly int _groupCount; + private readonly int _lexicalLevel; + + public LegacyIso8211FieldReader(Iso8211FieldDefinition fieldDefinition, ReadOnlySpan data, int lexicalLevel = 0) + { + _fieldDefinition = fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)); + _data = data.ToArray(); + _lexicalLevel = lexicalLevel; + (_parsedSubfields, _groupCount) = ParseSubfields(); + } + + public LegacyIso8211FieldReader(Iso8211FieldDefinition fieldDefinition, byte[] data, int lexicalLevel = 0) + { + _fieldDefinition = fieldDefinition ?? throw new ArgumentNullException(nameof(fieldDefinition)); + _data = data ?? throw new ArgumentNullException(nameof(data)); + _lexicalLevel = lexicalLevel; + (_parsedSubfields, _groupCount) = ParseSubfields(); + } + + public Iso8211FieldDefinition FieldDefinition => _fieldDefinition; + + public ReadOnlySpan Data => _data; + + public int SubfieldCount => _parsedSubfields.Length; + + public int GroupCount => _groupCount; + + public bool HasRepeatingGroups => _fieldDefinition.HasRepeatingGroup && _groupCount > 1; + + public T GetSubfield(string name) + { + if (!TryGetSubfield(name, out var value)) + { + throw new KeyNotFoundException($"Subfield '{name}' not found in field '{_fieldDefinition.Tag}'."); + } + return value; + } + + public bool TryGetSubfield(string name, out T value) + { + var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); + if (subfieldDef is null) + { + value = default!; + return false; + } + + // Find the first parsed subfield matching this definition + foreach (var parsed in _parsedSubfields) + { + if (parsed.DefinitionIndex == subfieldDef.Index) + { + value = ConvertValue(parsed, subfieldDef); + return true; + } + } + + value = default!; + return false; + } + + public ImmutableArray GetSubfieldValues(string name) + { + var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); + if (subfieldDef is null) + { + throw new KeyNotFoundException($"Subfield '{name}' not found in field definition '{_fieldDefinition.Tag}'."); + } + + var builder = ImmutableArray.CreateBuilder(); + + foreach (var parsed in _parsedSubfields) + { + if (parsed.DefinitionIndex == subfieldDef.Index) + { + builder.Add(ConvertValue(parsed, subfieldDef)); + } + } + + return builder.ToImmutable(); + } + + public T GetSubfieldAt(int index) + { + if (index < 0 || index >= _parsedSubfields.Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + var parsed = _parsedSubfields[index]; + var subfieldDef = _fieldDefinition.SubfieldDefinitions[parsed.DefinitionIndex]; + return ConvertValue(parsed, subfieldDef); + } + + public ReadOnlySpan GetSubfieldBytes(string name) + { + var subfieldDef = _fieldDefinition.GetSubfieldDefinition(name); + if (subfieldDef is null) + { + throw new KeyNotFoundException($"Subfield '{name}' not found in field '{_fieldDefinition.Tag}'."); + } + + foreach (var parsed in _parsedSubfields) + { + if (parsed.DefinitionIndex == subfieldDef.Index) + { + return _data.AsSpan(parsed.Offset, parsed.Length); + } + } + + throw new KeyNotFoundException($"Subfield '{name}' not found in parsed field data."); + } + + public ReadOnlySpan GetSubfieldBytesAt(int index) + { + if (index < 0 || index >= _parsedSubfields.Length) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + var parsed = _parsedSubfields[index]; + return _data.AsSpan(parsed.Offset, parsed.Length); + } + + private (ParsedSubfield[] subfields, int groupCount) ParseSubfields() + { + var subfields = _fieldDefinition.SubfieldDefinitions; + if (subfields.Count == 0) + { + return ([], 0); + } + + // Pre-size: for repeating groups, estimate ~4 bytes per subfield + int initialCapacity = subfields.Count; + if (_fieldDefinition.HasRepeatingGroup && _data.Length > 0) + { + initialCapacity = Math.Max(initialCapacity, _data.Length / 4); + } + + var parsed = new List(initialCapacity); + var data = _data.AsSpan(); + int offset = 0; + int subfieldIndex = 0; + int groupCount = 0; + bool inRepeatingGroup = false; + + while (offset < data.Length) + { + var subfieldDef = subfields[subfieldIndex]; + var format = subfieldDef.Format; + + // For variable-length subfields, check if we've hit a field terminator. + // Fixed-width binary subfields may contain 0x1E (30) as valid data, so we + // must not treat it as a terminator. + if (!format.IsFixedWidth && IsFieldTerminator(data, offset)) + { + break; + } + + // For fixed-width subfields in repeating groups, check if we've hit the end: + // If the remaining data is just the field terminator, we're done. + if (format.IsFixedWidth && _fieldDefinition.HasRepeatingGroup) + { + // If remaining bytes < expected width AND the first remaining byte is FT, stop + int remaining = data.Length - offset; + if (remaining < format.Width && remaining > 0 && data[offset] == FieldTerminator) + { + break; + } + } + + // Calculate the length of this subfield + int length = CalculateSubfieldLength(data, offset, format); + + parsed.Add(new ParsedSubfield + { + DefinitionIndex = subfieldIndex, + Offset = offset, + Length = length, + GroupIndex = groupCount + }); + + offset += length; + + // Skip unit terminator if present - only for variable-length subfields + // Fixed-width binary subfields (b11, b12, b14, b21, b22, b24) don't have UTs + if (!format.IsFixedWidth) + { + if (IsUnitTerminator(data, offset)) + { + offset += TerminatorWidth; + } + else if (IsFieldTerminator(data, offset)) + { + // Field terminator acts as the final unit terminator; don't advance + // past it so the next iteration's FT check can break the loop. + } + } + + // Advance to next subfield definition + subfieldIndex++; + + // Handle repeating groups + if (subfieldIndex >= subfields.Count) + { + if (_fieldDefinition.HasRepeatingGroup) + { + // Reset to the start of the repeating group + subfieldIndex = _fieldDefinition.RepeatingSubfieldStartIndex; + groupCount++; + inRepeatingGroup = true; + } + else + { + // No repeating group, we're done with definitions + break; + } + } + else if (!inRepeatingGroup && subfieldIndex == _fieldDefinition.RepeatingSubfieldStartIndex && _fieldDefinition.HasRepeatingGroup) + { + // Just entered the repeating group for the first time. + // Set to 0; the wrap-around at line ~424 will increment to 1 + // after the first complete group is parsed. + groupCount = 0; + inRepeatingGroup = true; + } + } + + // If we never entered a repeating group but have subfields, we have 1 group + if (!inRepeatingGroup && parsed.Count > 0) + { + groupCount = 1; + } + + return (parsed.ToArray(), groupCount); + } + + private int TerminatorWidth => _lexicalLevel >= 2 ? 2 : 1; + + private bool IsFieldTerminator(ReadOnlySpan data, int offset) + { + if (offset >= data.Length) return false; + if (data[offset] != FieldTerminator) return false; + if (_lexicalLevel >= 2) + return offset + 1 < data.Length && data[offset + 1] == 0x00; + return true; + } + + private bool IsUnitTerminator(ReadOnlySpan data, int offset) + { + if (offset >= data.Length) return false; + if (data[offset] != UnitTerminator) return false; + if (_lexicalLevel >= 2) + return offset + 1 < data.Length && data[offset + 1] == 0x00; + return true; + } + + private int CalculateSubfieldLength(ReadOnlySpan data, int offset, Iso8211SubfieldFormat format) + { + if (format.IsFixedWidth) + { + // Fixed-width subfield + return Math.Min(format.Width, data.Length - offset); + } + + // Variable-length subfield - scan for unit terminator or field terminator + int length = 0; + while (offset + length < data.Length) + { + if (IsUnitTerminator(data, offset + length) || IsFieldTerminator(data, offset + length)) + { + break; + } + length++; + } + + return length; + } + + private T ConvertValue(ParsedSubfield parsed, Iso8211SubfieldDefinition subfieldDef) + { + var span = _data.AsSpan(parsed.Offset, parsed.Length); + var format = subfieldDef.Format; + + object result = format.FormatType switch + { + Iso8211SubfieldFormatType.CharacterData => ConvertCharacterData(span), + Iso8211SubfieldFormatType.Integer => ConvertAsciiInteger(span), + Iso8211SubfieldFormatType.Real => ConvertAsciiReal(span), + Iso8211SubfieldFormatType.UnsignedInteger => ConvertUnsignedBinary(span, format.Width), + Iso8211SubfieldFormatType.SignedInteger => ConvertSignedBinary(span, format.Width), + Iso8211SubfieldFormatType.FloatingPoint => ConvertFloatingBinary(span, format.Width), + Iso8211SubfieldFormatType.BitString => ConvertBitString(span), + _ => throw new InvalidOperationException($"Unknown format type: {format.FormatType}") + }; + + return (T)result; + } + + private object ConvertCharacterData(ReadOnlySpan data) + { + var encoding = _lexicalLevel >= 2 ? Encoding.Unicode : Encoding.UTF8; + var str = encoding.GetString(data).TrimEnd('\x1F', '\x1E', '\0', ' '); + + if (typeof(T) == typeof(string)) + { + return str; + } + if (typeof(T) == typeof(int)) + { + return int.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(long)) + { + return long.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(double)) + { + return double.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(float)) + { + return float.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(byte[])) + { + return data.ToArray(); + } + if (typeof(T) == typeof(byte)) + { + return byte.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(sbyte)) + { + return sbyte.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(short)) + { + return short.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(ushort)) + { + return ushort.Parse(str, CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert character data to type {typeof(T).Name}."); + } + + private static object ConvertAsciiInteger(ReadOnlySpan data) + { + var str = Encoding.ASCII.GetString(data).Trim(); + + if (string.IsNullOrEmpty(str)) + { + return default(T)!; + } + + if (typeof(T) == typeof(string)) + { + return str; + } + if (typeof(T) == typeof(int)) + { + return int.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(uint)) + { + return uint.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(long)) + { + return long.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(ulong)) + { + return ulong.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(short)) + { + return short.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(ushort)) + { + return ushort.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(byte)) + { + return byte.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(sbyte)) + { + return sbyte.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(double)) + { + return double.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(float)) + { + return float.Parse(str, CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert ASCII integer to type {typeof(T).Name}."); + } + + private static object ConvertAsciiReal(ReadOnlySpan data) + { + var str = Encoding.ASCII.GetString(data).Trim(); + + if (string.IsNullOrEmpty(str)) + { + return default(T)!; + } + + if (typeof(T) == typeof(string)) + { + return str; + } + if (typeof(T) == typeof(double)) + { + return double.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(float)) + { + return float.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(decimal)) + { + return decimal.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(int)) + { + return (int)double.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(long)) + { + return (long)double.Parse(str, CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert ASCII real to type {typeof(T).Name}."); + } + + private static object ConvertUnsignedBinary(ReadOnlySpan data, int width) + { + // Use actual data length if shorter than expected width + int actualWidth = Math.Min(width, data.Length); + + ulong value = actualWidth switch + { + 1 => data[0], + 2 => BinaryPrimitives.ReadUInt16LittleEndian(data), + 4 => BinaryPrimitives.ReadUInt32LittleEndian(data), + 8 => BinaryPrimitives.ReadUInt64LittleEndian(data), + _ => ReadUnsignedLittleEndian(data, actualWidth) + }; + + if (typeof(T) == typeof(byte)) + { + return (byte)value; + } + if (typeof(T) == typeof(ushort)) + { + return (ushort)value; + } + if (typeof(T) == typeof(uint)) + { + return (uint)value; + } + if (typeof(T) == typeof(ulong)) + { + return value; + } + if (typeof(T) == typeof(int)) + { + return (int)value; + } + if (typeof(T) == typeof(long)) + { + return (long)value; + } + if (typeof(T) == typeof(short)) + { + return (short)value; + } + if (typeof(T) == typeof(sbyte)) + { + return (sbyte)value; + } + if (typeof(T) == typeof(double)) + { + return (double)value; + } + if (typeof(T) == typeof(float)) + { + return (float)value; + } + if (typeof(T) == typeof(string)) + { + return value.ToString(CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert unsigned binary to type {typeof(T).Name}."); + } + + private static object ConvertSignedBinary(ReadOnlySpan data, int width) + { + // Use actual data length if shorter than expected width + int actualWidth = Math.Min(width, data.Length); + + long value = actualWidth switch + { + 1 => (sbyte)data[0], + 2 => BinaryPrimitives.ReadInt16LittleEndian(data), + 4 => BinaryPrimitives.ReadInt32LittleEndian(data), + 8 => BinaryPrimitives.ReadInt64LittleEndian(data), + _ => ReadSignedLittleEndian(data, actualWidth) + }; + + if (typeof(T) == typeof(sbyte)) + { + return (sbyte)value; + } + if (typeof(T) == typeof(short)) + { + return (short)value; + } + if (typeof(T) == typeof(int)) + { + return (int)value; + } + if (typeof(T) == typeof(long)) + { + return value; + } + if (typeof(T) == typeof(byte)) + { + return (byte)value; + } + if (typeof(T) == typeof(ushort)) + { + return (ushort)value; + } + if (typeof(T) == typeof(uint)) + { + return (uint)value; + } + if (typeof(T) == typeof(ulong)) + { + return (ulong)value; + } + if (typeof(T) == typeof(double)) + { + return (double)value; + } + if (typeof(T) == typeof(float)) + { + return (float)value; + } + if (typeof(T) == typeof(string)) + { + return value.ToString(CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert signed binary to type {typeof(T).Name}."); + } + + private static object ConvertFloatingBinary(ReadOnlySpan data, int width) + { + int actualWidth = Math.Min(width, data.Length); + + double value = actualWidth switch + { + 4 => BinaryPrimitives.ReadSingleLittleEndian(data), + 8 => BinaryPrimitives.ReadDoubleLittleEndian(data), + _ => throw new InvalidOperationException( + $"Unsupported floating-point binary width: {actualWidth} byte(s).") + }; + + if (typeof(T) == typeof(double)) + { + return value; + } + if (typeof(T) == typeof(float)) + { + return (float)value; + } + if (typeof(T) == typeof(decimal)) + { + return (decimal)value; + } + if (typeof(T) == typeof(int)) + { + return (int)value; + } + if (typeof(T) == typeof(long)) + { + return (long)value; + } + if (typeof(T) == typeof(string)) + { + return value.ToString(CultureInfo.InvariantCulture); + } + + throw new InvalidOperationException($"Cannot convert floating-point binary to type {typeof(T).Name}."); + } + + private static object ConvertBitString(ReadOnlySpan data) + { + if (typeof(T) == typeof(byte[])) + { + return data.ToArray(); + } + if (typeof(T) == typeof(string)) + { + // Return hex representation + return Convert.ToHexString(data); + } + + // For integer types, delegate to unsigned binary conversion + return ConvertUnsignedBinary(data, data.Length); + } + + private static ulong ReadUnsignedLittleEndian(ReadOnlySpan data, int width) + { + ulong value = 0; + for (int i = 0; i < width && i < data.Length; i++) + { + value |= (ulong)data[i] << (i * 8); + } + return value; + } + + private static long ReadSignedLittleEndian(ReadOnlySpan data, int width) + { + var unsigned = ReadUnsignedLittleEndian(data, width); + + // Sign extend if the high bit is set + var signBit = 1UL << (width * 8 - 1); + if ((unsigned & signBit) != 0) + { + // Set all bits above the width to 1 + var mask = ~((1UL << (width * 8)) - 1); + unsigned |= mask; + } + + return (long)unsigned; + } + + private readonly struct ParsedSubfield + { + public int DefinitionIndex { get; init; } + + public int Offset { get; init; } + + public int Length { get; init; } + + public int GroupIndex { get; init; } + } +} diff --git a/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs b/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs new file mode 100644 index 0000000..78a14e1 --- /dev/null +++ b/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs @@ -0,0 +1,98 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using EncDotNet.S57; + +namespace EncDotNet.UnitTests; + +/// +/// Reduces an to a SHA-256 hash of a canonical text dump of every +/// value it holds, so tests can pin a reader's exact output across internal rewrites. +/// +internal static class S57DocumentFingerprint +{ + public static string Compute(S57Document document) + { + var text = new StringBuilder(); + text.AppendLine(document.DataSetIdentification?.ToString() ?? "no DSID"); + text.AppendLine(document.DataSetParameters?.ToString() ?? "no DSPM"); + + foreach (var feature in document.FeatureRecords) + { + text.Append("F ").Append(Name(feature.RecordName)) + .Append(' ').Append((int)feature.Primitive) + .Append(' ').Append(feature.Group) + .Append(' ').Append((int)feature.ObjectCode) + .Append(' ').Append(feature.RecordVersion) + .Append(' ').Append((int)feature.UpdateInstruction) + .AppendLine(); + Attributes(text, "A", feature.Attributes); + Attributes(text, "N", feature.NationalAttributes); + foreach (var p in feature.SpatialPointers) + { + text.Append(" S ").Append(Name(p.Name)).Append(' ') + .Append((int)p.Orientation).Append(' ').Append((int)p.Usage).Append(' ').Append((int)p.Mask).AppendLine(); + } + + foreach (var p in feature.FeaturePointers) + { + text.Append(" P ").Append(Name(p.Name)).Append(' ') + .Append((int)p.Relationship).Append(' ').Append(p.Comment).AppendLine(); + } + + Control(text, "SC", feature.SpatialPointerControl); + Control(text, "PC", feature.FeaturePointerControl); + } + + foreach (var vector in document.VectorRecords) + { + text.Append("V ").Append(Name(vector.RecordName)) + .Append(' ').Append(vector.RecordVersion) + .Append(' ').Append((int)vector.UpdateInstruction) + .AppendLine(); + Attributes(text, "A", vector.Attributes); + foreach (var p in vector.VectorPointers) + { + text.Append(" V ").Append(Name(p.Name)).Append(' ') + .Append((int)p.Orientation).Append(' ').Append((int)p.Usage).Append(' ') + .Append((int)p.Topology).Append(' ').Append((int)p.Mask).AppendLine(); + } + + foreach (var c in vector.Coordinates2D) + { + text.Append(" C ").Append(c.Y).Append(' ').Append(c.X).AppendLine(); + } + + foreach (var s in vector.Soundings) + { + text.Append(" D ").Append(s.Y).Append(' ').Append(s.X).Append(' ').Append(s.Depth).AppendLine(); + } + + Control(text, "VC", vector.VectorPointerControl); + Control(text, "CC", vector.CoordinateControl); + } + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))); + } + + private static string Name(S57RecordName name) => string.Create( + CultureInfo.InvariantCulture, + $"{name.RecordNameCode}/{name.RecordId}/{name.AgencyCode}/{name.FeatureId}/{name.FeatureSubdivision}"); + + private static void Attributes(StringBuilder text, string kind, IReadOnlyList attributes) + { + foreach (var a in attributes) + { + text.Append(' ').Append(kind).Append(' ').Append(a.AttributeCode).Append('=').Append(a.Value).AppendLine(); + } + } + + private static void Control(StringBuilder text, string kind, S57FieldUpdateControl? control) + { + if (control is { } c) + { + text.Append(' ').Append(kind).Append(' ') + .Append((int)c.UpdateInstruction).Append(' ').Append(c.Index).Append(' ').Append(c.Count).AppendLine(); + } + } +} diff --git a/tests/EncDotNet.UnitTests/S57DocumentReaderFingerprintTests.cs b/tests/EncDotNet.UnitTests/S57DocumentReaderFingerprintTests.cs new file mode 100644 index 0000000..41d62a3 --- /dev/null +++ b/tests/EncDotNet.UnitTests/S57DocumentReaderFingerprintTests.cs @@ -0,0 +1,60 @@ +using EncDotNet.S57; +using EncDotNet.Testing; + +namespace EncDotNet.UnitTests; + +/// +/// Pins 's exact output for the synthetic cells, so internal +/// rewrites of the reader and the ISO 8211 layer beneath it can't change what it returns. +/// +/// +/// The expected hashes were recorded from the reader before its performance rewrite. If a +/// change to the reader's output is intended, or the synthetic generator changes, re-record +/// them from the failure message and explain why in the pull request. +/// +public class S57DocumentReaderFingerprintTests +{ + public static TheoryData Cells => new() + { + { nameof(SyntheticCellProfile.Harbour), "84052F8318DC1187D28970F80DA5DFD945FE87B7D380EB8816EF6DC3D8CE53F0" }, + { nameof(SyntheticCellProfile.Approach), "6605571B43635891A1A561731C0D71C9B8ABA83E403EF04D1BECC0C1C27A8799" }, + { nameof(SyntheticCellProfile.Large), "BDF0F099C0081528FF1A2AF49AEC2C5FC2A2FD4150F09D9E33802FAAE3DF9C3F" }, + }; + + public static TheoryData Updated => new() + { + { nameof(SyntheticCellProfile.Harbour), "E8C34F98BE73E8C537A2777AFACAB1CE32EA75EB3CBC0A2051F2A487B5078345" }, + { nameof(SyntheticCellProfile.Approach), "E8B1157B3BC0D167B6B32CDB220BE23EFCB7F184DA84F3DB4AF117C555C931B2" }, + }; + + [Theory] + [MemberData(nameof(Cells))] + public void Read_SyntheticCell_MatchesRecordedOutput(string profileName, string expected) + { + var document = S57DocumentReader.Read(SyntheticS57Cell.Create(Profile(profileName))); + + Assert.Equal(expected, S57DocumentFingerprint.Compute(document)); + } + + [Theory] + [MemberData(nameof(Updated))] + public void ReadAndApplyUpdates_SyntheticCell_MatchesRecordedOutput(string profileName, string expected) + { + var profile = Profile(profileName); + var document = S57DocumentReader.Read(SyntheticS57Cell.Create(profile)); + for (var n = 1; n <= 3; n++) + { + document = document.ApplyChanges(S57DocumentReader.Read(SyntheticS57Cell.CreateUpdate(profile, n))); + } + + Assert.Equal(expected, S57DocumentFingerprint.Compute(document)); + } + + private static SyntheticCellProfile Profile(string name) => name switch + { + nameof(SyntheticCellProfile.Harbour) => SyntheticCellProfile.Harbour, + nameof(SyntheticCellProfile.Approach) => SyntheticCellProfile.Approach, + nameof(SyntheticCellProfile.Large) => SyntheticCellProfile.Large, + _ => throw new ArgumentOutOfRangeException(nameof(name)), + }; +} From fb3181e89a3267a075dc64ac1b6a231dfbafe65d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 22:24:47 +0000 Subject: [PATCH 2/2] Make the S-57 fingerprint independent of OS and culture The fingerprint tests failed on Windows: StringBuilder.AppendLine writes Environment.NewLine, so the hashed text had CRLF line endings there. End lines with '\n' and format numbers with the invariant culture, which also keeps a culture with a different minus sign (sv-SE uses U+2212) from changing the hash. The hashes are unchanged on Linux. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YagvELyJKBANk61DyFGUNJ --- .../S57DocumentFingerprint.cs | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs b/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs index 78a14e1..991d6f7 100644 --- a/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs +++ b/tests/EncDotNet.UnitTests/S57DocumentFingerprint.cs @@ -7,15 +7,33 @@ namespace EncDotNet.UnitTests; /// /// Reduces an to a SHA-256 hash of a canonical text dump of every -/// value it holds, so tests can pin a reader's exact output across internal rewrites. +/// value it holds, so tests can pin a reader's exact output across internal rewrites. The hash +/// is the same on every OS and in every culture. /// internal static class S57DocumentFingerprint { public static string Compute(S57Document document) + { + // The dump must be the same on every OS and in every culture: lines end in '\n' rather + // than Environment.NewLine, and numbers (including those in the records' ToString) + // format with the invariant culture. + var culture = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = CultureInfo.InvariantCulture; + try + { + return Hash(document); + } + finally + { + CultureInfo.CurrentCulture = culture; + } + } + + private static string Hash(S57Document document) { var text = new StringBuilder(); - text.AppendLine(document.DataSetIdentification?.ToString() ?? "no DSID"); - text.AppendLine(document.DataSetParameters?.ToString() ?? "no DSPM"); + text.Append(document.DataSetIdentification?.ToString() ?? "no DSID").Append('\n'); + text.Append(document.DataSetParameters?.ToString() ?? "no DSPM").Append('\n'); foreach (var feature in document.FeatureRecords) { @@ -25,19 +43,19 @@ public static string Compute(S57Document document) .Append(' ').Append((int)feature.ObjectCode) .Append(' ').Append(feature.RecordVersion) .Append(' ').Append((int)feature.UpdateInstruction) - .AppendLine(); + .Append('\n'); Attributes(text, "A", feature.Attributes); Attributes(text, "N", feature.NationalAttributes); foreach (var p in feature.SpatialPointers) { text.Append(" S ").Append(Name(p.Name)).Append(' ') - .Append((int)p.Orientation).Append(' ').Append((int)p.Usage).Append(' ').Append((int)p.Mask).AppendLine(); + .Append((int)p.Orientation).Append(' ').Append((int)p.Usage).Append(' ').Append((int)p.Mask).Append('\n'); } foreach (var p in feature.FeaturePointers) { text.Append(" P ").Append(Name(p.Name)).Append(' ') - .Append((int)p.Relationship).Append(' ').Append(p.Comment).AppendLine(); + .Append((int)p.Relationship).Append(' ').Append(p.Comment).Append('\n'); } Control(text, "SC", feature.SpatialPointerControl); @@ -49,23 +67,23 @@ public static string Compute(S57Document document) text.Append("V ").Append(Name(vector.RecordName)) .Append(' ').Append(vector.RecordVersion) .Append(' ').Append((int)vector.UpdateInstruction) - .AppendLine(); + .Append('\n'); Attributes(text, "A", vector.Attributes); foreach (var p in vector.VectorPointers) { text.Append(" V ").Append(Name(p.Name)).Append(' ') .Append((int)p.Orientation).Append(' ').Append((int)p.Usage).Append(' ') - .Append((int)p.Topology).Append(' ').Append((int)p.Mask).AppendLine(); + .Append((int)p.Topology).Append(' ').Append((int)p.Mask).Append('\n'); } foreach (var c in vector.Coordinates2D) { - text.Append(" C ").Append(c.Y).Append(' ').Append(c.X).AppendLine(); + text.Append(" C ").Append(c.Y).Append(' ').Append(c.X).Append('\n'); } foreach (var s in vector.Soundings) { - text.Append(" D ").Append(s.Y).Append(' ').Append(s.X).Append(' ').Append(s.Depth).AppendLine(); + text.Append(" D ").Append(s.Y).Append(' ').Append(s.X).Append(' ').Append(s.Depth).Append('\n'); } Control(text, "VC", vector.VectorPointerControl); @@ -83,7 +101,7 @@ private static void Attributes(StringBuilder text, string kind, IReadOnlyList