UnicodeMath: fewer allocations reading, and the writer reads back only when needed

AGENTS.md

@@ -68,7 +68,7 @@ init-only record: `Display`, `Strict`, `PreserveFormatting`, `LaTeXFlavor` stand
   error shows as PARLOT001). Its JavaScript translator is ported line by line over `JsObject`, with strings indexed by
   code points (`Js.Ch`, `Js.Len`, `Js.Sub`) where the JavaScript indexes UTF-16 units; Parlot parses `char`s, so the
   input maps the characters outside the basic plane to the private use area first (`MapToPrivate`). The writer checks
-  what it writes by reading it back.
+  what it writes by reading it back, when reading could give another equation (`NeedsReadBack`).
 - `Internal/LaTeX/`: LaTeX is read as Word reads it, through UnicodeMath (`LaTeXReader`, a port of TeX2UnicodeMath, with
   `RewriteTeXOnly` for the commands UnicodeMath has no control word for, and for what LaTeX means otherwise than
   UnicodeMath: a slash is a slash, not a fraction, and a bar without its pair is a bar, not an absolute value), and

src/SharpEquations/Internal/Text/CodePoints.cs

@@ -15,6 +15,79 @@ public static IEnumerable<Rune> Of(string s)
                 yield return rune;
         }
 
+        /// <summary>The number of runes of the text.</summary>
+        public static int Count(string s)
+        {
+            int count = s.Length;
+            for (int i = 1; i < s.Length; i++)
+            {
+                if (char.IsLowSurrogate(s[i]) && char.IsHighSurrogate(s[i - 1]))
+                    count--;
+            }
+            return count;
+        }
+
+        /// <summary>The number of UTF-16 units of the rune at an index in UTF-16 units.</summary>
+        public static int LengthAt(string s, int at) =>
+            char.IsHighSurrogate(s[at]) && at + 1 < s.Length && char.IsLowSurrogate(s[at + 1]) ? 2 : 1;
+
+        /// <summary>The index in UTF-16 units of the rune at an index in runes: the length of the text for the index after the last rune, -1 beyond.</summary>
+        public static int Offset(string s, int index)
+        {
+            int at = 0;
+            while (at < s.Length)
+            {
+                if (index-- == 0)
+                    return at;
+                at += LengthAt(s, at);
+            }
+            return index == 0 ? at : -1;
+        }
+
+        /// <summary>The value of the rune at an index in UTF-16 units: U+FFFD for a lone surrogate.</summary>
+        public static int ValueAt(string s, int at)
+        {
+            var c = s[at];
+            if (!char.IsSurrogate(c))
+                return c;
+            return LengthAt(s, at) == 2 ? char.ConvertToUtf32(c, s[at + 1]) : 0xFFFD;
+        }
+
+        private static readonly string[] Ascii = AsciiStrings();
+
+        private static string[] AsciiStrings()
+        {
+            var strings = new string[128];
+            for (int i = 0; i < strings.Length; i++)
+                strings[i] = ((char)i).ToString();
+            return strings;
+        }
+
+        /// <summary>The rune at an index in UTF-16 units, as a string: U+FFFD for a lone surrogate.</summary>
+        public static string TextAt(string s, int at)
+        {
+            var c = s[at];
+            if (c < 128)
+                return Ascii[c];
+            if (!char.IsSurrogate(c))
+                return c.ToString();
+            return LengthAt(s, at) == 2 ? s.Substring(at, 2) : "\uFFFD";
+        }
+
+        /// <summary>Whether the text holds a surrogate without its pair, which reads as U+FFFD.</summary>
+        public static bool HasLoneSurrogate(string s)
+        {
+            for (int i = 0; i < s.Length; i++)
+            {
+                if (!char.IsSurrogate(s[i]))
+                    continue;
+                if (LengthAt(s, i) == 1)
+                    return true;
+                i++;
+            }
+            return false;
+        }
+
         /// <summary>Whether the text holds the rune.</summary>
         public static bool Contains(string s, Rune rune)
         {

src/SharpEquations/Internal/Text/Rune.cs

@@ -28,6 +28,9 @@ public Rune(int value)
 
         public override string ToString() => char.ConvertFromUtf32(value);
 
+        public static System.Globalization.UnicodeCategory GetUnicodeCategory(Rune value) =>
+            System.Globalization.CharUnicodeInfo.GetUnicodeCategory(value.ToString(), 0);
+
         public bool Equals(Rune other) => value == other.value;
 
         public override bool Equals(object obj) => obj is Rune other && Equals(other);

src/SharpEquations/Internal/UnicodeMath/JsObject.cs

@@ -155,39 +155,34 @@ public static string Ch(string s, int i)
         {
             if (s == null || i < 0)
                 return null;
-            foreach (var codePoint in CodePoints.Of(s))
-            {
-                if (i-- == 0)
-                    return codePoint.ToString();
-            }
-            return null;
+            int at = CodePoints.Offset(s, i);
+            return at >= 0 && at < s.Length ? CodePoints.TextAt(s, at) : null;
         }
 
         /// <summary>The number of code points of a string.</summary>
-        public static int Len(string s) => s == null ? 0 : CodePoints.Of(s).Count();
+        public static int Len(string s) => s == null ? 0 : CodePoints.Count(s);
 
         /// <summary>The code point value at index i (by code points) of a string; -1 out of range.</summary>
         public static int CodeAt(string s, int i)
         {
             if (s == null || i < 0)
                 return -1;
-            foreach (var codePoint in CodePoints.Of(s))
-            {
-                if (i-- == 0)
-                    return codePoint.Value;
-            }
-            return -1;
+            int at = CodePoints.Offset(s, i);
+            return at >= 0 && at < s.Length ? CodePoints.ValueAt(s, at) : -1;
         }
 
         /// <summary>str.substring(start, end) by code points: indexes clamped, and swapped when start is after end.</summary>
         public static string Sub(string s, int start, int? end = null)
         {
-            var codePoints = CodePoints.Of(s).ToList();
-            int from = Math.Max(0, Math.Min(start, codePoints.Count));
-            int to = Math.Max(0, Math.Min(end ?? codePoints.Count, codePoints.Count));
+            int count = CodePoints.Count(s);
+            int from = Math.Max(0, Math.Min(start, count));
+            int to = Math.Max(0, Math.Min(end ?? count, count));
             if (from > to)
                 (from, to) = (to, from);
-            return string.Concat(codePoints.Skip(from).Take(to - from).Select(c => c.ToString()));
+            if (CodePoints.HasLoneSurrogate(s))
+                return string.Concat(CodePoints.Of(s).Skip(from).Take(to - from).Select(c => c.ToString()));
+            int first = CodePoints.Offset(s, from);
+            return s.Substring(first, CodePoints.Offset(s, to) - first);
         }
 
         /// <summary>value.length of an array or a string; null (undefined) for other values.</summary>

src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs

@@ -27,6 +27,8 @@ internal static partial class UnicodeMathInput
         /// </summary>
         public static string SetBuilderBars(string unicodemath)
         {
+            if (unicodemath.IndexOf('|') < 0 || unicodemath.IndexOf('{') < 0)
+                return unicodemath;
             var chars = Characters(unicodemath);
             var frames = new Stack<(string Open, List<int> Bars)>();
             var separators = new List<int>();
@@ -250,6 +252,8 @@ private static readonly (int AstralBegin, int AstralEnd, int PrivateBegin, int P
         /// </summary>
         public static string MapToPrivate(string s)
         {
+            if (!NeedsMapToPrivate(s))
+                return s;
             var cps = CodePoints.Of(s).Select(c => c.Value).ToList();
             var u = new List<int>();
             for (int i = 0; i < cps.Count; i++)
@@ -306,9 +310,31 @@ public static string MapToPrivate(string s)
             return string.Concat(u.Select(FromCodePoint));
         }
 
+        private static bool NeedsMapToPrivate(string s)
+        {
+            foreach (var c in s)
+            {
+                if (char.IsSurrogate(c) || c >= '₀' && c <= '₉')
+                    return true;
+            }
+            return false;
+        }
+
+        private static bool NeedsMapFromPrivate(string s)
+        {
+            foreach (var c in s)
+            {
+                if (c >= 0xD800 && c <= 0xF8FF)
+                    return true;
+            }
+            return false;
+        }
+
         /// <summary>mapFromPrivate: the inverse of <see cref="MapToPrivate"/> for the private use area.</summary>
         public static string MapFromPrivate(string s)
         {
+            if (!NeedsMapFromPrivate(s))
+                return s;
             var sb = new StringBuilder();
             foreach (var c in Characters(s))
             {
@@ -344,8 +370,14 @@ public static object AstMapFromPrivate(object ast)
                 case string s:
                     return MapFromPrivate(s);
                 case JsObject o:
-                    foreach (var key in new List<string>(o.Keys))
-                        o[key] = AstMapFromPrivate(o[key]);
+                    for (int i = 0; i < o.Keys.Count; i++)
+                    {
+                        var key = o.Keys[i];
+                        var value = o[key];
+                        var mapped = AstMapFromPrivate(value);
+                        if (!ReferenceEquals(mapped, value))
+                            o[key] = mapped;
+                    }
                     return o;
                 default:
                     return ast;

src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs

@@ -1,6 +1,7 @@
 namespace SharpEquations.Internal.UnicodeMath
 {
     using System.Collections.Generic;
+    using System.Globalization;
     using System.Linq;
     using System.Text.RegularExpressions;
     using System.Xml.Linq;
@@ -84,11 +85,64 @@ private static bool IsPlainLetter(XElement mi)
             return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= 0x3B1 && c <= 0x3C9;
         }
 
-        public static string Serialize(XElement math, bool keepSelectionInfo = false)
+        /// <summary>
+        /// Whether reading what is written could give another equation, which <see cref="FindLostIntents"/> looks for:
+        /// an intent or an argument name the equation holds, an n-ary operator, or an operator with a character that
+        /// reading gives identifiers with (a letter, ∞, an emoji).
+        /// </summary>
+        internal static bool NeedsReadBack(XElement math)
+        {
+            foreach (var element in math.DescendantsAndSelf())
+            {
+                if (IsHeld(element.Attribute("intent")) || IsHeld(element.Attribute("arg")))
+                    return true;
+                if (Name(element) == "mo" && !IsPlainOperator(Text(element)))
+                    return true;
+            }
+            return false;
+        }
+
+        private static bool IsPlainOperator(string op)
+        {
+            if (op.Length == 0)
+                return false;
+            foreach (var rune in CodePoints.Of(op))
+            {
+                switch (rune.Value)
+                {
+                    case > 0x20 and < 0x7F:
+                        if (rune.Value is >= '0' and <= '9' or >= 'A' and <= 'Z' or >= 'a' and <= 'z')
+                            return false;
+                        break;
+                    case 0x2118 or 0x221E or 0x22BA or 0x25FD or 0x25FE or > 0xFFFF:
+                        return false;
+                    case var _ when IsNaryOperator(rune.ToString()):
+                        return false;
+                    default:
+                        switch (System.Text.Rune.GetUnicodeCategory(rune))
+                        {
+                            case UnicodeCategory.MathSymbol or UnicodeCategory.DashPunctuation or UnicodeCategory.OpenPunctuation
+                                or UnicodeCategory.ClosePunctuation or UnicodeCategory.OtherPunctuation or UnicodeCategory.InitialQuotePunctuation
+                                or UnicodeCategory.FinalQuotePunctuation or UnicodeCategory.SpaceSeparator or UnicodeCategory.Format:
+                                break;
+                            default:
+                                return false;
+                        }
+                        break;
+                }
+            }
+            return true;
+        }
+
+        public static string Serialize(XElement math, bool keepSelectionInfo = false) => Serialize(math, keepSelectionInfo, false);
+
+        internal static string Serialize(XElement math, bool keepSelectionInfo, bool alwaysReadBack)
         {
             var italicized = MathML.DefaultIntents.Restore(Italicized(math));
             var serializer = new UnicodeMathSerializer(keepSelectionInfo) { display = (string)math.Attribute("display") == "block" };
             var text = serializer.Write(italicized);
+            if (!alwaysReadBack && !NeedsReadBack(italicized))
+                return text;
             // the intents and argument names that reading does not give back are written: ⓘ("name"…) and ⓐ(name …)
             bool display = (string)math.Attribute("display") == "block";
             for (int pass = 0; pass < 3; pass++)

src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs

@@ -118,7 +118,13 @@ public static string ItalicizeCharacter(string c)
         }
 
         /// <summary>Array.from(chars): the code points of a string, as strings.</summary>
-        public static List<string> Characters(string chars) => CodePoints.Of(chars).Select(c => c.ToString()).ToList();
+        public static List<string> Characters(string chars)
+        {
+            var characters = new List<string>(chars.Length);
+            for (int at = 0; at < chars.Length; at += CodePoints.LengthAt(chars, at))
+                characters.Add(CodePoints.TextAt(chars, at));
+            return characters;
+        }
 
         public static string ItalicizeCharacters(string chars) => string.Concat(Characters(chars).Select(ItalicizeCharacter));
 

tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs

@@ -66,6 +66,62 @@ private static Dictionary<string, string> Failures(Func<Equation, string?> check
 
         private static List<string> Listed(Dictionary<string, string> failures) => failures.Select(f => f.Key + "\n  " + f.Value).ToList();
 
+        /// <summary>
+        /// The UnicodeMath writer reads back what it writes only when reading could give another equation
+        /// (UnicodeMathSerializer.NeedsReadBack): what it writes without reading back is what it writes reading back.
+        /// </summary>
+        [Test]
+        public async Task UnicodeMathWrittenWithoutReadingBackIsTheSame()
+        {
+            static string? Check(Equation equation)
+            {
+                var math = equation.ToMathML();
+                var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false);
+                var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true);
+                return written == checkedBack ? null : written + " instead of " + checkedBack;
+            }
+
+            await Assert.That(Listed(Failures(Check))).IsEmpty();
+            await Assert.That(Listed(Failures(Check, WordEquations()))).IsEmpty();
+        }
+
+        /// <summary>
+        /// The same for an operator of each character of the basic plane that is not a letter or a digit, alone, between
+        /// two identifiers, before one and in a script: the characters that reading gives identifiers with are read back.
+        /// </summary>
+        [Test]
+        public async Task OperatorsWrittenWithoutReadingBackAreTheSame()
+        {
+            var failures = new List<string>();
+            for (int codePoint = 0x21; codePoint <= 0xFFFF; codePoint++)
+            {
+                if (codePoint is >= 0xD800 and <= 0xF8FF || char.IsLetterOrDigit((char)codePoint) || char.IsControl((char)codePoint))
+                    continue;
+                var mo = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mo", ((char)codePoint).ToString());
+                var a = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "a");
+                var b = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "b");
+                foreach (var content in new object[] { mo, new object[] { a, mo, b }, new object[] { mo, a }, new object[] { a, mo },
+                    new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "msub", a, mo) })
+                {
+                    Equation equation;
+                    try
+                    {
+                        equation = Equation.Parse(new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "math", content), EquationFormat.MathML);
+                    }
+                    catch (FormatException)
+                    {
+                        continue;
+                    }
+                    var math = equation.ToMathML();
+                    var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false);
+                    var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true);
+                    if (written != checkedBack)
+                        failures.Add("U+" + codePoint.ToString("X4") + ": " + written + " instead of " + checkedBack);
+                }
+            }
+            await Assert.That(failures).IsEmpty();
+        }
+
         [Test]
         public async Task WritingNeverThrows()
         {