From d04da98ac0a3f9ff6c2cad9e34ced5ec8a84de30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 19:51:41 +0000 Subject: [PATCH] UnicodeMath: strings read by code points without lists, and read back only when reading could differ [no-new-version] Js.Ch, Js.Len, Js.Sub and Characters walk the string instead of building lists of runes; MapToPrivate, MapFromPrivate and SetBuilderBars return their input when there is nothing to map. The UnicodeMath writer reads back what it wrote only for an equation with an intent or argument name to keep, an n-ary operator or an operator of a character that reading gives identifiers with; two tests compare what it writes with and without reading back, on the corpora and for every operator character of the basic plane. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01Q8QQiasbLr6WtXXCEP1zrJ --- AGENTS.md | 2 +- .../Internal/Text/CodePoints.cs | 73 +++++++++++++++++++ src/SharpEquations/Internal/Text/Rune.cs | 3 + .../Internal/UnicodeMath/JsObject.cs | 29 +++----- .../Internal/UnicodeMath/UnicodeMathInput.cs | 36 ++++++++- .../UnicodeMath/UnicodeMathSerializer.cs | 56 +++++++++++++- .../Internal/UnicodeMath/UnicodeMathText.cs | 8 +- .../Property/RoundTripPropertyTests.cs | 56 ++++++++++++++ 8 files changed, 241 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f546913..0a203d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,7 @@ init-only record: `Display`, `Strict`, `PreserveFormatting`, `LaTeXFlavor` stand error shows as PARLOT001). Its JavaScript translator is ported line by line over `JsObject`, with strings indexed by code points (`Js.Ch`, `Js.Len`, `Js.Sub`) where the JavaScript indexes UTF-16 units; Parlot parses `char`s, so the input maps the characters outside the basic plane to the private use area first (`MapToPrivate`). The writer checks - what it writes by reading it back. + what it writes by reading it back, when reading could give another equation (`NeedsReadBack`). - `Internal/LaTeX/`: LaTeX is read as Word reads it, through UnicodeMath (`LaTeXReader`, a port of TeX2UnicodeMath, with `RewriteTeXOnly` for the commands UnicodeMath has no control word for, and for what LaTeX means otherwise than UnicodeMath: a slash is a slash, not a fraction, and a bar without its pair is a bar, not an absolute value), and diff --git a/src/SharpEquations/Internal/Text/CodePoints.cs b/src/SharpEquations/Internal/Text/CodePoints.cs index 2a4e3a3..fd14013 100644 --- a/src/SharpEquations/Internal/Text/CodePoints.cs +++ b/src/SharpEquations/Internal/Text/CodePoints.cs @@ -15,6 +15,79 @@ public static IEnumerable Of(string s) yield return rune; } + /// The number of runes of the text. + public static int Count(string s) + { + int count = s.Length; + for (int i = 1; i < s.Length; i++) + { + if (char.IsLowSurrogate(s[i]) && char.IsHighSurrogate(s[i - 1])) + count--; + } + return count; + } + + /// The number of UTF-16 units of the rune at an index in UTF-16 units. + public static int LengthAt(string s, int at) => + char.IsHighSurrogate(s[at]) && at + 1 < s.Length && char.IsLowSurrogate(s[at + 1]) ? 2 : 1; + + /// The index in UTF-16 units of the rune at an index in runes: the length of the text for the index after the last rune, -1 beyond. + public static int Offset(string s, int index) + { + int at = 0; + while (at < s.Length) + { + if (index-- == 0) + return at; + at += LengthAt(s, at); + } + return index == 0 ? at : -1; + } + + /// The value of the rune at an index in UTF-16 units: U+FFFD for a lone surrogate. + public static int ValueAt(string s, int at) + { + var c = s[at]; + if (!char.IsSurrogate(c)) + return c; + return LengthAt(s, at) == 2 ? char.ConvertToUtf32(c, s[at + 1]) : 0xFFFD; + } + + private static readonly string[] Ascii = AsciiStrings(); + + private static string[] AsciiStrings() + { + var strings = new string[128]; + for (int i = 0; i < strings.Length; i++) + strings[i] = ((char)i).ToString(); + return strings; + } + + /// The rune at an index in UTF-16 units, as a string: U+FFFD for a lone surrogate. + public static string TextAt(string s, int at) + { + var c = s[at]; + if (c < 128) + return Ascii[c]; + if (!char.IsSurrogate(c)) + return c.ToString(); + return LengthAt(s, at) == 2 ? s.Substring(at, 2) : "\uFFFD"; + } + + /// Whether the text holds a surrogate without its pair, which reads as U+FFFD. + public static bool HasLoneSurrogate(string s) + { + for (int i = 0; i < s.Length; i++) + { + if (!char.IsSurrogate(s[i])) + continue; + if (LengthAt(s, i) == 1) + return true; + i++; + } + return false; + } + /// Whether the text holds the rune. public static bool Contains(string s, Rune rune) { diff --git a/src/SharpEquations/Internal/Text/Rune.cs b/src/SharpEquations/Internal/Text/Rune.cs index 1cf2a1e..fecbbad 100644 --- a/src/SharpEquations/Internal/Text/Rune.cs +++ b/src/SharpEquations/Internal/Text/Rune.cs @@ -28,6 +28,9 @@ public Rune(int value) public override string ToString() => char.ConvertFromUtf32(value); + public static System.Globalization.UnicodeCategory GetUnicodeCategory(Rune value) => + System.Globalization.CharUnicodeInfo.GetUnicodeCategory(value.ToString(), 0); + public bool Equals(Rune other) => value == other.value; public override bool Equals(object obj) => obj is Rune other && Equals(other); diff --git a/src/SharpEquations/Internal/UnicodeMath/JsObject.cs b/src/SharpEquations/Internal/UnicodeMath/JsObject.cs index 4e1a648..257dda5 100644 --- a/src/SharpEquations/Internal/UnicodeMath/JsObject.cs +++ b/src/SharpEquations/Internal/UnicodeMath/JsObject.cs @@ -155,39 +155,34 @@ public static string Ch(string s, int i) { if (s == null || i < 0) return null; - foreach (var codePoint in CodePoints.Of(s)) - { - if (i-- == 0) - return codePoint.ToString(); - } - return null; + int at = CodePoints.Offset(s, i); + return at >= 0 && at < s.Length ? CodePoints.TextAt(s, at) : null; } /// The number of code points of a string. - public static int Len(string s) => s == null ? 0 : CodePoints.Of(s).Count(); + public static int Len(string s) => s == null ? 0 : CodePoints.Count(s); /// The code point value at index i (by code points) of a string; -1 out of range. public static int CodeAt(string s, int i) { if (s == null || i < 0) return -1; - foreach (var codePoint in CodePoints.Of(s)) - { - if (i-- == 0) - return codePoint.Value; - } - return -1; + int at = CodePoints.Offset(s, i); + return at >= 0 && at < s.Length ? CodePoints.ValueAt(s, at) : -1; } /// str.substring(start, end) by code points: indexes clamped, and swapped when start is after end. public static string Sub(string s, int start, int? end = null) { - var codePoints = CodePoints.Of(s).ToList(); - int from = Math.Max(0, Math.Min(start, codePoints.Count)); - int to = Math.Max(0, Math.Min(end ?? codePoints.Count, codePoints.Count)); + int count = CodePoints.Count(s); + int from = Math.Max(0, Math.Min(start, count)); + int to = Math.Max(0, Math.Min(end ?? count, count)); if (from > to) (from, to) = (to, from); - return string.Concat(codePoints.Skip(from).Take(to - from).Select(c => c.ToString())); + if (CodePoints.HasLoneSurrogate(s)) + return string.Concat(CodePoints.Of(s).Skip(from).Take(to - from).Select(c => c.ToString())); + int first = CodePoints.Offset(s, from); + return s.Substring(first, CodePoints.Offset(s, to) - first); } /// value.length of an array or a string; null (undefined) for other values. diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs index 17090da..4dee605 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs @@ -27,6 +27,8 @@ internal static partial class UnicodeMathInput /// public static string SetBuilderBars(string unicodemath) { + if (unicodemath.IndexOf('|') < 0 || unicodemath.IndexOf('{') < 0) + return unicodemath; var chars = Characters(unicodemath); var frames = new Stack<(string Open, List Bars)>(); var separators = new List(); @@ -250,6 +252,8 @@ private static readonly (int AstralBegin, int AstralEnd, int PrivateBegin, int P /// public static string MapToPrivate(string s) { + if (!NeedsMapToPrivate(s)) + return s; var cps = CodePoints.Of(s).Select(c => c.Value).ToList(); var u = new List(); for (int i = 0; i < cps.Count; i++) @@ -306,9 +310,31 @@ public static string MapToPrivate(string s) return string.Concat(u.Select(FromCodePoint)); } + private static bool NeedsMapToPrivate(string s) + { + foreach (var c in s) + { + if (char.IsSurrogate(c) || c >= '₀' && c <= '₉') + return true; + } + return false; + } + + private static bool NeedsMapFromPrivate(string s) + { + foreach (var c in s) + { + if (c >= 0xD800 && c <= 0xF8FF) + return true; + } + return false; + } + /// mapFromPrivate: the inverse of for the private use area. public static string MapFromPrivate(string s) { + if (!NeedsMapFromPrivate(s)) + return s; var sb = new StringBuilder(); foreach (var c in Characters(s)) { @@ -344,8 +370,14 @@ public static object AstMapFromPrivate(object ast) case string s: return MapFromPrivate(s); case JsObject o: - foreach (var key in new List(o.Keys)) - o[key] = AstMapFromPrivate(o[key]); + for (int i = 0; i < o.Keys.Count; i++) + { + var key = o.Keys[i]; + var value = o[key]; + var mapped = AstMapFromPrivate(value); + if (!ReferenceEquals(mapped, value)) + o[key] = mapped; + } return o; default: return ast; diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs index c4b0d88..7efcd34 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs @@ -1,6 +1,7 @@ namespace SharpEquations.Internal.UnicodeMath { using System.Collections.Generic; + using System.Globalization; using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; @@ -84,11 +85,64 @@ private static bool IsPlainLetter(XElement mi) return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= 0x3B1 && c <= 0x3C9; } - public static string Serialize(XElement math, bool keepSelectionInfo = false) + /// + /// Whether reading what is written could give another equation, which looks for: + /// an intent or an argument name the equation holds, an n-ary operator, or an operator with a character that + /// reading gives identifiers with (a letter, ∞, an emoji). + /// + internal static bool NeedsReadBack(XElement math) + { + foreach (var element in math.DescendantsAndSelf()) + { + if (IsHeld(element.Attribute("intent")) || IsHeld(element.Attribute("arg"))) + return true; + if (Name(element) == "mo" && !IsPlainOperator(Text(element))) + return true; + } + return false; + } + + private static bool IsPlainOperator(string op) + { + if (op.Length == 0) + return false; + foreach (var rune in CodePoints.Of(op)) + { + switch (rune.Value) + { + case > 0x20 and < 0x7F: + if (rune.Value is >= '0' and <= '9' or >= 'A' and <= 'Z' or >= 'a' and <= 'z') + return false; + break; + case 0x2118 or 0x221E or 0x22BA or 0x25FD or 0x25FE or > 0xFFFF: + return false; + case var _ when IsNaryOperator(rune.ToString()): + return false; + default: + switch (System.Text.Rune.GetUnicodeCategory(rune)) + { + case UnicodeCategory.MathSymbol or UnicodeCategory.DashPunctuation or UnicodeCategory.OpenPunctuation + or UnicodeCategory.ClosePunctuation or UnicodeCategory.OtherPunctuation or UnicodeCategory.InitialQuotePunctuation + or UnicodeCategory.FinalQuotePunctuation or UnicodeCategory.SpaceSeparator or UnicodeCategory.Format: + break; + default: + return false; + } + break; + } + } + return true; + } + + public static string Serialize(XElement math, bool keepSelectionInfo = false) => Serialize(math, keepSelectionInfo, false); + + internal static string Serialize(XElement math, bool keepSelectionInfo, bool alwaysReadBack) { var italicized = MathML.DefaultIntents.Restore(Italicized(math)); var serializer = new UnicodeMathSerializer(keepSelectionInfo) { display = (string)math.Attribute("display") == "block" }; var text = serializer.Write(italicized); + if (!alwaysReadBack && !NeedsReadBack(italicized)) + return text; // the intents and argument names that reading does not give back are written: ⓘ("name"…) and ⓐ(name …) bool display = (string)math.Attribute("display") == "block"; for (int pass = 0; pass < 3; pass++) diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs index 5134e13..4469e93 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs @@ -118,7 +118,13 @@ public static string ItalicizeCharacter(string c) } /// Array.from(chars): the code points of a string, as strings. - public static List Characters(string chars) => CodePoints.Of(chars).Select(c => c.ToString()).ToList(); + public static List Characters(string chars) + { + var characters = new List(chars.Length); + for (int at = 0; at < chars.Length; at += CodePoints.LengthAt(chars, at)) + characters.Add(CodePoints.TextAt(chars, at)); + return characters; + } public static string ItalicizeCharacters(string chars) => string.Concat(Characters(chars).Select(ItalicizeCharacter)); diff --git a/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs b/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs index 947a844..b0ac7bf 100644 --- a/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs +++ b/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs @@ -66,6 +66,62 @@ private static Dictionary Failures(Func check private static List Listed(Dictionary failures) => failures.Select(f => f.Key + "\n " + f.Value).ToList(); + /// + /// The UnicodeMath writer reads back what it writes only when reading could give another equation + /// (UnicodeMathSerializer.NeedsReadBack): what it writes without reading back is what it writes reading back. + /// + [Test] + public async Task UnicodeMathWrittenWithoutReadingBackIsTheSame() + { + static string? Check(Equation equation) + { + var math = equation.ToMathML(); + var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false); + var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true); + return written == checkedBack ? null : written + " instead of " + checkedBack; + } + + await Assert.That(Listed(Failures(Check))).IsEmpty(); + await Assert.That(Listed(Failures(Check, WordEquations()))).IsEmpty(); + } + + /// + /// The same for an operator of each character of the basic plane that is not a letter or a digit, alone, between + /// two identifiers, before one and in a script: the characters that reading gives identifiers with are read back. + /// + [Test] + public async Task OperatorsWrittenWithoutReadingBackAreTheSame() + { + var failures = new List(); + for (int codePoint = 0x21; codePoint <= 0xFFFF; codePoint++) + { + if (codePoint is >= 0xD800 and <= 0xF8FF || char.IsLetterOrDigit((char)codePoint) || char.IsControl((char)codePoint)) + continue; + var mo = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mo", ((char)codePoint).ToString()); + var a = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "a"); + var b = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "b"); + foreach (var content in new object[] { mo, new object[] { a, mo, b }, new object[] { mo, a }, new object[] { a, mo }, + new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "msub", a, mo) }) + { + Equation equation; + try + { + equation = Equation.Parse(new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "math", content), EquationFormat.MathML); + } + catch (FormatException) + { + continue; + } + var math = equation.ToMathML(); + var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false); + var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true); + if (written != checkedBack) + failures.Add("U+" + codePoint.ToString("X4") + ": " + written + " instead of " + checkedBack); + } + } + await Assert.That(failures).IsEmpty(); + } + [Test] public async Task WritingNeverThrows() {