diff --git a/AGENTS.md b/AGENTS.md index f546913..0a203d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,7 @@ init-only record: `Display`, `Strict`, `PreserveFormatting`, `LaTeXFlavor` stand error shows as PARLOT001). Its JavaScript translator is ported line by line over `JsObject`, with strings indexed by code points (`Js.Ch`, `Js.Len`, `Js.Sub`) where the JavaScript indexes UTF-16 units; Parlot parses `char`s, so the input maps the characters outside the basic plane to the private use area first (`MapToPrivate`). The writer checks - what it writes by reading it back. + what it writes by reading it back, when reading could give another equation (`NeedsReadBack`). - `Internal/LaTeX/`: LaTeX is read as Word reads it, through UnicodeMath (`LaTeXReader`, a port of TeX2UnicodeMath, with `RewriteTeXOnly` for the commands UnicodeMath has no control word for, and for what LaTeX means otherwise than UnicodeMath: a slash is a slash, not a fraction, and a bar without its pair is a bar, not an absolute value), and diff --git a/src/SharpEquations/Internal/Text/CodePoints.cs b/src/SharpEquations/Internal/Text/CodePoints.cs index 2a4e3a3..fd14013 100644 --- a/src/SharpEquations/Internal/Text/CodePoints.cs +++ b/src/SharpEquations/Internal/Text/CodePoints.cs @@ -15,6 +15,79 @@ public static IEnumerable Of(string s) yield return rune; } + /// The number of runes of the text. + public static int Count(string s) + { + int count = s.Length; + for (int i = 1; i < s.Length; i++) + { + if (char.IsLowSurrogate(s[i]) && char.IsHighSurrogate(s[i - 1])) + count--; + } + return count; + } + + /// The number of UTF-16 units of the rune at an index in UTF-16 units. + public static int LengthAt(string s, int at) => + char.IsHighSurrogate(s[at]) && at + 1 < s.Length && char.IsLowSurrogate(s[at + 1]) ? 2 : 1; + + /// The index in UTF-16 units of the rune at an index in runes: the length of the text for the index after the last rune, -1 beyond. + public static int Offset(string s, int index) + { + int at = 0; + while (at < s.Length) + { + if (index-- == 0) + return at; + at += LengthAt(s, at); + } + return index == 0 ? at : -1; + } + + /// The value of the rune at an index in UTF-16 units: U+FFFD for a lone surrogate. + public static int ValueAt(string s, int at) + { + var c = s[at]; + if (!char.IsSurrogate(c)) + return c; + return LengthAt(s, at) == 2 ? char.ConvertToUtf32(c, s[at + 1]) : 0xFFFD; + } + + private static readonly string[] Ascii = AsciiStrings(); + + private static string[] AsciiStrings() + { + var strings = new string[128]; + for (int i = 0; i < strings.Length; i++) + strings[i] = ((char)i).ToString(); + return strings; + } + + /// The rune at an index in UTF-16 units, as a string: U+FFFD for a lone surrogate. + public static string TextAt(string s, int at) + { + var c = s[at]; + if (c < 128) + return Ascii[c]; + if (!char.IsSurrogate(c)) + return c.ToString(); + return LengthAt(s, at) == 2 ? s.Substring(at, 2) : "\uFFFD"; + } + + /// Whether the text holds a surrogate without its pair, which reads as U+FFFD. + public static bool HasLoneSurrogate(string s) + { + for (int i = 0; i < s.Length; i++) + { + if (!char.IsSurrogate(s[i])) + continue; + if (LengthAt(s, i) == 1) + return true; + i++; + } + return false; + } + /// Whether the text holds the rune. public static bool Contains(string s, Rune rune) { diff --git a/src/SharpEquations/Internal/Text/Rune.cs b/src/SharpEquations/Internal/Text/Rune.cs index 1cf2a1e..fecbbad 100644 --- a/src/SharpEquations/Internal/Text/Rune.cs +++ b/src/SharpEquations/Internal/Text/Rune.cs @@ -28,6 +28,9 @@ public Rune(int value) public override string ToString() => char.ConvertFromUtf32(value); + public static System.Globalization.UnicodeCategory GetUnicodeCategory(Rune value) => + System.Globalization.CharUnicodeInfo.GetUnicodeCategory(value.ToString(), 0); + public bool Equals(Rune other) => value == other.value; public override bool Equals(object obj) => obj is Rune other && Equals(other); diff --git a/src/SharpEquations/Internal/UnicodeMath/JsObject.cs b/src/SharpEquations/Internal/UnicodeMath/JsObject.cs index 4e1a648..257dda5 100644 --- a/src/SharpEquations/Internal/UnicodeMath/JsObject.cs +++ b/src/SharpEquations/Internal/UnicodeMath/JsObject.cs @@ -155,39 +155,34 @@ public static string Ch(string s, int i) { if (s == null || i < 0) return null; - foreach (var codePoint in CodePoints.Of(s)) - { - if (i-- == 0) - return codePoint.ToString(); - } - return null; + int at = CodePoints.Offset(s, i); + return at >= 0 && at < s.Length ? CodePoints.TextAt(s, at) : null; } /// The number of code points of a string. - public static int Len(string s) => s == null ? 0 : CodePoints.Of(s).Count(); + public static int Len(string s) => s == null ? 0 : CodePoints.Count(s); /// The code point value at index i (by code points) of a string; -1 out of range. public static int CodeAt(string s, int i) { if (s == null || i < 0) return -1; - foreach (var codePoint in CodePoints.Of(s)) - { - if (i-- == 0) - return codePoint.Value; - } - return -1; + int at = CodePoints.Offset(s, i); + return at >= 0 && at < s.Length ? CodePoints.ValueAt(s, at) : -1; } /// str.substring(start, end) by code points: indexes clamped, and swapped when start is after end. public static string Sub(string s, int start, int? end = null) { - var codePoints = CodePoints.Of(s).ToList(); - int from = Math.Max(0, Math.Min(start, codePoints.Count)); - int to = Math.Max(0, Math.Min(end ?? codePoints.Count, codePoints.Count)); + int count = CodePoints.Count(s); + int from = Math.Max(0, Math.Min(start, count)); + int to = Math.Max(0, Math.Min(end ?? count, count)); if (from > to) (from, to) = (to, from); - return string.Concat(codePoints.Skip(from).Take(to - from).Select(c => c.ToString())); + if (CodePoints.HasLoneSurrogate(s)) + return string.Concat(CodePoints.Of(s).Skip(from).Take(to - from).Select(c => c.ToString())); + int first = CodePoints.Offset(s, from); + return s.Substring(first, CodePoints.Offset(s, to) - first); } /// value.length of an array or a string; null (undefined) for other values. diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs index 17090da..4dee605 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathInput.cs @@ -27,6 +27,8 @@ internal static partial class UnicodeMathInput /// public static string SetBuilderBars(string unicodemath) { + if (unicodemath.IndexOf('|') < 0 || unicodemath.IndexOf('{') < 0) + return unicodemath; var chars = Characters(unicodemath); var frames = new Stack<(string Open, List Bars)>(); var separators = new List(); @@ -250,6 +252,8 @@ private static readonly (int AstralBegin, int AstralEnd, int PrivateBegin, int P /// public static string MapToPrivate(string s) { + if (!NeedsMapToPrivate(s)) + return s; var cps = CodePoints.Of(s).Select(c => c.Value).ToList(); var u = new List(); for (int i = 0; i < cps.Count; i++) @@ -306,9 +310,31 @@ public static string MapToPrivate(string s) return string.Concat(u.Select(FromCodePoint)); } + private static bool NeedsMapToPrivate(string s) + { + foreach (var c in s) + { + if (char.IsSurrogate(c) || c >= '₀' && c <= '₉') + return true; + } + return false; + } + + private static bool NeedsMapFromPrivate(string s) + { + foreach (var c in s) + { + if (c >= 0xD800 && c <= 0xF8FF) + return true; + } + return false; + } + /// mapFromPrivate: the inverse of for the private use area. public static string MapFromPrivate(string s) { + if (!NeedsMapFromPrivate(s)) + return s; var sb = new StringBuilder(); foreach (var c in Characters(s)) { @@ -344,8 +370,14 @@ public static object AstMapFromPrivate(object ast) case string s: return MapFromPrivate(s); case JsObject o: - foreach (var key in new List(o.Keys)) - o[key] = AstMapFromPrivate(o[key]); + for (int i = 0; i < o.Keys.Count; i++) + { + var key = o.Keys[i]; + var value = o[key]; + var mapped = AstMapFromPrivate(value); + if (!ReferenceEquals(mapped, value)) + o[key] = mapped; + } return o; default: return ast; diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs index c4b0d88..7efcd34 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathSerializer.cs @@ -1,6 +1,7 @@ namespace SharpEquations.Internal.UnicodeMath { using System.Collections.Generic; + using System.Globalization; using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; @@ -84,11 +85,64 @@ private static bool IsPlainLetter(XElement mi) return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= 0x3B1 && c <= 0x3C9; } - public static string Serialize(XElement math, bool keepSelectionInfo = false) + /// + /// Whether reading what is written could give another equation, which looks for: + /// an intent or an argument name the equation holds, an n-ary operator, or an operator with a character that + /// reading gives identifiers with (a letter, ∞, an emoji). + /// + internal static bool NeedsReadBack(XElement math) + { + foreach (var element in math.DescendantsAndSelf()) + { + if (IsHeld(element.Attribute("intent")) || IsHeld(element.Attribute("arg"))) + return true; + if (Name(element) == "mo" && !IsPlainOperator(Text(element))) + return true; + } + return false; + } + + private static bool IsPlainOperator(string op) + { + if (op.Length == 0) + return false; + foreach (var rune in CodePoints.Of(op)) + { + switch (rune.Value) + { + case > 0x20 and < 0x7F: + if (rune.Value is >= '0' and <= '9' or >= 'A' and <= 'Z' or >= 'a' and <= 'z') + return false; + break; + case 0x2118 or 0x221E or 0x22BA or 0x25FD or 0x25FE or > 0xFFFF: + return false; + case var _ when IsNaryOperator(rune.ToString()): + return false; + default: + switch (System.Text.Rune.GetUnicodeCategory(rune)) + { + case UnicodeCategory.MathSymbol or UnicodeCategory.DashPunctuation or UnicodeCategory.OpenPunctuation + or UnicodeCategory.ClosePunctuation or UnicodeCategory.OtherPunctuation or UnicodeCategory.InitialQuotePunctuation + or UnicodeCategory.FinalQuotePunctuation or UnicodeCategory.SpaceSeparator or UnicodeCategory.Format: + break; + default: + return false; + } + break; + } + } + return true; + } + + public static string Serialize(XElement math, bool keepSelectionInfo = false) => Serialize(math, keepSelectionInfo, false); + + internal static string Serialize(XElement math, bool keepSelectionInfo, bool alwaysReadBack) { var italicized = MathML.DefaultIntents.Restore(Italicized(math)); var serializer = new UnicodeMathSerializer(keepSelectionInfo) { display = (string)math.Attribute("display") == "block" }; var text = serializer.Write(italicized); + if (!alwaysReadBack && !NeedsReadBack(italicized)) + return text; // the intents and argument names that reading does not give back are written: ⓘ("name"…) and ⓐ(name …) bool display = (string)math.Attribute("display") == "block"; for (int pass = 0; pass < 3; pass++) diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs index 5134e13..4469e93 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathText.cs @@ -118,7 +118,13 @@ public static string ItalicizeCharacter(string c) } /// Array.from(chars): the code points of a string, as strings. - public static List Characters(string chars) => CodePoints.Of(chars).Select(c => c.ToString()).ToList(); + public static List Characters(string chars) + { + var characters = new List(chars.Length); + for (int at = 0; at < chars.Length; at += CodePoints.LengthAt(chars, at)) + characters.Add(CodePoints.TextAt(chars, at)); + return characters; + } public static string ItalicizeCharacters(string chars) => string.Concat(Characters(chars).Select(ItalicizeCharacter)); diff --git a/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs b/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs index 947a844..b0ac7bf 100644 --- a/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs +++ b/tests/SharpEquations.Tests/Property/RoundTripPropertyTests.cs @@ -66,6 +66,62 @@ private static Dictionary Failures(Func check private static List Listed(Dictionary failures) => failures.Select(f => f.Key + "\n " + f.Value).ToList(); + /// + /// The UnicodeMath writer reads back what it writes only when reading could give another equation + /// (UnicodeMathSerializer.NeedsReadBack): what it writes without reading back is what it writes reading back. + /// + [Test] + public async Task UnicodeMathWrittenWithoutReadingBackIsTheSame() + { + static string? Check(Equation equation) + { + var math = equation.ToMathML(); + var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false); + var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true); + return written == checkedBack ? null : written + " instead of " + checkedBack; + } + + await Assert.That(Listed(Failures(Check))).IsEmpty(); + await Assert.That(Listed(Failures(Check, WordEquations()))).IsEmpty(); + } + + /// + /// The same for an operator of each character of the basic plane that is not a letter or a digit, alone, between + /// two identifiers, before one and in a script: the characters that reading gives identifiers with are read back. + /// + [Test] + public async Task OperatorsWrittenWithoutReadingBackAreTheSame() + { + var failures = new List(); + for (int codePoint = 0x21; codePoint <= 0xFFFF; codePoint++) + { + if (codePoint is >= 0xD800 and <= 0xF8FF || char.IsLetterOrDigit((char)codePoint) || char.IsControl((char)codePoint)) + continue; + var mo = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mo", ((char)codePoint).ToString()); + var a = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "a"); + var b = new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "mi", "b"); + foreach (var content in new object[] { mo, new object[] { a, mo, b }, new object[] { mo, a }, new object[] { a, mo }, + new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "msub", a, mo) }) + { + Equation equation; + try + { + equation = Equation.Parse(new System.Xml.Linq.XElement(MathMLNamespaces.MathML + "math", content), EquationFormat.MathML); + } + catch (FormatException) + { + continue; + } + var math = equation.ToMathML(); + var written = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, false); + var checkedBack = Internal.UnicodeMath.UnicodeMathSerializer.Serialize(math, false, true); + if (written != checkedBack) + failures.Add("U+" + codePoint.ToString("X4") + ": " + written + " instead of " + checkedBack); + } + } + await Assert.That(failures).IsEmpty(); + } + [Test] public async Task WritingNeverThrows() {