From e7f27c9d178dc9ad97c07cec3a534cd5000803e4 Mon Sep 17 00:00:00 2001 From: Simone Bortolin Date: Mon, 9 Nov 2026 19:12:30 +0100 Subject: [PATCH] LaTeX reader: the space commands, \ outside environments, \middle beside empty sides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - \hspace, \hskip, \kern, \mkern, \mskip and \mspace read with their length in any form (1em, {10ex}, -1.5pt, 1em plus 1fil) as the UnicodeMath space nearest to it; \thinspace, \negthinspace, \negmedspace, \negthickspace, \nobreakspace and \space as the spaces they are - \ outside an environment, a line break an equation does not keep, left out - two bars with only zero spaces between them (|\!|) are no pair, which UnicodeMath would read as a double bar; the writer parts a bar alone from the next bar with \! - \middle with an empty side holds the placeholder there (├{⬚│⬚┤}) - the UnicodeMath grammar takes a no-break space after a bracket ((a)~b), reserved only before a diacritic - latex2mathml cases spaces and middle read; the corpus test compares the separator │ as | Co-Authored-By: Claude Opus 5.5 --- .../Internal/LaTeX/LaTeXReader.cs | 229 +++++++++++++++--- .../Internal/LaTeX/LaTeXSerializer.cs | 5 + .../UnicodeMath/UnicodeMathGrammar.parlot.cs | 2 +- .../LaTeX/LaTeXReaderTests.cs | 15 +- .../LaTeX/Latex2MathMLCorpusTests.cs | 11 +- 5 files changed, 225 insertions(+), 37 deletions(-) diff --git a/src/SharpEquations/Internal/LaTeX/LaTeXReader.cs b/src/SharpEquations/Internal/LaTeX/LaTeXReader.cs index 50174f4..987acb6 100644 --- a/src/SharpEquations/Internal/LaTeX/LaTeXReader.cs +++ b/src/SharpEquations/Internal/LaTeX/LaTeXReader.cs @@ -484,6 +484,15 @@ private static string RewriteTeXOnly(string latex) i += 2; continue; } + // \\ outside an environment, a line break, which an equation does not keep: \!, no space + if (tex[i + 1] == "\\" && !InEnvironment(tex, i)) + { + result.Append("\\!"); + for (i += 2; i < tex.Length && tex[i] == " "; i++) + { + } + continue; + } // \_ held, which the passes would read as a subscript of what follows (\_\{\}) if (tex[i + 1] == "_") { @@ -507,36 +516,30 @@ private static string RewriteTeXOnly(string latex) k++; while (result.Length > 0 && result[result.Length - 1] == ' ') result.Length--; - if (tex[k] == "\\" && tex[k + 1] == "|") - { - result.Append("\\‖"); - for (i = k + 2; i < tex.Length && tex[i] == " "; i++) - { - } - continue; - } - if (tex[k] == "|") - { - result.Append('│'); - i = k + 1; - continue; - } - if (tex.Sub(k, k + 5) == "\\vert") - { - result.Append('│'); - i = k + 5; - continue; - } // another delimiter in the middle is the delimiter itself, a slash the slash of UnicodeMath - if (tex[k] == "/") - { - result.Append("\\/"); - for (i = k + 1; i < tex.Length && tex[i] == " "; i++) - { - } + var (middle, afterMiddle) = tex[k] == "\\" && tex[k + 1] == "|" ? ("\\‖", k + 2) + : tex[k] == "|" ? ("│", k + 1) + : tex.Sub(k, k + 5) == "\\vert" && !IsAsciiAlphabetic(tex[k + 5]) ? ("│", k + 5) + : tex[k] == "/" ? ("\\/", k + 1) + : (null, k); + i = afterMiddle; + if (middle == null) continue; + if (middle != "│") + { + while (i < tex.Length && tex[i] == " ") + i++; } - i = k; + // UnicodeMath wants something on each side of the bar (├{│┤} is no UnicodeMath): an empty side holds + // the placeholder, which the writer writes as {} + if (EndsWithOpening(result)) + result.Append('⬚'); + result.Append(middle); + int afterBlanks = i; + while (tex[afterBlanks] == " ") + afterBlanks++; + if (tex[afterBlanks] == "\\" && ControlWord(tex, afterBlanks + 1) is "right" or "middle") + result.Append('⬚'); continue; } if (cw == "vert" && angles > 0) @@ -615,9 +618,20 @@ private static string RewriteTeXOnly(string latex) i = afterModulus; continue; } - if (cw == "hspace" && BracedArgument(tex, tex[j] == "*" ? j + 1 : j) is var (_, afterSpace) && afterSpace >= 0) + if (SpaceCommands.TryGetValue(cw, out var spaceCommand)) { - result.Append("\\quad "); + result.Append(spaceCommand); + i = j; + while (i < tex.Length && tex[i] == " ") + i++; + continue; + } + if (LengthCommands.Contains(cw)) + { + // \hspace{10ex}, \kern-1.5pt, \mkern10mu: the space of UnicodeMath nearest to the length, an em one when + // the length is not a number with a unit + var (length, afterSpace) = LengthArgument(tex, cw == "hspace" && tex[j] == "*" ? j + 1 : j); + result.Append(Space(length ?? 18)); i = afterSpace; continue; } @@ -900,13 +914,37 @@ private static string EscapeOddBars(string latex) frames.Push(new List()); var escaped = new List(); var kets = new HashSet(); + bool ZeroSpacesBetween(int first, int second) + { + bool zero = false; + for (int k = first + 1; k < second; k++) + { + if (tex[k] == "\\" && tex[k + 1] == "!") + { + zero = true; + k++; + } + else if (tex[k] != " ") + return false; + } + return zero; + } void Close(List bars, bool inBrackets) { foreach (var bar in new[] { "|", "‖", "∣" }) { var some = bars.Where(b => tex[b] == bar).ToList(); - if (some.Count % 2 == 1 && !(inBrackets && some.Count == 1 && bar != "‖")) - escaped.Add(some[some.Count - 1]); + if (inBrackets && some.Count == 1 && bar != "‖") + continue; + // bars in pairs, but two with only zero spaces between them (|\!|, which UnicodeMath would read as + // ||, one double bar): the first is then a bar alone + for (int k = 0; k < some.Count; k++) + { + if (k + 1 < some.Count && !ZeroSpacesBetween(some[k], some[k + 1])) + k++; + else + escaped.Add(some[k]); + } } } for (int i = 0; i < tex.Length; i++) @@ -1746,5 +1784,134 @@ private static string NonBlank(Text tex, int i) i++; return tex[i]; } + + /// Whether the index is between a \begin and its \end, where \\ parts rows. + private static bool InEnvironment(Text tex, int i) + { + int depth = 0; + for (int k = 0; k < i; k++) + { + if (tex[k] == "\\") + { + var cw = ControlWord(tex, k + 1); + depth += cw switch { "begin" => 1, "end" => -1, _ => 0 }; + k += Math.Max(1, cw.Length); + } + } + return depth > 0; + } + + /// Whether the result ends with \left and its delimiter, with nothing after them. + private static bool EndsWithOpening(StringBuilder result) + { + var text = result.ToString(); + var at = text.LastIndexOf("\\left", StringComparison.Ordinal); + if (at < 0) + return false; + var delimiter = new Text(text.Substring(at + 5).Trim()); + return delimiter.Length <= 1 || delimiter[0] == "\\" && (delimiter.Length == 2 || ControlWord(delimiter, 1).Length == delimiter.Length - 1); + } + + // the commands of a space whose length follows them + private static readonly HashSet LengthCommands = new HashSet(StringComparer.Ordinal) + { + "hspace", "hskip", "kern", "mkern", "mskip", "mspace", + }; + + // the spaces of plain TeX and amsmath that UnicodeMath has no control word for; the negative ones none, as \! + private static readonly Dictionary SpaceCommands = new Dictionary(StringComparer.Ordinal) + { + ["thinspace"] = "\\,", ["negthinspace"] = "\\!", ["negmedspace"] = "\\!", ["negthickspace"] = "\\!", ["nobreakspace"] = "~", + ["space"] = "\\ ", + }; + + /// The space of UnicodeMath nearest to a length in mu (18 to the em), \! (none) for a negative one. + private static string Space(double mu) => mu switch + { + <= 0 => "\\!", + < 3.5 => "\\,", + < 4.5 => "\\:", + < 7 => "\\;", + < 13.5 => "\\enspace ", + < 27 => "\\quad ", + _ => "\\qquad ", + }; + + /// + /// The length after a space command, in mu, braced ({10ex}) or not (-1.5pt, 1em plus 2em), and the index after + /// it; null if it is no number with a unit (\hspace{\fill}). + /// + private static (double? Mu, int Next) LengthArgument(Text tex, int i) + { + while (tex[i] == " ") + i++; + if (tex[i] == "{") + { + var (braced, afterBraced) = BracedArgument(tex, i); + if (afterBraced < 0) + return (null, i); + var inner = new Text(braced); + var (mu, afterLength) = Dimension(inner, 0); + return (NonBlank(inner, afterLength) == null ? mu : null, afterBraced); + } + var (length, next) = Dimension(tex, i); + if (length == null) + return (null, i); + // the stretch and the shrink of glue, \hskip 1em plus 1fil minus 2pt + while (true) + { + int word = next; + while (tex[word] == " ") + word++; + var glue = tex.Sub(word, word + 4) == "plus" ? 4 : tex.Sub(word, word + 5) == "minus" ? 5 : 0; + if (glue == 0 || Dimension(tex, word + glue) is not (not null, var afterGlue)) + return (length, next); + next = afterGlue; + } + } + + /// A number with its unit (-1.5pt, 10 mu, 1fil) in mu, and the index after it; null if none. + private static (double? Mu, int Next) Dimension(Text tex, int i) + { + while (tex[i] == " ") + i++; + int start = i; + if (tex[i] is "-" or "+") + i++; + int digits = i; + while (tex[i] is { Length: 1 } d && (d[0] is >= '0' and <= '9' or '.')) + i++; + if (i == digits || !double.TryParse(tex.Sub(start, i), System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var number)) + return (null, start); + while (tex[i] == " ") + i++; + // a unit is two letters (1ptx is 1pt and x), but for the infinite ones of glue (fil, fill, filll) + int unit = i; + i += 2; + if (tex.Sub(unit, i) == "fi") + { + while (i < unit + 5 && tex[i] == "l") + i++; + } + double? mu = tex.Sub(unit, i) switch + { + "em" => 18, + "ex" => 7.75, + "mu" => 1, + "pt" => 1.8, + "bp" => 1.81, + "dd" => 1.93, + "px" => 1.35, + "mm" => 5.12, + "pc" => 21.6, + "cc" => 23.1, + "cm" => 51.2, + "in" => 130.1, + "sp" => 1.8 / 65536, + "fil" or "fill" or "filll" => 0, + _ => null, + }; + return mu == null ? (null, start) : (number * mu, i); + } } } diff --git a/src/SharpEquations/Internal/LaTeX/LaTeXSerializer.cs b/src/SharpEquations/Internal/LaTeX/LaTeXSerializer.cs index d01c194..41b1b11 100644 --- a/src/SharpEquations/Internal/LaTeX/LaTeXSerializer.cs +++ b/src/SharpEquations/Internal/LaTeX/LaTeXSerializer.cs @@ -568,6 +568,11 @@ private string TeX(XElement value, bool noAddParens = false) ret = "\\$"; break; } + // a bar alone before another bar parted from it by \!, a zero space: the reader pairs bars with + // nothing between them, which UnicodeMath reads as one double bar + if (val == "|" && Attr(value, "intent") == ":text" && + value.ElementsAfterSelf().FirstOrDefault()?.DescendantsAndSelf().FirstOrDefault(d => !d.HasElements) is { } nextLeaf && IsBar(nextLeaf) && Text(nextLeaf) == "|") + return "|\\!"; if (Attr(value, "intent") == ":text" && !standard) { // a bracket or a bar that pairs with nothing: the reader escapes the brackets that close nothing and diff --git a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathGrammar.parlot.cs b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathGrammar.parlot.cs index 2d4e38e..ab769e4 100644 --- a/src/SharpEquations/Internal/UnicodeMath/UnicodeMathGrammar.parlot.cs +++ b/src/SharpEquations/Internal/UnicodeMath/UnicodeMathGrammar.parlot.cs @@ -608,7 +608,7 @@ private static Parser Build() L("ⓐ(").SkipAnd(OneOrMany(alphaAscii)).AndSkip(Opt(__)).And(exp).AndSkip(L(")")) .Then(t => (object)O("intend", O(("op", "ⓐ"), ("intent", O("text", Join(t.Item1.ToList()))), ("content", t.Item2))))); entity.Parser = Always(null).When((context, x) => UnicodeMathGrammarWork.Step()).SkipAnd(OneOf( - expBracket.And(OneOf(Not(OneOf(L("\u00A0"), diacritic)).Then(x => (object)true), Always(false))) + expBracket.And(OneOf(Not(OneOf(L("\u00A0").SkipAnd(diacritic), diacritic)).Then(x => (object)true), Always(false))) .When((context, t) => (bool)t.Item2 || t.Item1 is DiacriticizedParen) .Then(t => t.Item1 is DiacriticizedParen d ? d.Atoms : t.Item1), absoluteValue, diff --git a/tests/SharpEquations.Tests/LaTeX/LaTeXReaderTests.cs b/tests/SharpEquations.Tests/LaTeX/LaTeXReaderTests.cs index cd4614c..1d1d227 100644 --- a/tests/SharpEquations.Tests/LaTeX/LaTeXReaderTests.cs +++ b/tests/SharpEquations.Tests/LaTeX/LaTeXReaderTests.cs @@ -157,6 +157,11 @@ public async Task ReadsLessCommonCommands(string latex, string expectedLaTeX, st [Arguments(@"(a+b)/2", @"(a+b)/2", @"(𝑎+𝑏)\/2")] [Arguments(@"\left( a \middle/ b \right)", @"(a/b)", @"(𝑎\/𝑏)")] [Arguments(@"\left( a \middle\| b \right)", @"(a\|b)", @"(𝑎\‖𝑏)")] + [Arguments(@"\left\{\middle|\right\}", @"\{{}|{}\}", @"{⬚│⬚}")] + [Arguments(@"\left( a \middle| \right)", @"(a|{})", @"(𝑎│⬚)")] + [Arguments(@"|a|\\|b|", @"|a||b|", @"|𝑎||𝑏|")] + [Arguments(@"|\,|\\|\!|\,|", @"|\,||\!|\,|", "|\u2009|\\||\u2009|")] + [Arguments(@"|a|~|b|", @"|a|~|b|", "|𝑎|\u00A0|𝑏|")] [Arguments(@"\langle a | b \rangle | c", @"\langle a|b\rangle|c", @"⟨𝑎│𝑏⟩\|𝑐")] [Arguments(@"a|b", @"a|b", @"𝑎\|𝑏")] [Arguments(@"(a\|b)", @"(a\|b)", @"(𝑎\‖𝑏)")] @@ -228,6 +233,15 @@ public async Task ReadsTextWithMathInside() [Arguments(@"a\!b", "ab")] [Arguments(@"a\>b", @"a\:b")] [Arguments(@"\hspace{1em}x", "x")] + [Arguments(@"a\hspace1em b", @"a\quad b")] + [Arguments(@"a\hspace{2cm}b", @"a\quad\quad b")] + [Arguments(@"a\kern-1.5pt b", "ab")] + [Arguments(@"a\mkern10mu b", @"a\enspace b")] + [Arguments(@"a\mspace{18mu}b", @"a\quad b")] + [Arguments(@"a\hskip 1em plus 1fil b", @"a\quad b")] + [Arguments(@"a\thinspace b\negthinspace c\nobreakspace d\space e", @"a\,bc~d~e")] + [Arguments(@"a\\b", "ab")] + [Arguments(@"(a)~b", @"(a)~b")] public async Task ReadsSpaces(string latex, string expected) { await Assert.That(Equation.Parse(latex, EquationFormat.LaTeX).ToString(EquationFormat.LaTeX)).IsEqualTo(expected); @@ -309,7 +323,6 @@ public async Task KeepsKnownControlWordsWhenStrict() } [Test] - [Arguments(@"a\\b")] [Arguments(@"\frac{a}")] [Arguments(@"x_\frac{1}")] [Arguments(@"\text{unclosed")] diff --git a/tests/SharpEquations.Tests/LaTeX/Latex2MathMLCorpusTests.cs b/tests/SharpEquations.Tests/LaTeX/Latex2MathMLCorpusTests.cs index 98efa05..6175b7e 100644 --- a/tests/SharpEquations.Tests/LaTeX/Latex2MathMLCorpusTests.cs +++ b/tests/SharpEquations.Tests/LaTeX/Latex2MathMLCorpusTests.cs @@ -76,7 +76,8 @@ private static bool IsSpace(string text) => text.Length > 0 && /// The equation as MathML, with what latex2mathml and SharpEquations write differently for the same equation /// made the same: attributes that say the default or set the style, spaces (formatting, which an equation does not /// keep), invisible function application and times, the middle dot of \cdot (· for latex2mathml, ⋅ for - /// SharpEquations), names of operators and primes (mo for latex2mathml, mi and mo for SharpEquations), spacing + /// SharpEquations), the bar of \middle (| for latex2mathml, the separator │ of UnicodeMath for SharpEquations), + /// names of operators and primes (mo for latex2mathml, mi and mo for SharpEquations), spacing /// accents (˙ for latex2mathml, the combining dot above for SharpEquations), bars (mover with ¯ for latex2mathml, /// menclose for SharpEquations), capital Greek letters (upright in TeX, which latex2mathml writes as mi without /// mathvariant), symbols that are neither letters nor digits (mi for latex2mathml, %, °, mo for SharpEquations, @@ -115,6 +116,9 @@ private static string Canonical(Equation equation) case "mo" when text == "·": e.Value = "⋅"; break; + case "mo" when text == "│": + e.Value = "|"; + break; case "mo" when text != null && Accents.TryGetValue(text, out var mark) && e.Parent!.Name.LocalName is "mover" && e.ElementsBeforeSelf().Any(): e.Value = mark; break; @@ -198,13 +202,12 @@ private static string Canonical(Equation equation) // LaTeX and amsmath that the reader does not read yet: an operator name with nothing after it (\lg, \liminf, // \injlim, \operatorname*{argmax}_x), which UnicodeMath reads as letters, and the names of amsmath that are two - // words (lim inf); \limits after \mathop; both scripts of \overbrace and \underbrace; \middle between empty - // delimiters and bars with nothing between them (|\,|); a font over \text and escaped braces (\mathrm{\{b\}}, + // words (lim inf); \limits after \mathop; both scripts of \overbrace and \underbrace; a font over \text and escaped braces (\mathrm{\{b\}}, // \mathsf{\text{}}, \boldsymbol{1}); \newenvironment and ## in a macro private static readonly HashSet NotReadYet = new HashSet { "lg", "liminf", "limsup", "injlim", "projlim", "varinjlim", "operatornamewithlimits", "operatorname-star", - "issue-125-4-limits", "overbrace-c", "underbrace-c", "middle", "spaces", "mathrm", "mathsf", "boldsymbol", + "issue-125-4-limits", "overbrace-c", "underbrace-c", "mathrm", "mathsf", "boldsymbol", "newenvironment-multi-node", "newcommand-escaped-hash", };