From 14f01c67246a0e466d8693e52efe323f58f5e6c9 Mon Sep 17 00:00:00 2001 From: C Deutsch Date: Thu, 12 Nov 2026 16:09:43 -0600 Subject: [PATCH 1/6] Decode numeric character references in scraped titles on iOS and macOS The Apple scraper deleted every decimal reference in a page's `` and left hex ones as written, so a title written as `It’s here` was stored as `Its here`. The title is written into the push's envelope and read by every device, so the garbled one stayed for good, and it disagreed with Android, whose Jsoup decodes them. The title extraction moves from `TitleScraperService` into `PushTitle` in `PushContent.swift`, which the tests already compile, and decodes in one pass from the left: decimal and hex references (the `;` optional, 0x80 to 0x9F read as Windows-1252, as browsers and Jsoup do), and the named ones it already knew, now to their own characters rather than ASCII stand-ins, so `—` and `—` give the same title. A reference that names no character is left as text. Decoding now comes before the whitespace is collapsed, as it does in Jsoup. `PushTitleTests` covers decimal, hex, invalid and mixed cases, with expected strings checked against Jsoup 1.23.2. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/ios/PushThat/Services/PushContent.swift | 121 ++++++++++++++++++ .../Services/TitleScraperService.swift | 60 +-------- apps/ios/PushThatTests/E2eeVectorTests.swift | 47 +++++++ .../macos/PushThat/Services/PushContent.swift | 121 ++++++++++++++++++ .../Services/TitleScraperService.swift | 60 +-------- .../macos/PushThatTests/E2eeVectorTests.swift | 47 +++++++ 6 files changed, 338 insertions(+), 118 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index 52d32b0..7fb318b 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -492,3 +492,124 @@ enum PushIcon { return cache }() } + +/// A page's title as every client stores it: the text of its first `<title>`, character +/// references decoded, whitespace collapsed to single spaces and trimmed. +/// +/// Here rather than in `TitleScraperService` for the reason `PushIcon` is: this file is +/// compiled by both apps and by the tests, so the decoding is written once per app and +/// checked. The title is stored in the envelope and read by every device, so the aim is +/// the title Android's Jsoup gives for the same page (`doc.title()`): references decoded +/// before the whitespace is collapsed, as a parser does, and numeric ones decoded as a +/// browser decodes them. +enum PushTitle { + static func extract(from html: String) -> String? { + guard let match = titlePattern.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)), + let range = Range(match.range(at: 1), in: html) + else { return nil } + // Decoded first, so a ` ` or ` ` is collapsed with the rest of the + // whitespace, as Jsoup collapses the text it has already decoded. + return decodingEntities(String(html[range])) + .trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + } + + private static let titlePattern = try! NSRegularExpression( + pattern: "<title[^>]*>([^<]+)", options: .caseInsensitive + ) + + /// Decodes the named references in `named` and every decimal (`’`) and hex + /// (`’`) one, in a single pass from the left, so the text a reference decodes + /// to is never read again: `&#39;` is the text `'`, not an apostrophe. A + /// reference this does not know, or a numeric one that names no character (zero, a + /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. + /// + /// Jsoup parts from this on those: it makes U+FFFD of a number past U+10FFFF and a lone + /// surrogate of one in that range (which a Swift `String` cannot hold), and it reads the + /// handful of legacy names that need no `;` (`&`). No page means any of them, so + /// the text as written was kept over copying each of Jsoup's answers. + /// + /// The numeric ones used to be deleted, so a title with a typographic apostrophe lost + /// it, and every device then read the title with the letter missing. + static func decodingEntities(_ text: String) -> String { + guard text.contains("&") else { return text } + let scalars = Array(text.unicodeScalars) + var out = String.UnicodeScalarView() + var index = 0 + while index < scalars.count { + if scalars[index] == "&", let (decoded, length) = reference(in: scalars, at: index) { + out.append(contentsOf: decoded.unicodeScalars) + index += length + } else { + out.append(scalars[index]) + index += 1 + } + } + return String(out) + } + + /// The references a title is decoded for by name. Each decodes to its own character, + /// as on Android: these once became ASCII stand-ins (`-` for both dashes, `(c)`), + /// which disagreed with Jsoup and with the same character written as a number. + private static let named: [String: String] = [ + "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", + "nbsp": "\u{00A0}", "ndash": "\u{2013}", "mdash": "\u{2014}", "copy": "\u{00A9}", "reg": "\u{00AE}", + ] + + /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as + /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an + /// apostrophe is one. Those that Windows-1252 leaves undefined stay as they are. + private static let windows1252: [UInt32] = [ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, + ] + + /// The reference starting at the `&` at `start`: what it decodes to and how many + /// scalars it takes, or nil to leave the `&` as text. + private static func reference(in scalars: [Unicode.Scalar], at start: Int) -> (String, Int)? { + var index = start + 1 + guard index < scalars.count else { return nil } + guard scalars[index] == "#" else { + // A named reference needs its `;`, as before; the longest name above is 5. + guard let semicolon = scalars[index...].prefix(8).firstIndex(of: ";"), + let text = named[String(String.UnicodeScalarView(scalars[index.. digitsStart else { return nil } + // The `;` is optional, as it is for a browser and for Jsoup: `’s` is an + // apostrophe and an s. + if index < scalars.count, scalars[index] == ";" { index += 1 } + if (0x80...0x9F).contains(value) { value = windows1252[Int(value - 0x80)] } + // `Unicode.Scalar` refuses surrogates and anything past U+10FFFF. + guard value != 0, let scalar = Unicode.Scalar(value) else { return nil } + return (String(Character(scalar)), index - start) + } + + /// ASCII digits only: `Character.wholeNumberValue` would also take other scripts' digits, + /// which no parser reads in a reference. + private static func digitValue(_ scalar: Unicode.Scalar, radix: UInt32) -> UInt32? { + switch scalar { + case "0"..."9": return scalar.value - 0x30 + case "a"..."f" where radix == 16: return scalar.value - 0x61 + 10 + case "A"..."F" where radix == 16: return scalar.value - 0x41 + 10 + default: return nil + } + } +} diff --git a/apps/ios/PushThat/Services/TitleScraperService.swift b/apps/ios/PushThat/Services/TitleScraperService.swift index 593e528..0e2793a 100644 --- a/apps/ios/PushThat/Services/TitleScraperService.swift +++ b/apps/ios/PushThat/Services/TitleScraperService.swift @@ -97,7 +97,7 @@ actor TitleScraperService { // Icons resolve against the URL the page came from after redirects, as a // browser resolves them, not against the one that was pushed. let icon = withIcon ? await fetchIcon(from: PushIcon.candidates(html: html, pageURL: httpResponse.url ?? url)) : nil - return (extractTitle(from: html), icon) + return (PushTitle.extract(from: html), icon) } catch { // The error's type only: a URLError carries the page's full URL. print("Failed to scrape title: \(type(of: error))") @@ -149,62 +149,4 @@ actor TitleScraperService { return nil } } - - /// Extract title from HTML using regex - /// Note: For production, consider using SwiftSoup for proper HTML parsing - private func extractTitle(from html: String) -> String? { - // Try to find tag - let titlePattern = "<title[^>]*>([^<]+)" - - guard let regex = try? NSRegularExpression(pattern: titlePattern, options: .caseInsensitive), - let match = regex.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)), - let titleRange = Range(match.range(at: 1), in: html) else { - return nil - } - - let title = String(html[titleRange]) - .trimmingCharacters(in: .whitespacesAndNewlines) - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) - - // Decode HTML entities - return title.decodingHTMLEntities() - } -} - -// MARK: - String HTML Entity Decoding -extension String { - func decodingHTMLEntities() -> String { - var result = self - - let entities: [String: String] = [ - "&": "&", - "<": "<", - ">": ">", - """: "\"", - "'": "'", - "'": "'", - " ": " ", - "–": "-", - "—": "-", - "©": "(c)", - "®": "(r)" - ] - - for (entity, replacement) in entities { - result = result.replacingOccurrences(of: entity, with: replacement) - } - - // Handle numeric entities like ’ - let numericPattern = "&#(\\d+);" - if let regex = try? NSRegularExpression(pattern: numericPattern) { - let range = NSRange(result.startIndex..., in: result) - result = regex.stringByReplacingMatches( - in: result, - range: range, - withTemplate: "" - ) - } - - return result - } } diff --git a/apps/ios/PushThatTests/E2eeVectorTests.swift b/apps/ios/PushThatTests/E2eeVectorTests.swift index 2792582..888a917 100644 --- a/apps/ios/PushThatTests/E2eeVectorTests.swift +++ b/apps/ios/PushThatTests/E2eeVectorTests.swift @@ -677,3 +677,50 @@ final class PushIconTests: XCTestCase { return data as Data } } + +/// The title a scrape stores. Every device reads the one stored, so the expected strings +/// are what Android's Jsoup 1.23.2 gives for the same ``, checked against it, except +/// in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). +final class PushTitleTests: XCTestCase { + func testDecimalReferences() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready — at 4:30"), "Tea\u{2019}s ready \u{2014} at 4:30") + XCTAssertEqual(PushTitle.decodingEntities("😀 smile"), "\u{1F600} smile") + } + + func testHexReferences() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s — 😀"), "Tea\u{2019}s \u{2014} \u{1F600}") + } + + func testTheSemicolonIsOptionalOnANumber() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready"), "Tea\u{2019}s ready") + XCTAssertEqual(PushTitle.decodingEntities("䆼"), "\u{41BC}") + } + + func testWindows1252Range() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s – €5"), "Tea\u{2019}s \u{2013} \u{20AC}5") + XCTAssertEqual(PushTitle.decodingEntities(""), "\u{0081}") + } + + func testInvalidReferencesStayAsText() { + for text in ["&#;", "&#x;", "&#xZZ;", "�", "�", "�", "�", "�", + "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "&", "a & b", "&;"] { + XCTAssertEqual(PushTitle.decodingEntities(text), text, text) + } + } + + func testNamedReferences() { + XCTAssertEqual(PushTitle.decodingEntities("<b> "Q" 'A' © ® – — &"), + "<b> \"Q\" 'A' \u{00A9} \u{00AE} \u{2013} \u{2014} &") + } + + func testMixedAndDecodedOnce() { + XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), + "Fish & Chips\u{2122} \u{2014} It's ' &") + } + + func testExtractDecodesThenCollapsesWhitespace() { + let html = "<html><head><TITLE lang=\"en\">\n  Tea’s ready now  " + XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") + XCTAssertNil(PushTitle.extract(from: "")) + } +} diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index 52d32b0..7fb318b 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -492,3 +492,124 @@ enum PushIcon { return cache }() } + +/// A page's title as every client stores it: the text of its first ``, character +/// references decoded, whitespace collapsed to single spaces and trimmed. +/// +/// Here rather than in `TitleScraperService` for the reason `PushIcon` is: this file is +/// compiled by both apps and by the tests, so the decoding is written once per app and +/// checked. The title is stored in the envelope and read by every device, so the aim is +/// the title Android's Jsoup gives for the same page (`doc.title()`): references decoded +/// before the whitespace is collapsed, as a parser does, and numeric ones decoded as a +/// browser decodes them. +enum PushTitle { + static func extract(from html: String) -> String? { + guard let match = titlePattern.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)), + let range = Range(match.range(at: 1), in: html) + else { return nil } + // Decoded first, so a ` ` or ` ` is collapsed with the rest of the + // whitespace, as Jsoup collapses the text it has already decoded. + return decodingEntities(String(html[range])) + .trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + } + + private static let titlePattern = try! NSRegularExpression( + pattern: "<title[^>]*>([^<]+)", options: .caseInsensitive + ) + + /// Decodes the named references in `named` and every decimal (`’`) and hex + /// (`’`) one, in a single pass from the left, so the text a reference decodes + /// to is never read again: `&#39;` is the text `'`, not an apostrophe. A + /// reference this does not know, or a numeric one that names no character (zero, a + /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. + /// + /// Jsoup parts from this on those: it makes U+FFFD of a number past U+10FFFF and a lone + /// surrogate of one in that range (which a Swift `String` cannot hold), and it reads the + /// handful of legacy names that need no `;` (`&`). No page means any of them, so + /// the text as written was kept over copying each of Jsoup's answers. + /// + /// The numeric ones used to be deleted, so a title with a typographic apostrophe lost + /// it, and every device then read the title with the letter missing. + static func decodingEntities(_ text: String) -> String { + guard text.contains("&") else { return text } + let scalars = Array(text.unicodeScalars) + var out = String.UnicodeScalarView() + var index = 0 + while index < scalars.count { + if scalars[index] == "&", let (decoded, length) = reference(in: scalars, at: index) { + out.append(contentsOf: decoded.unicodeScalars) + index += length + } else { + out.append(scalars[index]) + index += 1 + } + } + return String(out) + } + + /// The references a title is decoded for by name. Each decodes to its own character, + /// as on Android: these once became ASCII stand-ins (`-` for both dashes, `(c)`), + /// which disagreed with Jsoup and with the same character written as a number. + private static let named: [String: String] = [ + "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", + "nbsp": "\u{00A0}", "ndash": "\u{2013}", "mdash": "\u{2014}", "copy": "\u{00A9}", "reg": "\u{00AE}", + ] + + /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as + /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an + /// apostrophe is one. Those that Windows-1252 leaves undefined stay as they are. + private static let windows1252: [UInt32] = [ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, + ] + + /// The reference starting at the `&` at `start`: what it decodes to and how many + /// scalars it takes, or nil to leave the `&` as text. + private static func reference(in scalars: [Unicode.Scalar], at start: Int) -> (String, Int)? { + var index = start + 1 + guard index < scalars.count else { return nil } + guard scalars[index] == "#" else { + // A named reference needs its `;`, as before; the longest name above is 5. + guard let semicolon = scalars[index...].prefix(8).firstIndex(of: ";"), + let text = named[String(String.UnicodeScalarView(scalars[index.. digitsStart else { return nil } + // The `;` is optional, as it is for a browser and for Jsoup: `’s` is an + // apostrophe and an s. + if index < scalars.count, scalars[index] == ";" { index += 1 } + if (0x80...0x9F).contains(value) { value = windows1252[Int(value - 0x80)] } + // `Unicode.Scalar` refuses surrogates and anything past U+10FFFF. + guard value != 0, let scalar = Unicode.Scalar(value) else { return nil } + return (String(Character(scalar)), index - start) + } + + /// ASCII digits only: `Character.wholeNumberValue` would also take other scripts' digits, + /// which no parser reads in a reference. + private static func digitValue(_ scalar: Unicode.Scalar, radix: UInt32) -> UInt32? { + switch scalar { + case "0"..."9": return scalar.value - 0x30 + case "a"..."f" where radix == 16: return scalar.value - 0x61 + 10 + case "A"..."F" where radix == 16: return scalar.value - 0x41 + 10 + default: return nil + } + } +} diff --git a/apps/macos/PushThat/Services/TitleScraperService.swift b/apps/macos/PushThat/Services/TitleScraperService.swift index 0e2dcdb..e2bc401 100644 --- a/apps/macos/PushThat/Services/TitleScraperService.swift +++ b/apps/macos/PushThat/Services/TitleScraperService.swift @@ -97,7 +97,7 @@ actor TitleScraperService { // Icons resolve against the URL the page came from after redirects, as a // browser resolves them, not against the one that was pushed. let icon = withIcon ? await fetchIcon(from: PushIcon.candidates(html: html, pageURL: httpResponse.url ?? url)) : nil - return (extractTitle(from: html), icon) + return (PushTitle.extract(from: html), icon) } catch { // The error's type only: a URLError carries the page's full URL. print("Failed to scrape title: \(type(of: error))") @@ -149,62 +149,4 @@ actor TitleScraperService { return nil } } - - /// Extract title from HTML using regex - /// Note: For production, consider using SwiftSoup for proper HTML parsing - private func extractTitle(from html: String) -> String? { - // Try to find tag - let titlePattern = "<title[^>]*>([^<]+)" - - guard let regex = try? NSRegularExpression(pattern: titlePattern, options: .caseInsensitive), - let match = regex.firstMatch(in: html, range: NSRange(html.startIndex..., in: html)), - let titleRange = Range(match.range(at: 1), in: html) else { - return nil - } - - let title = String(html[titleRange]) - .trimmingCharacters(in: .whitespacesAndNewlines) - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) - - // Decode HTML entities - return title.decodingHTMLEntities() - } -} - -// MARK: - String HTML Entity Decoding -extension String { - func decodingHTMLEntities() -> String { - var result = self - - let entities: [String: String] = [ - "&": "&", - "<": "<", - ">": ">", - """: "\"", - "'": "'", - "'": "'", - " ": " ", - "–": "-", - "—": "-", - "©": "(c)", - "®": "(r)" - ] - - for (entity, replacement) in entities { - result = result.replacingOccurrences(of: entity, with: replacement) - } - - // Handle numeric entities like ’ - let numericPattern = "&#(\\d+);" - if let regex = try? NSRegularExpression(pattern: numericPattern) { - let range = NSRange(result.startIndex..., in: result) - result = regex.stringByReplacingMatches( - in: result, - range: range, - withTemplate: "" - ) - } - - return result - } } diff --git a/apps/macos/PushThatTests/E2eeVectorTests.swift b/apps/macos/PushThatTests/E2eeVectorTests.swift index 2792582..888a917 100644 --- a/apps/macos/PushThatTests/E2eeVectorTests.swift +++ b/apps/macos/PushThatTests/E2eeVectorTests.swift @@ -677,3 +677,50 @@ final class PushIconTests: XCTestCase { return data as Data } } + +/// The title a scrape stores. Every device reads the one stored, so the expected strings +/// are what Android's Jsoup 1.23.2 gives for the same ``, checked against it, except +/// in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). +final class PushTitleTests: XCTestCase { + func testDecimalReferences() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready — at 4:30"), "Tea\u{2019}s ready \u{2014} at 4:30") + XCTAssertEqual(PushTitle.decodingEntities("😀 smile"), "\u{1F600} smile") + } + + func testHexReferences() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s — 😀"), "Tea\u{2019}s \u{2014} \u{1F600}") + } + + func testTheSemicolonIsOptionalOnANumber() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready"), "Tea\u{2019}s ready") + XCTAssertEqual(PushTitle.decodingEntities("䆼"), "\u{41BC}") + } + + func testWindows1252Range() { + XCTAssertEqual(PushTitle.decodingEntities("Tea’s – €5"), "Tea\u{2019}s \u{2013} \u{20AC}5") + XCTAssertEqual(PushTitle.decodingEntities(""), "\u{0081}") + } + + func testInvalidReferencesStayAsText() { + for text in ["&#;", "&#x;", "&#xZZ;", "�", "�", "�", "�", "�", + "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "&", "a & b", "&;"] { + XCTAssertEqual(PushTitle.decodingEntities(text), text, text) + } + } + + func testNamedReferences() { + XCTAssertEqual(PushTitle.decodingEntities("<b> "Q" 'A' © ® – — &"), + "<b> \"Q\" 'A' \u{00A9} \u{00AE} \u{2013} \u{2014} &") + } + + func testMixedAndDecodedOnce() { + XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), + "Fish & Chips\u{2122} \u{2014} It's ' &") + } + + func testExtractDecodesThenCollapsesWhitespace() { + let html = "<html><head><TITLE lang=\"en\">\n  Tea’s ready now  " + XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") + XCTAssertNil(PushTitle.extract(from: "")) + } +} From e5bc03162989727a99a1e71ce577122022f6b8ed Mon Sep 17 00:00:00 2001 From: C Deutsch Date: Thu, 12 Nov 2026 16:22:28 -0600 Subject: [PATCH 2/6] Decode HTML 4's named references in scraped titles on iOS and macOS The Apple scraper knew ten names, so a title holding `’`, `…` or `é` was stored with the reference as literal text, while Android's Jsoup decoded it. The title is stored for every device, so the two disagreed for good. `PushTitle` now knows HTML 4's 252 names and `'`, with `⟨` and `⟩` at HTML5's code points as Jsoup has them. That covers what titles use without carrying HTML5's two thousand, nearly all mathematical symbols. It also drops the soft hyphen and the zero-width space from the title, as Jsoup does when it collapses whitespace. Every name was checked against Jsoup 1.23.2. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 --- apps/ios/PushThat/Services/PushContent.swift | 81 ++++++++++++++++--- apps/ios/PushThatTests/E2eeVectorTests.swift | 9 +++ .../macos/PushThat/Services/PushContent.swift | 81 ++++++++++++++++--- .../macos/PushThatTests/E2eeVectorTests.swift | 9 +++ 4 files changed, 158 insertions(+), 22 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index 7fb318b..7242dff 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -508,8 +508,11 @@ enum PushTitle { let range = Range(match.range(at: 1), in: html) else { return nil } // Decoded first, so a ` ` or ` ` is collapsed with the rest of the - // whitespace, as Jsoup collapses the text it has already decoded. + // whitespace, as Jsoup collapses the text it has already decoded. Jsoup also drops + // the soft hyphen and the zero-width space as it collapses, however they were + // written, so they are dropped here too. return decodingEntities(String(html[range])) + .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) } @@ -548,14 +551,70 @@ enum PushTitle { return String(out) } - /// The references a title is decoded for by name. Each decodes to its own character, - /// as on Android: these once became ASCII stand-ins (`-` for both dashes, `(c)`), - /// which disagreed with Jsoup and with the same character written as a number. - private static let named: [String: String] = [ - "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", - "nbsp": "\u{00A0}", "ndash": "\u{2013}", "mdash": "\u{2014}", "copy": "\u{00A9}", "reg": "\u{00AE}", + /// The references a title is decoded for by name: HTML 4's 252 and `'`, by code + /// point. Jsoup knows HTML5's two thousand, but nearly all of those are mathematical + /// symbols no title uses, while these hold every one a page writes in practice + /// (`’`, `…`, `é`). `⟨` and `⟩` take HTML5's code points, + /// which replaced HTML 4's, as Jsoup does. Each decodes to its own character: a few once + /// became ASCII stand-ins (`-` for both dashes, `(c)`), which disagreed with Jsoup and + /// with the same character written as a number. + private static let named: [String: Unicode.Scalar] = [ + "quot": "\u{0022}", "amp": "\u{0026}", "apos": "\u{0027}", "lt": "\u{003C}", "gt": "\u{003E}", + "nbsp": "\u{00A0}", "iexcl": "\u{00A1}", "cent": "\u{00A2}", "pound": "\u{00A3}", "curren": "\u{00A4}", + "yen": "\u{00A5}", "brvbar": "\u{00A6}", "sect": "\u{00A7}", "uml": "\u{00A8}", "copy": "\u{00A9}", + "ordf": "\u{00AA}", "laquo": "\u{00AB}", "not": "\u{00AC}", "shy": "\u{00AD}", "reg": "\u{00AE}", + "macr": "\u{00AF}", "deg": "\u{00B0}", "plusmn": "\u{00B1}", "sup2": "\u{00B2}", "sup3": "\u{00B3}", + "acute": "\u{00B4}", "micro": "\u{00B5}", "para": "\u{00B6}", "middot": "\u{00B7}", "cedil": "\u{00B8}", + "sup1": "\u{00B9}", "ordm": "\u{00BA}", "raquo": "\u{00BB}", "frac14": "\u{00BC}", "frac12": "\u{00BD}", + "frac34": "\u{00BE}", "iquest": "\u{00BF}", "Agrave": "\u{00C0}", "Aacute": "\u{00C1}", "Acirc": "\u{00C2}", + "Atilde": "\u{00C3}", "Auml": "\u{00C4}", "Aring": "\u{00C5}", "AElig": "\u{00C6}", "Ccedil": "\u{00C7}", + "Egrave": "\u{00C8}", "Eacute": "\u{00C9}", "Ecirc": "\u{00CA}", "Euml": "\u{00CB}", "Igrave": "\u{00CC}", + "Iacute": "\u{00CD}", "Icirc": "\u{00CE}", "Iuml": "\u{00CF}", "ETH": "\u{00D0}", "Ntilde": "\u{00D1}", + "Ograve": "\u{00D2}", "Oacute": "\u{00D3}", "Ocirc": "\u{00D4}", "Otilde": "\u{00D5}", "Ouml": "\u{00D6}", + "times": "\u{00D7}", "Oslash": "\u{00D8}", "Ugrave": "\u{00D9}", "Uacute": "\u{00DA}", "Ucirc": "\u{00DB}", + "Uuml": "\u{00DC}", "Yacute": "\u{00DD}", "THORN": "\u{00DE}", "szlig": "\u{00DF}", "agrave": "\u{00E0}", + "aacute": "\u{00E1}", "acirc": "\u{00E2}", "atilde": "\u{00E3}", "auml": "\u{00E4}", "aring": "\u{00E5}", + "aelig": "\u{00E6}", "ccedil": "\u{00E7}", "egrave": "\u{00E8}", "eacute": "\u{00E9}", "ecirc": "\u{00EA}", + "euml": "\u{00EB}", "igrave": "\u{00EC}", "iacute": "\u{00ED}", "icirc": "\u{00EE}", "iuml": "\u{00EF}", + "eth": "\u{00F0}", "ntilde": "\u{00F1}", "ograve": "\u{00F2}", "oacute": "\u{00F3}", "ocirc": "\u{00F4}", + "otilde": "\u{00F5}", "ouml": "\u{00F6}", "divide": "\u{00F7}", "oslash": "\u{00F8}", "ugrave": "\u{00F9}", + "uacute": "\u{00FA}", "ucirc": "\u{00FB}", "uuml": "\u{00FC}", "yacute": "\u{00FD}", "thorn": "\u{00FE}", + "yuml": "\u{00FF}", "OElig": "\u{0152}", "oelig": "\u{0153}", "Scaron": "\u{0160}", "scaron": "\u{0161}", + "Yuml": "\u{0178}", "fnof": "\u{0192}", "circ": "\u{02C6}", "tilde": "\u{02DC}", "Alpha": "\u{0391}", + "Beta": "\u{0392}", "Gamma": "\u{0393}", "Delta": "\u{0394}", "Epsilon": "\u{0395}", "Zeta": "\u{0396}", + "Eta": "\u{0397}", "Theta": "\u{0398}", "Iota": "\u{0399}", "Kappa": "\u{039A}", "Lambda": "\u{039B}", + "Mu": "\u{039C}", "Nu": "\u{039D}", "Xi": "\u{039E}", "Omicron": "\u{039F}", "Pi": "\u{03A0}", + "Rho": "\u{03A1}", "Sigma": "\u{03A3}", "Tau": "\u{03A4}", "Upsilon": "\u{03A5}", "Phi": "\u{03A6}", + "Chi": "\u{03A7}", "Psi": "\u{03A8}", "Omega": "\u{03A9}", "alpha": "\u{03B1}", "beta": "\u{03B2}", + "gamma": "\u{03B3}", "delta": "\u{03B4}", "epsilon": "\u{03B5}", "zeta": "\u{03B6}", "eta": "\u{03B7}", + "theta": "\u{03B8}", "iota": "\u{03B9}", "kappa": "\u{03BA}", "lambda": "\u{03BB}", "mu": "\u{03BC}", + "nu": "\u{03BD}", "xi": "\u{03BE}", "omicron": "\u{03BF}", "pi": "\u{03C0}", "rho": "\u{03C1}", + "sigmaf": "\u{03C2}", "sigma": "\u{03C3}", "tau": "\u{03C4}", "upsilon": "\u{03C5}", "phi": "\u{03C6}", + "chi": "\u{03C7}", "psi": "\u{03C8}", "omega": "\u{03C9}", "thetasym": "\u{03D1}", "upsih": "\u{03D2}", + "piv": "\u{03D6}", "ensp": "\u{2002}", "emsp": "\u{2003}", "thinsp": "\u{2009}", "zwnj": "\u{200C}", + "zwj": "\u{200D}", "lrm": "\u{200E}", "rlm": "\u{200F}", "ndash": "\u{2013}", "mdash": "\u{2014}", + "lsquo": "\u{2018}", "rsquo": "\u{2019}", "sbquo": "\u{201A}", "ldquo": "\u{201C}", "rdquo": "\u{201D}", + "bdquo": "\u{201E}", "dagger": "\u{2020}", "Dagger": "\u{2021}", "bull": "\u{2022}", "hellip": "\u{2026}", + "permil": "\u{2030}", "prime": "\u{2032}", "Prime": "\u{2033}", "lsaquo": "\u{2039}", "rsaquo": "\u{203A}", + "oline": "\u{203E}", "frasl": "\u{2044}", "euro": "\u{20AC}", "image": "\u{2111}", "weierp": "\u{2118}", + "real": "\u{211C}", "trade": "\u{2122}", "alefsym": "\u{2135}", "larr": "\u{2190}", "uarr": "\u{2191}", + "rarr": "\u{2192}", "darr": "\u{2193}", "harr": "\u{2194}", "crarr": "\u{21B5}", "lArr": "\u{21D0}", + "uArr": "\u{21D1}", "rArr": "\u{21D2}", "dArr": "\u{21D3}", "hArr": "\u{21D4}", "forall": "\u{2200}", + "part": "\u{2202}", "exist": "\u{2203}", "empty": "\u{2205}", "nabla": "\u{2207}", "isin": "\u{2208}", + "notin": "\u{2209}", "ni": "\u{220B}", "prod": "\u{220F}", "sum": "\u{2211}", "minus": "\u{2212}", + "lowast": "\u{2217}", "radic": "\u{221A}", "prop": "\u{221D}", "infin": "\u{221E}", "ang": "\u{2220}", + "and": "\u{2227}", "or": "\u{2228}", "cap": "\u{2229}", "cup": "\u{222A}", "int": "\u{222B}", + "there4": "\u{2234}", "sim": "\u{223C}", "cong": "\u{2245}", "asymp": "\u{2248}", "ne": "\u{2260}", + "equiv": "\u{2261}", "le": "\u{2264}", "ge": "\u{2265}", "sub": "\u{2282}", "sup": "\u{2283}", + "nsub": "\u{2284}", "sube": "\u{2286}", "supe": "\u{2287}", "oplus": "\u{2295}", "otimes": "\u{2297}", + "perp": "\u{22A5}", "sdot": "\u{22C5}", "lceil": "\u{2308}", "rceil": "\u{2309}", "lfloor": "\u{230A}", + "rfloor": "\u{230B}", "loz": "\u{25CA}", "spades": "\u{2660}", "clubs": "\u{2663}", "hearts": "\u{2665}", + "diams": "\u{2666}", "lang": "\u{27E8}", "rang": "\u{27E9}", ] + /// How far past an `&` a name's `;` can be: the longest name, `thetasym`, and its `;`. + private static let longestReference = named.keys.map(\.count).max()! + 1 + /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an /// apostrophe is one. Those that Windows-1252 leaves undefined stay as they are. @@ -572,11 +631,11 @@ enum PushTitle { var index = start + 1 guard index < scalars.count else { return nil } guard scalars[index] == "#" else { - // A named reference needs its `;`, as before; the longest name above is 5. - guard let semicolon = scalars[index...].prefix(8).firstIndex(of: ";"), - let text = named[String(String.UnicodeScalarView(scalars[index.. \"Q\" 'A' \u{00A9} \u{00AE} \u{2013} \u{2014} &") } + func testHTML4Names() { + XCTAssertEqual(PushTitle.decodingEntities("‘Café’ “menu”… «€5» · • Brand™"), + "\u{2018}Caf\u{00E9}\u{2019} \u{201C}menu\u{201D}\u{2026} \u{00AB}\u{20AC}5\u{00BB} \u{00B7} \u{2022} Brand\u{2122}") + // The longest name, case kept apart, and the two HTML5 moved. + XCTAssertEqual(PushTitle.decodingEntities("ϑ Éé ⟨x⟩"), "\u{03D1} \u{00C9}\u{00E9} \u{27E8}x\u{27E9}") + XCTAssertEqual(PushTitle.decodingEntities("&EACUTE; &thetasymx;"), "&EACUTE; &thetasymx;") + } + func testMixedAndDecodedOnce() { XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), "Fish & Chips\u{2122} \u{2014} It's ' &") @@ -721,6 +729,7 @@ final class PushTitleTests: XCTestCase { func testExtractDecodesThenCollapsesWhitespace() { let html = "\n  Tea’s ready now  " XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") + XCTAssertEqual(PushTitle.extract(from: "Re­use​d\u{00AD} it"), "Reused it") XCTAssertNil(PushTitle.extract(from: "")) } } diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index 7fb318b..7242dff 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -508,8 +508,11 @@ enum PushTitle { let range = Range(match.range(at: 1), in: html) else { return nil } // Decoded first, so a ` ` or ` ` is collapsed with the rest of the - // whitespace, as Jsoup collapses the text it has already decoded. + // whitespace, as Jsoup collapses the text it has already decoded. Jsoup also drops + // the soft hyphen and the zero-width space as it collapses, however they were + // written, so they are dropped here too. return decodingEntities(String(html[range])) + .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) } @@ -548,14 +551,70 @@ enum PushTitle { return String(out) } - /// The references a title is decoded for by name. Each decodes to its own character, - /// as on Android: these once became ASCII stand-ins (`-` for both dashes, `(c)`), - /// which disagreed with Jsoup and with the same character written as a number. - private static let named: [String: String] = [ - "amp": "&", "lt": "<", "gt": ">", "quot": "\"", "apos": "'", - "nbsp": "\u{00A0}", "ndash": "\u{2013}", "mdash": "\u{2014}", "copy": "\u{00A9}", "reg": "\u{00AE}", + /// The references a title is decoded for by name: HTML 4's 252 and `'`, by code + /// point. Jsoup knows HTML5's two thousand, but nearly all of those are mathematical + /// symbols no title uses, while these hold every one a page writes in practice + /// (`’`, `…`, `é`). `⟨` and `⟩` take HTML5's code points, + /// which replaced HTML 4's, as Jsoup does. Each decodes to its own character: a few once + /// became ASCII stand-ins (`-` for both dashes, `(c)`), which disagreed with Jsoup and + /// with the same character written as a number. + private static let named: [String: Unicode.Scalar] = [ + "quot": "\u{0022}", "amp": "\u{0026}", "apos": "\u{0027}", "lt": "\u{003C}", "gt": "\u{003E}", + "nbsp": "\u{00A0}", "iexcl": "\u{00A1}", "cent": "\u{00A2}", "pound": "\u{00A3}", "curren": "\u{00A4}", + "yen": "\u{00A5}", "brvbar": "\u{00A6}", "sect": "\u{00A7}", "uml": "\u{00A8}", "copy": "\u{00A9}", + "ordf": "\u{00AA}", "laquo": "\u{00AB}", "not": "\u{00AC}", "shy": "\u{00AD}", "reg": "\u{00AE}", + "macr": "\u{00AF}", "deg": "\u{00B0}", "plusmn": "\u{00B1}", "sup2": "\u{00B2}", "sup3": "\u{00B3}", + "acute": "\u{00B4}", "micro": "\u{00B5}", "para": "\u{00B6}", "middot": "\u{00B7}", "cedil": "\u{00B8}", + "sup1": "\u{00B9}", "ordm": "\u{00BA}", "raquo": "\u{00BB}", "frac14": "\u{00BC}", "frac12": "\u{00BD}", + "frac34": "\u{00BE}", "iquest": "\u{00BF}", "Agrave": "\u{00C0}", "Aacute": "\u{00C1}", "Acirc": "\u{00C2}", + "Atilde": "\u{00C3}", "Auml": "\u{00C4}", "Aring": "\u{00C5}", "AElig": "\u{00C6}", "Ccedil": "\u{00C7}", + "Egrave": "\u{00C8}", "Eacute": "\u{00C9}", "Ecirc": "\u{00CA}", "Euml": "\u{00CB}", "Igrave": "\u{00CC}", + "Iacute": "\u{00CD}", "Icirc": "\u{00CE}", "Iuml": "\u{00CF}", "ETH": "\u{00D0}", "Ntilde": "\u{00D1}", + "Ograve": "\u{00D2}", "Oacute": "\u{00D3}", "Ocirc": "\u{00D4}", "Otilde": "\u{00D5}", "Ouml": "\u{00D6}", + "times": "\u{00D7}", "Oslash": "\u{00D8}", "Ugrave": "\u{00D9}", "Uacute": "\u{00DA}", "Ucirc": "\u{00DB}", + "Uuml": "\u{00DC}", "Yacute": "\u{00DD}", "THORN": "\u{00DE}", "szlig": "\u{00DF}", "agrave": "\u{00E0}", + "aacute": "\u{00E1}", "acirc": "\u{00E2}", "atilde": "\u{00E3}", "auml": "\u{00E4}", "aring": "\u{00E5}", + "aelig": "\u{00E6}", "ccedil": "\u{00E7}", "egrave": "\u{00E8}", "eacute": "\u{00E9}", "ecirc": "\u{00EA}", + "euml": "\u{00EB}", "igrave": "\u{00EC}", "iacute": "\u{00ED}", "icirc": "\u{00EE}", "iuml": "\u{00EF}", + "eth": "\u{00F0}", "ntilde": "\u{00F1}", "ograve": "\u{00F2}", "oacute": "\u{00F3}", "ocirc": "\u{00F4}", + "otilde": "\u{00F5}", "ouml": "\u{00F6}", "divide": "\u{00F7}", "oslash": "\u{00F8}", "ugrave": "\u{00F9}", + "uacute": "\u{00FA}", "ucirc": "\u{00FB}", "uuml": "\u{00FC}", "yacute": "\u{00FD}", "thorn": "\u{00FE}", + "yuml": "\u{00FF}", "OElig": "\u{0152}", "oelig": "\u{0153}", "Scaron": "\u{0160}", "scaron": "\u{0161}", + "Yuml": "\u{0178}", "fnof": "\u{0192}", "circ": "\u{02C6}", "tilde": "\u{02DC}", "Alpha": "\u{0391}", + "Beta": "\u{0392}", "Gamma": "\u{0393}", "Delta": "\u{0394}", "Epsilon": "\u{0395}", "Zeta": "\u{0396}", + "Eta": "\u{0397}", "Theta": "\u{0398}", "Iota": "\u{0399}", "Kappa": "\u{039A}", "Lambda": "\u{039B}", + "Mu": "\u{039C}", "Nu": "\u{039D}", "Xi": "\u{039E}", "Omicron": "\u{039F}", "Pi": "\u{03A0}", + "Rho": "\u{03A1}", "Sigma": "\u{03A3}", "Tau": "\u{03A4}", "Upsilon": "\u{03A5}", "Phi": "\u{03A6}", + "Chi": "\u{03A7}", "Psi": "\u{03A8}", "Omega": "\u{03A9}", "alpha": "\u{03B1}", "beta": "\u{03B2}", + "gamma": "\u{03B3}", "delta": "\u{03B4}", "epsilon": "\u{03B5}", "zeta": "\u{03B6}", "eta": "\u{03B7}", + "theta": "\u{03B8}", "iota": "\u{03B9}", "kappa": "\u{03BA}", "lambda": "\u{03BB}", "mu": "\u{03BC}", + "nu": "\u{03BD}", "xi": "\u{03BE}", "omicron": "\u{03BF}", "pi": "\u{03C0}", "rho": "\u{03C1}", + "sigmaf": "\u{03C2}", "sigma": "\u{03C3}", "tau": "\u{03C4}", "upsilon": "\u{03C5}", "phi": "\u{03C6}", + "chi": "\u{03C7}", "psi": "\u{03C8}", "omega": "\u{03C9}", "thetasym": "\u{03D1}", "upsih": "\u{03D2}", + "piv": "\u{03D6}", "ensp": "\u{2002}", "emsp": "\u{2003}", "thinsp": "\u{2009}", "zwnj": "\u{200C}", + "zwj": "\u{200D}", "lrm": "\u{200E}", "rlm": "\u{200F}", "ndash": "\u{2013}", "mdash": "\u{2014}", + "lsquo": "\u{2018}", "rsquo": "\u{2019}", "sbquo": "\u{201A}", "ldquo": "\u{201C}", "rdquo": "\u{201D}", + "bdquo": "\u{201E}", "dagger": "\u{2020}", "Dagger": "\u{2021}", "bull": "\u{2022}", "hellip": "\u{2026}", + "permil": "\u{2030}", "prime": "\u{2032}", "Prime": "\u{2033}", "lsaquo": "\u{2039}", "rsaquo": "\u{203A}", + "oline": "\u{203E}", "frasl": "\u{2044}", "euro": "\u{20AC}", "image": "\u{2111}", "weierp": "\u{2118}", + "real": "\u{211C}", "trade": "\u{2122}", "alefsym": "\u{2135}", "larr": "\u{2190}", "uarr": "\u{2191}", + "rarr": "\u{2192}", "darr": "\u{2193}", "harr": "\u{2194}", "crarr": "\u{21B5}", "lArr": "\u{21D0}", + "uArr": "\u{21D1}", "rArr": "\u{21D2}", "dArr": "\u{21D3}", "hArr": "\u{21D4}", "forall": "\u{2200}", + "part": "\u{2202}", "exist": "\u{2203}", "empty": "\u{2205}", "nabla": "\u{2207}", "isin": "\u{2208}", + "notin": "\u{2209}", "ni": "\u{220B}", "prod": "\u{220F}", "sum": "\u{2211}", "minus": "\u{2212}", + "lowast": "\u{2217}", "radic": "\u{221A}", "prop": "\u{221D}", "infin": "\u{221E}", "ang": "\u{2220}", + "and": "\u{2227}", "or": "\u{2228}", "cap": "\u{2229}", "cup": "\u{222A}", "int": "\u{222B}", + "there4": "\u{2234}", "sim": "\u{223C}", "cong": "\u{2245}", "asymp": "\u{2248}", "ne": "\u{2260}", + "equiv": "\u{2261}", "le": "\u{2264}", "ge": "\u{2265}", "sub": "\u{2282}", "sup": "\u{2283}", + "nsub": "\u{2284}", "sube": "\u{2286}", "supe": "\u{2287}", "oplus": "\u{2295}", "otimes": "\u{2297}", + "perp": "\u{22A5}", "sdot": "\u{22C5}", "lceil": "\u{2308}", "rceil": "\u{2309}", "lfloor": "\u{230A}", + "rfloor": "\u{230B}", "loz": "\u{25CA}", "spades": "\u{2660}", "clubs": "\u{2663}", "hearts": "\u{2665}", + "diams": "\u{2666}", "lang": "\u{27E8}", "rang": "\u{27E9}", ] + /// How far past an `&` a name's `;` can be: the longest name, `thetasym`, and its `;`. + private static let longestReference = named.keys.map(\.count).max()! + 1 + /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an /// apostrophe is one. Those that Windows-1252 leaves undefined stay as they are. @@ -572,11 +631,11 @@ enum PushTitle { var index = start + 1 guard index < scalars.count else { return nil } guard scalars[index] == "#" else { - // A named reference needs its `;`, as before; the longest name above is 5. - guard let semicolon = scalars[index...].prefix(8).firstIndex(of: ";"), - let text = named[String(String.UnicodeScalarView(scalars[index.. \"Q\" 'A' \u{00A9} \u{00AE} \u{2013} \u{2014} &") } + func testHTML4Names() { + XCTAssertEqual(PushTitle.decodingEntities("‘Café’ “menu”… «€5» · • Brand™"), + "\u{2018}Caf\u{00E9}\u{2019} \u{201C}menu\u{201D}\u{2026} \u{00AB}\u{20AC}5\u{00BB} \u{00B7} \u{2022} Brand\u{2122}") + // The longest name, case kept apart, and the two HTML5 moved. + XCTAssertEqual(PushTitle.decodingEntities("ϑ Éé ⟨x⟩"), "\u{03D1} \u{00C9}\u{00E9} \u{27E8}x\u{27E9}") + XCTAssertEqual(PushTitle.decodingEntities("&EACUTE; &thetasymx;"), "&EACUTE; &thetasymx;") + } + func testMixedAndDecodedOnce() { XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), "Fish & Chips\u{2122} \u{2014} It's ' &") @@ -721,6 +729,7 @@ final class PushTitleTests: XCTestCase { func testExtractDecodesThenCollapsesWhitespace() { let html = "\n  Tea’s ready now  " XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") + XCTAssertEqual(PushTitle.extract(from: "Re­use​d\u{00AD} it"), "Reused it") XCTAssertNil(PushTitle.extract(from: "")) } } From 0dba73553a030e308aac83313057639de7ec7da1 Mon Sep 17 00:00:00 2001 From: C Deutsch Date: Thu, 12 Nov 2026 16:27:20 -0600 Subject: [PATCH 3/6] Collapse only Jsoup's whitespace in scraped titles on iOS and macOS `PushTitle` collapsed and trimmed with ICU's `\s` and `.whitespacesAndNewlines`, which take every Unicode space. Jsoup, which Android scrapes with, collapses only space, tab, newline, form feed, carriage return and the no-break space, and trims what is at or below U+0020. Now that references are decoded, a title with ` `, the narrow no-break space French writes before `!`, or an ideographic space was stored as a plain space on Apple and as written on Android, and the first device to scrape decided which every device saw. The title is now collapsed over Jsoup's six and trimmed as Java's `trim()` does, checked against Jsoup 1.23.2. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 --- apps/ios/PushThat/Services/PushContent.swift | 12 ++++++++++-- apps/ios/PushThatTests/E2eeVectorTests.swift | 5 +++++ apps/macos/PushThat/Services/PushContent.swift | 12 ++++++++++-- apps/macos/PushThatTests/E2eeVectorTests.swift | 5 +++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index 7242dff..dfe484b 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -511,12 +511,20 @@ enum PushTitle { // whitespace, as Jsoup collapses the text it has already decoded. Jsoup also drops // the soft hyphen and the zero-width space as it collapses, however they were // written, so they are dropped here too. + // + // Jsoup's whitespace, not Unicode's: it collapses only these six, so a thin space + // (` `), the narrow no-break space French puts before `!` or an ideographic + // space is kept as written. `\s` and `.whitespacesAndNewlines` would flatten them to a + // space and store a different title from Android's. Then Java's `trim()`, which + // takes everything up to U+0020 off the ends and nothing above it. return decodingEntities(String(html[range])) .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) - .trimmingCharacters(in: .whitespacesAndNewlines) - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + .replacingOccurrences(of: "[ \\t\\n\\f\\r\\u00A0]+", with: " ", options: .regularExpression) + .trimmingCharacters(in: javaTrimmed) } + private static let javaTrimmed = CharacterSet(charactersIn: "\u{0000}"..."\u{0020}") + private static let titlePattern = try! NSRegularExpression( pattern: "]*>([^<]+)", options: .caseInsensitive ) diff --git a/apps/ios/PushThatTests/E2eeVectorTests.swift b/apps/ios/PushThatTests/E2eeVectorTests.swift index 7d4b84d..61e5bb7 100644 --- a/apps/ios/PushThatTests/E2eeVectorTests.swift +++ b/apps/ios/PushThatTests/E2eeVectorTests.swift @@ -730,6 +730,11 @@ final class PushTitleTests: XCTestCase { let html = "\n  Tea’s ready now  " XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") XCTAssertEqual(PushTitle.extract(from: "Re­use​d\u{00AD} it"), "Reused it") + // Only Jsoup's six are collapsed: a thin, narrow or ideographic space, a line + // separator or a vertical tab stays as it is. + XCTAssertEqual(PushTitle.extract(from: "
10 km  ! x\u{3000}y z"), + "\u{2028}10\u{2009}km \u{202F}! x\u{3000}y\u{000B}z") + XCTAssertEqual(PushTitle.extract(from: "\u{0001} a "), "a") XCTAssertNil(PushTitle.extract(from: "")) } } diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index 7242dff..dfe484b 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -511,12 +511,20 @@ enum PushTitle { // whitespace, as Jsoup collapses the text it has already decoded. Jsoup also drops // the soft hyphen and the zero-width space as it collapses, however they were // written, so they are dropped here too. + // + // Jsoup's whitespace, not Unicode's: it collapses only these six, so a thin space + // (` `), the narrow no-break space French puts before `!` or an ideographic + // space is kept as written. `\s` and `.whitespacesAndNewlines` would flatten them to a + // space and store a different title from Android's. Then Java's `trim()`, which + // takes everything up to U+0020 off the ends and nothing above it. return decodingEntities(String(html[range])) .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) - .trimmingCharacters(in: .whitespacesAndNewlines) - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + .replacingOccurrences(of: "[ \\t\\n\\f\\r\\u00A0]+", with: " ", options: .regularExpression) + .trimmingCharacters(in: javaTrimmed) } + private static let javaTrimmed = CharacterSet(charactersIn: "\u{0000}"..."\u{0020}") + private static let titlePattern = try! NSRegularExpression( pattern: "]*>([^<]+)", options: .caseInsensitive ) diff --git a/apps/macos/PushThatTests/E2eeVectorTests.swift b/apps/macos/PushThatTests/E2eeVectorTests.swift index 7d4b84d..61e5bb7 100644 --- a/apps/macos/PushThatTests/E2eeVectorTests.swift +++ b/apps/macos/PushThatTests/E2eeVectorTests.swift @@ -730,6 +730,11 @@ final class PushTitleTests: XCTestCase { let html = "\n  Tea’s ready now  " XCTAssertEqual(PushTitle.extract(from: html), "Tea\u{2019}s ready now") XCTAssertEqual(PushTitle.extract(from: "Re­use​d\u{00AD} it"), "Reused it") + // Only Jsoup's six are collapsed: a thin, narrow or ideographic space, a line + // separator or a vertical tab stays as it is. + XCTAssertEqual(PushTitle.extract(from: "
10 km  ! x\u{3000}y z"), + "\u{2028}10\u{2009}km \u{202F}! x\u{3000}y\u{000B}z") + XCTAssertEqual(PushTitle.extract(from: "\u{0001} a "), "a") XCTAssertNil(PushTitle.extract(from: "")) } } From 7a5d4e3c4b1aad62ef0c349b3e2962bb1c7be90a Mon Sep 17 00:00:00 2001 From: C Deutsch Date: Thu, 12 Nov 2026 16:29:06 -0600 Subject: [PATCH 4/6] Read HTML's legacy names without their semicolon in titles on iOS and macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PushTitle` required a `;` after every name, on the claim that no page writes one without it. Hand-written titles do: `© 2024` and `Fish & Chips` were stored as written on Apple, while Android's Jsoup decodes them as HTML says to, and the first device to scrape decided which every device saw. The uppercase aliases (`&`, `©`) were missing too. A name now decodes with its `;` from the table, or else as the longest start of it that is one of HTML's 106 legacy names, the rest left as text (`¬it;` is `¬it;`), which is HTML's rule and Jsoup's. The aliases join the table. Checked against Jsoup 1.23.2. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 --- apps/ios/PushThat/Services/PushContent.swift | 66 +++++++++++++++---- apps/ios/PushThatTests/E2eeVectorTests.swift | 16 ++++- .../macos/PushThat/Services/PushContent.swift | 66 +++++++++++++++---- .../macos/PushThatTests/E2eeVectorTests.swift | 16 ++++- 4 files changed, 132 insertions(+), 32 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index dfe484b..593412d 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -535,10 +535,12 @@ enum PushTitle { /// reference this does not know, or a numeric one that names no character (zero, a /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. /// - /// Jsoup parts from this on those: it makes U+FFFD of a number past U+10FFFF and a lone - /// surrogate of one in that range (which a Swift `String` cannot hold), and it reads the - /// handful of legacy names that need no `;` (`&`). No page means any of them, so - /// the text as written was kept over copying each of Jsoup's answers. + /// Jsoup parts from this on those numbers: it makes U+FFFD of one past U+10FFFF and a + /// lone surrogate of one in that range (which a Swift `String` cannot hold). No page + /// means either, so the text as written was kept over copying Jsoup's answer. + /// + /// A name needs its `;`, except the 106 in `legacy`, which HTML reads without one, as + /// Jsoup and browsers do: hand-written titles hold `© 2024` and `Fish & Chips`. /// /// The numeric ones used to be deleted, so a title with a typographic apostrophe lost /// it, and every device then read the title with the letter missing. @@ -565,8 +567,10 @@ enum PushTitle { /// (`’`, `…`, `é`). `⟨` and `⟩` take HTML5's code points, /// which replaced HTML 4's, as Jsoup does. Each decodes to its own character: a few once /// became ASCII stand-ins (`-` for both dashes, `(c)`), which disagreed with Jsoup and - /// with the same character written as a number. + /// with the same character written as a number. HTML5's uppercase aliases of six of + /// them (`&`, `©`) are here too, being among the legacy names in `legacy`. private static let named: [String: Unicode.Scalar] = [ + "AMP": "\u{0026}", "COPY": "\u{00A9}", "GT": "\u{003E}", "LT": "\u{003C}", "QUOT": "\u{0022}", "REG": "\u{00AE}", "quot": "\u{0022}", "amp": "\u{0026}", "apos": "\u{0027}", "lt": "\u{003C}", "gt": "\u{003E}", "nbsp": "\u{00A0}", "iexcl": "\u{00A1}", "cent": "\u{00A2}", "pound": "\u{00A3}", "curren": "\u{00A4}", "yen": "\u{00A5}", "brvbar": "\u{00A6}", "sect": "\u{00A7}", "uml": "\u{00A8}", "copy": "\u{00A9}", @@ -620,8 +624,24 @@ enum PushTitle { "diams": "\u{2666}", "lang": "\u{27E8}", "rang": "\u{27E9}", ] - /// How far past an `&` a name's `;` can be: the longest name, `thetasym`, and its `;`. - private static let longestReference = named.keys.map(\.count).max()! + 1 + /// The names HTML reads without their `;`, kept from before HTML required one: Latin-1 + /// and the four that `&` and markup need, with the uppercase aliases. Every one is in + /// `named`. HTML5 lists them, and Jsoup reads them so. + private static let legacy: Set = [ + "Aacute", "aacute", "Acirc", "acirc", "acute", "AElig", "aelig", "Agrave", "agrave", "AMP", + "amp", "Aring", "aring", "Atilde", "atilde", "Auml", "auml", "brvbar", "Ccedil", "ccedil", + "cedil", "cent", "COPY", "copy", "curren", "deg", "divide", "Eacute", "eacute", "Ecirc", + "ecirc", "Egrave", "egrave", "ETH", "eth", "Euml", "euml", "frac12", "frac14", "frac34", + "GT", "gt", "Iacute", "iacute", "Icirc", "icirc", "iexcl", "Igrave", "igrave", "iquest", + "Iuml", "iuml", "laquo", "LT", "lt", "macr", "micro", "middot", "nbsp", "not", "Ntilde", + "ntilde", "Oacute", "oacute", "Ocirc", "ocirc", "Ograve", "ograve", "ordf", "ordm", + "Oslash", "oslash", "Otilde", "otilde", "Ouml", "ouml", "para", "plusmn", "pound", "QUOT", + "quot", "raquo", "REG", "reg", "sect", "shy", "sup1", "sup2", "sup3", "szlig", "THORN", + "thorn", "times", "Uacute", "uacute", "Ucirc", "ucirc", "Ugrave", "ugrave", "uml", "Uuml", + "uuml", "Yacute", "yacute", "yen", "yuml", + ] + + private static let longestLegacyName = legacy.map(\.count).max()! /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an @@ -638,13 +658,7 @@ enum PushTitle { private static func reference(in scalars: [Unicode.Scalar], at start: Int) -> (String, Int)? { var index = start + 1 guard index < scalars.count else { return nil } - guard scalars[index] == "#" else { - // A named reference needs its `;`, as before. - guard let semicolon = scalars[index...].prefix(longestReference).firstIndex(of: ";"), - let scalar = named[String(String.UnicodeScalarView(scalars[index.. (String, Int)? { + var end = start + 1 + while end < scalars.count, isASCIIAlphanumeric(scalars[end]) { end += 1 } + func name(_ length: Int) -> String { String(String.UnicodeScalarView(scalars[(start + 1)..<(start + 1 + length)])) } + let length = end - start - 1 + if end < scalars.count, scalars[end] == ";", let scalar = named[name(length)] { + return (String(Character(scalar)), length + 2) + } + for prefix in stride(from: min(length, longestLegacyName), through: 2, by: -1) { + if legacy.contains(name(prefix)), let scalar = named[name(prefix)] { + return (String(Character(scalar)), prefix + 1) + } + } + return nil + } + + private static func isASCIIAlphanumeric(_ scalar: Unicode.Scalar) -> Bool { + ("a"..."z").contains(scalar) || ("A"..."Z").contains(scalar) || ("0"..."9").contains(scalar) + } + /// ASCII digits only: `Character.wholeNumberValue` would also take other scripts' digits, /// which no parser reads in a reference. private static func digitValue(_ scalar: Unicode.Scalar, radix: UInt32) -> UInt32? { diff --git a/apps/ios/PushThatTests/E2eeVectorTests.swift b/apps/ios/PushThatTests/E2eeVectorTests.swift index 61e5bb7..fd976f2 100644 --- a/apps/ios/PushThatTests/E2eeVectorTests.swift +++ b/apps/ios/PushThatTests/E2eeVectorTests.swift @@ -680,7 +680,7 @@ final class PushIconTests: XCTestCase { /// The title a scrape stores. Every device reads the one stored, so the expected strings /// are what Android's Jsoup 1.23.2 gives for the same ``, checked against it, except -/// in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). +/// for the numbers in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). final class PushTitleTests: XCTestCase { func testDecimalReferences() { XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready — at 4:30"), "Tea\u{2019}s ready \u{2014} at 4:30") @@ -703,7 +703,7 @@ final class PushTitleTests: XCTestCase { func testInvalidReferencesStayAsText() { for text in ["&#;", "&#x;", "&#xZZ;", "�", "�", "�", "�", "�", - "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "&", "a & b", "&;"] { + "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "a & b", "&;"] { XCTAssertEqual(PushTitle.decodingEntities(text), text, text) } } @@ -721,6 +721,18 @@ final class PushTitleTests: XCTestCase { XCTAssertEqual(PushTitle.decodingEntities("&EACUTE; &thetasymx;"), "&EACUTE; &thetasymx;") } + func testLegacyNamesNeedNoSemicolon() { + XCTAssertEqual(PushTitle.decodingEntities("© 2024 Example & Co & © É"), + "\u{00A9} 2024 Example & Co & \u{00A9} \u{00C9}") + // The longest legacy name the run starts with, the rest left as text. + XCTAssertEqual(PushTitle.decodingEntities("∉ ¬it; ¬in ©2024 ½x ²x &x <5 >="), + "\u{2209} \u{00AC}it; \u{00AC}in \u{00A9}2024 \u{00BD}x \u{00B2}x &x <5 >=") + // Not legacy: these need their `;`. + for text in ["&hellip", "&rsquo", "&thetasym", "&sup", "&supx", "&frac1", "&Copy", "&a", "&am"] { + XCTAssertEqual(PushTitle.decodingEntities(text), text, text) + } + } + func testMixedAndDecodedOnce() { XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), "Fish & Chips\u{2122} \u{2014} It's ' &") diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index dfe484b..593412d 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -535,10 +535,12 @@ enum PushTitle { /// reference this does not know, or a numeric one that names no character (zero, a /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. /// - /// Jsoup parts from this on those: it makes U+FFFD of a number past U+10FFFF and a lone - /// surrogate of one in that range (which a Swift `String` cannot hold), and it reads the - /// handful of legacy names that need no `;` (`&`). No page means any of them, so - /// the text as written was kept over copying each of Jsoup's answers. + /// Jsoup parts from this on those numbers: it makes U+FFFD of one past U+10FFFF and a + /// lone surrogate of one in that range (which a Swift `String` cannot hold). No page + /// means either, so the text as written was kept over copying Jsoup's answer. + /// + /// A name needs its `;`, except the 106 in `legacy`, which HTML reads without one, as + /// Jsoup and browsers do: hand-written titles hold `© 2024` and `Fish & Chips`. /// /// The numeric ones used to be deleted, so a title with a typographic apostrophe lost /// it, and every device then read the title with the letter missing. @@ -565,8 +567,10 @@ enum PushTitle { /// (`’`, `…`, `é`). `⟨` and `⟩` take HTML5's code points, /// which replaced HTML 4's, as Jsoup does. Each decodes to its own character: a few once /// became ASCII stand-ins (`-` for both dashes, `(c)`), which disagreed with Jsoup and - /// with the same character written as a number. + /// with the same character written as a number. HTML5's uppercase aliases of six of + /// them (`&`, `©`) are here too, being among the legacy names in `legacy`. private static let named: [String: Unicode.Scalar] = [ + "AMP": "\u{0026}", "COPY": "\u{00A9}", "GT": "\u{003E}", "LT": "\u{003C}", "QUOT": "\u{0022}", "REG": "\u{00AE}", "quot": "\u{0022}", "amp": "\u{0026}", "apos": "\u{0027}", "lt": "\u{003C}", "gt": "\u{003E}", "nbsp": "\u{00A0}", "iexcl": "\u{00A1}", "cent": "\u{00A2}", "pound": "\u{00A3}", "curren": "\u{00A4}", "yen": "\u{00A5}", "brvbar": "\u{00A6}", "sect": "\u{00A7}", "uml": "\u{00A8}", "copy": "\u{00A9}", @@ -620,8 +624,24 @@ enum PushTitle { "diams": "\u{2666}", "lang": "\u{27E8}", "rang": "\u{27E9}", ] - /// How far past an `&` a name's `;` can be: the longest name, `thetasym`, and its `;`. - private static let longestReference = named.keys.map(\.count).max()! + 1 + /// The names HTML reads without their `;`, kept from before HTML required one: Latin-1 + /// and the four that `&` and markup need, with the uppercase aliases. Every one is in + /// `named`. HTML5 lists them, and Jsoup reads them so. + private static let legacy: Set<String> = [ + "Aacute", "aacute", "Acirc", "acirc", "acute", "AElig", "aelig", "Agrave", "agrave", "AMP", + "amp", "Aring", "aring", "Atilde", "atilde", "Auml", "auml", "brvbar", "Ccedil", "ccedil", + "cedil", "cent", "COPY", "copy", "curren", "deg", "divide", "Eacute", "eacute", "Ecirc", + "ecirc", "Egrave", "egrave", "ETH", "eth", "Euml", "euml", "frac12", "frac14", "frac34", + "GT", "gt", "Iacute", "iacute", "Icirc", "icirc", "iexcl", "Igrave", "igrave", "iquest", + "Iuml", "iuml", "laquo", "LT", "lt", "macr", "micro", "middot", "nbsp", "not", "Ntilde", + "ntilde", "Oacute", "oacute", "Ocirc", "ocirc", "Ograve", "ograve", "ordf", "ordm", + "Oslash", "oslash", "Otilde", "otilde", "Ouml", "ouml", "para", "plusmn", "pound", "QUOT", + "quot", "raquo", "REG", "reg", "sect", "shy", "sup1", "sup2", "sup3", "szlig", "THORN", + "thorn", "times", "Uacute", "uacute", "Ucirc", "ucirc", "Ugrave", "ugrave", "uml", "Uuml", + "uuml", "Yacute", "yacute", "yen", "yuml", + ] + + private static let longestLegacyName = legacy.map(\.count).max()! /// What the code points 0x80 to 0x9F mean in a numeric reference: HTML reads them as /// Windows-1252, as Jsoup and every browser do, so the `’` an old page wrote for an @@ -638,13 +658,7 @@ enum PushTitle { private static func reference(in scalars: [Unicode.Scalar], at start: Int) -> (String, Int)? { var index = start + 1 guard index < scalars.count else { return nil } - guard scalars[index] == "#" else { - // A named reference needs its `;`, as before. - guard let semicolon = scalars[index...].prefix(longestReference).firstIndex(of: ";"), - let scalar = named[String(String.UnicodeScalarView(scalars[index..<semicolon]))] - else { return nil } - return (String(Character(scalar)), semicolon + 1 - start) - } + guard scalars[index] == "#" else { return namedReference(in: scalars, at: start) } index += 1 var radix: UInt32 = 10 if index < scalars.count, scalars[index] == "x" || scalars[index] == "X" { @@ -669,6 +683,30 @@ enum PushTitle { return (String(Character(scalar)), index - start) } + /// A name and its `;` from `named`, or else the longest start of the name that is in + /// `legacy`, the rest of it left as text. That is how HTML reads a name in a title, and + /// how Jsoup does: `∉` is the one character, `¬it;` is `¬it;`, `©2024` is + /// `©2024`, while `&hellip` without its `;` stays as written. + private static func namedReference(in scalars: [Unicode.Scalar], at start: Int) -> (String, Int)? { + var end = start + 1 + while end < scalars.count, isASCIIAlphanumeric(scalars[end]) { end += 1 } + func name(_ length: Int) -> String { String(String.UnicodeScalarView(scalars[(start + 1)..<(start + 1 + length)])) } + let length = end - start - 1 + if end < scalars.count, scalars[end] == ";", let scalar = named[name(length)] { + return (String(Character(scalar)), length + 2) + } + for prefix in stride(from: min(length, longestLegacyName), through: 2, by: -1) { + if legacy.contains(name(prefix)), let scalar = named[name(prefix)] { + return (String(Character(scalar)), prefix + 1) + } + } + return nil + } + + private static func isASCIIAlphanumeric(_ scalar: Unicode.Scalar) -> Bool { + ("a"..."z").contains(scalar) || ("A"..."Z").contains(scalar) || ("0"..."9").contains(scalar) + } + /// ASCII digits only: `Character.wholeNumberValue` would also take other scripts' digits, /// which no parser reads in a reference. private static func digitValue(_ scalar: Unicode.Scalar, radix: UInt32) -> UInt32? { diff --git a/apps/macos/PushThatTests/E2eeVectorTests.swift b/apps/macos/PushThatTests/E2eeVectorTests.swift index 61e5bb7..fd976f2 100644 --- a/apps/macos/PushThatTests/E2eeVectorTests.swift +++ b/apps/macos/PushThatTests/E2eeVectorTests.swift @@ -680,7 +680,7 @@ final class PushIconTests: XCTestCase { /// The title a scrape stores. Every device reads the one stored, so the expected strings /// are what Android's Jsoup 1.23.2 gives for the same `<title>`, checked against it, except -/// in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). +/// for the numbers in `testInvalidReferencesStayAsText` (see `PushTitle.decodingEntities`). final class PushTitleTests: XCTestCase { func testDecimalReferences() { XCTAssertEqual(PushTitle.decodingEntities("Tea’s ready — at 4:30"), "Tea\u{2019}s ready \u{2014} at 4:30") @@ -703,7 +703,7 @@ final class PushTitleTests: XCTestCase { func testInvalidReferencesStayAsText() { for text in ["&#;", "&#x;", "&#xZZ;", "�", "�", "�", "�", "�", - "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "&", "a & b", "&;"] { + "�", "&#-5;", "&#\u{0663};", "&bogus;", "&", "a & b", "&;"] { XCTAssertEqual(PushTitle.decodingEntities(text), text, text) } } @@ -721,6 +721,18 @@ final class PushTitleTests: XCTestCase { XCTAssertEqual(PushTitle.decodingEntities("&EACUTE; &thetasymx;"), "&EACUTE; &thetasymx;") } + func testLegacyNamesNeedNoSemicolon() { + XCTAssertEqual(PushTitle.decodingEntities("© 2024 Example & Co & © É"), + "\u{00A9} 2024 Example & Co & \u{00A9} \u{00C9}") + // The longest legacy name the run starts with, the rest left as text. + XCTAssertEqual(PushTitle.decodingEntities("∉ ¬it; ¬in ©2024 ½x ²x &x <5 >="), + "\u{2209} \u{00AC}it; \u{00AC}in \u{00A9}2024 \u{00BD}x \u{00B2}x &x <5 >=") + // Not legacy: these need their `;`. + for text in ["&hellip", "&rsquo", "&thetasym", "&sup", "&supx", "&frac1", "&Copy", "&a", "&am"] { + XCTAssertEqual(PushTitle.decodingEntities(text), text, text) + } + } + func testMixedAndDecodedOnce() { XCTAssertEqual(PushTitle.decodingEntities("Fish & Chips™ — It's &#39; &amp;"), "Fish & Chips\u{2122} \u{2014} It's ' &") From 30f3e14ab430e52f610be20a2a0200b26e3e4f0d Mon Sep 17 00:00:00 2001 From: C Deutsch <git@cdeutsch.com> Date: Thu, 12 Nov 2026 16:32:40 -0600 Subject: [PATCH 5/6] Turn a NUL in a scraped title into U+FFFD on iOS and macOS An HTML parser reads a NUL in a `<title>` as U+FFFD, and Jsoup on Android does. `PushTitle` kept it, and since the title is now trimmed as Java's `trim()` does, one at either end was dropped instead. The title is stored for every device, so the two clients stored different titles for the same page. The NUL now becomes U+FFFD before anything else, checked against Jsoup 1.23.2. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/ios/PushThat/Services/PushContent.swift | 5 ++++- apps/ios/PushThatTests/E2eeVectorTests.swift | 1 + apps/macos/PushThat/Services/PushContent.swift | 5 ++++- apps/macos/PushThatTests/E2eeVectorTests.swift | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index 593412d..880a709 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -517,7 +517,10 @@ enum PushTitle { // space is kept as written. `\s` and `.whitespacesAndNewlines` would flatten them to a // space and store a different title from Android's. Then Java's `trim()`, which // takes everything up to U+0020 off the ends and nothing above it. - return decodingEntities(String(html[range])) + // + // A NUL written into the page is U+FFFD to an HTML parser, Jsoup's included, so it + // is here too rather than kept, or trimmed off an end as Java's `trim()` would. + return decodingEntities(String(html[range]).replacingOccurrences(of: "\u{0000}", with: "\u{FFFD}")) .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) .replacingOccurrences(of: "[ \\t\\n\\f\\r\\u00A0]+", with: " ", options: .regularExpression) .trimmingCharacters(in: javaTrimmed) diff --git a/apps/ios/PushThatTests/E2eeVectorTests.swift b/apps/ios/PushThatTests/E2eeVectorTests.swift index fd976f2..fa765f0 100644 --- a/apps/ios/PushThatTests/E2eeVectorTests.swift +++ b/apps/ios/PushThatTests/E2eeVectorTests.swift @@ -747,6 +747,7 @@ final class PushTitleTests: XCTestCase { XCTAssertEqual(PushTitle.extract(from: "<title>
10 km  ! x\u{3000}y z"), "\u{2028}10\u{2009}km \u{202F}! x\u{3000}y\u{000B}z") XCTAssertEqual(PushTitle.extract(from: "\u{0001} a "), "a") + XCTAssertEqual(PushTitle.extract(from: "\u{0000}a\u{0000}b"), "\u{FFFD}a\u{FFFD}b") XCTAssertNil(PushTitle.extract(from: "")) } } diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index 593412d..880a709 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -517,7 +517,10 @@ enum PushTitle { // space is kept as written. `\s` and `.whitespacesAndNewlines` would flatten them to a // space and store a different title from Android's. Then Java's `trim()`, which // takes everything up to U+0020 off the ends and nothing above it. - return decodingEntities(String(html[range])) + // + // A NUL written into the page is U+FFFD to an HTML parser, Jsoup's included, so it + // is here too rather than kept, or trimmed off an end as Java's `trim()` would. + return decodingEntities(String(html[range]).replacingOccurrences(of: "\u{0000}", with: "\u{FFFD}")) .replacingOccurrences(of: "[\\u00AD\\u200B]", with: "", options: .regularExpression) .replacingOccurrences(of: "[ \\t\\n\\f\\r\\u00A0]+", with: " ", options: .regularExpression) .trimmingCharacters(in: javaTrimmed) diff --git a/apps/macos/PushThatTests/E2eeVectorTests.swift b/apps/macos/PushThatTests/E2eeVectorTests.swift index fd976f2..fa765f0 100644 --- a/apps/macos/PushThatTests/E2eeVectorTests.swift +++ b/apps/macos/PushThatTests/E2eeVectorTests.swift @@ -747,6 +747,7 @@ final class PushTitleTests: XCTestCase { XCTAssertEqual(PushTitle.extract(from: "
10 km  ! x\u{3000}y z"), "\u{2028}10\u{2009}km \u{202F}! x\u{3000}y\u{000B}z") XCTAssertEqual(PushTitle.extract(from: "\u{0001} a "), "a") + XCTAssertEqual(PushTitle.extract(from: "\u{0000}a\u{0000}b"), "\u{FFFD}a\u{FFFD}b") XCTAssertNil(PushTitle.extract(from: "")) } } From bed83d042d168589ff101352a864a5c8715444c0 Mon Sep 17 00:00:00 2001 From: C Deutsch Date: Thu, 12 Nov 2026 16:32:54 -0600 Subject: [PATCH 6/6] Name `�` among the references Jsoup reads differently, on iOS and macOS The comment on `PushTitle.decodingEntities` listed where Jsoup's answer for an invalid numeric reference differs from keeping it as written, and left out `�`, which Jsoup drops. A reader checking the two clients against each other would take that difference for a bug. Comment only. This makes the same change in both apps: `apps/macos` is a fork of `apps/ios`. Co-Authored-By: Claude Opus 5.5 --- apps/ios/PushThat/Services/PushContent.swift | 7 ++++--- apps/macos/PushThat/Services/PushContent.swift | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/ios/PushThat/Services/PushContent.swift b/apps/ios/PushThat/Services/PushContent.swift index 880a709..00c9e00 100644 --- a/apps/ios/PushThat/Services/PushContent.swift +++ b/apps/ios/PushThat/Services/PushContent.swift @@ -538,9 +538,10 @@ enum PushTitle { /// reference this does not know, or a numeric one that names no character (zero, a /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. /// - /// Jsoup parts from this on those numbers: it makes U+FFFD of one past U+10FFFF and a - /// lone surrogate of one in that range (which a Swift `String` cannot hold). No page - /// means either, so the text as written was kept over copying Jsoup's answer. + /// Jsoup parts from this on those numbers: it drops `�`, makes U+FFFD of one past + /// U+10FFFF and a lone surrogate of one in that range (which a Swift `String` cannot + /// hold). No page means any of them, so the text as written was kept over copying + /// Jsoup's answers. /// /// A name needs its `;`, except the 106 in `legacy`, which HTML reads without one, as /// Jsoup and browsers do: hand-written titles hold `© 2024` and `Fish & Chips`. diff --git a/apps/macos/PushThat/Services/PushContent.swift b/apps/macos/PushThat/Services/PushContent.swift index 880a709..00c9e00 100644 --- a/apps/macos/PushThat/Services/PushContent.swift +++ b/apps/macos/PushThat/Services/PushContent.swift @@ -538,9 +538,10 @@ enum PushTitle { /// reference this does not know, or a numeric one that names no character (zero, a /// surrogate, past U+10FFFF, or no digits at all), is left as it was written. /// - /// Jsoup parts from this on those numbers: it makes U+FFFD of one past U+10FFFF and a - /// lone surrogate of one in that range (which a Swift `String` cannot hold). No page - /// means either, so the text as written was kept over copying Jsoup's answer. + /// Jsoup parts from this on those numbers: it drops `�`, makes U+FFFD of one past + /// U+10FFFF and a lone surrogate of one in that range (which a Swift `String` cannot + /// hold). No page means any of them, so the text as written was kept over copying + /// Jsoup's answers. /// /// A name needs its `;`, except the 106 in `legacy`, which HTML reads without one, as /// Jsoup and browsers do: hand-written titles hold `© 2024` and `Fish & Chips`.