diff --git a/.claude/docs/hard-rules.md b/.claude/docs/hard-rules.md index cd9584819..8d9e44627 100644 --- a/.claude/docs/hard-rules.md +++ b/.claude/docs/hard-rules.md @@ -73,7 +73,11 @@ app can fix itself belongs in the service files that wrap the generated code. migrations — when the version number increases, the database is deleted and rebuilt from server data on next login (`SessionAuthenticator.initializeDatabase`). This means: -- Adding/removing tables or columns → bump version +- Removing a table, or adding/removing/changing columns on an existing table → bump version +- **Adding a new table does not bump**, as long as it starts empty and fills from the server. + `createTablesIfNeeded()` runs on every open with `create(ifNotExists: true)`, so an existing store + gains the table in place and keeps its data. Code reading the new table must treat "empty" as + "never synced". Any change to an existing table's columns still bumps. - Changing which table a query reads from → bump version if the old schema can't satisfy the new query - **Changing the encoding of a value already stored in a column → bump version.** The column's SQL type is unchanged, so nothing about `Schema.swift` looks different, but rows written by an earlier diff --git a/CLAUDE.md b/CLAUDE.md index 5036d06cd..954ae3569 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,7 +105,7 @@ it before touching the relevant area. - **Exhaustive switches** — Prefer `switch` over `if case` for enums so the compiler flags new cases. - **Modernize incrementally** — Use modern Swift/SwiftUI APIs in net-new/isolated code; don't refactor working code just to modernize. One observation system per class. - **Generated protos** — `FlipcashAPI` only re-exports the published contract packages; there is no generated code to edit here. Change the wrapping service files instead. -- **Database schema** — Bump `Database.schemaVersion` on every change to what the store *writes*, including the encoding of a persisted blob (no migrations; DB is rebuilt from server). +- **Database schema** — Bump `Database.schemaVersion` on every change to what the store *writes*, including the encoding of a persisted blob (no migrations; DB is rebuilt from server). A change that only adds new tables, which start empty and fill from the server, does not bump. - **Logging** — Message string is a constant; every variable goes in structured `metadata`. Never log proto blobs whole. - **Error reporting** — Call `ErrorReporting.captureError(...)` unconditionally; classify via `ServerError.reportingLevel`, never gate at the call site. Best-effort chatter never reports. - **Form validation** — Validate free-form input through the `Validator` family and submit the validator's `Output`, never inline regex/trim; keypad amounts parse only via `AmountValidator`. diff --git a/Flipcash/Core/Controllers/ConversationController.swift b/Flipcash/Core/Controllers/ConversationController.swift index f0fcc5671..c2d9d58a1 100644 --- a/Flipcash/Core/Controllers/ConversationController.swift +++ b/Flipcash/Core/Controllers/ConversationController.swift @@ -261,6 +261,10 @@ final class ConversationController { /// The emoji reactions concern: the user's taps, their calls, and the stream's updates. let reactions: ConversationReactions + /// Keeps opened groups' full rosters on device from the stream's roster updates. Set by the + /// session after init; `nil` leaves roster updates unpersisted. + @ObservationIgnored var roster: RosterSync? + init( fetching: any ConversationFetching, membership: any ConversationMembership, @@ -408,6 +412,7 @@ final class ConversationController { self.logCounterpartRead(event) self.applyTyping(event) self.applyReactions(event) + self.roster?.apply(event) if case .needsCatchUp(let conversationID, _) = gap { self.scheduleGapCatchUp(conversationID) } diff --git a/Flipcash/Core/Controllers/FlipClient+Protocols.swift b/Flipcash/Core/Controllers/FlipClient+Protocols.swift index 915f94229..454c51db0 100644 --- a/Flipcash/Core/Controllers/FlipClient+Protocols.swift +++ b/Flipcash/Core/Controllers/FlipClient+Protocols.swift @@ -67,6 +67,12 @@ protocol ConversationFetching: AnyObject, Sendable { func getChat(owner: KeyPair, conversationID: ConversationID) async throws -> Conversation } +/// Roster read surface used by `RosterSync`. Maps 1:1 to `flipcash.chat.v1.Chat.GetRoster`. +protocol RosterFetching: AnyObject, Sendable { + /// One page of a chat's roster, most recently joined first; see ``FlipClient/RosterPage``. + func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> FlipClient.RosterPage +} + /// Group membership surface used by `ConversationController`. Separate from ``ConversationFetching`` /// because these write: they are the only chat RPCs that change what the caller is a member of. protocol ConversationMembership: AnyObject, Sendable { @@ -166,5 +172,5 @@ extension FlipClient { } extension FlipClient: ContactVerifying, OnrampAuthorizing, ContactSyncing, - ConversationFetching, ConversationMembership, + ConversationFetching, ConversationMembership, RosterFetching, ConversationViewerSettings, ConversationEventStreaming {} diff --git a/Flipcash/Core/Controllers/RosterSearch.swift b/Flipcash/Core/Controllers/RosterSearch.swift new file mode 100644 index 000000000..d8ae39838 --- /dev/null +++ b/Flipcash/Core/Controllers/RosterSearch.swift @@ -0,0 +1,115 @@ +// +// RosterSearch.swift +// Flipcash +// + +import Foundation +import FlipcashCore +import FlipcashStore + +/// A group member a roster search matched. +nonisolated struct MemberMatch: Identifiable, Hashable, Sendable { + let member: ConversationMember + let id: UserID +} + +/// Searches a group's members by name or username, for the mention picker. +/// +/// Callers depend on this rather than on ``LocalRosterSearch`` so a server-side search can replace +/// or back the local one without them changing. +protocol RosterSearchSource: AnyObject { + /// Readies the source to answer for a group; called when the picker opens. + func prepare(chatID: ConversationID) async + + /// Returns up to `limit` members of a group matching `query`, best match first, never the + /// signed-in user or a blocked one. An empty query returns the members who spoke most recently. + func search(chatID: ConversationID, query: String, limit: Int) async throws -> [MemberMatch] +} + +extension RosterSearchSource { + /// Returns up to 20 members of a group matching `query`; see ``search(chatID:query:limit:)``. + func search(chatID: ConversationID, query: String) async throws -> [MemberMatch] { + try await search(chatID: chatID, query: query, limit: 20) + } +} + +/// Answers roster searches from the roster ``RosterSync`` holds on device. +/// +/// A member matches when every word of the query is a prefix of one of their tokens, the words of +/// their display name and their username, compared after ``RosterSearchText/normalize(_:)``. Matches +/// rank in three tiers: +/// 1. members who sent one of the chat's newest held messages, most recent first; +/// 2. the member whose username is exactly the query; +/// 3. everyone else, by normalized display name, then raw display name, both in code point order. +/// Ties fall to the lowercase user id. The signed-in user and blocked users never match. +final class LocalRosterSearch: RosterSearchSource { + + /// How many of the chat's newest held messages count toward "spoke recently". + static let recentMessageWindow = 50 + + private let database: Database + private let roster: RosterSync + private let selfUserID: UserID + private let blockedUserIDs: () -> Set + + init(database: Database, roster: RosterSync, selfUserID: UserID, blockedUserIDs: @escaping () -> Set = { [] }) { + self.database = database + self.roster = roster + self.selfUserID = selfUserID + self.blockedUserIDs = blockedUserIDs + } + + func prepare(chatID: ConversationID) async { + await roster.refreshFirstPage(chatID) + } + + func search(chatID: ConversationID, query: String, limit: Int) async throws -> [MemberMatch] { + let words = RosterSearchText.queryWords(query) + let recent = try database.recentSenders(conversationID: chatID, window: Self.recentMessageWindow) + + let candidates: Set + if words.isEmpty { + candidates = Set(recent) + } else { + candidates = try database.rosterMemberIDs(matchingPrefixes: words, conversationID: chatID) + } + guard !candidates.isEmpty, limit > 0 else { return [] } + + let recency = Dictionary(recent.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) + let exactUsername = words.count == 1 ? words[0] : nil + let blocked = blockedUserIDs() + + let ranked = try database.rosterEntries(conversationID: chatID, userIDs: candidates) + .compactMap { entry -> (Rank, RosterEntry, UserID)? in + guard let userID = entry.member.userID, userID != selfUserID, !blocked.contains(userID) else { return nil } + let username = entry.member.username.map { RosterSearchText.normalize($0.value) } + let rank: Rank + if let position = recency[userID] { + rank = .recentSpeaker(position) + } else if let exactUsername, username == exactUsername { + rank = .exactUsername + } else { + rank = .alphabetical + } + return (rank, entry, userID) + } + .sorted { lhs, rhs in + if lhs.0 != rhs.0 { return lhs.0 < rhs.0 } + if lhs.1.sortKey != rhs.1.sortKey { + return lhs.1.sortKey.unicodeScalars.lexicographicallyPrecedes(rhs.1.sortKey.unicodeScalars) + } + if lhs.1.member.displayName != rhs.1.member.displayName { + return lhs.1.member.displayName.unicodeScalars.lexicographicallyPrecedes(rhs.1.member.displayName.unicodeScalars) + } + return lhs.2.uuidString.lowercased() < rhs.2.uuidString.lowercased() + } + + return ranked.prefix(limit).map { MemberMatch(member: $0.1.member, id: $0.2) } + } + + private enum Rank: Comparable { + case recentSpeaker(Int) + case exactUsername + case alphabetical + } +} diff --git a/Flipcash/Core/Controllers/RosterSync.swift b/Flipcash/Core/Controllers/RosterSync.swift new file mode 100644 index 000000000..b7aba7f35 --- /dev/null +++ b/Flipcash/Core/Controllers/RosterSync.swift @@ -0,0 +1,196 @@ +// +// RosterSync.swift +// Flipcash +// + +import Foundation +import FlipcashCore +import FlipcashStore + +nonisolated private let logger = Logger(label: "flipcash.roster-sync") + +/// Keeps the full roster of each group the user opens on device, for member search. +/// +/// A group is tracked from its first open. The first open reads `Chat.GetRoster` to the end. Later +/// opens catch up only when the roster moved past the last version a read applied: they re-read +/// from the first page, newest joins first, and stop at a member already covered. ``apply(_:)`` +/// keeps a tracked group current from the stream in between. Groups never opened are not fetched. +/// Every path merges per member by roster version, greater winning. +actor RosterSync { + + /// Pages a read fetches before it stops, `has_more` or not; at 100 per page, 2,000 members. + static let defaultPageCap = 20 + + private let fetching: any RosterFetching + private let database: Database + private let owner: KeyPair + private let pageCap: Int + + /// Reads in flight, so a second trigger for the same group joins the first rather than reading twice. + private var inFlight: [ConversationID: Task] = [:] + + init(fetching: any RosterFetching, database: Database, owner: KeyPair, pageCap: Int = RosterSync.defaultPageCap) { + self.fetching = fetching + self.database = database + self.owner = owner + self.pageCap = pageCap + } + + /// Starts tracking a group and brings its held roster up to date: a full read when none has + /// finished or a reconcile is owed, a catch-up when the roster moved past the watermark, nothing + /// otherwise. Returns once any read is done. + func syncIfNeeded(_ conversationID: ConversationID) async { + await run(conversationID, refreshFirstPage: false) + } + + /// As ``syncIfNeeded(_:)``, but always re-reads at least the first page, so the newest members' + /// names and pictures are current; profile changes don't move the roster version. + func refreshFirstPage(_ conversationID: ConversationID) async { + await run(conversationID, refreshFirstPage: true) + } + + /// Writes a stream roster update into the held roster of a tracked group; other events are ignored. + @MainActor + func apply(_ event: ConversationStreamEvent) { + guard case .rosterChanged(let conversationID, let updates) = event else { return } + do { + try database.applyRosterUpdates(updates, conversationID: conversationID) + } catch { + logger.error("Failed to apply roster updates", metadata: [ + "conversationID": "\(conversationID)", + "error": "\(error)", + ]) + ErrorReporting.captureError(error, reason: "Failed to apply roster updates") + } + } + + // MARK: - Private - + + private func run(_ conversationID: ConversationID, refreshFirstPage: Bool) async { + if let running = inFlight[conversationID] { + await running.value + return + } + let task = Task { await self.bringUpToDate(conversationID, refreshFirstPage: refreshFirstPage) } + inFlight[conversationID] = task + await task.value + inFlight[conversationID] = nil + } + + private func bringUpToDate(_ conversationID: ConversationID, refreshFirstPage: Bool) async { + let state: RosterSyncState + do { + try database.beginTrackingRoster(conversationID: conversationID) + guard let tracked = try database.rosterSyncState(conversationID: conversationID) else { return } + state = tracked + } catch { + await report(error, reason: "Failed to read roster sync state", conversationID: conversationID) + return + } + + if state.needsFullRead { + await fullRead(conversationID, reconciling: state.reconcilePending) + return + } + guard state.needsCatchUp || refreshFirstPage else { return } + + do { + switch try await catchUp(conversationID, watermark: state.watermark) { + case .caughtUp: + break + case .reconcileNeeded: + await fullRead(conversationID, reconciling: true) + } + } catch { + await report(error, reason: "Failed to catch up roster", conversationID: conversationID) + } + } + + /// Reads from the first page until it reaches a member the watermark already covers. + private func catchUp(_ conversationID: ConversationID, watermark: UInt64) async throws -> RosterCatchUpOutcome { + // Pages list the most recently joined first, and a member's version is the roster version of + // their join, so everyone after the first member at or below the watermark is already held. + // `Member.version` is documented to move on future member changes too (a role change, say); + // once one does, a member can sit above the watermark out of join order, and this stop rule + // needs revisiting. + let read = try await readPages(conversationID) { page in + page.members.contains { $0.version <= watermark } + } + + guard read.stoppedEarly else { + // The read ran to the end or the cap without reaching the watermark: it's a full read. + try database.applyFullRosterRead(read.members, summaries: read.summaries, isComplete: read.reachedEnd, conversationID: conversationID) + logRead("Roster catch-up became a full read", read, conversationID: conversationID) + return .caughtUp + } + + guard let summary = read.summaries.min(by: { $0.version < $1.version }) else { return .caughtUp } + let outcome = try database.applyRosterCatchUp(read.members, summary: summary, conversationID: conversationID) + logRead("Caught up roster", read, conversationID: conversationID) + return outcome + } + + /// Reads the whole roster, up to the page cap. A reconcile runs at background priority, since it + /// only removes members who left; a failure leaves it pending for the next open. + private func fullRead(_ conversationID: ConversationID, reconciling: Bool) async { + let task = Task(priority: reconciling ? .background : nil) { + let read = try await self.readPages(conversationID) { _ in false } + try self.database.applyFullRosterRead(read.members, summaries: read.summaries, isComplete: read.reachedEnd, conversationID: conversationID) + self.logRead("Read full roster", read, conversationID: conversationID) + } + do { + try await task.value + } catch { + await report(error, reason: "Failed to read full roster", conversationID: conversationID) + } + } + + private struct Read { + var members: [ConversationMember] = [] + var summaries: [ConversationRosterSummary] = [] + var pages = 0 + var reachedEnd = false + var stoppedEarly = false + } + + /// Pages from the first page until `stop` returns `true` for a page, the last page, or the cap. + /// Throws without returning anything read when a page fails, so a partial read records nothing. + private func readPages(_ conversationID: ConversationID, stop: (FlipClient.RosterPage) -> Bool) async throws -> Read { + var read = Read() + var pagingToken: Data? + while read.pages < pageCap { + let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: pagingToken) + read.pages += 1 + read.members.append(contentsOf: page.members) + read.summaries.append(page.rosterSummary) + if stop(page) { + read.stoppedEarly = true + break + } + guard let next = page.nextPagingToken else { + read.reachedEnd = true + break + } + pagingToken = next + } + return read + } + + private func logRead(_ message: Logger.Message, _ read: Read, conversationID: ConversationID) { + logger.info(message, metadata: [ + "conversationID": "\(conversationID)", + "members": "\(read.members.count)", + "pages": "\(read.pages)", + "reachedEnd": "\(read.reachedEnd)", + ]) + } + + private func report(_ error: Error, reason: String, conversationID: ConversationID) async { + logger.error("Roster sync failed", metadata: [ + "reason": "\(reason)", + "conversationID": "\(conversationID)", + "error": "\(error)", + ]) + await ErrorReporting.captureError(error, reason: reason) + } +} diff --git a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift index 82dac0e4f..bb0d98ef6 100644 --- a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift +++ b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift @@ -680,6 +680,12 @@ struct ConversationScreen: View { guard groupConversation != nil else { return } await sessionContainer.knownAuthors.reload() } + // Page the group's full roster into the store for member search. Detached from this task + // inside `RosterSync`, so leaving the chat mid-sync doesn't discard the pages fetched. + .task(id: groupConversation?.id) { + guard let groupID = groupConversation?.id else { return } + await conversationController.roster?.syncIfNeeded(groupID) + } // Name the senders the chat's roster and the local cache both leave out. Keyed on that set, // so it runs when a page of older messages reveals a sender nothing here can name — and not // again once the fetch has landed them. diff --git a/Flipcash/Core/Session/DatabaseStore.swift b/Flipcash/Core/Session/DatabaseStore.swift index 08f3fccd7..d42488f4e 100644 --- a/Flipcash/Core/Session/DatabaseStore.swift +++ b/Flipcash/Core/Session/DatabaseStore.swift @@ -67,12 +67,8 @@ final class DatabaseStore { // Currently we don't do migrations so every time // the user version is outdated, we'll rebuild the // database during sync. - let userVersion = (try? Database.userVersion(files: files)) ?? 0 - let currentVersion = Database.schemaVersion - if currentVersion > userVersion { - try Database.deleteStore(files: files) + if try Database.discardStoreIfOutdated(files: files) { logger.error("Outdated user version, deleted database.") - try Database.setUserVersion(version: currentVersion, files: files) } let database = try Database(url: files.database) diff --git a/Flipcash/Core/Session/SessionAuthenticator.swift b/Flipcash/Core/Session/SessionAuthenticator.swift index 2c72c929d..bf2fd0387 100644 --- a/Flipcash/Core/Session/SessionAuthenticator.swift +++ b/Flipcash/Core/Session/SessionAuthenticator.swift @@ -530,6 +530,8 @@ final class SessionContainer { let profileAvatars: ProfileAvatarStore /// Identities for chat senders the chat's own roster leaves out — see ``KnownAuthorDirectory``. let knownAuthors: KnownAuthorDirectory + /// Group member search for the mention picker — see ``RosterSearchSource``. + let rosterSearch: any RosterSearchSource /// Fills cash link cards in for chat transcripts. Container-scoped so a link shared into two /// conversations is looked up once, and holding a `GiftCardAccountReading` rather than the /// `Client` it was built from — rendering a card must never reach a path that claims it. @@ -667,6 +669,8 @@ final class SessionContainer { owner: session.ownerKeyPair, selfUserID: session.userID ) + let rosterSync = RosterSync(fetching: flipClient, database: database, owner: session.ownerKeyPair) + conversationController.roster = rosterSync // Chat cash arrives in the sender's native currency; the counters are USD. // Wired before `start()` so the first delivered message already normalises. conversationController.receipts.usdRate = { [weak ratesController] currency in @@ -698,6 +702,14 @@ final class SessionContainer { database: database ) self.blocklistController = blocklistController + self.rosterSearch = LocalRosterSearch( + database: database, + roster: rosterSync, + selfUserID: session.userID, + blockedUserIDs: { [weak blocklistController] in + Set(blocklistController?.blockedUsers.map(\.userID) ?? []) + } + ) // The blocklist drives which conversations the feed hides: it supplies the // current set, reconciles the hidden flags on every change, and reconciles diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift index 66e517cf0..3def0a2bb 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift @@ -98,7 +98,7 @@ extension FlipClient { } /// Represents one paged read of a chat's roster: the members returned, most recently joined - /// first, and the roster summary as of that page's read. + /// first, the roster summary as of that page's read, and how to continue. /// /// This is a single pinned read, not a merged view — a large group's roster can page from an /// index that trails the event stream (see `chat.v1.GetRoster`'s staleness contract), so the @@ -108,28 +108,29 @@ extension FlipClient { public struct RosterPage: Sendable { public let members: [ConversationMember] public let rosterSummary: ConversationRosterSummary - } - - /// Pages a chat's roster to exhaustion against a single pinned read. The caller must already be - /// consuming its `subscribeConversationStream` events — as with `getDmChatFeed`/ - /// `getGroupChatFeed` — and must merge the result against them by ``ConversationMember/version`` - /// rather than treating it as an authoritative snapshot; see ``RosterPage``. - public func getRoster(owner: KeyPair, conversationID: ConversationID) async throws -> RosterPage { - var members: [ConversationMember] = [] - var rosterSummary = ConversationRosterSummary(memberCount: 0, version: 0) - var pagingToken: Data? + /// The token to pass for the next page; `nil` on the last page. + public let nextPagingToken: Data? - while true { - let page = try await withCheckedThrowingContinuation { c in - chatService.getRoster(owner: owner, conversationID: conversationID, pagingToken: pagingToken) { c.resume(with: $0) } - } - members.append(contentsOf: page.members) - rosterSummary = page.rosterSummary - if !page.hasMore { break } - pagingToken = page.pagingToken + public init(members: [ConversationMember], rosterSummary: ConversationRosterSummary, nextPagingToken: Data?) { + self.members = members + self.rosterSummary = rosterSummary + self.nextPagingToken = nextPagingToken } + } - return RosterPage(members: members, rosterSummary: rosterSummary) + /// Fetches one page of a chat's roster, up to 100 members. Pass `nil` for the first page and the + /// previous page's ``RosterPage/nextPagingToken`` after that. The caller must already be consuming + /// its `subscribeConversationStream` events and must merge the result against them by + /// ``ConversationMember/version``; see ``RosterPage``. + public func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> RosterPage { + let page = try await withCheckedThrowingContinuation { c in + chatService.getRoster(owner: owner, conversationID: conversationID, pagingToken: pagingToken) { c.resume(with: $0) } + } + return RosterPage( + members: page.members, + rosterSummary: page.rosterSummary, + nextPagingToken: page.hasMore ? page.pagingToken : nil + ) } /// Edits a group chat's title and/or picture; every parameter left `nil` leaves that field diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift index 8baf36108..9958e2877 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift @@ -287,7 +287,7 @@ final class ChatService: Sendable { /// staleness contract) — callers must merge each page against what the event stream has already /// told them, by ``ConversationMember/version``, greater winning, rather than trusting a page to /// be a complete, current snapshot. - func getRoster(owner: KeyPair, conversationID: ConversationID, pageSize: Int = 50, pagingToken: Data?, completion: @Sendable @escaping (Result) -> Void) { + func getRoster(owner: KeyPair, conversationID: ConversationID, pageSize: Int = 100, pagingToken: Data?, completion: @Sendable @escaping (Result) -> Void) { let request = Flipcash_Chat_V1_GetRosterRequest.with { $0.chatID = conversationID.proto $0.queryOptions = .with { diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift new file mode 100644 index 000000000..88fa84669 --- /dev/null +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift @@ -0,0 +1,71 @@ +// +// RosterSearchText.swift +// FlipcashCore +// + +import Foundation + +/// The text rules roster search indexes and queries by. Android applies the same rules, so a query +/// matches the same members on both platforms. +public enum RosterSearchText { + + /// Returns `text` NFKD-decomposed, lowercased, and stripped of nonspacing marks (general category + /// `Mn`), so "Érica" and "erica" compare equal. Lowercasing after NFKD folds "İ" to "i" plus a + /// combining dot, which the strip then drops. Spacing marks (`Mc`) are kept: in scripts such as + /// Devanagari they carry the vowel. + public static func normalize(_ text: String) -> String { + let folded = text.decomposedStringWithCompatibilityMapping.lowercased() + var scalars = String.UnicodeScalarView() + for scalar in folded.unicodeScalars where scalar.properties.generalCategory != .nonspacingMark { + scalars.append(scalar) + } + return String(scalars) + } + + /// Returns the normalized words of `text`, split on Unicode `White_Space` and separators + /// (general category `Z`), empty words dropped. + public static func words(_ text: String) -> [String] { + normalize(text) + .unicodeScalars + .split(whereSeparator: isSeparator) + .map { String(String.UnicodeScalarView($0)) } + } + + /// Returns the distinct search tokens for a member: each word of the display name, plus the + /// username without its `@`. + public static func tokens(displayName: String, username: String?) -> Set { + var tokens = Set(words(displayName)) + if let username { + tokens.formUnion(words(username).map(droppingAt).filter { !$0.isEmpty }) + } + return tokens + } + + /// Returns the words of a typed query, with a leading `@` (either width) dropped from each word. + public static func queryWords(_ query: String) -> [String] { + words(query).map(droppingAt).filter { !$0.isEmpty } + } + + /// Returns the exclusive upper bound of a prefix range, so `token >= prefix AND token < bound` + /// matches every token starting with `prefix`. + /// + /// U+10FFFF rather than U+FFFF: SQLite compares UTF-8 bytes, and a token whose next character is + /// outside the BMP (an emoji after a name, say) sorts above `prefix + U+FFFF`. + public static func prefixUpperBound(_ prefix: String) -> String { + prefix + "\u{10FFFF}" + } + + // NFKD folds the fullwidth "＠" to "@", so one check covers both. + private static func droppingAt(_ word: String) -> String { + word.hasPrefix("@") ? String(word.dropFirst()) : word + } + + private static func isSeparator(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.properties.generalCategory { + case .spaceSeparator, .lineSeparator, .paragraphSeparator: + return true + default: + return scalar.properties.isWhitespace + } + } +} diff --git a/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift b/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift index d6f051cea..c2f7efe1a 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift @@ -334,6 +334,7 @@ nonisolated extension Database { try writer.run(c.table.filter(doomed.contains(c.id)).delete()) try writer.run(m.table.filter(doomed.contains(m.conversationId)).delete()) try writer.run(g.table.filter(doomed.contains(g.conversationId)).delete()) + try deleteRosterRows(conversationIDs: doomed) } for group in groups where group.type == .group { try writeConversation(group) @@ -409,7 +410,7 @@ nonisolated extension Database { } /// Removes a conversation the signed-in user has left (or been removed from): its row and member - /// rows. Messages are left in place, orphaned but unread — the same treatment + /// rows and any synced roster. Messages are left in place, orphaned but unread — the same treatment /// `replaceConversationFeed` gives a conversation that drops out of a feed snapshot. public func deleteConversation(conversationID: ConversationID) throws { let c = ConversationTable() @@ -417,6 +418,7 @@ nonisolated extension Database { try writer.transaction { try writer.run(c.table.filter(c.id == conversationID.data).delete()) try writer.run(m.table.filter(m.conversationId == conversationID.data).delete()) + try deleteRosterRows(conversationIDs: [conversationID.data]) } } diff --git a/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift new file mode 100644 index 000000000..32d12182f --- /dev/null +++ b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift @@ -0,0 +1,423 @@ +// +// Database+Roster.swift +// FlipcashStore +// + +import Foundation +import FlipcashCore +import SQLite + +/// Where a group's locally held roster stands against the server's. +public struct RosterSyncState: Sendable, Equatable { + /// The last roster version a `Chat.GetRoster` read fully applied. + public let watermark: UInt64 + /// The greatest roster version seen from the stream, the chat feed, or a read. + public let observedVersion: UInt64 + /// The group's member count, as the latest roster summary reported it. + public let memberCount: UInt64 + /// How many current members are held locally. + public let heldCount: Int + /// Whether a full read has ever finished. + public let fullySynced: Bool + /// Whether the last full read stopped at the page cap. + public let truncated: Bool + /// Whether a full read is owed, from an unseen leave or a read that couldn't drop members. + public let reconcilePending: Bool + + /// Whether the roster should be read from the first page to the end. + public var needsFullRead: Bool { !fullySynced || reconcilePending } + + /// Whether the roster moved past what a read has applied, so the newest pages should be re-read. + public var needsCatchUp: Bool { observedVersion > watermark } +} + +/// What a catch-up read found once merged. +public enum RosterCatchUpOutcome: Sendable, Equatable { + /// The held roster agrees with the page's count; the watermark moved to the page's version. + case caughtUp + /// More members are held than the page counts, so someone left unseen; a full read is owed. + case reconcileNeeded +} + +/// A current roster member as stored, with the key roster search sorts it by. +public struct RosterEntry: Sendable, Equatable { + public let member: ConversationMember + /// ``RosterSearchText/normalize(_:)`` of the display name. + public let sortKey: String +} + +nonisolated extension Database { + + // MARK: - Sync state - + + /// Returns the roster sync state for a group, or `nil` when its roster is not tracked yet. + public func rosterSyncState(conversationID: ConversationID) throws -> RosterSyncState? { + let s = RosterSyncTable() + let r = RosterMemberTable() + let c = ConversationTable() + guard let row = try reader.pluck(s.table.filter(s.conversationId == conversationID.data)) else { + return nil + } + let held = try reader.scalar(r.table.filter(r.conversationId == conversationID.data && r.isMember == true).count) + let feedVersion = try reader.pluck(c.table.select(c.rosterVersion).filter(c.id == conversationID.data))?[c.rosterVersion] ?? 0 + return RosterSyncState( + watermark: row[s.watermark], + observedVersion: max(row[s.observedVersion], feedVersion), + memberCount: row[s.memberCount], + heldCount: held, + fullySynced: row[s.fullySynced], + truncated: row[s.truncated], + reconcilePending: row[s.reconcilePending] + ) + } + + /// Starts tracking a group's roster so stream updates are written from now on. A no-op when the + /// group is already tracked. + public func beginTrackingRoster(conversationID: ConversationID) throws { + let s = RosterSyncTable() + let c = ConversationTable() + try writer.transaction { + let chat = try writer.pluck(c.table.select(c.rosterVersion, c.rosterMemberCount).filter(c.id == conversationID.data)) + try writer.run(s.table.insert( + or: .ignore, + s.conversationId <- conversationID.data, + s.watermark <- 0, + s.observedVersion <- chat?[c.rosterVersion] ?? 0, + s.memberCount <- chat?[c.rosterMemberCount] ?? 0, + s.fullySynced <- false, + s.truncated <- false, + s.reconcilePending <- false + )) + } + } + + // MARK: - Writes - + + /// Merges a read from the first page to the end, or to the page cap, into a tracked group. + /// + /// Per member, the greater version wins against what the stream already wrote. A held member the + /// read lacks is dropped only when the read reached the end, every page reported the same roster + /// version, and the held row's version is not above it. When the pages disagree, nothing is + /// dropped and a reconcile stays pending. A truncated read drops nothing and owes nothing: the + /// members past the cap were never held. + public func applyFullRosterRead( + _ members: [ConversationMember], + summaries: [ConversationRosterSummary], + isComplete: Bool, + conversationID: ConversationID + ) throws { + guard let oldest = summaries.min(by: { $0.version < $1.version }), + let newest = summaries.max(by: { $0.version < $1.version }) else { return } + let isConsistent = oldest.version == newest.version + let s = RosterSyncTable() + try writer.transaction { + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + var held = try mergeRosterMembers(members, conversationID: conversationID) + + if isComplete && isConsistent { + let readIDs = Set(members.compactMap(\.userID)) + for (userID, version) in held where !readIDs.contains(userID) && version <= oldest.version { + try deleteRosterMember(userID: userID, conversationID: conversationID) + held[userID] = nil + } + } + + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.watermark <- max(sync[s.watermark], oldest.version), + s.observedVersion <- max(sync[s.observedVersion], newest.version), + s.memberCount <- oldest.memberCount, + s.fullySynced <- true, + s.truncated <- !isComplete, + s.reconcilePending <- !isConsistent + )) + } + } + + /// Merges a read of the newest pages into a tracked group and checks the page's count for leaves + /// the read can't show. + /// + /// `summary` is the page's roster summary; with several pages, the one with the lowest version. + /// When no more members are held than it counts, the watermark moves to its version: a join the + /// read missed is newer than the page and arrives on the stream. When more are held, someone left + /// unseen, so a reconcile is recorded and the watermark stays put. + public func applyRosterCatchUp( + _ members: [ConversationMember], + summary: ConversationRosterSummary, + conversationID: ConversationID + ) throws -> RosterCatchUpOutcome { + let s = RosterSyncTable() + let r = RosterMemberTable() + var outcome = RosterCatchUpOutcome.caughtUp + try writer.transaction { + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + try mergeRosterMembers(members, conversationID: conversationID) + let held = try writer.scalar(r.table.filter(r.conversationId == conversationID.data && r.isMember == true).count) + let observed = max(sync[s.observedVersion], summary.version) + + if UInt64(held) > summary.memberCount { + outcome = .reconcileNeeded + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.observedVersion <- observed, + s.reconcilePending <- true + )) + } else { + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.watermark <- max(sync[s.watermark], summary.version), + s.observedVersion <- observed, + s.memberCount <- summary.memberCount + )) + } + } + return outcome + } + + /// Applies live roster updates to a tracked group, returning `false` when the group is not tracked. + /// + /// Each update wins against the member's row only with a greater version. The watermark does not + /// move: the stream can't show whether an update was skipped, so the next open catches up from + /// the roster's first page instead. + @discardableResult + public func applyRosterUpdates(_ updates: [DecodedRosterUpdate], conversationID: ConversationID) throws -> Bool { + let s = RosterSyncTable() + var isTracked = false + try writer.transaction { + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + isTracked = true + var observed = sync[s.observedVersion] + var memberCount = sync[s.memberCount] + let held = try heldVersions(conversationID: conversationID) + + for update in updates.sorted(by: { $0.rosterSummary.version < $1.rosterSummary.version }) { + let updateVersion = update.rosterSummary.version + if updateVersion > observed { + observed = updateVersion + memberCount = update.rosterSummary.memberCount + } + + switch update.change { + case .joined(let member, _): + guard let userID = member.userID else { continue } + if let existing = held[userID], existing >= updateVersion { continue } + try writeRosterMember(member, userID: userID, version: updateVersion, conversationID: conversationID) + case .left(let userID): + if let existing = held[userID], existing >= updateVersion { continue } + try writeRosterDeparture(userID: userID, version: updateVersion, conversationID: conversationID) + } + } + + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.observedVersion <- observed, + s.memberCount <- memberCount + )) + } + return isTracked + } + + /// Must be called inside a `writer.transaction`. Rewrites a user's name, username, picture, and + /// search tokens in every group that holds them as a current member, keeping each row's version, + /// since a profile change doesn't move the roster version. + func refreshRosterProfile(_ profile: Profile, userID: UserID) throws { + let r = RosterMemberTable() + let t = RosterTokenTable() + let rows = try Array(writer.prepareRowIterator( + r.table.filter(r.userId == userID && r.isMember == true) + )) + for row in rows { + let conversationData = row[r.conversationId] + let displayName = profile.displayName ?? row[r.displayName] + try writer.run(r.table.filter(r.conversationId == conversationData && r.userId == userID).update( + r.displayName <- displayName, + r.username <- profile.username?.value, + r.profilePictureBlobID <- profile.profilePicture?.blobID.data, + r.profilePictureThumbnailBlobID <- profile.profilePicture?.thumbnailBlobID.data, + r.profilePictureThumbnailBlurhash <- profile.profilePicture?.thumbnailBlurhash, + r.sortKey <- RosterSearchText.normalize(displayName) + )) + try writer.run(t.table.filter(t.conversationId == conversationData && t.userId == userID).delete()) + for token in RosterSearchText.tokens(displayName: displayName, username: profile.username?.value) { + try writer.run(t.table.insert(or: .ignore, t.conversationId <- conversationData, t.token <- token, t.userId <- userID)) + } + } + } + + /// Removes everything held for a group's roster, tracking included. + public func deleteRoster(conversationID: ConversationID) throws { + try writer.transaction { + try deleteRosterRows(conversationIDs: [conversationID.data]) + } + } + + // MARK: - Reads - + + /// Returns the ids of current members with, for every word in `prefixes`, a token starting with it. + /// Each word is one index range scan over `(conversationId, token)`. + public func rosterMemberIDs(matchingPrefixes prefixes: [String], conversationID: ConversationID) throws -> Set { + let t = RosterTokenTable() + var matched: Set? + for prefix in prefixes { + let query = t.table + .select(distinct: t.userId) + .filter(t.conversationId == conversationID.data + && t.token >= prefix + && t.token < RosterSearchText.prefixUpperBound(prefix)) + let ids = Set(try reader.prepareRowIterator(query).map { $0[t.userId] }) + matched = matched.map { $0.intersection(ids) } ?? ids + if matched?.isEmpty == true { break } + } + return matched ?? [] + } + + /// Returns the current members of a group, limited to `userIDs` when given. + public func rosterEntries(conversationID: ConversationID, userIDs: Set? = nil) throws -> [RosterEntry] { + let r = RosterMemberTable() + var query = r.table.filter(r.conversationId == conversationID.data && r.isMember == true) + if let userIDs { + query = query.filter(userIDs.contains(r.userId)) + } + return try reader.prepareRowIterator(query).map { row in + RosterEntry( + member: ConversationMember( + userID: row[r.userId], + displayName: row[r.displayName], + profilePicture: rosterProfilePicture(from: row), + username: row[r.username].flatMap(Username.init), + joinedAt: row[r.joinedAt].map(Date.init(timeIntervalSinceReferenceDate:)), + version: row[r.version] + ), + sortKey: row[r.sortKey] + ) + } + } + + /// Returns who sent the newest `window` held messages of a conversation, most recent sender first, + /// each once. + public func recentSenders(conversationID: ConversationID, window: Int) throws -> [UserID] { + let m = ConversationMessageTable() + let query = m.table + .select(m.senderId) + .filter(m.conversationId == conversationID.data) + .order(m.id.desc) + .limit(window) + var seen: Set = [] + var senders: [UserID] = [] + for row in try Array(reader.prepareRowIterator(query)) { + guard let sender = row[m.senderId], seen.insert(sender).inserted else { continue } + senders.append(sender) + } + return senders + } + + // MARK: - Private - + + /// Must be called inside a `writer.transaction`. Deletes every roster row for the given chats. + func deleteRosterRows(conversationIDs: [Data]) throws { + guard !conversationIDs.isEmpty else { return } + let r = RosterMemberTable() + let t = RosterTokenTable() + let s = RosterSyncTable() + try writer.run(r.table.filter(conversationIDs.contains(r.conversationId)).delete()) + try writer.run(t.table.filter(conversationIDs.contains(t.conversationId)).delete()) + try writer.run(s.table.filter(conversationIDs.contains(s.conversationId)).delete()) + } + + /// Must be called inside a `writer.transaction`. Every held row's version, departures included. + private func heldVersions(conversationID: ConversationID) throws -> [UserID: UInt64] { + let r = RosterMemberTable() + let query = r.table.select(r.userId, r.version).filter(r.conversationId == conversationID.data) + var versions: [UserID: UInt64] = [:] + for row in try Array(writer.prepareRowIterator(query)) { + versions[row[r.userId]] = row[r.version] + } + return versions + } + + /// Must be called inside a `writer.transaction`. Writes each member whose version is not below the + /// held row's, so an equal version refreshes a name or picture. Returns every held row's version. + @discardableResult + private func mergeRosterMembers(_ members: [ConversationMember], conversationID: ConversationID) throws -> [UserID: UInt64] { + var held = try heldVersions(conversationID: conversationID) + for member in members { + guard let userID = member.userID else { continue } + if let existing = held[userID], existing > member.version { continue } + try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) + held[userID] = member.version + } + return held + } + + /// Must be called inside a `writer.transaction`. + private func writeRosterMember(_ member: ConversationMember, userID: UserID, version: UInt64, conversationID: ConversationID) throws { + let r = RosterMemberTable() + let t = RosterTokenTable() + try writer.run(r.table.insert( + or: .replace, + r.conversationId <- conversationID.data, + r.userId <- userID, + r.displayName <- member.displayName, + r.username <- member.username?.value, + r.profilePictureBlobID <- member.profilePicture?.blobID.data, + r.profilePictureThumbnailBlobID <- member.profilePicture?.thumbnailBlobID.data, + r.profilePictureThumbnailBlurhash <- member.profilePicture?.thumbnailBlurhash, + r.joinedAt <- member.joinedAt?.timeIntervalSinceReferenceDate, + r.version <- version, + r.isMember <- true, + r.sortKey <- RosterSearchText.normalize(member.displayName) + )) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + for token in RosterSearchText.tokens(displayName: member.displayName, username: member.username?.value) { + try writer.run(t.table.insert( + or: .ignore, + t.conversationId <- conversationID.data, + t.token <- token, + t.userId <- userID + )) + } + } + + /// Must be called inside a `writer.transaction`. Keeps a departed member's row, so a trailing + /// roster page can't bring them back, and drops them from the index. + private func writeRosterDeparture(userID: UserID, version: UInt64, conversationID: ConversationID) throws { + let r = RosterMemberTable() + try writer.run(r.table.insert( + or: .replace, + r.conversationId <- conversationID.data, + r.userId <- userID, + r.displayName <- "", + r.username <- nil, + r.profilePictureBlobID <- nil, + r.profilePictureThumbnailBlobID <- nil, + r.profilePictureThumbnailBlurhash <- nil, + r.joinedAt <- nil, + r.version <- version, + r.isMember <- false, + r.sortKey <- "" + )) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + } + + /// Must be called inside a `writer.transaction`. + private func deleteRosterMember(userID: UserID, conversationID: ConversationID) throws { + let r = RosterMemberTable() + try writer.run(r.table.filter(r.conversationId == conversationID.data && r.userId == userID).delete()) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + } + + /// Must be called inside a `writer.transaction`. + private func deleteRosterTokens(userID: UserID, conversationID: ConversationID) throws { + let t = RosterTokenTable() + try writer.run(t.table.filter(t.conversationId == conversationID.data && t.userId == userID).delete()) + } + + private func rosterProfilePicture(from row: RowIterator.Element) -> ProfilePicture? { + let r = RosterMemberTable() + guard let blobID = row[r.profilePictureBlobID], + let thumbnailBlobID = row[r.profilePictureThumbnailBlobID] else { + return nil + } + return ProfilePicture( + blobID: BlobID(data: blobID), + thumbnailBlobID: BlobID(data: thumbnailBlobID), + thumbnailBlurhash: row[r.profilePictureThumbnailBlurhash] + ) + } +} diff --git a/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift b/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift index ef4325b7a..d95dae903 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift @@ -29,11 +29,14 @@ nonisolated extension Database { public func upsertUserProfile(_ profile: Profile, userID: UserID) throws { let t = UserProfileTable() let data = try JSONEncoder().encode(profile) - try writer.run(t.table.upsert( - t.userID <- userID, - t.data <- data, - onConflictOf: t.userID - )) + try writer.transaction { + try writer.run(t.table.upsert( + t.userID <- userID, + t.data <- data, + onConflictOf: t.userID + )) + try refreshRosterProfile(profile, userID: userID) + } } // MARK: - Delete - diff --git a/FlipcashCore/Sources/FlipcashStore/Database.swift b/FlipcashCore/Sources/FlipcashStore/Database.swift index d1d35bba8..b5690a849 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database.swift @@ -187,12 +187,15 @@ nonisolated open class Database: @unchecked Sendable { /// The schema version this build writes. /// /// A launch that finds a lower version recorded beside the store deletes the store and rebuilds - /// it from sync, which is the project's substitute for schema migrations. Bump this whenever a - /// table definition in `Schema.swift` changes, and equally whenever the *encoding* of a value + /// it from sync, which is the project's substitute for schema migrations. Bump this whenever an + /// existing table definition in `Schema.swift` changes, and equally whenever the *encoding* of a value /// already stored in a column changes — a new non-optional property on a `Codable` persisted as /// a JSON blob leaves the column's SQL type untouched, but rows written by an earlier build stop /// decoding, and the `try?` at the read sites turns that into a silent nil for the whole value. /// + /// A brand-new table that starts empty and fills from the server doesn't need a bump: + /// `createTablesIfNeeded()` adds it to an existing store on the next open. + /// /// This used to be the `SQLiteVersion` key in the app's `Info.plist`. It moved into code because /// the notification service extension needs the same number to decide whether the store on disk /// is one it understands, and an extension cannot read the app's `Info.plist` — separate bundles. @@ -225,6 +228,17 @@ nonisolated open class Database: @unchecked Sendable { ) } + /// Deletes the store when its recorded version is below `schemaVersion` and records the new one, + /// returning whether it deleted. A store at or above `schemaVersion` is left as it is. + @discardableResult + public static func discardStoreIfOutdated(files: StoreLocation.Files, schemaVersion: Int = Database.schemaVersion) throws -> Bool { + let recorded = (try? userVersion(files: files)) ?? 0 + guard schemaVersion > recorded else { return false } + try deleteStore(files: files) + try setUserVersion(version: schemaVersion, files: files) + return true + } + public static func userVersion(files: StoreLocation.Files) throws -> Int? { let versionString = try String( contentsOf: files.version, diff --git a/FlipcashCore/Sources/FlipcashStore/Schema.swift b/FlipcashCore/Sources/FlipcashStore/Schema.swift index b31225e64..4e86a9957 100644 --- a/FlipcashCore/Sources/FlipcashStore/Schema.swift +++ b/FlipcashCore/Sources/FlipcashStore/Schema.swift @@ -324,6 +324,69 @@ nonisolated public struct ConversationMemberTable: Sendable { public let username = Expression ("username") } +// A group's full roster, as `RosterSync` pages it from `Chat.GetRoster` and keeps it current from the +// stream. Its own table rather than `conversation_member`, which holds only the members the chat +// metadata embeds and is replaced wholesale on every metadata write. +// +// A leave keeps the row with `isMember` false and the leave's version, so a roster page that trails +// the stream cannot bring the member back: per user, the row with the greater `version` wins. +nonisolated public struct RosterMemberTable: Sendable { + public static let name = "roster_member" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + public let userId = Expression ("userId") + public let displayName = Expression ("displayName") + public let username = Expression ("username") + public let profilePictureBlobID = Expression ("profilePictureBlobID") + public let profilePictureThumbnailBlobID = Expression ("profilePictureThumbnailBlobID") + public let profilePictureThumbnailBlurhash = Expression ("profilePictureThumbnailBlurhash") + public let joinedAt = Expression ("joinedAt") + // The roster version of the member's latest join, or of their leave when `isMember` is false. + public let version = Expression ("version") + public let isMember = Expression ("isMember") + // `RosterSearchText.normalize(displayName)`, the alphabetical sort key. + public let sortKey = Expression ("sortKey") +} + +// The roster search index: one row per normalized word of a member's display name, plus their +// username. The primary key leads with `(conversationId, token)`, so a prefix lookup is a range scan. +nonisolated public struct RosterTokenTable: Sendable { + public static let name = "roster_token" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + public let token = Expression ("token") + public let userId = Expression ("userId") +} + +// Per-group roster sync bookkeeping. A group with no row is not tracked: its stream roster updates +// are not written to `roster_member` until the chat is first opened. +nonisolated public struct RosterSyncTable: Sendable { + public static let name = "roster_sync" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + // The last roster version a `Chat.GetRoster` read fully applied. Stream updates never move it, + // because the version is opaque and a skipped update can't be detected from the stream alone. + public let watermark = Expression ("watermark") + // The greatest roster version seen from any source; above `watermark` means a catch-up is due. + public let observedVersion = Expression ("observedVersion") + public let memberCount = Expression ("memberCount") + // Whether a read from the first page to the end (or to the page cap) has ever finished. + public let fullySynced = Expression ("fullySynced") + // Whether the last full read stopped at the page cap before `has_more` went false. + public let truncated = Expression ("truncated") + // A leave went unseen, or a full read couldn't be trusted to drop members; the next open re-reads. + public let reconcilePending = Expression ("reconcilePending") +} + // One row per message; cash content is decomposed across the amount columns // the same way `activity` stores ExchangedFiat. nonisolated public struct ConversationMessageTable: Sendable { @@ -620,6 +683,44 @@ nonisolated extension Database { }) } + let rosterMemberTable = RosterMemberTable() + let rosterTokenTable = RosterTokenTable() + let rosterSyncTable = RosterSyncTable() + + try writer.transaction { + try writer.run(rosterMemberTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterMemberTable.conversationId) + t.column(rosterMemberTable.userId) + t.column(rosterMemberTable.displayName) + t.column(rosterMemberTable.username) + t.column(rosterMemberTable.profilePictureBlobID) + t.column(rosterMemberTable.profilePictureThumbnailBlobID) + t.column(rosterMemberTable.profilePictureThumbnailBlurhash) + t.column(rosterMemberTable.joinedAt) + t.column(rosterMemberTable.version) + t.column(rosterMemberTable.isMember) + t.column(rosterMemberTable.sortKey) + t.primaryKey(rosterMemberTable.conversationId, rosterMemberTable.userId) + }) + try writer.run(rosterTokenTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterTokenTable.conversationId) + t.column(rosterTokenTable.token) + t.column(rosterTokenTable.userId) + t.primaryKey(rosterTokenTable.conversationId, rosterTokenTable.token, rosterTokenTable.userId) + }) + // Rewriting one member's tokens looks them up by user, which the primary key can't serve. + try writer.run(rosterTokenTable.table.createIndex(rosterTokenTable.conversationId, rosterTokenTable.userId, ifNotExists: true)) + try writer.run(rosterSyncTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterSyncTable.conversationId, primaryKey: true) + t.column(rosterSyncTable.watermark) + t.column(rosterSyncTable.observedVersion) + t.column(rosterSyncTable.memberCount) + t.column(rosterSyncTable.fullySynced) + t.column(rosterSyncTable.truncated) + t.column(rosterSyncTable.reconcilePending) + }) + } + let conversationMessageTable = ConversationMessageTable() try writer.transaction { diff --git a/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift new file mode 100644 index 000000000..cbaf0222a --- /dev/null +++ b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift @@ -0,0 +1,72 @@ +// +// RosterSearchTextTests.swift +// FlipcashCoreTests +// + +import Testing +import Foundation +@testable import FlipcashCore + +/// The text rules roster search indexes by. Android runs the same cases, so a change here is a +/// cross-platform change. +@Suite("Roster search text normalization") +struct RosterSearchTextTests { + + @Test("Lowercases and strips diacritics", arguments: [ + ("Érica", "erica"), + ("JOSÉ", "jose"), + ("Zoë", "zoe"), + ("Ångström", "angstrom"), + ("Nguyễn", "nguyen"), + ]) + func stripsDiacritics(input: String, expected: String) { + #expect(RosterSearchText.normalize(input) == expected) + } + + @Test("Compatibility forms fold to their plain letters") + func foldsCompatibilityForms() { + #expect(RosterSearchText.normalize("Ｅｒｉｃａ") == "erica") + #expect(RosterSearchText.normalize("fiona") == "fiona") + } + + @Test("Dotted capital I folds to a plain i") + func dottedCapitalI() { + #expect(RosterSearchText.normalize("İstanbul") == "istanbul") + } + + @Test("Spacing marks are kept, so Devanagari vowels survive") + func keepsSpacingMarks() { + // U+093F DEVANAGARI VOWEL SIGN I is category Mc. + #expect(RosterSearchText.normalize("कि") == "कि") + } + + @Test("Tokens are each display name word plus the username") + func tokens() { + let tokens = RosterSearchText.tokens(displayName: " Érica de la Cruz ", username: "ecruz") + #expect(tokens == ["erica", "de", "la", "cruz", "ecruz"]) + #expect(RosterSearchText.tokens(displayName: "Bo", username: "＠bo_b") == ["bo", "bo_b"]) + } + + @Test("Any whitespace or separator splits words", arguments: ["\u{00A0}", "\u{2028}", "\u{3000}", "\t"]) + func separatorsSplit(separator: String) { + #expect(RosterSearchText.words("Ana\(separator)Lima") == ["ana", "lima"]) + } + + @Test("A query drops a leading @ of either width") + func queryDropsAt() { + #expect(RosterSearchText.queryWords("@Eri") == ["eri"]) + #expect(RosterSearchText.queryWords("＠eri") == ["eri"]) + #expect(RosterSearchText.queryWords("@") == []) + #expect(RosterSearchText.queryWords("@@eri") == ["@eri"]) + #expect(RosterSearchText.queryWords("") == []) + } + + @Test("The prefix bound sorts above a name continued by an emoji") + func prefixBoundCoversAstralCharacters() { + let prefix = "bob" + let token = RosterSearchText.normalize("Bob🔥") + #expect(token >= prefix) + // SQLite compares UTF-8 bytes, so compare the encodings rather than Swift strings. + #expect(Array(token.utf8).lexicographicallyPrecedes(Array(RosterSearchText.prefixUpperBound(prefix).utf8))) + } +} diff --git a/FlipcashTests/Chat/RosterSearchTests.swift b/FlipcashTests/Chat/RosterSearchTests.swift new file mode 100644 index 000000000..309ee8192 --- /dev/null +++ b/FlipcashTests/Chat/RosterSearchTests.swift @@ -0,0 +1,568 @@ +// +// RosterSearchTests.swift +// FlipcashTests +// + +import Testing +import Foundation +import FlipcashCore +import FlipcashStore +import SQLite +@testable import Flipcash + +@Suite("Roster sync and search") +@MainActor +struct RosterSearchTests { + + // MARK: - Upgrade - + + @Test("A store from before the roster tables syncs a group on first open and searches it") + func upgradedStoreSyncsOnFirstOpen() async throws { + let url = try RosterStoreFixture.makeStoreWithoutRosterTables() + defer { Database.removeTemp(at: url) } + let database = try Database(url: url) + defer { try? database.close() } + + let members = [ + ConversationMember(userID: UUID(), displayName: "Érica Souza", version: 1), + ConversationMember(userID: UUID(), displayName: "Luis García", version: 1), + ] + let harness = Harness(database: database, members: members) + #expect(try database.rosterSyncState(conversationID: harness.chatID) == nil) + + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 1) + #expect(try harness.state().fullySynced) + #expect(try await harness.names("eri") == ["Érica Souza"]) + } + + // MARK: - Full read - + + @Test("A first open pages until has_more is false and persists every member") + func pagesToEnd() async throws { + let harness = try Harness(pages: 3, perPage: 100) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 3) + let state = try harness.state() + #expect(state.heldCount == 300) + #expect(state.fullySynced) + #expect(state.truncated == false) + #expect(state.watermark == 2) + #expect(state.needsFullRead == false) + } + + @Test("The page cap stops a read, and search uses what is held") + func capStopsPaging() async throws { + let harness = try Harness(pages: 5, perPage: 10, pageCap: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 2) + let state = try harness.state() + #expect(state.heldCount == 20) + #expect(state.truncated) + #expect(state.needsFullRead == false) + + #expect(try await harness.search.search(chatID: harness.chatID, query: "member", limit: 100).count == 20) + + await harness.sync.syncIfNeeded(harness.chatID) + #expect(harness.fetching.calls == 2) + } + + @Test("A synced group is not read again while the roster hasn't moved") + func noReadWhenCurrent() async throws { + let harness = try Harness(pages: 1, perPage: 5) + await harness.sync.syncIfNeeded(harness.chatID) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(harness.fetching.calls == 1) + } + + @Test("A failed page records nothing") + func failedPageRecordsNothing() async throws { + let harness = try Harness(pages: 2, perPage: 5) + harness.fetching.failingCalls = [2] + await harness.sync.syncIfNeeded(harness.chatID) + + let state = try harness.state() + #expect(state.heldCount == 0) + #expect(state.fullySynced == false) + } + + @Test("A read whose pages report different versions drops nothing and owes a reconcile") + func midReadVersionChange() async throws { + let harness = try Harness(members: [member("Ann"), member("Bea")], perPage: 1) + await harness.sync.syncIfNeeded(harness.chatID) + + let departed = harness.fetching.members[1] + harness.fetching.members = [harness.fetching.members[0], member("Cal")] + harness.fetching.pageVersions = [3, 4] + try harness.database.markRosterReconcilePending(harness.chatID) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try harness.heldIDs().contains(departed.userID!)) + #expect(try harness.state().reconcilePending) + } + + // MARK: - Catch-up - + + @Test("Missed joins are recovered from the first page, stopping at the watermark") + func catchUpRecoversJoins() async throws { + let harness = try Harness(pages: 3, perPage: 100) + await harness.sync.syncIfNeeded(harness.chatID) + + let missed = (0..<4).map { member("New \($0)", version: 5 + UInt64($0)) } + let streamed = member("Streamed", version: 9) + harness.fetching.members = [streamed] + missed.reversed() + harness.fetching.members + harness.fetching.version = 9 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(streamed, count: 305, version: 9)])) + + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 4) + let state = try harness.state() + #expect(state.heldCount == 305) + #expect(state.watermark == 9) + #expect(state.needsCatchUp == false) + } + + @Test("An equal count moves the watermark to the page's version, not the stream's") + func equalCountUsesPageVersion() async throws { + let ann = member("Ann") + let max = member("Max") + let harness = try Harness(members: [ann, max]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 10) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 9), change: .left(userID: max.userID!)), + joined(yan, count: 2, version: 10), + ])) + + // The page trails the stream: it is at version 8, before Max left and Yan joined. + let wes = member("Wes", version: 8) + harness.fetching.members = [wes, ann, max] + harness.fetching.version = 8 + await harness.sync.syncIfNeeded(harness.chatID) + + let state = try harness.state() + #expect(state.heldCount == 3) + #expect(state.watermark == 8) + #expect(try Set(harness.heldIDs()) == [ann.userID!, wes.userID!, yan.userID!]) + } + + @Test("More members held than counted owes a reconcile that survives a failed read") + func unseenLeaveSetsPending() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 4) + harness.fetching.members = [yan] + harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(yan, count: 3, version: 4)])) + + harness.fetching.failingCalls = [3] + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try harness.state().reconcilePending) + #expect(try harness.heldIDs().contains(cal.userID!)) + + harness.fetching.failingCalls = [] + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try harness.state().reconcilePending == false) + #expect(try harness.heldIDs().contains(cal.userID!) == false) + } + + @Test("An unseen leave runs one reconcile even when triggered twice") + func reconcileRunsOnce() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 4) + harness.fetching.members = [yan] + harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(yan, count: 3, version: 4)])) + + async let first: Void = harness.sync.syncIfNeeded(harness.chatID) + async let second: Void = harness.sync.syncIfNeeded(harness.chatID) + _ = await (first, second) + + // One for the initial read, one for the catch-up, one for the reconcile. + #expect(harness.fetching.calls == 3) + #expect(try harness.heldIDs().contains(cal.userID!) == false) + } + + @Test("A reconcile drops departed members but keeps one the stream added after the page") + func reconcileKeepsNewerMembers() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = member("Zed", version: 6) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 6)])) + + // Cal left at version 3, unseen; the pages are at version 4, before Zed joined. + harness.fetching.members = harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.syncIfNeeded(harness.chatID) + + let held = try harness.heldIDs() + #expect(held.contains(cal.userID!) == false) + #expect(held.contains(zed.userID!)) + #expect(try harness.state().reconcilePending == false) + } + + // MARK: - Stream - + + @Test("A stream join adds the member to the index and a leave removes them") + func streamJoinAndLeave() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = member("Zed Quill", version: 3) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 3)])) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").map(\.id) == [zed.userID!]) + + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 4), change: .left(userID: zed.userID!)), + ])) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").isEmpty) + } + + @Test("A page trailing the stream cannot bring back a member who left") + func trailingPageDoesNotResurrect() async throws { + let harness = try Harness(pages: 1, perPage: 2) + try harness.database.beginTrackingRoster(conversationID: harness.chatID) + let departed = harness.fetching.members[0] + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 3), change: .left(userID: departed.userID!)), + ])) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try harness.heldIDs().contains(departed.userID!) == false) + } + + @Test("A stale leave, older than the held join, changes nothing") + func staleLeaveIgnored() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = member("Zed Quill", version: 5) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 5)])) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 3), change: .left(userID: zed.userID!)), + ])) + + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").map(\.id) == [zed.userID!]) + #expect(try harness.state().heldCount == 3) + } + + @Test("A profile write after a leave doesn't bring the member back") + func profileAfterLeaveStaysOut() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + let departed = harness.fetching.members[0] + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 3), change: .left(userID: departed.userID!)), + ])) + + try harness.database.upsertUserProfile( + Profile(displayName: "Zed Quill", phone: Phone?.none, email: nil, username: Username("zedq")), + userID: departed.userID! + ) + + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").isEmpty) + #expect(try await harness.search.search(chatID: harness.chatID, query: "zedq").isEmpty) + #expect(try harness.state().heldCount == 1) + #expect(try harness.heldIDs().contains(departed.userID!) == false) + } + + // MARK: - Profiles - + + @Test("Opening the picker re-reads the first page and re-tokenizes a rename") + func refreshRenames() async throws { + let original = member("Old Name") + let harness = try Harness(members: [original]) + await harness.sync.syncIfNeeded(harness.chatID) + + harness.fetching.members = [ConversationMember(userID: original.userID, displayName: "New Name", version: original.version)] + await harness.search.prepare(chatID: harness.chatID) + + #expect(try await harness.names("new") == ["New Name"]) + #expect(try await harness.names("old") == []) + } + + @Test("A cached profile re-tokenizes the user in every group") + func profileRetokenizesEveryChat() async throws { + let shared = member("Old Name") + let harness = try Harness(members: [shared]) + let otherChat = ConversationID(data: Data(repeating: 0x08, count: 32)) + await harness.sync.syncIfNeeded(harness.chatID) + await harness.sync.syncIfNeeded(otherChat) + + try harness.database.upsertUserProfile( + Profile(displayName: "Nadia Renamed", phone: Phone?.none, email: nil, username: Username("nadia")), + userID: shared.userID! + ) + + for chat in [harness.chatID, otherChat] { + #expect(try await harness.search.search(chatID: chat, query: "ren").map(\.member.displayName) == ["Nadia Renamed"]) + #expect(try await harness.search.search(chatID: chat, query: "nadia").count == 1) + #expect(try await harness.search.search(chatID: chat, query: "old").isEmpty) + } + } + + // MARK: - Matching - + + @Test("A prefix matches any word of the display name, and the username") + func prefixMatching() async throws { + let harness = try Harness(members: [ + member("Ana María Lopez", username: "anita"), + member("Bob Stone", username: "rocky"), + member("María García"), + member("Luis García"), + ]) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try await harness.names("ana") == ["Ana María Lopez"]) + #expect(try await harness.names("lop") == ["Ana María Lopez"]) + #expect(try await harness.names("roc") == ["Bob Stone"]) + #expect(try await harness.names("＠roc") == ["Bob Stone"]) + #expect(try await harness.names("ana lo") == ["Ana María Lopez"]) + #expect(try await harness.names("ma garc") == ["María García"]) + #expect(try await harness.names("tone") == []) + } + + @Test("Matching ignores case and diacritics") + func caseAndDiacritics() async throws { + let harness = try Harness(members: [member("Érica Souza"), member("ERIK")]) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try await harness.names("eri") == ["Érica Souza", "ERIK"]) + #expect(try await harness.names("ÉRI").count == 2) + #expect(try await harness.names("souz") == ["Érica Souza"]) + } + + @Test("An emoji right after the prefix still matches") + func astralCharacterAfterPrefix() async throws { + let harness = try Harness(members: [member("Bob🔥")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("bob") == ["Bob🔥"]) + } + + // MARK: - Ranking - + + @Test("A recent speaker outranks an exact username, which outranks the rest by name") + func ranking() async throws { + let speaker = member("Sam Zulu", username: "samz") + let exact = member("Yara Sam", username: "sam") + let alphaA = member("Samantha Adams") + let alphaB = member("Samuel Brown") + let harness = try Harness(members: [alphaB, exact, alphaA, speaker]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: speaker.userID!) + + #expect(try await harness.names("sam") == ["Sam Zulu", "Yara Sam", "Samantha Adams", "Samuel Brown"]) + } + + @Test("The most recent speaker ranks above an earlier one") + func recencyOrder() async throws { + let first = member("Ann One") + let second = member("Ann Two") + let harness = try Harness(members: [first, second]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: first.userID!) + try harness.postMessage(from: second.userID!) + + #expect(try await harness.names("ann") == ["Ann Two", "Ann One"]) + } + + @Test("Only the newest 50 messages count toward recent speakers") + func recencyWindow() async throws { + let old = member("Ann Old") + let chatty = member("Zed Chatty") + let harness = try Harness(members: [old, chatty]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: old.userID!) + for _ in 0..<50 { try harness.postMessage(from: chatty.userID!) } + + #expect(try await harness.names("") == ["Zed Chatty"]) + } + + @Test("Folded name ties fall back to the raw display name") + func rawNameTiebreak() async throws { + let harness = try Harness(members: [member("éva"), member("Eva")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("eva") == ["Eva", "éva"]) + } + + @Test("Names sort by code point, so U+FFFD comes before an emoji") + func codePointOrder() async throws { + // UTF-16 order would put the emoji's surrogate (D83D) first; neither character decomposes. + let harness = try Harness(members: [member("Ann \u{1F600}"), member("Ann \u{FFFD}")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("ann") == ["Ann \u{FFFD}", "Ann \u{1F600}"]) + } + + @Test("An empty query returns recent speakers only") + func emptyQuery() async throws { + let speaker = member("Talker") + let harness = try Harness(members: [speaker, member("Quiet")]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: speaker.userID!) + + #expect(try await harness.names("") == ["Talker"]) + #expect(try await harness.names("＠") == ["Talker"]) + } + + @Test("The signed-in user is never a match, even after speaking") + func excludesSelf() async throws { + let me = member("Sam Me", username: "me") + let harness = try Harness(members: [me, member("Sam Other")], selfUserID: me.userID!) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: me.userID!) + + #expect(try await harness.names("sam") == ["Sam Other"]) + #expect(try await harness.names("") == []) + } + + @Test("A blocked user is never a match") + func excludesBlocked() async throws { + let blocked = member("Sam Blocked") + let harness = try Harness(members: [blocked, member("Sam Other")], blocked: [blocked.userID!]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: blocked.userID!) + + #expect(try await harness.names("sam") == ["Sam Other"]) + } +} + +// MARK: - Support - + +private func member(_ displayName: String, username: String? = nil, version: UInt64 = 1) -> ConversationMember { + ConversationMember(userID: UUID(), displayName: displayName, username: username.flatMap(Username.init), version: version) +} + +private func joined(_ member: ConversationMember, count: UInt64, version: UInt64) -> DecodedRosterUpdate { + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: count, version: version), change: .joined(member: member, chat: nil)) +} + +@MainActor +private struct Harness { + let database: Database + let fetching: FakeRosterFetching + let sync: RosterSync + let search: LocalRosterSearch + let chatID = ConversationID(data: Data(repeating: 0x07, count: 32)) + private let messageCounter = Counter() + + init(members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID(), blocked: Set = []) throws { + let (database, _) = try Database.makeTemp() + self.init(database: database, members: members, perPage: perPage, pageCap: pageCap, selfUserID: selfUserID, blocked: blocked) + } + + init(database: Database, members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID(), blocked: Set = []) { + self.database = database + self.fetching = FakeRosterFetching(members: members, perPage: perPage) + self.sync = RosterSync(fetching: fetching, database: database, owner: .generate()!, pageCap: pageCap) + self.search = LocalRosterSearch(database: database, roster: sync, selfUserID: selfUserID, blockedUserIDs: { blocked }) + } + + init(pages: Int, perPage: Int, pageCap: Int = RosterSync.defaultPageCap) throws { + let members = (0..<(pages * perPage)).map { ConversationMember(userID: UUID(), displayName: "Member \($0)", version: 1) } + try self.init(members: members, perPage: perPage, pageCap: pageCap) + } + + func state() throws -> RosterSyncState { + try #require(try database.rosterSyncState(conversationID: chatID)) + } + + func heldIDs() throws -> [UserID] { + try database.rosterEntries(conversationID: chatID).compactMap(\.member.userID) + } + + func names(_ query: String) async throws -> [String] { + try await search.search(chatID: chatID, query: query, limit: 10).map(\.member.displayName) + } + + func postMessage(from sender: UserID) throws { + let id = messageCounter.next() + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: id), senderID: sender, content: .text("hi"), date: .now, unreadSeq: id), + ], conversationID: chatID) + } +} + +private extension Database { + /// Stands in for a catch-up that found more members held than counted. + func markRosterReconcilePending(_ conversationID: ConversationID) throws { + let s = RosterSyncTable() + try writer.run(s.table.filter(s.conversationId == conversationID.data).update(s.reconcilePending <- true)) + } +} + +nonisolated private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var value: UInt64 = 0 + func next() -> UInt64 { lock.withLock { value += 1; return value } } +} + +/// Serves a roster in pages, most recently joined first, like `Chat.GetRoster`. +nonisolated private final class FakeRosterFetching: RosterFetching, @unchecked Sendable { + struct Failure: Error {} + + private let lock = NSLock() + private let perPage: Int + private var _members: [ConversationMember] + private var _version: UInt64 = 2 + private var _pageVersions: [UInt64]? + private var _failingCalls: Set = [] + private var _calls = 0 + + init(members: [ConversationMember], perPage: Int) { + self._members = members + self.perPage = perPage + } + + var members: [ConversationMember] { + get { lock.withLock { _members } } + set { lock.withLock { _members = newValue } } + } + + /// The roster version every page reports, unless ``pageVersions`` overrides it per page. + var version: UInt64 { + get { lock.withLock { _version } } + set { lock.withLock { _version = newValue } } + } + + var pageVersions: [UInt64]? { + get { lock.withLock { _pageVersions } } + set { lock.withLock { _pageVersions = newValue } } + } + + /// 1-based call numbers that throw instead of returning a page. + var failingCalls: Set { + get { lock.withLock { _failingCalls } } + set { lock.withLock { _failingCalls = newValue } } + } + + var calls: Int { lock.withLock { _calls } } + + func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> FlipClient.RosterPage { + try lock.withLock { + _calls += 1 + if _failingCalls.contains(_calls) { throw Failure() } + let start = pagingToken.map { Int(String(decoding: $0, as: UTF8.self))! } ?? 0 + let end = min(start + perPage, _members.count) + let next = end < _members.count ? Data(String(end).utf8) : nil + let pageIndex = start / perPage + let version = _pageVersions.flatMap { pageIndex < $0.count ? $0[pageIndex] : nil } ?? _version + return FlipClient.RosterPage( + members: Array(_members[start.. (StoreLocation.Files, URL) { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("roster-upgrade-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let files = StoreLocation(directory: root, legacyDirectory: root, isShared: true).files(owner: try PublicKey(Data(repeating: 9, count: 32))) + let database = try Database(url: files.database) + try RosterStoreFixture.seed(database) + try database.close() + try Database.setUserVersion(version: recordedVersion, files: files) + return (files, root) + } +} + +/// Builds stores shaped like the ones already on devices, which predate the roster tables. +enum RosterStoreFixture { + + struct RowCounts: Equatable { + let conversations: Int + let messages: Int + } + + /// A store with two conversations and three messages, and the roster tables dropped. + static func makeStoreWithoutRosterTables() throws -> URL { + let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("roster-v43-\(UUID().uuidString).sqlite") + let database = try Database(url: url) + try seed(database) + try database.close() + + let connection = try Connection(url.path) + for table in [RosterMemberTable.name, RosterTokenTable.name, RosterSyncTable.name] { + try connection.run("DROP TABLE \(table)") + } + return url + } + + static func seed(_ database: Database) throws { + for (index, id) in [ConversationID.test(1), ConversationID.test(2)].enumerated() { + try database.upsertConversation(Conversation(id: id, members: [], lastMessage: nil, lastActivity: Date(timeIntervalSince1970: Double(index)))) + } + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: 1), senderID: nil, content: .text("one"), date: Date(timeIntervalSince1970: 10), unreadSeq: 1), + ConversationMessage(id: MessageID(value: 2), senderID: nil, content: .text("two"), date: Date(timeIntervalSince1970: 20), unreadSeq: 2), + ], conversationID: .test(1)) + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: 3), senderID: nil, content: .text("three"), date: Date(timeIntervalSince1970: 30), unreadSeq: 1), + ], conversationID: .test(2)) + } + + static func rowCounts(at url: URL) throws -> RowCounts { + let connection = try Connection(url.path, readonly: true) + return RowCounts( + conversations: Int(try connection.scalar("SELECT count(*) FROM \(ConversationTable.name)") as! Int64), + messages: Int(try connection.scalar("SELECT count(*) FROM \(ConversationMessageTable.name)") as! Int64) + ) + } + + static func rosterRowCount(at url: URL) throws -> Int { + let connection = try Connection(url.path, readonly: true) + var total: Int64 = 0 + for table in [RosterMemberTable.name, RosterTokenTable.name, RosterSyncTable.name] { + total += try connection.scalar("SELECT count(*) FROM \(table)") as! Int64 + } + return Int(total) + } + + static func tableNames(at url: URL) throws -> Set { + let connection = try Connection(url.path, readonly: true) + return Set(try connection.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").compactMap { $0[0] as? String }) + } + + static func schemaSQL(at url: URL) throws -> [String] { + let connection = try Connection(url.path, readonly: true) + return try connection.prepare("SELECT sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name").compactMap { $0[0] as? String } + } +}