From 530dbc46813ecf3c10478c9ad1351dd7a4abc2a5 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Tue, 29 Sep 2026 22:23:34 -0400 Subject: [PATCH 1/7] feat(store): index group rosters for member search The mention picker needs to search a group's members as the user types, and the only roster held today is the few members chat metadata embeds. Add three tables, bump the schema to 44: - roster_member (conversationId, userId): the full roster. A leave keeps the row with isMember false and the leave's version, so a GetRoster page that trails the stream can't bring the member back. - roster_token (conversationId, token, userId): one row per normalized display-name word plus the username. The primary key leads with (conversationId, token), so a prefix lookup is a range scan. - roster_sync: per-group version, member count, last full sync, cap and gap state. A group with no row is untracked. RosterSearchText holds the normalization Android mirrors: lowercase, NFKD, drop general category M. The prefix upper bound is prefix + U+10FFFF rather than U+FFFF, because SQLite compares UTF-8 bytes and a name followed by an emoji sorts above prefix + U+FFFF. FlipClient.getRoster paged to the end in one call; replace it with getRosterPage so the caller can cap the loop, and raise the page size from 50 to 100. --- .../Clients/Flip API/FlipClient+Chat.swift | 41 +- .../Flip API/Services/ChatService.swift | 2 +- .../Conversation/RosterSearchText.swift | 66 ++++ .../Database+Conversations.swift | 4 +- .../FlipcashStore/Database+Roster.swift | 355 ++++++++++++++++++ .../Sources/FlipcashStore/Database.swift | 2 +- .../Sources/FlipcashStore/Schema.swift | 97 +++++ .../RosterSearchTextTests.swift | 59 +++ 8 files changed, 603 insertions(+), 23 deletions(-) create mode 100644 FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift create mode 100644 FlipcashCore/Sources/FlipcashStore/Database+Roster.swift create mode 100644 FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift index 66e517cf0..3def0a2bb 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift @@ -98,7 +98,7 @@ extension FlipClient { } /// Represents one paged read of a chat's roster: the members returned, most recently joined - /// first, and the roster summary as of that page's read. + /// first, the roster summary as of that page's read, and how to continue. /// /// This is a single pinned read, not a merged view — a large group's roster can page from an /// index that trails the event stream (see `chat.v1.GetRoster`'s staleness contract), so the @@ -108,28 +108,29 @@ extension FlipClient { public struct RosterPage: Sendable { public let members: [ConversationMember] public let rosterSummary: ConversationRosterSummary - } - - /// Pages a chat's roster to exhaustion against a single pinned read. The caller must already be - /// consuming its `subscribeConversationStream` events — as with `getDmChatFeed`/ - /// `getGroupChatFeed` — and must merge the result against them by ``ConversationMember/version`` - /// rather than treating it as an authoritative snapshot; see ``RosterPage``. - public func getRoster(owner: KeyPair, conversationID: ConversationID) async throws -> RosterPage { - var members: [ConversationMember] = [] - var rosterSummary = ConversationRosterSummary(memberCount: 0, version: 0) - var pagingToken: Data? + /// The token to pass for the next page; `nil` on the last page. + public let nextPagingToken: Data? - while true { - let page = try await withCheckedThrowingContinuation { c in - chatService.getRoster(owner: owner, conversationID: conversationID, pagingToken: pagingToken) { c.resume(with: $0) } - } - members.append(contentsOf: page.members) - rosterSummary = page.rosterSummary - if !page.hasMore { break } - pagingToken = page.pagingToken + public init(members: [ConversationMember], rosterSummary: ConversationRosterSummary, nextPagingToken: Data?) { + self.members = members + self.rosterSummary = rosterSummary + self.nextPagingToken = nextPagingToken } + } - return RosterPage(members: members, rosterSummary: rosterSummary) + /// Fetches one page of a chat's roster, up to 100 members. Pass `nil` for the first page and the + /// previous page's ``RosterPage/nextPagingToken`` after that. The caller must already be consuming + /// its `subscribeConversationStream` events and must merge the result against them by + /// ``ConversationMember/version``; see ``RosterPage``. + public func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> RosterPage { + let page = try await withCheckedThrowingContinuation { c in + chatService.getRoster(owner: owner, conversationID: conversationID, pagingToken: pagingToken) { c.resume(with: $0) } + } + return RosterPage( + members: page.members, + rosterSummary: page.rosterSummary, + nextPagingToken: page.hasMore ? page.pagingToken : nil + ) } /// Edits a group chat's title and/or picture; every parameter left `nil` leaves that field diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift index 8baf36108..9958e2877 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatService.swift @@ -287,7 +287,7 @@ final class ChatService: Sendable { /// staleness contract) — callers must merge each page against what the event stream has already /// told them, by ``ConversationMember/version``, greater winning, rather than trusting a page to /// be a complete, current snapshot. - func getRoster(owner: KeyPair, conversationID: ConversationID, pageSize: Int = 50, pagingToken: Data?, completion: @Sendable @escaping (Result) -> Void) { + func getRoster(owner: KeyPair, conversationID: ConversationID, pageSize: Int = 100, pagingToken: Data?, completion: @Sendable @escaping (Result) -> Void) { let request = Flipcash_Chat_V1_GetRosterRequest.with { $0.chatID = conversationID.proto $0.queryOptions = .with { diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift new file mode 100644 index 000000000..3c6cc81e0 --- /dev/null +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift @@ -0,0 +1,66 @@ +// +// RosterSearchText.swift +// FlipcashCore +// + +import Foundation + +/// The text rules roster search indexes and queries by. Android applies the same rules, so a query +/// matches the same members on both platforms. +public enum RosterSearchText { + + /// Returns `text` lowercased, NFKD-decomposed, and stripped of every combining mark + /// (general category `M`: `Mn`, `Mc`, `Me`), so "Érica" and "erica" compare equal. + public static func normalize(_ text: String) -> String { + let decomposed = text.lowercased().decomposedStringWithCompatibilityMapping + var scalars = String.UnicodeScalarView() + for scalar in decomposed.unicodeScalars where !isMark(scalar) { + scalars.append(scalar) + } + return String(scalars) + } + + /// Returns the normalized words of `text`, split on runs of Unicode `White_Space`, empty words dropped. + public static func words(_ text: String) -> [String] { + normalize(text) + .unicodeScalars + .split(whereSeparator: \.properties.isWhitespace) + .map { String(String.UnicodeScalarView($0)) } + } + + /// Returns the distinct search tokens for a member: each word of the display name, plus the + /// username without its `@`. + public static func tokens(displayName: String, username: String?) -> Set { + var tokens = Set(words(displayName)) + if let username { + tokens.formUnion(words(username)) + } + return tokens + } + + /// Returns the words of a typed query, with a leading `@` (either width) dropped from each word. + public static func queryWords(_ query: String) -> [String] { + // NFKD folds the fullwidth "＠" to "@", so one check covers both. + words(query) + .map { $0.hasPrefix("@") ? String($0.drop(while: { $0 == "@" })) : $0 } + .filter { !$0.isEmpty } + } + + /// Returns the exclusive upper bound of a prefix range, so `token >= prefix AND token < bound` + /// matches every token starting with `prefix`. + /// + /// U+10FFFF rather than U+FFFF: SQLite compares UTF-8 bytes, and a token whose next character is + /// outside the BMP (an emoji after a name, say) sorts above `prefix + U+FFFF`. + public static func prefixUpperBound(_ prefix: String) -> String { + prefix + "\u{10FFFF}" + } + + private static func isMark(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.properties.generalCategory { + case .nonspacingMark, .spacingMark, .enclosingMark: + return true + default: + return false + } + } +} diff --git a/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift b/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift index d6f051cea..c2f7efe1a 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database+Conversations.swift @@ -334,6 +334,7 @@ nonisolated extension Database { try writer.run(c.table.filter(doomed.contains(c.id)).delete()) try writer.run(m.table.filter(doomed.contains(m.conversationId)).delete()) try writer.run(g.table.filter(doomed.contains(g.conversationId)).delete()) + try deleteRosterRows(conversationIDs: doomed) } for group in groups where group.type == .group { try writeConversation(group) @@ -409,7 +410,7 @@ nonisolated extension Database { } /// Removes a conversation the signed-in user has left (or been removed from): its row and member - /// rows. Messages are left in place, orphaned but unread — the same treatment + /// rows and any synced roster. Messages are left in place, orphaned but unread — the same treatment /// `replaceConversationFeed` gives a conversation that drops out of a feed snapshot. public func deleteConversation(conversationID: ConversationID) throws { let c = ConversationTable() @@ -417,6 +418,7 @@ nonisolated extension Database { try writer.transaction { try writer.run(c.table.filter(c.id == conversationID.data).delete()) try writer.run(m.table.filter(m.conversationId == conversationID.data).delete()) + try deleteRosterRows(conversationIDs: [conversationID.data]) } } diff --git a/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift new file mode 100644 index 000000000..81229139b --- /dev/null +++ b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift @@ -0,0 +1,355 @@ +// +// Database+Roster.swift +// FlipcashStore +// + +import Foundation +import FlipcashCore +import SQLite + +/// Where a group's locally held roster stands against the server's. +public struct RosterSyncState: Sendable, Equatable { + /// The greatest roster version applied, from a sync or the stream. + public let version: UInt64 + /// The group's true member count, as the latest roster summary reported it. + public let memberCount: UInt64 + /// How many current members are held locally. + public let heldCount: Int + /// When the last full sync finished, or `nil` if none has. + public let syncedAt: Date? + /// Whether the last full sync stopped at the page cap. + public let isCapped: Bool + /// The stream version that revealed a skipped version, or `nil` when the stream has been contiguous. + public let gapVersion: UInt64? + + /// Whether the group's roster should be paged again from the start. + public var needsFullSync: Bool { + syncedAt == nil || gapVersion != nil || (!isCapped && UInt64(heldCount) < memberCount) + } +} + +/// A current roster member as stored, with the key roster search sorts it by. +public struct RosterEntry: Sendable, Equatable { + public let member: ConversationMember + /// ``RosterSearchText/normalize(_:)`` of the display name. + public let sortKey: String +} + +nonisolated extension Database { + + // MARK: - Sync state - + + /// Returns the roster sync state for a group, or `nil` when its roster is not tracked yet. + public func rosterSyncState(conversationID: ConversationID) throws -> RosterSyncState? { + let s = RosterSyncTable() + let r = RosterMemberTable() + let c = ConversationTable() + guard let row = try reader.pluck(s.table.filter(s.conversationId == conversationID.data)) else { + return nil + } + let held = try reader.scalar(r.table.filter(r.conversationId == conversationID.data && r.isMember == true).count) + let feedCount = try reader.pluck(c.table.select(c.rosterMemberCount).filter(c.id == conversationID.data))?[c.rosterMemberCount] ?? 0 + return RosterSyncState( + version: row[s.version], + memberCount: max(row[s.memberCount], feedCount), + heldCount: held, + syncedAt: row[s.syncedAt].map(Date.init(timeIntervalSinceReferenceDate:)), + isCapped: row[s.isCapped], + gapVersion: row[s.gapVersion] + ) + } + + /// Starts tracking a group's roster so stream updates are written from now on. Seeds the version + /// and count from the cached chat row; a no-op when the group is already tracked. + public func beginTrackingRoster(conversationID: ConversationID) throws { + let s = RosterSyncTable() + let c = ConversationTable() + try writer.transaction { + let chat = try writer.pluck(c.table.select(c.rosterVersion, c.rosterMemberCount).filter(c.id == conversationID.data)) + try writer.run(s.table.insert( + or: .ignore, + s.conversationId <- conversationID.data, + s.version <- chat?[c.rosterVersion] ?? 0, + s.memberCount <- chat?[c.rosterMemberCount] ?? 0, + s.syncedAt <- nil, + s.isCapped <- false, + s.gapVersion <- nil + )) + } + } + + // MARK: - Writes - + + /// Merges a paged roster read into a tracked group's roster. + /// + /// Per member, the greater version wins against what the stream already wrote. When `isComplete`, + /// a held row the snapshot lacks is dropped unless its version is newer than `summary.version`, + /// the version the snapshot is at least as fresh as. When the read stopped early, absent rows are + /// kept, because they may be on the pages not fetched. + public func mergeRosterSnapshot( + _ members: [ConversationMember], + summary: ConversationRosterSummary, + isComplete: Bool, + conversationID: ConversationID, + now: Date = .now + ) throws { + let s = RosterSyncTable() + let r = RosterMemberTable() + try writer.transaction { + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + var held = try heldVersions(conversationID: conversationID) + + for member in members { + guard let userID = member.userID else { continue } + if let existing = held[userID], existing > member.version { continue } + try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) + held[userID] = member.version + } + + if isComplete { + let snapshotIDs = Set(members.compactMap(\.userID)) + for (userID, version) in held where !snapshotIDs.contains(userID) && version <= summary.version { + try deleteRosterMember(userID: userID, conversationID: conversationID) + } + } + + let version = sync[s.version] + let gap = sync[s.gapVersion].flatMap { $0 > summary.version + 1 ? $0 : nil } + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.version <- max(version, summary.version), + s.memberCount <- summary.version >= version ? summary.memberCount : sync[s.memberCount], + s.syncedAt <- now.timeIntervalSinceReferenceDate, + s.isCapped <- !isComplete, + s.gapVersion <- gap + )) + } + } + + /// Rewrites the names and pictures of members a fresh roster page carries, for a tracked group. + /// A row the stream has since moved past is left alone. Nothing is removed. + public func refreshRosterMembers(_ members: [ConversationMember], conversationID: ConversationID) throws { + let s = RosterSyncTable() + try writer.transaction { + guard try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) != nil else { return } + let held = try heldVersions(conversationID: conversationID) + for member in members { + guard let userID = member.userID else { continue } + if let existing = held[userID], existing > member.version { continue } + try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) + } + } + } + + /// Applies live roster updates to a tracked group, returning `false` when the group is not tracked. + /// + /// Each update wins against the member's row only with a greater version. An update more than one + /// past the greatest applied version records a gap, which ``RosterSyncState/needsFullSync`` reports. + @discardableResult + public func applyRosterUpdates(_ updates: [DecodedRosterUpdate], conversationID: ConversationID) throws -> Bool { + let s = RosterSyncTable() + var isTracked = false + try writer.transaction { + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + isTracked = true + var version = sync[s.version] + var memberCount = sync[s.memberCount] + var gap = sync[s.gapVersion] + let held = try heldVersions(conversationID: conversationID) + + for update in updates.sorted(by: { $0.rosterSummary.version < $1.rosterSummary.version }) { + let updateVersion = update.rosterSummary.version + if updateVersion > version + 1 { + gap = max(gap ?? 0, updateVersion) + } + if updateVersion > version { + version = updateVersion + memberCount = update.rosterSummary.memberCount + } + + switch update.change { + case .joined(let member, _): + guard let userID = member.userID else { continue } + if let existing = held[userID], existing >= updateVersion { continue } + try writeRosterMember(member, userID: userID, version: updateVersion, conversationID: conversationID) + case .left(let userID): + if let existing = held[userID], existing >= updateVersion { continue } + try writeRosterDeparture(userID: userID, version: updateVersion, conversationID: conversationID) + } + } + + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.version <- version, + s.memberCount <- memberCount, + s.gapVersion <- gap + )) + } + return isTracked + } + + /// Removes everything held for a group's roster, tracking included. + public func deleteRoster(conversationID: ConversationID) throws { + try writer.transaction { + try deleteRosterRows(conversationIDs: [conversationID.data]) + } + } + + // MARK: - Reads - + + /// Returns the ids of current members with, for every word in `prefixes`, a token starting with it. + /// Each word is one index range scan over `(conversationId, token)`. + public func rosterMemberIDs(matchingPrefixes prefixes: [String], conversationID: ConversationID) throws -> Set { + let t = RosterTokenTable() + var matched: Set? + for prefix in prefixes { + let query = t.table + .select(distinct: t.userId) + .filter(t.conversationId == conversationID.data + && t.token >= prefix + && t.token < RosterSearchText.prefixUpperBound(prefix)) + let ids = Set(try reader.prepareRowIterator(query).map { $0[t.userId] }) + matched = matched.map { $0.intersection(ids) } ?? ids + if matched?.isEmpty == true { break } + } + return matched ?? [] + } + + /// Returns the current members of a group, limited to `userIDs` when given. + public func rosterEntries(conversationID: ConversationID, userIDs: Set? = nil) throws -> [RosterEntry] { + let r = RosterMemberTable() + var query = r.table.filter(r.conversationId == conversationID.data && r.isMember == true) + if let userIDs { + query = query.filter(userIDs.contains(r.userId)) + } + return try reader.prepareRowIterator(query).map { row in + RosterEntry( + member: ConversationMember( + userID: row[r.userId], + displayName: row[r.displayName], + profilePicture: rosterProfilePicture(from: row), + username: row[r.username].flatMap(Username.init), + joinedAt: row[r.joinedAt].map(Date.init(timeIntervalSinceReferenceDate:)), + version: row[r.version] + ), + sortKey: row[r.sortKey] + ) + } + } + + /// Returns who sent the newest `window` held messages of a conversation, most recent sender first, + /// each once. + public func recentSenders(conversationID: ConversationID, window: Int) throws -> [UserID] { + let m = ConversationMessageTable() + let query = m.table + .select(m.senderId) + .filter(m.conversationId == conversationID.data) + .order(m.id.desc) + .limit(window) + var seen: Set = [] + var senders: [UserID] = [] + for row in try Array(reader.prepareRowIterator(query)) { + guard let sender = row[m.senderId], seen.insert(sender).inserted else { continue } + senders.append(sender) + } + return senders + } + + // MARK: - Private - + + /// Must be called inside a `writer.transaction`. Deletes every roster row for the given chats. + func deleteRosterRows(conversationIDs: [Data]) throws { + guard !conversationIDs.isEmpty else { return } + let r = RosterMemberTable() + let t = RosterTokenTable() + let s = RosterSyncTable() + try writer.run(r.table.filter(conversationIDs.contains(r.conversationId)).delete()) + try writer.run(t.table.filter(conversationIDs.contains(t.conversationId)).delete()) + try writer.run(s.table.filter(conversationIDs.contains(s.conversationId)).delete()) + } + + /// Must be called inside a `writer.transaction`. Every held row's version, departures included. + private func heldVersions(conversationID: ConversationID) throws -> [UserID: UInt64] { + let r = RosterMemberTable() + let query = r.table.select(r.userId, r.version).filter(r.conversationId == conversationID.data) + var versions: [UserID: UInt64] = [:] + for row in try Array(writer.prepareRowIterator(query)) { + versions[row[r.userId]] = row[r.version] + } + return versions + } + + /// Must be called inside a `writer.transaction`. + private func writeRosterMember(_ member: ConversationMember, userID: UserID, version: UInt64, conversationID: ConversationID) throws { + let r = RosterMemberTable() + let t = RosterTokenTable() + try writer.run(r.table.insert( + or: .replace, + r.conversationId <- conversationID.data, + r.userId <- userID, + r.displayName <- member.displayName, + r.username <- member.username?.value, + r.profilePictureBlobID <- member.profilePicture?.blobID.data, + r.profilePictureThumbnailBlobID <- member.profilePicture?.thumbnailBlobID.data, + r.profilePictureThumbnailBlurhash <- member.profilePicture?.thumbnailBlurhash, + r.joinedAt <- member.joinedAt?.timeIntervalSinceReferenceDate, + r.version <- version, + r.isMember <- true, + r.sortKey <- RosterSearchText.normalize(member.displayName) + )) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + for token in RosterSearchText.tokens(displayName: member.displayName, username: member.username?.value) { + try writer.run(t.table.insert( + or: .ignore, + t.conversationId <- conversationID.data, + t.token <- token, + t.userId <- userID + )) + } + } + + /// Must be called inside a `writer.transaction`. Keeps a departed member's row, so a trailing + /// roster page can't bring them back, and drops them from the index. + private func writeRosterDeparture(userID: UserID, version: UInt64, conversationID: ConversationID) throws { + let r = RosterMemberTable() + try writer.run(r.table.insert( + or: .replace, + r.conversationId <- conversationID.data, + r.userId <- userID, + r.displayName <- "", + r.username <- nil, + r.profilePictureBlobID <- nil, + r.profilePictureThumbnailBlobID <- nil, + r.profilePictureThumbnailBlurhash <- nil, + r.joinedAt <- nil, + r.version <- version, + r.isMember <- false, + r.sortKey <- "" + )) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + } + + /// Must be called inside a `writer.transaction`. + private func deleteRosterMember(userID: UserID, conversationID: ConversationID) throws { + let r = RosterMemberTable() + try writer.run(r.table.filter(r.conversationId == conversationID.data && r.userId == userID).delete()) + try deleteRosterTokens(userID: userID, conversationID: conversationID) + } + + /// Must be called inside a `writer.transaction`. + private func deleteRosterTokens(userID: UserID, conversationID: ConversationID) throws { + let t = RosterTokenTable() + try writer.run(t.table.filter(t.conversationId == conversationID.data && t.userId == userID).delete()) + } + + private func rosterProfilePicture(from row: RowIterator.Element) -> ProfilePicture? { + let r = RosterMemberTable() + guard let blobID = row[r.profilePictureBlobID], + let thumbnailBlobID = row[r.profilePictureThumbnailBlobID] else { + return nil + } + return ProfilePicture( + blobID: BlobID(data: blobID), + thumbnailBlobID: BlobID(data: thumbnailBlobID), + thumbnailBlurhash: row[r.profilePictureThumbnailBlurhash] + ) + } +} diff --git a/FlipcashCore/Sources/FlipcashStore/Database.swift b/FlipcashCore/Sources/FlipcashStore/Database.swift index d1d35bba8..7c0ae939f 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database.swift @@ -197,7 +197,7 @@ nonisolated open class Database: @unchecked Sendable { /// the notification service extension needs the same number to decide whether the store on disk /// is one it understands, and an extension cannot read the app's `Info.plist` — separate bundles. /// Both targets link this module, so they cannot disagree. - public static let schemaVersion = 43 + public static let schemaVersion = 44 /// Removes the store and the write-ahead log files beside it. /// diff --git a/FlipcashCore/Sources/FlipcashStore/Schema.swift b/FlipcashCore/Sources/FlipcashStore/Schema.swift index b31225e64..4a10fe6f7 100644 --- a/FlipcashCore/Sources/FlipcashStore/Schema.swift +++ b/FlipcashCore/Sources/FlipcashStore/Schema.swift @@ -324,6 +324,66 @@ nonisolated public struct ConversationMemberTable: Sendable { public let username = Expression ("username") } +// A group's full roster, as `RosterSync` pages it from `Chat.GetRoster` and keeps it current from the +// stream. Its own table rather than `conversation_member`, which holds only the members the chat +// metadata embeds and is replaced wholesale on every metadata write. +// +// A leave keeps the row with `isMember` false and the leave's version, so a roster page that trails +// the stream cannot bring the member back: per user, the row with the greater `version` wins. +nonisolated public struct RosterMemberTable: Sendable { + public static let name = "roster_member" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + public let userId = Expression ("userId") + public let displayName = Expression ("displayName") + public let username = Expression ("username") + public let profilePictureBlobID = Expression ("profilePictureBlobID") + public let profilePictureThumbnailBlobID = Expression ("profilePictureThumbnailBlobID") + public let profilePictureThumbnailBlurhash = Expression ("profilePictureThumbnailBlurhash") + public let joinedAt = Expression ("joinedAt") + // The roster version of the member's latest join, or of their leave when `isMember` is false. + public let version = Expression ("version") + public let isMember = Expression ("isMember") + // `RosterSearchText.normalize(displayName)`, the alphabetical sort key. + public let sortKey = Expression ("sortKey") +} + +// The roster search index: one row per normalized word of a member's display name, plus their +// username. The primary key leads with `(conversationId, token)`, so a prefix lookup is a range scan. +nonisolated public struct RosterTokenTable: Sendable { + public static let name = "roster_token" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + public let token = Expression ("token") + public let userId = Expression ("userId") +} + +// Per-group roster sync bookkeeping. A group with no row is not tracked: its stream roster updates +// are not written to `roster_member` until the chat is first opened. +nonisolated public struct RosterSyncTable: Sendable { + public static let name = "roster_sync" + + public init() {} + + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + // The greatest roster version applied, from a sync or the stream. + public let version = Expression ("version") + public let memberCount = Expression ("memberCount") + // When the last full sync finished; `nil` until one has. + public let syncedAt = Expression ("syncedAt") + // Whether the last full sync stopped at the page cap before `has_more` went false. + public let isCapped = Expression ("isCapped") + // The greatest stream version that arrived after a skipped version; set means a re-sync is due. + public let gapVersion = Expression ("gapVersion") +} + // One row per message; cash content is decomposed across the amount columns // the same way `activity` stores ExchangedFiat. nonisolated public struct ConversationMessageTable: Sendable { @@ -620,6 +680,43 @@ nonisolated extension Database { }) } + let rosterMemberTable = RosterMemberTable() + let rosterTokenTable = RosterTokenTable() + let rosterSyncTable = RosterSyncTable() + + try writer.transaction { + try writer.run(rosterMemberTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterMemberTable.conversationId) + t.column(rosterMemberTable.userId) + t.column(rosterMemberTable.displayName) + t.column(rosterMemberTable.username) + t.column(rosterMemberTable.profilePictureBlobID) + t.column(rosterMemberTable.profilePictureThumbnailBlobID) + t.column(rosterMemberTable.profilePictureThumbnailBlurhash) + t.column(rosterMemberTable.joinedAt) + t.column(rosterMemberTable.version) + t.column(rosterMemberTable.isMember) + t.column(rosterMemberTable.sortKey) + t.primaryKey(rosterMemberTable.conversationId, rosterMemberTable.userId) + }) + try writer.run(rosterTokenTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterTokenTable.conversationId) + t.column(rosterTokenTable.token) + t.column(rosterTokenTable.userId) + t.primaryKey(rosterTokenTable.conversationId, rosterTokenTable.token, rosterTokenTable.userId) + }) + // Rewriting one member's tokens looks them up by user, which the primary key can't serve. + try writer.run(rosterTokenTable.table.createIndex(rosterTokenTable.conversationId, rosterTokenTable.userId, ifNotExists: true)) + try writer.run(rosterSyncTable.table.create(ifNotExists: true, withoutRowid: true) { t in + t.column(rosterSyncTable.conversationId, primaryKey: true) + t.column(rosterSyncTable.version) + t.column(rosterSyncTable.memberCount) + t.column(rosterSyncTable.syncedAt) + t.column(rosterSyncTable.isCapped) + t.column(rosterSyncTable.gapVersion) + }) + } + let conversationMessageTable = ConversationMessageTable() try writer.transaction { diff --git a/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift new file mode 100644 index 000000000..2d2bf430c --- /dev/null +++ b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift @@ -0,0 +1,59 @@ +// +// RosterSearchTextTests.swift +// FlipcashCoreTests +// + +import Testing +import Foundation +@testable import FlipcashCore + +/// The text rules roster search indexes by. Android runs the same cases, so a change here is a +/// cross-platform change. +@Suite("Roster search text normalization") +struct RosterSearchTextTests { + + @Test("Lowercases and strips diacritics", arguments: [ + ("Érica", "erica"), + ("JOSÉ", "jose"), + ("Zoë", "zoe"), + ("Ångström", "angstrom"), + ("Nguyễn", "nguyen"), + ]) + func stripsDiacritics(input: String, expected: String) { + #expect(RosterSearchText.normalize(input) == expected) + } + + @Test("Compatibility forms fold to their plain letters") + func foldsCompatibilityForms() { + #expect(RosterSearchText.normalize("Ｅｒｉｃａ") == "erica") + #expect(RosterSearchText.normalize("fiona") == "fiona") + } + + @Test("Tokens are each display name word plus the username") + func tokens() { + let tokens = RosterSearchText.tokens(displayName: " Érica de la Cruz ", username: "ecruz") + #expect(tokens == ["erica", "de", "la", "cruz", "ecruz"]) + } + + @Test("A no-break space splits words like a space") + func noBreakSpaceSplits() { + #expect(RosterSearchText.words("Ana\u{00A0}Lima") == ["ana", "lima"]) + } + + @Test("A query drops a leading @ of either width") + func queryDropsAt() { + #expect(RosterSearchText.queryWords("@Eri") == ["eri"]) + #expect(RosterSearchText.queryWords("＠eri") == ["eri"]) + #expect(RosterSearchText.queryWords("@") == []) + #expect(RosterSearchText.queryWords("") == []) + } + + @Test("The prefix bound sorts above a name continued by an emoji") + func prefixBoundCoversAstralCharacters() { + let prefix = "bob" + let token = RosterSearchText.normalize("Bob🔥") + #expect(token >= prefix) + // SQLite compares UTF-8 bytes, so compare the encodings rather than Swift strings. + #expect(Array(token.utf8).lexicographicallyPrecedes(Array(RosterSearchText.prefixUpperBound(prefix).utf8))) + } +} From 8f01838bb4525fc0515b943676fbc52029c03474 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Tue, 29 Sep 2026 22:23:35 -0400 Subject: [PATCH 2/7] feat(chat): sync opened group rosters and search them locally RosterSync pages Chat.GetRoster into the store when a group is opened and it has never been fully synced, holds fewer members than member_count, or the stream skipped a roster version. It stops after 20 pages (2,000 members). Groups are not synced at launch; stream roster updates are written only for groups already opened. LocalRosterSearch sits behind RosterSearchSource so a server search can replace it later. A member matches when every query word prefixes one of their tokens. Ranking: senders of the chat's newest 200 held messages, most recent first; then an exact username match; then by normalized display name. The signed-in user is excluded, and an empty query returns recent speakers. prepare(chatID:) refreshes the first roster page, since profile edits don't bump the roster version. --- .../Controllers/ConversationController.swift | 5 + .../Controllers/FlipClient+Protocols.swift | 8 +- Flipcash/Core/Controllers/RosterSearch.swift | 102 +++++++ Flipcash/Core/Controllers/RosterSync.swift | 134 +++++++++ .../Conversation/ConversationScreen.swift | 6 + .../Core/Session/SessionAuthenticator.swift | 5 + FlipcashTests/Chat/RosterSearchTests.swift | 275 ++++++++++++++++++ 7 files changed, 534 insertions(+), 1 deletion(-) create mode 100644 Flipcash/Core/Controllers/RosterSearch.swift create mode 100644 Flipcash/Core/Controllers/RosterSync.swift create mode 100644 FlipcashTests/Chat/RosterSearchTests.swift diff --git a/Flipcash/Core/Controllers/ConversationController.swift b/Flipcash/Core/Controllers/ConversationController.swift index f0fcc5671..c2d9d58a1 100644 --- a/Flipcash/Core/Controllers/ConversationController.swift +++ b/Flipcash/Core/Controllers/ConversationController.swift @@ -261,6 +261,10 @@ final class ConversationController { /// The emoji reactions concern: the user's taps, their calls, and the stream's updates. let reactions: ConversationReactions + /// Keeps opened groups' full rosters on device from the stream's roster updates. Set by the + /// session after init; `nil` leaves roster updates unpersisted. + @ObservationIgnored var roster: RosterSync? + init( fetching: any ConversationFetching, membership: any ConversationMembership, @@ -408,6 +412,7 @@ final class ConversationController { self.logCounterpartRead(event) self.applyTyping(event) self.applyReactions(event) + self.roster?.apply(event) if case .needsCatchUp(let conversationID, _) = gap { self.scheduleGapCatchUp(conversationID) } diff --git a/Flipcash/Core/Controllers/FlipClient+Protocols.swift b/Flipcash/Core/Controllers/FlipClient+Protocols.swift index 915f94229..454c51db0 100644 --- a/Flipcash/Core/Controllers/FlipClient+Protocols.swift +++ b/Flipcash/Core/Controllers/FlipClient+Protocols.swift @@ -67,6 +67,12 @@ protocol ConversationFetching: AnyObject, Sendable { func getChat(owner: KeyPair, conversationID: ConversationID) async throws -> Conversation } +/// Roster read surface used by `RosterSync`. Maps 1:1 to `flipcash.chat.v1.Chat.GetRoster`. +protocol RosterFetching: AnyObject, Sendable { + /// One page of a chat's roster, most recently joined first; see ``FlipClient/RosterPage``. + func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> FlipClient.RosterPage +} + /// Group membership surface used by `ConversationController`. Separate from ``ConversationFetching`` /// because these write: they are the only chat RPCs that change what the caller is a member of. protocol ConversationMembership: AnyObject, Sendable { @@ -166,5 +172,5 @@ extension FlipClient { } extension FlipClient: ContactVerifying, OnrampAuthorizing, ContactSyncing, - ConversationFetching, ConversationMembership, + ConversationFetching, ConversationMembership, RosterFetching, ConversationViewerSettings, ConversationEventStreaming {} diff --git a/Flipcash/Core/Controllers/RosterSearch.swift b/Flipcash/Core/Controllers/RosterSearch.swift new file mode 100644 index 000000000..f5f7aab98 --- /dev/null +++ b/Flipcash/Core/Controllers/RosterSearch.swift @@ -0,0 +1,102 @@ +// +// RosterSearch.swift +// Flipcash +// + +import Foundation +import FlipcashCore +import FlipcashStore + +/// A group member a roster search matched. +nonisolated struct MemberMatch: Identifiable, Hashable, Sendable { + let member: ConversationMember + let id: UserID +} + +/// Searches a group's members by name or username, for the mention picker. +/// +/// Callers depend on this rather than on ``LocalRosterSearch`` so a server-side search can replace +/// or back the local one without them changing. +protocol RosterSearchSource: AnyObject { + /// Readies the source to answer for a group; called when the picker opens. + func prepare(chatID: ConversationID) async + + /// Returns up to `limit` members of a group matching `query`, best match first, never the + /// signed-in user. An empty query returns the members who spoke most recently. + func search(chatID: ConversationID, query: String, limit: Int) async throws -> [MemberMatch] +} + +/// Answers roster searches from the roster ``RosterSync`` holds on device. +/// +/// A member matches when every word of the query is a prefix of one of their tokens, the words of +/// their display name and their username, compared after ``RosterSearchText/normalize(_:)``. Matches +/// rank in three tiers: +/// 1. members who sent one of the chat's newest held messages, most recent first; +/// 2. the member whose username is exactly the query; +/// 3. everyone else, by normalized display name in code point order. +/// Ties fall to the lowercase user id. +final class LocalRosterSearch: RosterSearchSource { + + /// How many of the chat's newest held messages count toward "spoke recently". + static let recentMessageWindow = 200 + + private let database: Database + private let roster: RosterSync + private let selfUserID: UserID + + init(database: Database, roster: RosterSync, selfUserID: UserID) { + self.database = database + self.roster = roster + self.selfUserID = selfUserID + } + + func prepare(chatID: ConversationID) async { + await roster.refreshFirstPage(chatID) + } + + func search(chatID: ConversationID, query: String, limit: Int) async throws -> [MemberMatch] { + let words = RosterSearchText.queryWords(query) + let recent = try database.recentSenders(conversationID: chatID, window: Self.recentMessageWindow) + + let candidates: Set + if words.isEmpty { + candidates = Set(recent) + } else { + candidates = try database.rosterMemberIDs(matchingPrefixes: words, conversationID: chatID) + } + guard !candidates.isEmpty, limit > 0 else { return [] } + + let recency = Dictionary(recent.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) + let exactUsername = words.count == 1 ? words[0] : nil + + let ranked = try database.rosterEntries(conversationID: chatID, userIDs: candidates) + .compactMap { entry -> (Rank, RosterEntry, UserID)? in + guard let userID = entry.member.userID, userID != selfUserID else { return nil } + let username = entry.member.username.map { RosterSearchText.normalize($0.value) } + let rank: Rank + if let position = recency[userID] { + rank = .recentSpeaker(position) + } else if let exactUsername, username == exactUsername { + rank = .exactUsername + } else { + rank = .alphabetical + } + return (rank, entry, userID) + } + .sorted { lhs, rhs in + if lhs.0 != rhs.0 { return lhs.0 < rhs.0 } + if lhs.1.sortKey != rhs.1.sortKey { + return lhs.1.sortKey.unicodeScalars.lexicographicallyPrecedes(rhs.1.sortKey.unicodeScalars) + } + return lhs.2.uuidString.lowercased() < rhs.2.uuidString.lowercased() + } + + return ranked.prefix(limit).map { MemberMatch(member: $0.1.member, id: $0.2) } + } + + private enum Rank: Comparable { + case recentSpeaker(Int) + case exactUsername + case alphabetical + } +} diff --git a/Flipcash/Core/Controllers/RosterSync.swift b/Flipcash/Core/Controllers/RosterSync.swift new file mode 100644 index 000000000..7dd0ef02c --- /dev/null +++ b/Flipcash/Core/Controllers/RosterSync.swift @@ -0,0 +1,134 @@ +// +// RosterSync.swift +// Flipcash +// + +import Foundation +import FlipcashCore +import FlipcashStore + +nonisolated private let logger = Logger(label: "flipcash.roster-sync") + +/// Keeps the full roster of each group the user opens on device, for member search. +/// +/// A group is tracked from its first open: ``syncIfNeeded(_:)`` pages `Chat.GetRoster` into the +/// store, and ``apply(_:)`` keeps it current from the event stream's roster updates. Groups never +/// opened are not fetched. Both paths merge per member by roster version, greater winning, so a page +/// that trails the stream and a stream update that races a sync converge on the same roster. +actor RosterSync { + + /// Pages fetched before a sync stops, `has_more` or not; at 100 per page, 2,000 members. + static let defaultPageCap = 20 + + private let fetching: any RosterFetching + private let database: Database + private let owner: KeyPair + private let pageCap: Int + + /// Full syncs in flight, so a second open of the same group joins the first rather than paging twice. + private var inFlight: [ConversationID: Task] = [:] + + init(fetching: any RosterFetching, database: Database, owner: KeyPair, pageCap: Int = RosterSync.defaultPageCap) { + self.fetching = fetching + self.database = database + self.owner = owner + self.pageCap = pageCap + } + + /// Starts tracking a group and pages its full roster when it has never been fully synced, holds + /// fewer members than it has, or the stream skipped a version. Returns once any sync is done. + func syncIfNeeded(_ conversationID: ConversationID) async { + if let running = inFlight[conversationID] { + await running.value + return + } + do { + try database.beginTrackingRoster(conversationID: conversationID) + guard try database.rosterSyncState(conversationID: conversationID)?.needsFullSync ?? true else { return } + } catch { + await report(error, reason: "Failed to read roster sync state", conversationID: conversationID) + return + } + let task = Task { await self.pageFullRoster(conversationID) } + inFlight[conversationID] = task + await task.value + inFlight[conversationID] = nil + } + + /// Rewrites the held names and pictures of a group's most recently joined members from a fresh + /// first page, since profile changes don't bump the roster version. Runs a full sync instead when + /// one is due. + func refreshFirstPage(_ conversationID: ConversationID) async { + do { + try database.beginTrackingRoster(conversationID: conversationID) + let needsFullSync = try database.rosterSyncState(conversationID: conversationID)?.needsFullSync ?? true + if needsFullSync || inFlight[conversationID] != nil { + await syncIfNeeded(conversationID) + return + } + let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: nil) + try database.refreshRosterMembers(page.members, conversationID: conversationID) + } catch { + await report(error, reason: "Failed to refresh roster first page", conversationID: conversationID) + } + } + + /// Writes a stream roster update into the held roster of a tracked group; other events are ignored. + @MainActor + func apply(_ event: ConversationStreamEvent) { + guard case .rosterChanged(let conversationID, let updates) = event else { return } + do { + try database.applyRosterUpdates(updates, conversationID: conversationID) + } catch { + logger.error("Failed to apply roster updates", metadata: [ + "conversationID": "\(conversationID)", + "error": "\(error)", + ]) + ErrorReporting.captureError(error, reason: "Failed to apply roster updates") + } + } + + private func pageFullRoster(_ conversationID: ConversationID) async { + var members: [ConversationMember] = [] + // The least fresh page's summary: the snapshot as a whole is only as current as that. + var summary: ConversationRosterSummary? + var pagingToken: Data? + var pages = 0 + var isComplete = false + + do { + while pages < pageCap { + let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: pagingToken) + pages += 1 + members.append(contentsOf: page.members) + if page.rosterSummary.version < summary?.version ?? .max { + summary = page.rosterSummary + } + guard let next = page.nextPagingToken else { + isComplete = true + break + } + pagingToken = next + } + guard let summary else { return } + try database.mergeRosterSnapshot(members, summary: summary, isComplete: isComplete, conversationID: conversationID) + logger.info("Synced roster", metadata: [ + "conversationID": "\(conversationID)", + "members": "\(members.count)", + "pages": "\(pages)", + "isComplete": "\(isComplete)", + ]) + } catch { + await report(error, reason: "Failed to sync roster", conversationID: conversationID) + } + } + + private func report(_ error: Error, reason: String, conversationID: ConversationID) async { + logger.error("Roster sync failed", metadata: [ + "reason": "\(reason)", + "conversationID": "\(conversationID)", + "error": "\(error)", + ]) + await ErrorReporting.captureError(error, reason: reason) + } +} diff --git a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift index 82dac0e4f..bb0d98ef6 100644 --- a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift +++ b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift @@ -680,6 +680,12 @@ struct ConversationScreen: View { guard groupConversation != nil else { return } await sessionContainer.knownAuthors.reload() } + // Page the group's full roster into the store for member search. Detached from this task + // inside `RosterSync`, so leaving the chat mid-sync doesn't discard the pages fetched. + .task(id: groupConversation?.id) { + guard let groupID = groupConversation?.id else { return } + await conversationController.roster?.syncIfNeeded(groupID) + } // Name the senders the chat's roster and the local cache both leave out. Keyed on that set, // so it runs when a page of older messages reveals a sender nothing here can name — and not // again once the fetch has landed them. diff --git a/Flipcash/Core/Session/SessionAuthenticator.swift b/Flipcash/Core/Session/SessionAuthenticator.swift index 2c72c929d..69b0afffa 100644 --- a/Flipcash/Core/Session/SessionAuthenticator.swift +++ b/Flipcash/Core/Session/SessionAuthenticator.swift @@ -530,6 +530,8 @@ final class SessionContainer { let profileAvatars: ProfileAvatarStore /// Identities for chat senders the chat's own roster leaves out — see ``KnownAuthorDirectory``. let knownAuthors: KnownAuthorDirectory + /// Group member search for the mention picker — see ``RosterSearchSource``. + let rosterSearch: any RosterSearchSource /// Fills cash link cards in for chat transcripts. Container-scoped so a link shared into two /// conversations is looked up once, and holding a `GiftCardAccountReading` rather than the /// `Client` it was built from — rendering a card must never reach a path that claims it. @@ -667,6 +669,9 @@ final class SessionContainer { owner: session.ownerKeyPair, selfUserID: session.userID ) + let rosterSync = RosterSync(fetching: flipClient, database: database, owner: session.ownerKeyPair) + conversationController.roster = rosterSync + self.rosterSearch = LocalRosterSearch(database: database, roster: rosterSync, selfUserID: session.userID) // Chat cash arrives in the sender's native currency; the counters are USD. // Wired before `start()` so the first delivered message already normalises. conversationController.receipts.usdRate = { [weak ratesController] currency in diff --git a/FlipcashTests/Chat/RosterSearchTests.swift b/FlipcashTests/Chat/RosterSearchTests.swift new file mode 100644 index 000000000..5dbc7866e --- /dev/null +++ b/FlipcashTests/Chat/RosterSearchTests.swift @@ -0,0 +1,275 @@ +// +// RosterSearchTests.swift +// FlipcashTests +// + +import Testing +import Foundation +import FlipcashCore +import FlipcashStore +@testable import Flipcash + +@Suite("Roster sync and search") +@MainActor +struct RosterSearchTests { + + // MARK: - Paging - + + @Test("A sync pages until has_more is false and persists every member") + func pagesToEnd() async throws { + let harness = try Harness(pages: 3, perPage: 100) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 3) + let state = try #require(try harness.database.rosterSyncState(conversationID: harness.chatID)) + #expect(state.heldCount == 300) + #expect(state.isCapped == false) + #expect(state.needsFullSync == false) + } + + @Test("The page cap stops a sync and search uses what is held") + func capStopsPaging() async throws { + let harness = try Harness(pages: 5, perPage: 10, pageCap: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 2) + let state = try #require(try harness.database.rosterSyncState(conversationID: harness.chatID)) + #expect(state.heldCount == 20) + #expect(state.isCapped) + #expect(state.needsFullSync == false) + + let matches = try await harness.search.search(chatID: harness.chatID, query: "member", limit: 100) + #expect(matches.count == 20) + } + + @Test("A synced group is not paged again on the next open") + func noResyncWhenComplete() async throws { + let harness = try Harness(pages: 1, perPage: 5) + await harness.sync.syncIfNeeded(harness.chatID) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(harness.fetching.calls == 1) + } + + // MARK: - Stream - + + @Test("A stream join adds the member to the index and a leave removes them") + func streamJoinAndLeave() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = UUID() + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate( + rosterSummary: ConversationRosterSummary(memberCount: 3, version: 3), + change: .joined(member: ConversationMember(userID: zed, displayName: "Zed Quill", version: 3), chat: nil) + ), + ])) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui", limit: 10).map(\.id) == [zed]) + + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 4), change: .left(userID: zed)), + ])) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui", limit: 10).isEmpty) + } + + @Test("A page trailing the stream cannot bring back a member who left") + func trailingPageDoesNotResurrect() async throws { + let harness = try Harness(pages: 1, perPage: 2) + try harness.database.beginTrackingRoster(conversationID: harness.chatID) + let departed = harness.fetching.members[0] + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 3), change: .left(userID: departed.userID!)), + ])) + await harness.sync.syncIfNeeded(harness.chatID) + + let held = try harness.database.rosterEntries(conversationID: harness.chatID).compactMap(\.member.userID) + #expect(!held.contains(departed.userID!)) + } + + @Test("A skipped stream version marks the group for re-sync") + func versionGapMarksResync() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate( + rosterSummary: ConversationRosterSummary(memberCount: 3, version: 9), + change: .joined(member: ConversationMember(userID: UUID(), displayName: "Late", version: 9), chat: nil) + ), + ])) + #expect(try harness.database.rosterSyncState(conversationID: harness.chatID)?.needsFullSync == true) + } + + // MARK: - Matching - + + @Test("A prefix matches any word of the display name, and the username") + func prefixMatching() async throws { + let harness = try Harness(members: [ + member("Ana María Lopez", username: "anita"), + member("Bob Stone", username: "rocky"), + ]) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try await harness.names("ana") == ["Ana María Lopez"]) + #expect(try await harness.names("mar") == ["Ana María Lopez"]) + #expect(try await harness.names("lop") == ["Ana María Lopez"]) + #expect(try await harness.names("roc") == ["Bob Stone"]) + #expect(try await harness.names("@roc") == ["Bob Stone"]) + #expect(try await harness.names("ana lo") == ["Ana María Lopez"]) + #expect(try await harness.names("tone") == []) + } + + @Test("Matching ignores case and diacritics") + func caseAndDiacritics() async throws { + let harness = try Harness(members: [member("Érica Souza", username: nil), member("ERIK", username: nil)]) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try await harness.names("eri") == ["ERIK", "Érica Souza"].sorted { RosterSearchText.normalize($0) < RosterSearchText.normalize($1) }) + #expect(try await harness.names("ÉRI").count == 2) + #expect(try await harness.names("souz") == ["Érica Souza"]) + } + + // MARK: - Ranking - + + @Test("Recent speakers rank first, then an exact username, then by display name") + func ranking() async throws { + let speaker = member("Sam Zulu", username: "samz") + let exact = member("Yara Sam", username: "sam") + let alphaA = member("Samantha Adams", username: nil) + let alphaB = member("Samuel Brown", username: nil) + let harness = try Harness(members: [alphaB, exact, alphaA, speaker]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: speaker.userID!) + + #expect(try await harness.names("sam") == ["Sam Zulu", "Yara Sam", "Samantha Adams", "Samuel Brown"]) + } + + @Test("The most recent speaker ranks above an earlier one") + func recencyOrder() async throws { + let first = member("Ann One", username: nil) + let second = member("Ann Two", username: nil) + let harness = try Harness(members: [first, second]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: first.userID!) + try harness.postMessage(from: second.userID!) + + #expect(try await harness.names("ann") == ["Ann Two", "Ann One"]) + } + + @Test("An empty query returns recent speakers only") + func emptyQuery() async throws { + let speaker = member("Talker", username: nil) + let harness = try Harness(members: [speaker, member("Quiet", username: nil)]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: speaker.userID!) + + #expect(try await harness.names("") == ["Talker"]) + #expect(try await harness.names("@") == ["Talker"]) + } + + @Test("The signed-in user is never a match, even after speaking") + func excludesSelf() async throws { + let me = member("Sam Me", username: "me") + let harness = try Harness(members: [me, member("Sam Other", username: nil)], selfUserID: me.userID!) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: me.userID!) + + #expect(try await harness.names("sam") == ["Sam Other"]) + #expect(try await harness.names("") == []) + } + + @Test("A first-page refresh rewrites a renamed member's tokens") + func refreshRenames() async throws { + let original = member("Old Name", username: nil) + let harness = try Harness(members: [original]) + await harness.sync.syncIfNeeded(harness.chatID) + + harness.fetching.members = [ConversationMember(userID: original.userID, displayName: "New Name", version: original.version)] + await harness.search.prepare(chatID: harness.chatID) + + #expect(try await harness.names("new") == ["New Name"]) + #expect(try await harness.names("old") == []) + } +} + +// MARK: - Support - + +private func member(_ displayName: String, username: String?, version: UInt64 = 1) -> ConversationMember { + ConversationMember(userID: UUID(), displayName: displayName, username: username.flatMap(Username.init), version: version) +} + +@MainActor +private struct Harness { + let database: Database + let fetching: FakeRosterFetching + let sync: RosterSync + let search: LocalRosterSearch + let chatID = ConversationID(data: Data(repeating: 0x07, count: 32)) + private let url: URL + private let messageCounter = Counter() + + init(members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID()) throws { + let (database, url) = try Database.makeTemp() + self.database = database + self.url = url + self.fetching = FakeRosterFetching(members: members, perPage: perPage) + self.sync = RosterSync(fetching: fetching, database: database, owner: .generate()!, pageCap: pageCap) + self.search = LocalRosterSearch(database: database, roster: sync, selfUserID: selfUserID) + } + + init(pages: Int, perPage: Int, pageCap: Int = RosterSync.defaultPageCap) throws { + let members = (0..<(pages * perPage)).map { ConversationMember(userID: UUID(), displayName: "Member \($0)", version: 1) } + try self.init(members: members, perPage: perPage, pageCap: pageCap) + } + + func names(_ query: String) async throws -> [String] { + try await search.search(chatID: chatID, query: query, limit: 10).map(\.member.displayName) + } + + func postMessage(from sender: UserID) throws { + let id = messageCounter.next() + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: id), senderID: sender, content: .text("hi"), date: .now, unreadSeq: id), + ], conversationID: chatID) + } +} + +nonisolated private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var value: UInt64 = 0 + func next() -> UInt64 { lock.withLock { value += 1; return value } } +} + +/// Serves a fixed roster in pages, most recent first, like `Chat.GetRoster`. +nonisolated private final class FakeRosterFetching: RosterFetching, @unchecked Sendable { + private let lock = NSLock() + private let perPage: Int + private var _members: [ConversationMember] + private var _calls = 0 + + init(members: [ConversationMember], perPage: Int) { + self._members = members + self.perPage = perPage + } + + var members: [ConversationMember] { + get { lock.withLock { _members } } + set { lock.withLock { _members = newValue } } + } + + var calls: Int { lock.withLock { _calls } } + + func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> FlipClient.RosterPage { + lock.withLock { + _calls += 1 + let start = pagingToken.map { Int(String(decoding: $0, as: UTF8.self))! } ?? 0 + let end = min(start + perPage, _members.count) + let next = end < _members.count ? Data(String(end).utf8) : nil + return FlipClient.RosterPage( + members: Array(_members[start.. Date: Tue, 29 Sep 2026 22:37:07 -0400 Subject: [PATCH 3/7] feat(chat): catch up group rosters from a watermark A missed roster update used to force a full re-read. GetRoster pages newest joins first and each Member carries the roster version of its join, so a catch-up now reads from the first page and stops at the first member at or below the stored watermark; usually one page. Leaves don't show up that way, so the page's member_count is checked after the merge. If more members are held than counted, a reconcile is recorded and runs as a background full read; the persisted flag resumes it on the next open if the app dies first. A full read drops a missing member only when it reached the end, every page reported the same roster version, and the held row is not newer. A failed page records nothing. Search now matches Android: NFKD, then lowercase, then strip Mn; split on whitespace and Z; recent speakers come from the newest 50 held messages; names tie-break on folded then raw display name, then user id. Blocked users are excluded, and a cached profile re-tokenizes that user in every tracked group, since profile edits don't move the roster version. --- Flipcash/Core/Controllers/RosterSearch.swift | 25 +- Flipcash/Core/Controllers/RosterSync.swift | 190 ++++++---- .../Core/Session/SessionAuthenticator.swift | 9 +- .../Conversation/RosterSearchText.swift | 33 +- .../FlipcashStore/Database+Roster.swift | 218 +++++++---- .../FlipcashStore/Database+UserProfiles.swift | 13 +- .../Sources/FlipcashStore/Schema.swift | 34 +- .../RosterSearchTextTests.swift | 19 +- FlipcashTests/Chat/RosterSearchTests.swift | 355 ++++++++++++++---- 9 files changed, 647 insertions(+), 249 deletions(-) diff --git a/Flipcash/Core/Controllers/RosterSearch.swift b/Flipcash/Core/Controllers/RosterSearch.swift index f5f7aab98..d8ae39838 100644 --- a/Flipcash/Core/Controllers/RosterSearch.swift +++ b/Flipcash/Core/Controllers/RosterSearch.swift @@ -22,10 +22,17 @@ protocol RosterSearchSource: AnyObject { func prepare(chatID: ConversationID) async /// Returns up to `limit` members of a group matching `query`, best match first, never the - /// signed-in user. An empty query returns the members who spoke most recently. + /// signed-in user or a blocked one. An empty query returns the members who spoke most recently. func search(chatID: ConversationID, query: String, limit: Int) async throws -> [MemberMatch] } +extension RosterSearchSource { + /// Returns up to 20 members of a group matching `query`; see ``search(chatID:query:limit:)``. + func search(chatID: ConversationID, query: String) async throws -> [MemberMatch] { + try await search(chatID: chatID, query: query, limit: 20) + } +} + /// Answers roster searches from the roster ``RosterSync`` holds on device. /// /// A member matches when every word of the query is a prefix of one of their tokens, the words of @@ -33,21 +40,23 @@ protocol RosterSearchSource: AnyObject { /// rank in three tiers: /// 1. members who sent one of the chat's newest held messages, most recent first; /// 2. the member whose username is exactly the query; -/// 3. everyone else, by normalized display name in code point order. -/// Ties fall to the lowercase user id. +/// 3. everyone else, by normalized display name, then raw display name, both in code point order. +/// Ties fall to the lowercase user id. The signed-in user and blocked users never match. final class LocalRosterSearch: RosterSearchSource { /// How many of the chat's newest held messages count toward "spoke recently". - static let recentMessageWindow = 200 + static let recentMessageWindow = 50 private let database: Database private let roster: RosterSync private let selfUserID: UserID + private let blockedUserIDs: () -> Set - init(database: Database, roster: RosterSync, selfUserID: UserID) { + init(database: Database, roster: RosterSync, selfUserID: UserID, blockedUserIDs: @escaping () -> Set = { [] }) { self.database = database self.roster = roster self.selfUserID = selfUserID + self.blockedUserIDs = blockedUserIDs } func prepare(chatID: ConversationID) async { @@ -68,10 +77,11 @@ final class LocalRosterSearch: RosterSearchSource { let recency = Dictionary(recent.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) let exactUsername = words.count == 1 ? words[0] : nil + let blocked = blockedUserIDs() let ranked = try database.rosterEntries(conversationID: chatID, userIDs: candidates) .compactMap { entry -> (Rank, RosterEntry, UserID)? in - guard let userID = entry.member.userID, userID != selfUserID else { return nil } + guard let userID = entry.member.userID, userID != selfUserID, !blocked.contains(userID) else { return nil } let username = entry.member.username.map { RosterSearchText.normalize($0.value) } let rank: Rank if let position = recency[userID] { @@ -88,6 +98,9 @@ final class LocalRosterSearch: RosterSearchSource { if lhs.1.sortKey != rhs.1.sortKey { return lhs.1.sortKey.unicodeScalars.lexicographicallyPrecedes(rhs.1.sortKey.unicodeScalars) } + if lhs.1.member.displayName != rhs.1.member.displayName { + return lhs.1.member.displayName.unicodeScalars.lexicographicallyPrecedes(rhs.1.member.displayName.unicodeScalars) + } return lhs.2.uuidString.lowercased() < rhs.2.uuidString.lowercased() } diff --git a/Flipcash/Core/Controllers/RosterSync.swift b/Flipcash/Core/Controllers/RosterSync.swift index 7dd0ef02c..d649d334d 100644 --- a/Flipcash/Core/Controllers/RosterSync.swift +++ b/Flipcash/Core/Controllers/RosterSync.swift @@ -11,13 +11,14 @@ nonisolated private let logger = Logger(label: "flipcash.roster-sync") /// Keeps the full roster of each group the user opens on device, for member search. /// -/// A group is tracked from its first open: ``syncIfNeeded(_:)`` pages `Chat.GetRoster` into the -/// store, and ``apply(_:)`` keeps it current from the event stream's roster updates. Groups never -/// opened are not fetched. Both paths merge per member by roster version, greater winning, so a page -/// that trails the stream and a stream update that races a sync converge on the same roster. +/// A group is tracked from its first open. The first open reads `Chat.GetRoster` to the end. Later +/// opens catch up only when the roster moved past the last version a read applied: they re-read +/// from the first page, newest joins first, and stop at a member already covered. ``apply(_:)`` +/// keeps a tracked group current from the stream in between. Groups never opened are not fetched. +/// Every path merges per member by roster version, greater winning. actor RosterSync { - /// Pages fetched before a sync stops, `has_more` or not; at 100 per page, 2,000 members. + /// Pages a read fetches before it stops, `has_more` or not; at 100 per page, 2,000 members. static let defaultPageCap = 20 private let fetching: any RosterFetching @@ -25,7 +26,7 @@ actor RosterSync { private let owner: KeyPair private let pageCap: Int - /// Full syncs in flight, so a second open of the same group joins the first rather than paging twice. + /// Reads in flight, so a second trigger for the same group joins the first rather than reading twice. private var inFlight: [ConversationID: Task] = [:] init(fetching: any RosterFetching, database: Database, owner: KeyPair, pageCap: Int = RosterSync.defaultPageCap) { @@ -35,92 +36,153 @@ actor RosterSync { self.pageCap = pageCap } - /// Starts tracking a group and pages its full roster when it has never been fully synced, holds - /// fewer members than it has, or the stream skipped a version. Returns once any sync is done. + /// Starts tracking a group and brings its held roster up to date: a full read when none has + /// finished or a reconcile is owed, a catch-up when the roster moved past the watermark, nothing + /// otherwise. Returns once any read is done. func syncIfNeeded(_ conversationID: ConversationID) async { + await run(conversationID, refreshFirstPage: false) + } + + /// As ``syncIfNeeded(_:)``, but always re-reads at least the first page, so the newest members' + /// names and pictures are current; profile changes don't move the roster version. + func refreshFirstPage(_ conversationID: ConversationID) async { + await run(conversationID, refreshFirstPage: true) + } + + /// Writes a stream roster update into the held roster of a tracked group; other events are ignored. + @MainActor + func apply(_ event: ConversationStreamEvent) { + guard case .rosterChanged(let conversationID, let updates) = event else { return } + do { + try database.applyRosterUpdates(updates, conversationID: conversationID) + } catch { + logger.error("Failed to apply roster updates", metadata: [ + "conversationID": "\(conversationID)", + "error": "\(error)", + ]) + ErrorReporting.captureError(error, reason: "Failed to apply roster updates") + } + } + + // MARK: - Private - + + private func run(_ conversationID: ConversationID, refreshFirstPage: Bool) async { if let running = inFlight[conversationID] { await running.value return } + let task = Task { await self.bringUpToDate(conversationID, refreshFirstPage: refreshFirstPage) } + inFlight[conversationID] = task + await task.value + inFlight[conversationID] = nil + } + + private func bringUpToDate(_ conversationID: ConversationID, refreshFirstPage: Bool) async { + let state: RosterSyncState do { try database.beginTrackingRoster(conversationID: conversationID) - guard try database.rosterSyncState(conversationID: conversationID)?.needsFullSync ?? true else { return } + guard let tracked = try database.rosterSyncState(conversationID: conversationID) else { return } + state = tracked } catch { await report(error, reason: "Failed to read roster sync state", conversationID: conversationID) return } - let task = Task { await self.pageFullRoster(conversationID) } - inFlight[conversationID] = task - await task.value - inFlight[conversationID] = nil - } - /// Rewrites the held names and pictures of a group's most recently joined members from a fresh - /// first page, since profile changes don't bump the roster version. Runs a full sync instead when - /// one is due. - func refreshFirstPage(_ conversationID: ConversationID) async { + if state.needsFullRead { + await fullRead(conversationID, reconciling: state.reconcilePending) + return + } + guard state.needsCatchUp || refreshFirstPage else { return } + do { - try database.beginTrackingRoster(conversationID: conversationID) - let needsFullSync = try database.rosterSyncState(conversationID: conversationID)?.needsFullSync ?? true - if needsFullSync || inFlight[conversationID] != nil { - await syncIfNeeded(conversationID) - return + switch try await catchUp(conversationID, watermark: state.watermark) { + case .caughtUp: + break + case .reconcileNeeded: + await fullRead(conversationID, reconciling: true) } - let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: nil) - try database.refreshRosterMembers(page.members, conversationID: conversationID) } catch { - await report(error, reason: "Failed to refresh roster first page", conversationID: conversationID) + await report(error, reason: "Failed to catch up roster", conversationID: conversationID) } } - /// Writes a stream roster update into the held roster of a tracked group; other events are ignored. - @MainActor - func apply(_ event: ConversationStreamEvent) { - guard case .rosterChanged(let conversationID, let updates) = event else { return } + /// Reads from the first page until it reaches a member the watermark already covers. + private func catchUp(_ conversationID: ConversationID, watermark: UInt64) async throws -> RosterCatchUpOutcome { + // Pages list the most recently joined first, and a member's version is the roster version of + // their join, so everyone after the first member at or below the watermark is already held. + // `Member.version` is documented to move on future member changes too (a role change, say); + // once one does, a member can sit above the watermark out of join order, and this stop rule + // needs revisiting. + let read = try await readPages(conversationID) { page in + page.members.contains { $0.version <= watermark } + } + + guard read.stoppedEarly else { + // The read ran to the end or the cap without reaching the watermark: it's a full read. + try database.applyFullRosterRead(read.members, summaries: read.summaries, isComplete: read.reachedEnd, conversationID: conversationID) + logRead("Roster catch-up became a full read", read, conversationID: conversationID) + return .caughtUp + } + + guard let summary = read.summaries.min(by: { $0.version < $1.version }) else { return .caughtUp } + let outcome = try database.applyRosterCatchUp(read.members, summary: summary, conversationID: conversationID) + logRead("Caught up roster", read, conversationID: conversationID) + return outcome + } + + /// Reads the whole roster, up to the page cap. A reconcile runs at background priority, since it + /// only removes members who left; a failure leaves it pending for the next open. + private func fullRead(_ conversationID: ConversationID, reconciling: Bool) async { + let task = Task(priority: reconciling ? .background : nil) { + let read = try await self.readPages(conversationID) { _ in false } + try self.database.applyFullRosterRead(read.members, summaries: read.summaries, isComplete: read.reachedEnd, conversationID: conversationID) + await self.logRead("Read full roster", read, conversationID: conversationID) + } do { - try database.applyRosterUpdates(updates, conversationID: conversationID) + try await task.value } catch { - logger.error("Failed to apply roster updates", metadata: [ - "conversationID": "\(conversationID)", - "error": "\(error)", - ]) - ErrorReporting.captureError(error, reason: "Failed to apply roster updates") + await report(error, reason: "Failed to read full roster", conversationID: conversationID) } } - private func pageFullRoster(_ conversationID: ConversationID) async { + private struct Read { var members: [ConversationMember] = [] - // The least fresh page's summary: the snapshot as a whole is only as current as that. - var summary: ConversationRosterSummary? - var pagingToken: Data? + var summaries: [ConversationRosterSummary] = [] var pages = 0 - var isComplete = false + var reachedEnd = false + var stoppedEarly = false + } - do { - while pages < pageCap { - let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: pagingToken) - pages += 1 - members.append(contentsOf: page.members) - if page.rosterSummary.version < summary?.version ?? .max { - summary = page.rosterSummary - } - guard let next = page.nextPagingToken else { - isComplete = true - break - } - pagingToken = next + /// Pages from the first page until `stop` returns `true` for a page, the last page, or the cap. + /// Throws without returning anything read when a page fails, so a partial read records nothing. + private func readPages(_ conversationID: ConversationID, stop: (FlipClient.RosterPage) -> Bool) async throws -> Read { + var read = Read() + var pagingToken: Data? + while read.pages < pageCap { + let page = try await fetching.getRosterPage(owner: owner, conversationID: conversationID, pagingToken: pagingToken) + read.pages += 1 + read.members.append(contentsOf: page.members) + read.summaries.append(page.rosterSummary) + if stop(page) { + read.stoppedEarly = true + break } - guard let summary else { return } - try database.mergeRosterSnapshot(members, summary: summary, isComplete: isComplete, conversationID: conversationID) - logger.info("Synced roster", metadata: [ - "conversationID": "\(conversationID)", - "members": "\(members.count)", - "pages": "\(pages)", - "isComplete": "\(isComplete)", - ]) - } catch { - await report(error, reason: "Failed to sync roster", conversationID: conversationID) + guard let next = page.nextPagingToken else { + read.reachedEnd = true + break + } + pagingToken = next } + return read + } + + private func logRead(_ message: Logger.Message, _ read: Read, conversationID: ConversationID) { + logger.info(message, metadata: [ + "conversationID": "\(conversationID)", + "members": "\(read.members.count)", + "pages": "\(read.pages)", + "reachedEnd": "\(read.reachedEnd)", + ]) } private func report(_ error: Error, reason: String, conversationID: ConversationID) async { diff --git a/Flipcash/Core/Session/SessionAuthenticator.swift b/Flipcash/Core/Session/SessionAuthenticator.swift index 69b0afffa..bf2fd0387 100644 --- a/Flipcash/Core/Session/SessionAuthenticator.swift +++ b/Flipcash/Core/Session/SessionAuthenticator.swift @@ -671,7 +671,6 @@ final class SessionContainer { ) let rosterSync = RosterSync(fetching: flipClient, database: database, owner: session.ownerKeyPair) conversationController.roster = rosterSync - self.rosterSearch = LocalRosterSearch(database: database, roster: rosterSync, selfUserID: session.userID) // Chat cash arrives in the sender's native currency; the counters are USD. // Wired before `start()` so the first delivered message already normalises. conversationController.receipts.usdRate = { [weak ratesController] currency in @@ -703,6 +702,14 @@ final class SessionContainer { database: database ) self.blocklistController = blocklistController + self.rosterSearch = LocalRosterSearch( + database: database, + roster: rosterSync, + selfUserID: session.userID, + blockedUserIDs: { [weak blocklistController] in + Set(blocklistController?.blockedUsers.map(\.userID) ?? []) + } + ) // The blocklist drives which conversations the feed hides: it supplies the // current set, reconciles the hidden flags on every change, and reconciles diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift index 3c6cc81e0..88fa84669 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/RosterSearchText.swift @@ -9,22 +9,25 @@ import Foundation /// matches the same members on both platforms. public enum RosterSearchText { - /// Returns `text` lowercased, NFKD-decomposed, and stripped of every combining mark - /// (general category `M`: `Mn`, `Mc`, `Me`), so "Érica" and "erica" compare equal. + /// Returns `text` NFKD-decomposed, lowercased, and stripped of nonspacing marks (general category + /// `Mn`), so "Érica" and "erica" compare equal. Lowercasing after NFKD folds "İ" to "i" plus a + /// combining dot, which the strip then drops. Spacing marks (`Mc`) are kept: in scripts such as + /// Devanagari they carry the vowel. public static func normalize(_ text: String) -> String { - let decomposed = text.lowercased().decomposedStringWithCompatibilityMapping + let folded = text.decomposedStringWithCompatibilityMapping.lowercased() var scalars = String.UnicodeScalarView() - for scalar in decomposed.unicodeScalars where !isMark(scalar) { + for scalar in folded.unicodeScalars where scalar.properties.generalCategory != .nonspacingMark { scalars.append(scalar) } return String(scalars) } - /// Returns the normalized words of `text`, split on runs of Unicode `White_Space`, empty words dropped. + /// Returns the normalized words of `text`, split on Unicode `White_Space` and separators + /// (general category `Z`), empty words dropped. public static func words(_ text: String) -> [String] { normalize(text) .unicodeScalars - .split(whereSeparator: \.properties.isWhitespace) + .split(whereSeparator: isSeparator) .map { String(String.UnicodeScalarView($0)) } } @@ -33,17 +36,14 @@ public enum RosterSearchText { public static func tokens(displayName: String, username: String?) -> Set { var tokens = Set(words(displayName)) if let username { - tokens.formUnion(words(username)) + tokens.formUnion(words(username).map(droppingAt).filter { !$0.isEmpty }) } return tokens } /// Returns the words of a typed query, with a leading `@` (either width) dropped from each word. public static func queryWords(_ query: String) -> [String] { - // NFKD folds the fullwidth "＠" to "@", so one check covers both. - words(query) - .map { $0.hasPrefix("@") ? String($0.drop(while: { $0 == "@" })) : $0 } - .filter { !$0.isEmpty } + words(query).map(droppingAt).filter { !$0.isEmpty } } /// Returns the exclusive upper bound of a prefix range, so `token >= prefix AND token < bound` @@ -55,12 +55,17 @@ public enum RosterSearchText { prefix + "\u{10FFFF}" } - private static func isMark(_ scalar: Unicode.Scalar) -> Bool { + // NFKD folds the fullwidth "＠" to "@", so one check covers both. + private static func droppingAt(_ word: String) -> String { + word.hasPrefix("@") ? String(word.dropFirst()) : word + } + + private static func isSeparator(_ scalar: Unicode.Scalar) -> Bool { switch scalar.properties.generalCategory { - case .nonspacingMark, .spacingMark, .enclosingMark: + case .spaceSeparator, .lineSeparator, .paragraphSeparator: return true default: - return false + return scalar.properties.isWhitespace } } } diff --git a/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift index 81229139b..32d12182f 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database+Roster.swift @@ -9,23 +9,34 @@ import SQLite /// Where a group's locally held roster stands against the server's. public struct RosterSyncState: Sendable, Equatable { - /// The greatest roster version applied, from a sync or the stream. - public let version: UInt64 - /// The group's true member count, as the latest roster summary reported it. + /// The last roster version a `Chat.GetRoster` read fully applied. + public let watermark: UInt64 + /// The greatest roster version seen from the stream, the chat feed, or a read. + public let observedVersion: UInt64 + /// The group's member count, as the latest roster summary reported it. public let memberCount: UInt64 /// How many current members are held locally. public let heldCount: Int - /// When the last full sync finished, or `nil` if none has. - public let syncedAt: Date? - /// Whether the last full sync stopped at the page cap. - public let isCapped: Bool - /// The stream version that revealed a skipped version, or `nil` when the stream has been contiguous. - public let gapVersion: UInt64? - - /// Whether the group's roster should be paged again from the start. - public var needsFullSync: Bool { - syncedAt == nil || gapVersion != nil || (!isCapped && UInt64(heldCount) < memberCount) - } + /// Whether a full read has ever finished. + public let fullySynced: Bool + /// Whether the last full read stopped at the page cap. + public let truncated: Bool + /// Whether a full read is owed, from an unseen leave or a read that couldn't drop members. + public let reconcilePending: Bool + + /// Whether the roster should be read from the first page to the end. + public var needsFullRead: Bool { !fullySynced || reconcilePending } + + /// Whether the roster moved past what a read has applied, so the newest pages should be re-read. + public var needsCatchUp: Bool { observedVersion > watermark } +} + +/// What a catch-up read found once merged. +public enum RosterCatchUpOutcome: Sendable, Equatable { + /// The held roster agrees with the page's count; the watermark moved to the page's version. + case caughtUp + /// More members are held than the page counts, so someone left unseen; a full read is owed. + case reconcileNeeded } /// A current roster member as stored, with the key roster search sorts it by. @@ -48,19 +59,20 @@ nonisolated extension Database { return nil } let held = try reader.scalar(r.table.filter(r.conversationId == conversationID.data && r.isMember == true).count) - let feedCount = try reader.pluck(c.table.select(c.rosterMemberCount).filter(c.id == conversationID.data))?[c.rosterMemberCount] ?? 0 + let feedVersion = try reader.pluck(c.table.select(c.rosterVersion).filter(c.id == conversationID.data))?[c.rosterVersion] ?? 0 return RosterSyncState( - version: row[s.version], - memberCount: max(row[s.memberCount], feedCount), + watermark: row[s.watermark], + observedVersion: max(row[s.observedVersion], feedVersion), + memberCount: row[s.memberCount], heldCount: held, - syncedAt: row[s.syncedAt].map(Date.init(timeIntervalSinceReferenceDate:)), - isCapped: row[s.isCapped], - gapVersion: row[s.gapVersion] + fullySynced: row[s.fullySynced], + truncated: row[s.truncated], + reconcilePending: row[s.reconcilePending] ) } - /// Starts tracking a group's roster so stream updates are written from now on. Seeds the version - /// and count from the cached chat row; a no-op when the group is already tracked. + /// Starts tracking a group's roster so stream updates are written from now on. A no-op when the + /// group is already tracked. public func beginTrackingRoster(conversationID: ConversationID) throws { let s = RosterSyncTable() let c = ConversationTable() @@ -69,81 +81,101 @@ nonisolated extension Database { try writer.run(s.table.insert( or: .ignore, s.conversationId <- conversationID.data, - s.version <- chat?[c.rosterVersion] ?? 0, + s.watermark <- 0, + s.observedVersion <- chat?[c.rosterVersion] ?? 0, s.memberCount <- chat?[c.rosterMemberCount] ?? 0, - s.syncedAt <- nil, - s.isCapped <- false, - s.gapVersion <- nil + s.fullySynced <- false, + s.truncated <- false, + s.reconcilePending <- false )) } } // MARK: - Writes - - /// Merges a paged roster read into a tracked group's roster. + /// Merges a read from the first page to the end, or to the page cap, into a tracked group. /// - /// Per member, the greater version wins against what the stream already wrote. When `isComplete`, - /// a held row the snapshot lacks is dropped unless its version is newer than `summary.version`, - /// the version the snapshot is at least as fresh as. When the read stopped early, absent rows are - /// kept, because they may be on the pages not fetched. - public func mergeRosterSnapshot( + /// Per member, the greater version wins against what the stream already wrote. A held member the + /// read lacks is dropped only when the read reached the end, every page reported the same roster + /// version, and the held row's version is not above it. When the pages disagree, nothing is + /// dropped and a reconcile stays pending. A truncated read drops nothing and owes nothing: the + /// members past the cap were never held. + public func applyFullRosterRead( _ members: [ConversationMember], - summary: ConversationRosterSummary, + summaries: [ConversationRosterSummary], isComplete: Bool, - conversationID: ConversationID, - now: Date = .now + conversationID: ConversationID ) throws { + guard let oldest = summaries.min(by: { $0.version < $1.version }), + let newest = summaries.max(by: { $0.version < $1.version }) else { return } + let isConsistent = oldest.version == newest.version let s = RosterSyncTable() - let r = RosterMemberTable() try writer.transaction { guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } - var held = try heldVersions(conversationID: conversationID) + var held = try mergeRosterMembers(members, conversationID: conversationID) - for member in members { - guard let userID = member.userID else { continue } - if let existing = held[userID], existing > member.version { continue } - try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) - held[userID] = member.version - } - - if isComplete { - let snapshotIDs = Set(members.compactMap(\.userID)) - for (userID, version) in held where !snapshotIDs.contains(userID) && version <= summary.version { + if isComplete && isConsistent { + let readIDs = Set(members.compactMap(\.userID)) + for (userID, version) in held where !readIDs.contains(userID) && version <= oldest.version { try deleteRosterMember(userID: userID, conversationID: conversationID) + held[userID] = nil } } - let version = sync[s.version] - let gap = sync[s.gapVersion].flatMap { $0 > summary.version + 1 ? $0 : nil } try writer.run(s.table.filter(s.conversationId == conversationID.data).update( - s.version <- max(version, summary.version), - s.memberCount <- summary.version >= version ? summary.memberCount : sync[s.memberCount], - s.syncedAt <- now.timeIntervalSinceReferenceDate, - s.isCapped <- !isComplete, - s.gapVersion <- gap + s.watermark <- max(sync[s.watermark], oldest.version), + s.observedVersion <- max(sync[s.observedVersion], newest.version), + s.memberCount <- oldest.memberCount, + s.fullySynced <- true, + s.truncated <- !isComplete, + s.reconcilePending <- !isConsistent )) } } - /// Rewrites the names and pictures of members a fresh roster page carries, for a tracked group. - /// A row the stream has since moved past is left alone. Nothing is removed. - public func refreshRosterMembers(_ members: [ConversationMember], conversationID: ConversationID) throws { + /// Merges a read of the newest pages into a tracked group and checks the page's count for leaves + /// the read can't show. + /// + /// `summary` is the page's roster summary; with several pages, the one with the lowest version. + /// When no more members are held than it counts, the watermark moves to its version: a join the + /// read missed is newer than the page and arrives on the stream. When more are held, someone left + /// unseen, so a reconcile is recorded and the watermark stays put. + public func applyRosterCatchUp( + _ members: [ConversationMember], + summary: ConversationRosterSummary, + conversationID: ConversationID + ) throws -> RosterCatchUpOutcome { let s = RosterSyncTable() + let r = RosterMemberTable() + var outcome = RosterCatchUpOutcome.caughtUp try writer.transaction { - guard try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) != nil else { return } - let held = try heldVersions(conversationID: conversationID) - for member in members { - guard let userID = member.userID else { continue } - if let existing = held[userID], existing > member.version { continue } - try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) + guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } + try mergeRosterMembers(members, conversationID: conversationID) + let held = try writer.scalar(r.table.filter(r.conversationId == conversationID.data && r.isMember == true).count) + let observed = max(sync[s.observedVersion], summary.version) + + if UInt64(held) > summary.memberCount { + outcome = .reconcileNeeded + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.observedVersion <- observed, + s.reconcilePending <- true + )) + } else { + try writer.run(s.table.filter(s.conversationId == conversationID.data).update( + s.watermark <- max(sync[s.watermark], summary.version), + s.observedVersion <- observed, + s.memberCount <- summary.memberCount + )) } } + return outcome } /// Applies live roster updates to a tracked group, returning `false` when the group is not tracked. /// - /// Each update wins against the member's row only with a greater version. An update more than one - /// past the greatest applied version records a gap, which ``RosterSyncState/needsFullSync`` reports. + /// Each update wins against the member's row only with a greater version. The watermark does not + /// move: the stream can't show whether an update was skipped, so the next open catches up from + /// the roster's first page instead. @discardableResult public func applyRosterUpdates(_ updates: [DecodedRosterUpdate], conversationID: ConversationID) throws -> Bool { let s = RosterSyncTable() @@ -151,18 +183,14 @@ nonisolated extension Database { try writer.transaction { guard let sync = try writer.pluck(s.table.filter(s.conversationId == conversationID.data)) else { return } isTracked = true - var version = sync[s.version] + var observed = sync[s.observedVersion] var memberCount = sync[s.memberCount] - var gap = sync[s.gapVersion] let held = try heldVersions(conversationID: conversationID) for update in updates.sorted(by: { $0.rosterSummary.version < $1.rosterSummary.version }) { let updateVersion = update.rosterSummary.version - if updateVersion > version + 1 { - gap = max(gap ?? 0, updateVersion) - } - if updateVersion > version { - version = updateVersion + if updateVersion > observed { + observed = updateVersion memberCount = update.rosterSummary.memberCount } @@ -178,14 +206,40 @@ nonisolated extension Database { } try writer.run(s.table.filter(s.conversationId == conversationID.data).update( - s.version <- version, - s.memberCount <- memberCount, - s.gapVersion <- gap + s.observedVersion <- observed, + s.memberCount <- memberCount )) } return isTracked } + /// Must be called inside a `writer.transaction`. Rewrites a user's name, username, picture, and + /// search tokens in every group that holds them as a current member, keeping each row's version, + /// since a profile change doesn't move the roster version. + func refreshRosterProfile(_ profile: Profile, userID: UserID) throws { + let r = RosterMemberTable() + let t = RosterTokenTable() + let rows = try Array(writer.prepareRowIterator( + r.table.filter(r.userId == userID && r.isMember == true) + )) + for row in rows { + let conversationData = row[r.conversationId] + let displayName = profile.displayName ?? row[r.displayName] + try writer.run(r.table.filter(r.conversationId == conversationData && r.userId == userID).update( + r.displayName <- displayName, + r.username <- profile.username?.value, + r.profilePictureBlobID <- profile.profilePicture?.blobID.data, + r.profilePictureThumbnailBlobID <- profile.profilePicture?.thumbnailBlobID.data, + r.profilePictureThumbnailBlurhash <- profile.profilePicture?.thumbnailBlurhash, + r.sortKey <- RosterSearchText.normalize(displayName) + )) + try writer.run(t.table.filter(t.conversationId == conversationData && t.userId == userID).delete()) + for token in RosterSearchText.tokens(displayName: displayName, username: profile.username?.value) { + try writer.run(t.table.insert(or: .ignore, t.conversationId <- conversationData, t.token <- token, t.userId <- userID)) + } + } + } + /// Removes everything held for a group's roster, tracking included. public func deleteRoster(conversationID: ConversationID) throws { try writer.transaction { @@ -277,6 +331,20 @@ nonisolated extension Database { return versions } + /// Must be called inside a `writer.transaction`. Writes each member whose version is not below the + /// held row's, so an equal version refreshes a name or picture. Returns every held row's version. + @discardableResult + private func mergeRosterMembers(_ members: [ConversationMember], conversationID: ConversationID) throws -> [UserID: UInt64] { + var held = try heldVersions(conversationID: conversationID) + for member in members { + guard let userID = member.userID else { continue } + if let existing = held[userID], existing > member.version { continue } + try writeRosterMember(member, userID: userID, version: member.version, conversationID: conversationID) + held[userID] = member.version + } + return held + } + /// Must be called inside a `writer.transaction`. private func writeRosterMember(_ member: ConversationMember, userID: UserID, version: UInt64, conversationID: ConversationID) throws { let r = RosterMemberTable() diff --git a/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift b/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift index ef4325b7a..d95dae903 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database+UserProfiles.swift @@ -29,11 +29,14 @@ nonisolated extension Database { public func upsertUserProfile(_ profile: Profile, userID: UserID) throws { let t = UserProfileTable() let data = try JSONEncoder().encode(profile) - try writer.run(t.table.upsert( - t.userID <- userID, - t.data <- data, - onConflictOf: t.userID - )) + try writer.transaction { + try writer.run(t.table.upsert( + t.userID <- userID, + t.data <- data, + onConflictOf: t.userID + )) + try refreshRosterProfile(profile, userID: userID) + } } // MARK: - Delete - diff --git a/FlipcashCore/Sources/FlipcashStore/Schema.swift b/FlipcashCore/Sources/FlipcashStore/Schema.swift index 4a10fe6f7..4e86a9957 100644 --- a/FlipcashCore/Sources/FlipcashStore/Schema.swift +++ b/FlipcashCore/Sources/FlipcashStore/Schema.swift @@ -371,17 +371,20 @@ nonisolated public struct RosterSyncTable: Sendable { public init() {} - public let table = Table(Self.name) - public let conversationId = Expression ("conversationId") - // The greatest roster version applied, from a sync or the stream. - public let version = Expression ("version") - public let memberCount = Expression ("memberCount") - // When the last full sync finished; `nil` until one has. - public let syncedAt = Expression ("syncedAt") - // Whether the last full sync stopped at the page cap before `has_more` went false. - public let isCapped = Expression ("isCapped") - // The greatest stream version that arrived after a skipped version; set means a re-sync is due. - public let gapVersion = Expression ("gapVersion") + public let table = Table(Self.name) + public let conversationId = Expression ("conversationId") + // The last roster version a `Chat.GetRoster` read fully applied. Stream updates never move it, + // because the version is opaque and a skipped update can't be detected from the stream alone. + public let watermark = Expression ("watermark") + // The greatest roster version seen from any source; above `watermark` means a catch-up is due. + public let observedVersion = Expression ("observedVersion") + public let memberCount = Expression ("memberCount") + // Whether a read from the first page to the end (or to the page cap) has ever finished. + public let fullySynced = Expression ("fullySynced") + // Whether the last full read stopped at the page cap before `has_more` went false. + public let truncated = Expression ("truncated") + // A leave went unseen, or a full read couldn't be trusted to drop members; the next open re-reads. + public let reconcilePending = Expression ("reconcilePending") } // One row per message; cash content is decomposed across the amount columns @@ -709,11 +712,12 @@ nonisolated extension Database { try writer.run(rosterTokenTable.table.createIndex(rosterTokenTable.conversationId, rosterTokenTable.userId, ifNotExists: true)) try writer.run(rosterSyncTable.table.create(ifNotExists: true, withoutRowid: true) { t in t.column(rosterSyncTable.conversationId, primaryKey: true) - t.column(rosterSyncTable.version) + t.column(rosterSyncTable.watermark) + t.column(rosterSyncTable.observedVersion) t.column(rosterSyncTable.memberCount) - t.column(rosterSyncTable.syncedAt) - t.column(rosterSyncTable.isCapped) - t.column(rosterSyncTable.gapVersion) + t.column(rosterSyncTable.fullySynced) + t.column(rosterSyncTable.truncated) + t.column(rosterSyncTable.reconcilePending) }) } diff --git a/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift index 2d2bf430c..cbaf0222a 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/RosterSearchTextTests.swift @@ -29,15 +29,27 @@ struct RosterSearchTextTests { #expect(RosterSearchText.normalize("fiona") == "fiona") } + @Test("Dotted capital I folds to a plain i") + func dottedCapitalI() { + #expect(RosterSearchText.normalize("İstanbul") == "istanbul") + } + + @Test("Spacing marks are kept, so Devanagari vowels survive") + func keepsSpacingMarks() { + // U+093F DEVANAGARI VOWEL SIGN I is category Mc. + #expect(RosterSearchText.normalize("कि") == "कि") + } + @Test("Tokens are each display name word plus the username") func tokens() { let tokens = RosterSearchText.tokens(displayName: " Érica de la Cruz ", username: "ecruz") #expect(tokens == ["erica", "de", "la", "cruz", "ecruz"]) + #expect(RosterSearchText.tokens(displayName: "Bo", username: "＠bo_b") == ["bo", "bo_b"]) } - @Test("A no-break space splits words like a space") - func noBreakSpaceSplits() { - #expect(RosterSearchText.words("Ana\u{00A0}Lima") == ["ana", "lima"]) + @Test("Any whitespace or separator splits words", arguments: ["\u{00A0}", "\u{2028}", "\u{3000}", "\t"]) + func separatorsSplit(separator: String) { + #expect(RosterSearchText.words("Ana\(separator)Lima") == ["ana", "lima"]) } @Test("A query drops a leading @ of either width") @@ -45,6 +57,7 @@ struct RosterSearchTextTests { #expect(RosterSearchText.queryWords("@Eri") == ["eri"]) #expect(RosterSearchText.queryWords("＠eri") == ["eri"]) #expect(RosterSearchText.queryWords("@") == []) + #expect(RosterSearchText.queryWords("@@eri") == ["@eri"]) #expect(RosterSearchText.queryWords("") == []) } diff --git a/FlipcashTests/Chat/RosterSearchTests.swift b/FlipcashTests/Chat/RosterSearchTests.swift index 5dbc7866e..e2fcae980 100644 --- a/FlipcashTests/Chat/RosterSearchTests.swift +++ b/FlipcashTests/Chat/RosterSearchTests.swift @@ -7,49 +7,189 @@ import Testing import Foundation import FlipcashCore import FlipcashStore +import SQLite @testable import Flipcash @Suite("Roster sync and search") @MainActor struct RosterSearchTests { - // MARK: - Paging - + // MARK: - Full read - - @Test("A sync pages until has_more is false and persists every member") + @Test("A first open pages until has_more is false and persists every member") func pagesToEnd() async throws { let harness = try Harness(pages: 3, perPage: 100) await harness.sync.syncIfNeeded(harness.chatID) #expect(harness.fetching.calls == 3) - let state = try #require(try harness.database.rosterSyncState(conversationID: harness.chatID)) + let state = try harness.state() #expect(state.heldCount == 300) - #expect(state.isCapped == false) - #expect(state.needsFullSync == false) + #expect(state.fullySynced) + #expect(state.truncated == false) + #expect(state.watermark == 2) + #expect(state.needsFullRead == false) } - @Test("The page cap stops a sync and search uses what is held") + @Test("The page cap stops a read, and search uses what is held") func capStopsPaging() async throws { let harness = try Harness(pages: 5, perPage: 10, pageCap: 2) await harness.sync.syncIfNeeded(harness.chatID) #expect(harness.fetching.calls == 2) - let state = try #require(try harness.database.rosterSyncState(conversationID: harness.chatID)) + let state = try harness.state() #expect(state.heldCount == 20) - #expect(state.isCapped) - #expect(state.needsFullSync == false) + #expect(state.truncated) + #expect(state.needsFullRead == false) - let matches = try await harness.search.search(chatID: harness.chatID, query: "member", limit: 100) - #expect(matches.count == 20) + #expect(try await harness.search.search(chatID: harness.chatID, query: "member", limit: 100).count == 20) + + await harness.sync.syncIfNeeded(harness.chatID) + #expect(harness.fetching.calls == 2) } - @Test("A synced group is not paged again on the next open") - func noResyncWhenComplete() async throws { + @Test("A synced group is not read again while the roster hasn't moved") + func noReadWhenCurrent() async throws { let harness = try Harness(pages: 1, perPage: 5) await harness.sync.syncIfNeeded(harness.chatID) await harness.sync.syncIfNeeded(harness.chatID) #expect(harness.fetching.calls == 1) } + @Test("A failed page records nothing") + func failedPageRecordsNothing() async throws { + let harness = try Harness(pages: 2, perPage: 5) + harness.fetching.failingCalls = [2] + await harness.sync.syncIfNeeded(harness.chatID) + + let state = try harness.state() + #expect(state.heldCount == 0) + #expect(state.fullySynced == false) + } + + @Test("A read whose pages report different versions drops nothing and owes a reconcile") + func midReadVersionChange() async throws { + let harness = try Harness(members: [member("Ann"), member("Bea")], perPage: 1) + await harness.sync.syncIfNeeded(harness.chatID) + + let departed = harness.fetching.members[1] + harness.fetching.members = [harness.fetching.members[0], member("Cal")] + harness.fetching.pageVersions = [3, 4] + try harness.database.markRosterReconcilePending(harness.chatID) + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(try harness.heldIDs().contains(departed.userID!)) + #expect(try harness.state().reconcilePending) + } + + // MARK: - Catch-up - + + @Test("Missed joins are recovered from the first page, stopping at the watermark") + func catchUpRecoversJoins() async throws { + let harness = try Harness(pages: 3, perPage: 100) + await harness.sync.syncIfNeeded(harness.chatID) + + let missed = (0..<4).map { member("New \($0)", version: 5 + UInt64($0)) } + let streamed = member("Streamed", version: 9) + harness.fetching.members = [streamed] + missed.reversed() + harness.fetching.members + harness.fetching.version = 9 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(streamed, count: 305, version: 9)])) + + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 4) + let state = try harness.state() + #expect(state.heldCount == 305) + #expect(state.watermark == 9) + #expect(state.needsCatchUp == false) + } + + @Test("An equal count moves the watermark to the page's version, not the stream's") + func equalCountUsesPageVersion() async throws { + let ann = member("Ann") + let max = member("Max") + let harness = try Harness(members: [ann, max]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 10) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 9), change: .left(userID: max.userID!)), + joined(yan, count: 2, version: 10), + ])) + + // The page trails the stream: it is at version 8, before Max left and Yan joined. + let wes = member("Wes", version: 8) + harness.fetching.members = [wes, ann, max] + harness.fetching.version = 8 + await harness.sync.syncIfNeeded(harness.chatID) + + let state = try harness.state() + #expect(state.heldCount == 3) + #expect(state.watermark == 8) + #expect(try Set(harness.heldIDs()) == [ann.userID!, wes.userID!, yan.userID!]) + } + + @Test("More members held than counted owes a reconcile that survives a failed read") + func unseenLeaveSetsPending() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 4) + harness.fetching.members = [yan] + harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(yan, count: 3, version: 4)])) + + harness.fetching.failingCalls = [3] + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try harness.state().reconcilePending) + #expect(try harness.heldIDs().contains(cal.userID!)) + + harness.fetching.failingCalls = [] + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try harness.state().reconcilePending == false) + #expect(try harness.heldIDs().contains(cal.userID!) == false) + } + + @Test("An unseen leave runs one reconcile even when triggered twice") + func reconcileRunsOnce() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let yan = member("Yan", version: 4) + harness.fetching.members = [yan] + harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(yan, count: 3, version: 4)])) + + async let first: Void = harness.sync.syncIfNeeded(harness.chatID) + async let second: Void = harness.sync.syncIfNeeded(harness.chatID) + _ = await (first, second) + + // One for the initial read, one for the catch-up, one for the reconcile. + #expect(harness.fetching.calls == 3) + #expect(try harness.heldIDs().contains(cal.userID!) == false) + } + + @Test("A reconcile drops departed members but keeps one the stream added after the page") + func reconcileKeepsNewerMembers() async throws { + let cal = member("Cal") + let harness = try Harness(members: [member("Ann"), member("Bea"), cal]) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = member("Zed", version: 6) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 6)])) + + // Cal left at version 3, unseen; the pages are at version 4, before Zed joined. + harness.fetching.members = harness.fetching.members.filter { $0.userID != cal.userID } + harness.fetching.version = 4 + await harness.sync.syncIfNeeded(harness.chatID) + + let held = try harness.heldIDs() + #expect(held.contains(cal.userID!) == false) + #expect(held.contains(zed.userID!)) + #expect(try harness.state().reconcilePending == false) + } + // MARK: - Stream - @Test("A stream join adds the member to the index and a leave removes them") @@ -57,19 +197,14 @@ struct RosterSearchTests { let harness = try Harness(pages: 1, perPage: 2) await harness.sync.syncIfNeeded(harness.chatID) - let zed = UUID() - await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ - DecodedRosterUpdate( - rosterSummary: ConversationRosterSummary(memberCount: 3, version: 3), - change: .joined(member: ConversationMember(userID: zed, displayName: "Zed Quill", version: 3), chat: nil) - ), - ])) - #expect(try await harness.search.search(chatID: harness.chatID, query: "qui", limit: 10).map(\.id) == [zed]) + let zed = member("Zed Quill", version: 3) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 3)])) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").map(\.id) == [zed.userID!]) await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ - DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 4), change: .left(userID: zed)), + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 4), change: .left(userID: zed.userID!)), ])) - #expect(try await harness.search.search(chatID: harness.chatID, query: "qui", limit: 10).isEmpty) + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").isEmpty) } @Test("A page trailing the stream cannot bring back a member who left") @@ -82,22 +217,42 @@ struct RosterSearchTests { ])) await harness.sync.syncIfNeeded(harness.chatID) - let held = try harness.database.rosterEntries(conversationID: harness.chatID).compactMap(\.member.userID) - #expect(!held.contains(departed.userID!)) + #expect(try harness.heldIDs().contains(departed.userID!) == false) } - @Test("A skipped stream version marks the group for re-sync") - func versionGapMarksResync() async throws { - let harness = try Harness(pages: 1, perPage: 2) + // MARK: - Profiles - + + @Test("Opening the picker re-reads the first page and re-tokenizes a rename") + func refreshRenames() async throws { + let original = member("Old Name") + let harness = try Harness(members: [original]) await harness.sync.syncIfNeeded(harness.chatID) - await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ - DecodedRosterUpdate( - rosterSummary: ConversationRosterSummary(memberCount: 3, version: 9), - change: .joined(member: ConversationMember(userID: UUID(), displayName: "Late", version: 9), chat: nil) - ), - ])) - #expect(try harness.database.rosterSyncState(conversationID: harness.chatID)?.needsFullSync == true) + harness.fetching.members = [ConversationMember(userID: original.userID, displayName: "New Name", version: original.version)] + await harness.search.prepare(chatID: harness.chatID) + + #expect(try await harness.names("new") == ["New Name"]) + #expect(try await harness.names("old") == []) + } + + @Test("A cached profile re-tokenizes the user in every group") + func profileRetokenizesEveryChat() async throws { + let shared = member("Old Name") + let harness = try Harness(members: [shared]) + let otherChat = ConversationID(data: Data(repeating: 0x08, count: 32)) + await harness.sync.syncIfNeeded(harness.chatID) + await harness.sync.syncIfNeeded(otherChat) + + try harness.database.upsertUserProfile( + Profile(displayName: "Nadia Renamed", phone: Phone?.none, email: nil, username: Username("nadia")), + userID: shared.userID! + ) + + for chat in [harness.chatID, otherChat] { + #expect(try await harness.search.search(chatID: chat, query: "ren").map(\.member.displayName) == ["Nadia Renamed"]) + #expect(try await harness.search.search(chatID: chat, query: "nadia").count == 1) + #expect(try await harness.search.search(chatID: chat, query: "old").isEmpty) + } } // MARK: - Matching - @@ -107,36 +262,45 @@ struct RosterSearchTests { let harness = try Harness(members: [ member("Ana María Lopez", username: "anita"), member("Bob Stone", username: "rocky"), + member("María García"), + member("Luis García"), ]) await harness.sync.syncIfNeeded(harness.chatID) #expect(try await harness.names("ana") == ["Ana María Lopez"]) - #expect(try await harness.names("mar") == ["Ana María Lopez"]) #expect(try await harness.names("lop") == ["Ana María Lopez"]) #expect(try await harness.names("roc") == ["Bob Stone"]) - #expect(try await harness.names("@roc") == ["Bob Stone"]) + #expect(try await harness.names("＠roc") == ["Bob Stone"]) #expect(try await harness.names("ana lo") == ["Ana María Lopez"]) + #expect(try await harness.names("ma garc") == ["María García"]) #expect(try await harness.names("tone") == []) } @Test("Matching ignores case and diacritics") func caseAndDiacritics() async throws { - let harness = try Harness(members: [member("Érica Souza", username: nil), member("ERIK", username: nil)]) + let harness = try Harness(members: [member("Érica Souza"), member("ERIK")]) await harness.sync.syncIfNeeded(harness.chatID) - #expect(try await harness.names("eri") == ["ERIK", "Érica Souza"].sorted { RosterSearchText.normalize($0) < RosterSearchText.normalize($1) }) + #expect(try await harness.names("eri") == ["Érica Souza", "ERIK"]) #expect(try await harness.names("ÉRI").count == 2) #expect(try await harness.names("souz") == ["Érica Souza"]) } + @Test("An emoji right after the prefix still matches") + func astralCharacterAfterPrefix() async throws { + let harness = try Harness(members: [member("Bob🔥")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("bob") == ["Bob🔥"]) + } + // MARK: - Ranking - - @Test("Recent speakers rank first, then an exact username, then by display name") + @Test("A recent speaker outranks an exact username, which outranks the rest by name") func ranking() async throws { let speaker = member("Sam Zulu", username: "samz") let exact = member("Yara Sam", username: "sam") - let alphaA = member("Samantha Adams", username: nil) - let alphaB = member("Samuel Brown", username: nil) + let alphaA = member("Samantha Adams") + let alphaB = member("Samuel Brown") let harness = try Harness(members: [alphaB, exact, alphaA, speaker]) await harness.sync.syncIfNeeded(harness.chatID) try harness.postMessage(from: speaker.userID!) @@ -146,8 +310,8 @@ struct RosterSearchTests { @Test("The most recent speaker ranks above an earlier one") func recencyOrder() async throws { - let first = member("Ann One", username: nil) - let second = member("Ann Two", username: nil) + let first = member("Ann One") + let second = member("Ann Two") let harness = try Harness(members: [first, second]) await harness.sync.syncIfNeeded(harness.chatID) try harness.postMessage(from: first.userID!) @@ -156,21 +320,40 @@ struct RosterSearchTests { #expect(try await harness.names("ann") == ["Ann Two", "Ann One"]) } + @Test("Only the newest 50 messages count toward recent speakers") + func recencyWindow() async throws { + let old = member("Ann Old") + let chatty = member("Zed Chatty") + let harness = try Harness(members: [old, chatty]) + await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: old.userID!) + for _ in 0..<50 { try harness.postMessage(from: chatty.userID!) } + + #expect(try await harness.names("") == ["Zed Chatty"]) + } + + @Test("Folded name ties fall back to the raw display name") + func rawNameTiebreak() async throws { + let harness = try Harness(members: [member("éva"), member("Eva")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("eva") == ["Eva", "éva"]) + } + @Test("An empty query returns recent speakers only") func emptyQuery() async throws { - let speaker = member("Talker", username: nil) - let harness = try Harness(members: [speaker, member("Quiet", username: nil)]) + let speaker = member("Talker") + let harness = try Harness(members: [speaker, member("Quiet")]) await harness.sync.syncIfNeeded(harness.chatID) try harness.postMessage(from: speaker.userID!) #expect(try await harness.names("") == ["Talker"]) - #expect(try await harness.names("@") == ["Talker"]) + #expect(try await harness.names("＠") == ["Talker"]) } @Test("The signed-in user is never a match, even after speaking") func excludesSelf() async throws { let me = member("Sam Me", username: "me") - let harness = try Harness(members: [me, member("Sam Other", username: nil)], selfUserID: me.userID!) + let harness = try Harness(members: [me, member("Sam Other")], selfUserID: me.userID!) await harness.sync.syncIfNeeded(harness.chatID) try harness.postMessage(from: me.userID!) @@ -178,26 +361,27 @@ struct RosterSearchTests { #expect(try await harness.names("") == []) } - @Test("A first-page refresh rewrites a renamed member's tokens") - func refreshRenames() async throws { - let original = member("Old Name", username: nil) - let harness = try Harness(members: [original]) + @Test("A blocked user is never a match") + func excludesBlocked() async throws { + let blocked = member("Sam Blocked") + let harness = try Harness(members: [blocked, member("Sam Other")], blocked: [blocked.userID!]) await harness.sync.syncIfNeeded(harness.chatID) + try harness.postMessage(from: blocked.userID!) - harness.fetching.members = [ConversationMember(userID: original.userID, displayName: "New Name", version: original.version)] - await harness.search.prepare(chatID: harness.chatID) - - #expect(try await harness.names("new") == ["New Name"]) - #expect(try await harness.names("old") == []) + #expect(try await harness.names("sam") == ["Sam Other"]) } } // MARK: - Support - -private func member(_ displayName: String, username: String?, version: UInt64 = 1) -> ConversationMember { +private func member(_ displayName: String, username: String? = nil, version: UInt64 = 1) -> ConversationMember { ConversationMember(userID: UUID(), displayName: displayName, username: username.flatMap(Username.init), version: version) } +private func joined(_ member: ConversationMember, count: UInt64, version: UInt64) -> DecodedRosterUpdate { + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: count, version: version), change: .joined(member: member, chat: nil)) +} + @MainActor private struct Harness { let database: Database @@ -205,16 +389,14 @@ private struct Harness { let sync: RosterSync let search: LocalRosterSearch let chatID = ConversationID(data: Data(repeating: 0x07, count: 32)) - private let url: URL private let messageCounter = Counter() - init(members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID()) throws { - let (database, url) = try Database.makeTemp() + init(members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID(), blocked: Set = []) throws { + let (database, _) = try Database.makeTemp() self.database = database - self.url = url self.fetching = FakeRosterFetching(members: members, perPage: perPage) self.sync = RosterSync(fetching: fetching, database: database, owner: .generate()!, pageCap: pageCap) - self.search = LocalRosterSearch(database: database, roster: sync, selfUserID: selfUserID) + self.search = LocalRosterSearch(database: database, roster: sync, selfUserID: selfUserID, blockedUserIDs: { blocked }) } init(pages: Int, perPage: Int, pageCap: Int = RosterSync.defaultPageCap) throws { @@ -222,6 +404,14 @@ private struct Harness { try self.init(members: members, perPage: perPage, pageCap: pageCap) } + func state() throws -> RosterSyncState { + try #require(try database.rosterSyncState(conversationID: chatID)) + } + + func heldIDs() throws -> [UserID] { + try database.rosterEntries(conversationID: chatID).compactMap(\.member.userID) + } + func names(_ query: String) async throws -> [String] { try await search.search(chatID: chatID, query: query, limit: 10).map(\.member.displayName) } @@ -234,17 +424,30 @@ private struct Harness { } } +private extension Database { + /// Stands in for a catch-up that found more members held than counted. + func markRosterReconcilePending(_ conversationID: ConversationID) throws { + let s = RosterSyncTable() + try writer.run(s.table.filter(s.conversationId == conversationID.data).update(s.reconcilePending <- true)) + } +} + nonisolated private final class Counter: @unchecked Sendable { private let lock = NSLock() private var value: UInt64 = 0 func next() -> UInt64 { lock.withLock { value += 1; return value } } } -/// Serves a fixed roster in pages, most recent first, like `Chat.GetRoster`. +/// Serves a roster in pages, most recently joined first, like `Chat.GetRoster`. nonisolated private final class FakeRosterFetching: RosterFetching, @unchecked Sendable { + struct Failure: Error {} + private let lock = NSLock() private let perPage: Int private var _members: [ConversationMember] + private var _version: UInt64 = 2 + private var _pageVersions: [UInt64]? + private var _failingCalls: Set = [] private var _calls = 0 init(members: [ConversationMember], perPage: Int) { @@ -257,17 +460,37 @@ nonisolated private final class FakeRosterFetching: RosterFetching, @unchecked S set { lock.withLock { _members = newValue } } } + /// The roster version every page reports, unless ``pageVersions`` overrides it per page. + var version: UInt64 { + get { lock.withLock { _version } } + set { lock.withLock { _version = newValue } } + } + + var pageVersions: [UInt64]? { + get { lock.withLock { _pageVersions } } + set { lock.withLock { _pageVersions = newValue } } + } + + /// 1-based call numbers that throw instead of returning a page. + var failingCalls: Set { + get { lock.withLock { _failingCalls } } + set { lock.withLock { _failingCalls = newValue } } + } + var calls: Int { lock.withLock { _calls } } func getRosterPage(owner: KeyPair, conversationID: ConversationID, pagingToken: Data?) async throws -> FlipClient.RosterPage { - lock.withLock { + try lock.withLock { _calls += 1 + if _failingCalls.contains(_calls) { throw Failure() } let start = pagingToken.map { Int(String(decoding: $0, as: UTF8.self))! } ?? 0 let end = min(start + perPage, _members.count) let next = end < _members.count ? Data(String(end).utf8) : nil + let pageIndex = start / perPage + let version = _pageVersions.flatMap { pageIndex < $0.count ? $0[pageIndex] : nil } ?? _version return FlipClient.RosterPage( members: Array(_members[start.. Date: Tue, 29 Sep 2026 23:11:30 -0400 Subject: [PATCH 4/7] test(chat): cover stale leaves and profile writes after a leave Android found two leave gaps in its roster store. Neither exists here: refreshRosterProfile touches only rows with isMember set and never writes version or isMember, and applyRosterUpdates skips a change at or below the held version before it touches the row or its tokens. These tests hold both in place, and add a code-point sort case that doesn't depend on fullwidth letters, which NFKD folds to ASCII. --- FlipcashTests/Chat/RosterSearchTests.swift | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/FlipcashTests/Chat/RosterSearchTests.swift b/FlipcashTests/Chat/RosterSearchTests.swift index e2fcae980..342ed4d98 100644 --- a/FlipcashTests/Chat/RosterSearchTests.swift +++ b/FlipcashTests/Chat/RosterSearchTests.swift @@ -220,6 +220,41 @@ struct RosterSearchTests { #expect(try harness.heldIDs().contains(departed.userID!) == false) } + @Test("A stale leave, older than the held join, changes nothing") + func staleLeaveIgnored() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + + let zed = member("Zed Quill", version: 5) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [joined(zed, count: 3, version: 5)])) + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 2, version: 3), change: .left(userID: zed.userID!)), + ])) + + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").map(\.id) == [zed.userID!]) + #expect(try harness.state().heldCount == 3) + } + + @Test("A profile write after a leave doesn't bring the member back") + func profileAfterLeaveStaysOut() async throws { + let harness = try Harness(pages: 1, perPage: 2) + await harness.sync.syncIfNeeded(harness.chatID) + let departed = harness.fetching.members[0] + await harness.sync.apply(.rosterChanged(conversationID: harness.chatID, updates: [ + DecodedRosterUpdate(rosterSummary: ConversationRosterSummary(memberCount: 1, version: 3), change: .left(userID: departed.userID!)), + ])) + + try harness.database.upsertUserProfile( + Profile(displayName: "Zed Quill", phone: Phone?.none, email: nil, username: Username("zedq")), + userID: departed.userID! + ) + + #expect(try await harness.search.search(chatID: harness.chatID, query: "qui").isEmpty) + #expect(try await harness.search.search(chatID: harness.chatID, query: "zedq").isEmpty) + #expect(try harness.state().heldCount == 1) + #expect(try harness.heldIDs().contains(departed.userID!) == false) + } + // MARK: - Profiles - @Test("Opening the picker re-reads the first page and re-tokenizes a rename") @@ -339,6 +374,14 @@ struct RosterSearchTests { #expect(try await harness.names("eva") == ["Eva", "éva"]) } + @Test("Names sort by code point, so U+FFFD comes before an emoji") + func codePointOrder() async throws { + // UTF-16 order would put the emoji's surrogate (D83D) first; neither character decomposes. + let harness = try Harness(members: [member("Ann \u{1F600}"), member("Ann \u{FFFD}")]) + await harness.sync.syncIfNeeded(harness.chatID) + #expect(try await harness.names("ann") == ["Ann \u{FFFD}", "Ann \u{1F600}"]) + } + @Test("An empty query returns recent speakers only") func emptyQuery() async throws { let speaker = member("Talker") From 50b16abba9348115aad63478c1299603e715e8bb Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Wed, 30 Sep 2026 07:07:06 -0400 Subject: [PATCH 5/7] chore(chat): drop an unneeded await in the full roster read --- Flipcash/Core/Controllers/RosterSync.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Flipcash/Core/Controllers/RosterSync.swift b/Flipcash/Core/Controllers/RosterSync.swift index d649d334d..b7aba7f35 100644 --- a/Flipcash/Core/Controllers/RosterSync.swift +++ b/Flipcash/Core/Controllers/RosterSync.swift @@ -136,7 +136,7 @@ actor RosterSync { let task = Task(priority: reconciling ? .background : nil) { let read = try await self.readPages(conversationID) { _ in false } try self.database.applyFullRosterRead(read.members, summaries: read.summaries, isComplete: read.reachedEnd, conversationID: conversationID) - await self.logRead("Read full roster", read, conversationID: conversationID) + self.logRead("Read full roster", read, conversationID: conversationID) } do { try await task.value From 79cd9ac3be78b7a71b6fdb032a992115cd7560f8 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Wed, 30 Sep 2026 08:28:27 -0400 Subject: [PATCH 6/7] fix(store): add the roster tables without wiping existing stores The 43 -> 44 bump deleted every store on upgrade. The three roster tables are new and start empty, and createTablesIfNeeded() already creates them with IF NOT EXISTS on every open, so a v43 store gains them in place. Each group has no roster_sync row yet and does a full read the first time it opens. The version check moves into Database.discardStoreIfOutdated so its keep/delete decision is testable: a store at or above schemaVersion is kept, and a later bump still deletes the whole file, roster tables included. --- Flipcash/Core/Session/DatabaseStore.swift | 6 +- .../Sources/FlipcashStore/Database.swift | 13 +- FlipcashTests/Chat/RosterSearchTests.swift | 27 +++ .../Chat/RosterStoreUpgradeTests.swift | 160 ++++++++++++++++++ 4 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 FlipcashTests/Chat/RosterStoreUpgradeTests.swift diff --git a/Flipcash/Core/Session/DatabaseStore.swift b/Flipcash/Core/Session/DatabaseStore.swift index 08f3fccd7..d42488f4e 100644 --- a/Flipcash/Core/Session/DatabaseStore.swift +++ b/Flipcash/Core/Session/DatabaseStore.swift @@ -67,12 +67,8 @@ final class DatabaseStore { // Currently we don't do migrations so every time // the user version is outdated, we'll rebuild the // database during sync. - let userVersion = (try? Database.userVersion(files: files)) ?? 0 - let currentVersion = Database.schemaVersion - if currentVersion > userVersion { - try Database.deleteStore(files: files) + if try Database.discardStoreIfOutdated(files: files) { logger.error("Outdated user version, deleted database.") - try Database.setUserVersion(version: currentVersion, files: files) } let database = try Database(url: files.database) diff --git a/FlipcashCore/Sources/FlipcashStore/Database.swift b/FlipcashCore/Sources/FlipcashStore/Database.swift index 7c0ae939f..af7b25efe 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database.swift @@ -197,7 +197,7 @@ nonisolated open class Database: @unchecked Sendable { /// the notification service extension needs the same number to decide whether the store on disk /// is one it understands, and an extension cannot read the app's `Info.plist` — separate bundles. /// Both targets link this module, so they cannot disagree. - public static let schemaVersion = 44 + public static let schemaVersion = 43 /// Removes the store and the write-ahead log files beside it. /// @@ -225,6 +225,17 @@ nonisolated open class Database: @unchecked Sendable { ) } + /// Deletes the store when its recorded version is below `schemaVersion` and records the new one, + /// returning whether it deleted. A store at or above `schemaVersion` is left as it is. + @discardableResult + public static func discardStoreIfOutdated(files: StoreLocation.Files, schemaVersion: Int = Database.schemaVersion) throws -> Bool { + let recorded = (try? userVersion(files: files)) ?? 0 + guard schemaVersion > recorded else { return false } + try deleteStore(files: files) + try setUserVersion(version: schemaVersion, files: files) + return true + } + public static func userVersion(files: StoreLocation.Files) throws -> Int? { let versionString = try String( contentsOf: files.version, diff --git a/FlipcashTests/Chat/RosterSearchTests.swift b/FlipcashTests/Chat/RosterSearchTests.swift index 342ed4d98..309ee8192 100644 --- a/FlipcashTests/Chat/RosterSearchTests.swift +++ b/FlipcashTests/Chat/RosterSearchTests.swift @@ -14,6 +14,29 @@ import SQLite @MainActor struct RosterSearchTests { + // MARK: - Upgrade - + + @Test("A store from before the roster tables syncs a group on first open and searches it") + func upgradedStoreSyncsOnFirstOpen() async throws { + let url = try RosterStoreFixture.makeStoreWithoutRosterTables() + defer { Database.removeTemp(at: url) } + let database = try Database(url: url) + defer { try? database.close() } + + let members = [ + ConversationMember(userID: UUID(), displayName: "Érica Souza", version: 1), + ConversationMember(userID: UUID(), displayName: "Luis García", version: 1), + ] + let harness = Harness(database: database, members: members) + #expect(try database.rosterSyncState(conversationID: harness.chatID) == nil) + + await harness.sync.syncIfNeeded(harness.chatID) + + #expect(harness.fetching.calls == 1) + #expect(try harness.state().fullySynced) + #expect(try await harness.names("eri") == ["Érica Souza"]) + } + // MARK: - Full read - @Test("A first open pages until has_more is false and persists every member") @@ -436,6 +459,10 @@ private struct Harness { init(members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID(), blocked: Set = []) throws { let (database, _) = try Database.makeTemp() + self.init(database: database, members: members, perPage: perPage, pageCap: pageCap, selfUserID: selfUserID, blocked: blocked) + } + + init(database: Database, members: [ConversationMember], perPage: Int = 100, pageCap: Int = RosterSync.defaultPageCap, selfUserID: UserID = UUID(), blocked: Set = []) { self.database = database self.fetching = FakeRosterFetching(members: members, perPage: perPage) self.sync = RosterSync(fetching: fetching, database: database, owner: .generate()!, pageCap: pageCap) diff --git a/FlipcashTests/Chat/RosterStoreUpgradeTests.swift b/FlipcashTests/Chat/RosterStoreUpgradeTests.swift new file mode 100644 index 000000000..e7be240ff --- /dev/null +++ b/FlipcashTests/Chat/RosterStoreUpgradeTests.swift @@ -0,0 +1,160 @@ +// +// RosterStoreUpgradeTests.swift +// FlipcashTests +// + +import Testing +import Foundation +import FlipcashCore +import FlipcashStore +import SQLite +@testable import Flipcash + +/// The roster tables arrive without a schema bump, so an existing store must gain them on open +/// with its data intact, and the version gate must still behave as it did. +@Suite("Roster tables on an existing store") +struct RosterStoreUpgradeTests { + + private static let rosterTables = [RosterMemberTable.name, RosterTokenTable.name, RosterSyncTable.name] + + @Test("Opening a store from before the roster tables keeps its data and adds them") + func keepsDataAndAddsTables() throws { + let url = try RosterStoreFixture.makeStoreWithoutRosterTables() + defer { Database.removeTemp(at: url) } + let before = try RosterStoreFixture.rowCounts(at: url) + #expect(before.conversations == 2) + #expect(before.messages == 3) + #expect(try RosterStoreFixture.tableNames(at: url).isDisjoint(with: Self.rosterTables)) + + let database = try Database(url: url) + try database.close() + + #expect(try RosterStoreFixture.rowCounts(at: url) == before) + #expect(try RosterStoreFixture.tableNames(at: url).isSuperset(of: Self.rosterTables)) + #expect(try RosterStoreFixture.rosterRowCount(at: url) == 0) + } + + @Test("Opening the upgraded store again changes nothing") + func secondOpenIsANoOp() throws { + let url = try RosterStoreFixture.makeStoreWithoutRosterTables() + defer { Database.removeTemp(at: url) } + + try Database(url: url).close() + let first = try RosterStoreFixture.rowCounts(at: url) + let schema = try RosterStoreFixture.schemaSQL(at: url) + + try Database(url: url).close() + + #expect(try RosterStoreFixture.rowCounts(at: url) == first) + #expect(try RosterStoreFixture.schemaSQL(at: url) == schema) + } + + @Test("A store recorded at the current version is kept") + func currentVersionKeepsStore() throws { + let (files, root) = try Self.makeStore(recordedVersion: Database.schemaVersion) + defer { try? FileManager.default.removeItem(at: root) } + + #expect(try Database.discardStoreIfOutdated(files: files) == false) + #expect(FileManager.default.fileExists(atPath: files.database.path)) + } + + @Test("A store recorded above the code's version is kept, not deleted") + func newerVersionKeepsStore() throws { + let (files, root) = try Self.makeStore(recordedVersion: Database.schemaVersion + 1) + defer { try? FileManager.default.removeItem(at: root) } + + #expect(try Database.discardStoreIfOutdated(files: files) == false) + #expect(FileManager.default.fileExists(atPath: files.database.path)) + #expect(try Database.userVersion(files: files) == Database.schemaVersion + 1) + } + + @Test("A later bump still deletes the whole store, roster tables included") + func laterBumpDeletesStore() throws { + let (files, root) = try Self.makeStore(recordedVersion: Database.schemaVersion) + defer { try? FileManager.default.removeItem(at: root) } + let bumped = Database.schemaVersion + 1 + + #expect(try Database.discardStoreIfOutdated(files: files, schemaVersion: bumped)) + #expect(!FileManager.default.fileExists(atPath: files.database.path)) + #expect(try Database.userVersion(files: files) == bumped) + + let reopened = try Database(url: files.database) + try reopened.close() + #expect(try RosterStoreFixture.rowCounts(at: files.database) == .init(conversations: 0, messages: 0)) + } + + /// A full current-schema store with data in it, recorded at `recordedVersion`. + private static func makeStore(recordedVersion: Int) throws -> (StoreLocation.Files, URL) { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("roster-upgrade-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let files = StoreLocation(directory: root, legacyDirectory: root, isShared: true).files(owner: try PublicKey(Data(repeating: 9, count: 32))) + let database = try Database(url: files.database) + try RosterStoreFixture.seed(database) + try database.close() + try Database.setUserVersion(version: recordedVersion, files: files) + return (files, root) + } +} + +/// Builds stores shaped like the ones already on devices, which predate the roster tables. +enum RosterStoreFixture { + + struct RowCounts: Equatable { + let conversations: Int + let messages: Int + } + + /// A store with two conversations and three messages, and the roster tables dropped. + static func makeStoreWithoutRosterTables() throws -> URL { + let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("roster-v43-\(UUID().uuidString).sqlite") + let database = try Database(url: url) + try seed(database) + try database.close() + + let connection = try Connection(url.path) + for table in [RosterMemberTable.name, RosterTokenTable.name, RosterSyncTable.name] { + try connection.run("DROP TABLE \(table)") + } + return url + } + + static func seed(_ database: Database) throws { + for (index, id) in [ConversationID.test(1), ConversationID.test(2)].enumerated() { + try database.upsertConversation(Conversation(id: id, members: [], lastMessage: nil, lastActivity: Date(timeIntervalSince1970: Double(index)))) + } + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: 1), senderID: nil, content: .text("one"), date: Date(timeIntervalSince1970: 10), unreadSeq: 1), + ConversationMessage(id: MessageID(value: 2), senderID: nil, content: .text("two"), date: Date(timeIntervalSince1970: 20), unreadSeq: 2), + ], conversationID: .test(1)) + try database.upsertConversationMessages([ + ConversationMessage(id: MessageID(value: 3), senderID: nil, content: .text("three"), date: Date(timeIntervalSince1970: 30), unreadSeq: 1), + ], conversationID: .test(2)) + } + + static func rowCounts(at url: URL) throws -> RowCounts { + let connection = try Connection(url.path, readonly: true) + return RowCounts( + conversations: Int(try connection.scalar("SELECT count(*) FROM \(ConversationTable.name)") as! Int64), + messages: Int(try connection.scalar("SELECT count(*) FROM \(ConversationMessageTable.name)") as! Int64) + ) + } + + static func rosterRowCount(at url: URL) throws -> Int { + let connection = try Connection(url.path, readonly: true) + var total: Int64 = 0 + for table in [RosterMemberTable.name, RosterTokenTable.name, RosterSyncTable.name] { + total += try connection.scalar("SELECT count(*) FROM \(table)") as! Int64 + } + return Int(total) + } + + static func tableNames(at url: URL) throws -> Set { + let connection = try Connection(url.path, readonly: true) + return Set(try connection.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").compactMap { $0[0] as? String }) + } + + static func schemaSQL(at url: URL) throws -> [String] { + let connection = try Connection(url.path, readonly: true) + return try connection.prepare("SELECT sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name").compactMap { $0[0] as? String } + } +} From 7f18e655c94d509ba3bf83b3cb7d2b41501ce795 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Wed, 30 Sep 2026 09:49:17 -0400 Subject: [PATCH 7/7] docs(store): let a new empty table skip the schema bump createTablesIfNeeded() runs on every open with IF NOT EXISTS, so a new table that starts empty and fills from the server reaches existing stores without deleting them. Changes to an existing table's columns still bump. --- .claude/docs/hard-rules.md | 6 +++++- CLAUDE.md | 2 +- FlipcashCore/Sources/FlipcashStore/Database.swift | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.claude/docs/hard-rules.md b/.claude/docs/hard-rules.md index cd9584819..8d9e44627 100644 --- a/.claude/docs/hard-rules.md +++ b/.claude/docs/hard-rules.md @@ -73,7 +73,11 @@ app can fix itself belongs in the service files that wrap the generated code. migrations — when the version number increases, the database is deleted and rebuilt from server data on next login (`SessionAuthenticator.initializeDatabase`). This means: -- Adding/removing tables or columns → bump version +- Removing a table, or adding/removing/changing columns on an existing table → bump version +- **Adding a new table does not bump**, as long as it starts empty and fills from the server. + `createTablesIfNeeded()` runs on every open with `create(ifNotExists: true)`, so an existing store + gains the table in place and keeps its data. Code reading the new table must treat "empty" as + "never synced". Any change to an existing table's columns still bumps. - Changing which table a query reads from → bump version if the old schema can't satisfy the new query - **Changing the encoding of a value already stored in a column → bump version.** The column's SQL type is unchanged, so nothing about `Schema.swift` looks different, but rows written by an earlier diff --git a/CLAUDE.md b/CLAUDE.md index 5036d06cd..954ae3569 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,7 +105,7 @@ it before touching the relevant area. - **Exhaustive switches** — Prefer `switch` over `if case` for enums so the compiler flags new cases. - **Modernize incrementally** — Use modern Swift/SwiftUI APIs in net-new/isolated code; don't refactor working code just to modernize. One observation system per class. - **Generated protos** — `FlipcashAPI` only re-exports the published contract packages; there is no generated code to edit here. Change the wrapping service files instead. -- **Database schema** — Bump `Database.schemaVersion` on every change to what the store *writes*, including the encoding of a persisted blob (no migrations; DB is rebuilt from server). +- **Database schema** — Bump `Database.schemaVersion` on every change to what the store *writes*, including the encoding of a persisted blob (no migrations; DB is rebuilt from server). A change that only adds new tables, which start empty and fill from the server, does not bump. - **Logging** — Message string is a constant; every variable goes in structured `metadata`. Never log proto blobs whole. - **Error reporting** — Call `ErrorReporting.captureError(...)` unconditionally; classify via `ServerError.reportingLevel`, never gate at the call site. Best-effort chatter never reports. - **Form validation** — Validate free-form input through the `Validator` family and submit the validator's `Output`, never inline regex/trim; keypad amounts parse only via `AmountValidator`. diff --git a/FlipcashCore/Sources/FlipcashStore/Database.swift b/FlipcashCore/Sources/FlipcashStore/Database.swift index af7b25efe..b5690a849 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database.swift @@ -187,12 +187,15 @@ nonisolated open class Database: @unchecked Sendable { /// The schema version this build writes. /// /// A launch that finds a lower version recorded beside the store deletes the store and rebuilds - /// it from sync, which is the project's substitute for schema migrations. Bump this whenever a - /// table definition in `Schema.swift` changes, and equally whenever the *encoding* of a value + /// it from sync, which is the project's substitute for schema migrations. Bump this whenever an + /// existing table definition in `Schema.swift` changes, and equally whenever the *encoding* of a value /// already stored in a column changes — a new non-optional property on a `Codable` persisted as /// a JSON blob leaves the column's SQL type untouched, but rows written by an earlier build stop /// decoding, and the `try?` at the read sites turns that into a silent nil for the whole value. /// + /// A brand-new table that starts empty and fills from the server doesn't need a bump: + /// `createTablesIfNeeded()` adds it to an existing store on the next open. + /// /// This used to be the `SQLiteVersion` key in the app's `Info.plist`. It moved into code because /// the notification service extension needs the same number to decide whether the store on disk /// is one it understands, and an extension cannot read the app's `Info.plist` — separate bundles.