From 5a70b5d2218f02ed6f351f25791449f2ab569f2e Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Sat, 3 Oct 2026 13:15:23 -0400 Subject: [PATCH 1/2] feat(chat): cache photos on disk and encrypt them in encrypted DMs Downloaded photos go to a dedicated Kingfisher disk cache keyed by blob id, so they no longer fall back to the blur hash after five minutes. In an encrypted DM a photo is sealed with the shared-core chat cipher, stored with endToEndEncryptedFor set to the chat, and sent as EncryptedContent. Incoming encrypted photos are decrypted before they are drawn, and one that can't be opened shows as unavailable. Blob rejections now log the server's reason code instead of "unknown". --- .../Controllers/ConversationController.swift | 30 +- .../Controllers/EncryptedChatClient.swift | 66 ++- .../Controllers/FlipClient+Protocols.swift | 11 +- .../Screens/Conversation/AttachMenu.swift | 11 + .../Conversation/ChatItem+Conversation.swift | 5 +- .../Conversation/ChatMediaUploader.swift | 131 +++++- .../ChatScreenRepresentable.swift | 23 +- .../Screens/Conversation/ComposerChip.swift | 8 +- .../Conversation/ComposerReplyStrip.swift | 14 +- .../Conversation/ConversationBottomBar.swift | 4 +- .../Conversation/ConversationScreen.swift | 31 +- .../Profile/DialogItem+ProfilePicture.swift | 2 +- .../FlipcashCore/Blob/BlobUploader.swift | 74 ++- .../Clients/Flip API/FlipClient+Blob.swift | 8 + .../Clients/Flip API/FlipClient+Chat.swift | 9 + .../Flip API/Services/BlobService.swift | 19 +- .../Services/ChatMessagingService.swift | 13 + .../Sources/FlipcashCore/Models/Blob.swift | 57 ++- .../Models/Chat/ChatMediaStrings.swift | 4 + .../Models/Chat/ChatMessage.swift | 5 +- .../Models/Chat/ChatPreviewMapping.swift | 3 +- .../FlipcashCore/Models/Chat/ChatQuote.swift | 5 +- .../Models/Conversation/ChatSeal.swift | 209 ++++++++- .../Conversation/ConversationMessage.swift | 10 +- .../Models/Conversation/MediaAttachment.swift | 21 +- .../Sources/FlipcashStore/Database.swift | 2 +- .../BlobRejectionReasonTests.swift | 31 ++ .../FlipcashCoreTests/BlobUploaderTests.swift | 75 ++- .../ChatPreviewDecryptionTests.swift | 19 + .../FlipcashCoreTests/ChatSealTests.swift | 170 +++++++ .../FlipcashCoreTests/UploadPolicyTests.swift | 16 + .../ChatCipherBlobVectorTests.swift | 121 +++++ .../Fixtures/chat_cipher.json | 438 ++++++++++++++++++ FlipcashTests/Chat/ChatMediaCellTests.swift | 116 ++++- .../Chat/ChatMediaURLResolverTests.swift | 79 +++- .../Chat/ChatMediaUploaderTests.swift | 114 ++++- FlipcashTests/Chat/ChatMediaViewerTests.swift | 8 +- FlipcashTests/Chat/ChatQuoteBubbleTests.swift | 6 +- .../Chat/ConversationMediaSendTests.swift | 4 +- .../Database+ConversationsTests.swift | 21 + .../TestSupport/MockChatMediaBlobStore.swift | 11 + .../TestSupport/MockConversations.swift | 20 +- .../FlipcashUI/Chat/ChatBubbleView.swift | 6 +- .../FlipcashUI/Chat/ChatLinkMessageCell.swift | 6 +- .../FlipcashUI/Chat/ChatMediaCell.swift | 47 +- .../Chat/ChatMediaImageSource.swift | 110 ++++- .../Chat/ChatMediaURLResolver.swift | 113 +++-- .../Chat/ChatMediaViewerController.swift | 18 +- .../FlipcashUI/Chat/ChatMessageCell.swift | 6 +- .../FlipcashUI/Chat/ChatQuotePanelView.swift | 32 +- .../FlipcashUI/Chat/ChatViewController.swift | 14 +- .../FlipcashUI/Chat/LinkableBubbleView.swift | 6 +- .../Sources/FlipcashUI/Images/BlurHash.swift | 101 ++++ 53 files changed, 2265 insertions(+), 218 deletions(-) create mode 100644 FlipcashCore/Tests/FlipcashCoreTests/BlobRejectionReasonTests.swift create mode 100644 FlipcashCoreVectors/Tests/FlipcashCoreVectorsTests/ChatCipherBlobVectorTests.swift create mode 100644 FlipcashCoreVectors/Tests/FlipcashCoreVectorsTests/Fixtures/chat_cipher.json diff --git a/Flipcash/Core/Controllers/ConversationController.swift b/Flipcash/Core/Controllers/ConversationController.swift index a9eda1fff..bb06755e2 100644 --- a/Flipcash/Core/Controllers/ConversationController.swift +++ b/Flipcash/Core/Controllers/ConversationController.swift @@ -2068,6 +2068,30 @@ final class ConversationController { /// posted only once its own chip's upload settles, so a later photo never lands before an /// earlier one. A chip that fails marks only its own bubble `.failed`, and the next chip still /// gets its turn. + /// The seal a photo staged for `conversationID` is encrypted with, nil when the chat takes + /// plaintext photos; throws when the chat encrypts but its seal can't be built. + func photoSeal(for conversationID: ConversationID) async throws -> ChatSeal? { + try await messaging.photoSeal(owner: owner, conversationID: conversationID) + } + + /// Decrypts `conversationID`'s end-to-end encrypted photo blobs, or nil while the chat's key is + /// unknown or the chat doesn't encrypt. + func mediaBlobDecrypt(for conversationID: ConversationID) async -> (@Sendable (Data, BlobID, SealedBlob) throws -> Data)? { + guard let seal = await messaging.openingSeal(owner: owner, conversationID: conversationID) else { return nil } + return { blob, blobID, sealed in + do { + return try seal.decryptBlob(blob, blobID: blobID, sealed: sealed) + } catch { + logger.warning("Encrypted chat photo failed to open", metadata: [ + "conversationID": "\(conversationID)", + "blobID": "\(blobID)", + "error": "\(error)", + ]) + throw error + } + } + } + @discardableResult func sendMedia( _ chips: [ComposerChip], @@ -2163,9 +2187,9 @@ final class ConversationController { logger.error("Photo send has no upload to await", metadata: ["conversationID": "\(conversationID)"]) return false } - let blobID: BlobID + let photo: UploadedPhoto do { - blobID = try await upload.value + photo = try await upload.value } catch { store.markPending(clientMessageID: clientMessageID, status: .failed, in: conversationID) logger.error("Failed to upload conversation photo", metadata: [ @@ -2181,7 +2205,7 @@ final class ConversationController { let message = try await messaging.sendMediaMessage( owner: owner, conversationID: conversationID, - blobID: blobID, + photo: photo, caption: caption, repliedTo: repliedTo, clientMessageID: clientMessageID diff --git a/Flipcash/Core/Controllers/EncryptedChatClient.swift b/Flipcash/Core/Controllers/EncryptedChatClient.swift index 669cfc495..f798d1bd7 100644 --- a/Flipcash/Core/Controllers/EncryptedChatClient.swift +++ b/Flipcash/Core/Controllers/EncryptedChatClient.swift @@ -9,6 +9,8 @@ import Foundation import os import FlipcashCore +private let logger = Logger(label: "flipcash.encrypted-chat") + /// The chat surface `ConversationController` talks to, with DM end-to-end encryption applied at /// the edge: every message read from the server comes back decrypted (or marked with why it /// couldn't be), and every text sent into an encrypting chat goes out sealed. @@ -141,9 +143,67 @@ extension EncryptedChatClient: ConversationMessaging { } } - /// Media has no sealed form, so a photo goes out in plaintext whatever the chat's encryption. - func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, blobID: BlobID, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage { - try await client.sendMediaMessage(owner: owner, conversationID: conversationID, blobID: blobID, caption: caption, repliedTo: repliedTo, clientMessageID: clientMessageID) + /// Sends a photo the way it was uploaded, after checking the chat still decides the same way: a + /// plaintext blob never goes into a chat that encrypts, nor a sealed one into a chat that doesn't. + func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, photo: UploadedPhoto, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage { + try await refetchingOnRefusal(conversationID) { + let seal = try await sealForSending(conversationID, owner: owner) + switch Self.photoSend(photo, seal: seal, conversationID: conversationID) { + case .plain(let blobID): + return try await client.sendMediaMessage(owner: owner, conversationID: conversationID, blobID: blobID, caption: caption, repliedTo: repliedTo, clientMessageID: clientMessageID) + case .sealed(let sealed, let seal): + return try await client.sendSealedMediaMessage(owner: owner, conversationID: conversationID, photo: sealed, caption: caption, repliedTo: repliedTo, seal: seal, clientMessageID: clientMessageID) + case .mismatch: + logger.error("Photo was uploaded for a different encryption choice", metadata: [ + "conversationID": "\(conversationID)", + "sealed": "\(seal != nil)", + ]) + throw ErrorSendMessage.encryptionFailed + } + } + } + + /// How an uploaded photo goes out given the chat's current seal. + enum PhotoSend: Equatable { + /// Plaintext, into a chat that doesn't encrypt. + case plain(BlobID) + /// Sealed, into the chat it was encrypted for. + case sealed(SealedPhoto, ChatSeal) + /// The photo was uploaded for a different encryption choice than the chat now makes. + case mismatch + + static func == (lhs: PhotoSend, rhs: PhotoSend) -> Bool { + switch (lhs, rhs) { + case (.plain(let a), .plain(let b)): a == b + case (.sealed(let a, _), .sealed(let b, _)): a == b + case (.mismatch, .mismatch): true + case (.plain, _), (.sealed, _), (.mismatch, _): false + } + } + } + + /// Matches `photo` to `seal`: a plaintext blob never goes into a chat that encrypts, nor a + /// sealed one into a chat that doesn't or a different chat. + static func photoSend(_ photo: UploadedPhoto, seal: ChatSeal?, conversationID: ConversationID) -> PhotoSend { + switch (photo, seal) { + case (.plain(let blobID), nil): + .plain(blobID) + case (.sealed(let sealed), let seal?) where seal.conversationID == conversationID: + .sealed(sealed, seal) + case (.plain, _?), (.sealed, _): + .mismatch + } + } + + func photoSeal(owner: KeyPair, conversationID: ConversationID) async throws -> ChatSeal? { + try await sealForSending(conversationID, owner: owner) + } + + func openingSeal(owner: KeyPair, conversationID: ConversationID) async -> ChatSeal? { + switch await opener(conversationID, owner: owner) { + case .seal(let seal): seal + case .awaitingKey, .unsupported: nil + } } func editMessage(owner: KeyPair, conversationID: ConversationID, messageID: MessageID, text: String, repliedTo: MessageID?, expectedEventSequence: UInt64) async throws -> MessageMutation { diff --git a/Flipcash/Core/Controllers/FlipClient+Protocols.swift b/Flipcash/Core/Controllers/FlipClient+Protocols.swift index cdf13d4d8..8946dd06a 100644 --- a/Flipcash/Core/Controllers/FlipClient+Protocols.swift +++ b/Flipcash/Core/Controllers/FlipClient+Protocols.swift @@ -112,8 +112,15 @@ protocol ConversationMessaging: AnyObject, Sendable { onBatch: @MainActor @Sendable @escaping (_ messages: [ConversationMessage], _ checkpoint: UInt64?) -> Void ) async throws -> UInt64 func sendMessage(owner: KeyPair, conversationID: ConversationID, text: String, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage - /// Sends the finalized photo `blobID` as one media message, with `caption` under it. - func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, blobID: BlobID, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage + /// Sends the finalized `photo` as one media message, with `caption` under it; a sealed photo + /// goes out inside `EncryptedContent`. + func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, photo: UploadedPhoto, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage + /// The seal a photo sent into `conversationID` is encrypted with, nil when it goes out in + /// plaintext. Throws when the chat encrypts but its seal can't be built. + func photoSeal(owner: KeyPair, conversationID: ConversationID) async throws -> ChatSeal? + /// The seal `conversationID`'s messages open with, nil when it can't be built yet or the chat + /// doesn't encrypt. + func openingSeal(owner: KeyPair, conversationID: ConversationID) async -> ChatSeal? /// Replaces a message's text, keeping it a reply to `repliedTo` when set. `expectedEventSequence` is the optimistic-concurrency guard: the /// server applies the edit only if the message still carries that sequence, and reports a /// conflict with the winning state otherwise. diff --git a/Flipcash/Core/Screens/Conversation/AttachMenu.swift b/Flipcash/Core/Screens/Conversation/AttachMenu.swift index 1c676bf0c..2afa5bfe1 100644 --- a/Flipcash/Core/Screens/Conversation/AttachMenu.swift +++ b/Flipcash/Core/Screens/Conversation/AttachMenu.swift @@ -7,6 +7,7 @@ import SwiftUI import FlipcashUI +import FlipcashCore /// A row of the composer's attach menu. enum AttachMenuItem: Equatable { @@ -208,3 +209,13 @@ extension AttachMenuItem { } } } + +/// Decides whether a chat offers Camera and Photos. +enum ChatMediaGate { + + /// True for any chat this device has a record of: an encrypted DM's photos are encrypted for it, + /// a plaintext chat's go up in plaintext. False while the record is still loading. + static func acceptsMedia(_ conversation: Conversation?) -> Bool { + conversation != nil + } +} diff --git a/Flipcash/Core/Screens/Conversation/ChatItem+Conversation.swift b/Flipcash/Core/Screens/Conversation/ChatItem+Conversation.swift index a7d58db36..16d4ce9e3 100644 --- a/Flipcash/Core/Screens/Conversation/ChatItem+Conversation.swift +++ b/Flipcash/Core/Screens/Conversation/ChatItem+Conversation.swift @@ -273,7 +273,8 @@ extension ChatItem { height: attachment?.height ?? 0, blurhash: attachment?.blurhash, caption: caption, - isRedacted: message.redacted + isRedacted: message.redacted, + sealed: attachment?.sealed )) } @@ -446,7 +447,7 @@ extension ChatItem { stableID: original.stableID, authorName: authorName, snippet: ChatQuote.snippet(forText: ChatMediaStrings.quoteSnippet(caption: caption)), - kind: .media(thumbnailBlobID: original.redacted ? nil : attachments.first?.blobID), + kind: .media(thumbnailBlobID: original.redacted ? nil : attachments.first?.blobID, sealed: attachments.first?.sealed), authorID: original.senderID ) case .encrypted: diff --git a/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift b/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift index 7b3338dec..c2c6beb8d 100644 --- a/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift +++ b/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift @@ -8,6 +8,7 @@ import UIKit import ImageIO import FlipcashCore +import FlipcashUI private let logger = Logger(label: "flipcash.chat-media-upload") @@ -22,6 +23,10 @@ protocol ChatMediaBlobStoring { /// Stores `data` and returns its blob, before the server has finalized it. func storeBlob(_ data: Data, mimeType: String) async throws -> BlobID + /// Stores `image` end-to-end encrypted with `seal` for its DM and returns the blob, before the + /// server has finalized it. + func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload + /// Returns once the blob is servable, throwing `ErrorBlob.rejected` when it is refused and /// `ErrorBlob.timedOut` when it is still processing. func awaitBlobFinalization(blobID: BlobID) async throws @@ -41,15 +46,39 @@ struct SessionChatMediaBlobStore: ChatMediaBlobStoring { try await flipClient.storeBlob(data, mimeType: mimeType, owner: session.ownerKeyPair) } + func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload { + try await flipClient.storeEncryptedBlob(image, seal: seal, owner: session.ownerKeyPair) + } + func awaitBlobFinalization(blobID: BlobID) async throws { try await flipClient.awaitBlobFinalization(blobID: blobID, owner: session.ownerKeyPair) } } +/// A chat photo uploaded and finalized, in the form its message references it. +enum UploadedPhoto: Hashable, Sendable { + /// A plaintext blob, whose metadata the server derives. + case plain(BlobID) + /// A blob end-to-end encrypted for the chat, with the metadata its sealed message carries. + case sealed(SealedPhoto) + + /// The finalized blob. + var blobID: BlobID { + switch self { + case .plain(let blobID): blobID + case .sealed(let photo): photo.blobID + } + } +} + /// Why a chat photo did not upload. enum ChatMediaUploadError: Error { /// The upload policy accepts no JPEG. case noMatchingConstraint + /// The upload policy allows the owner no end-to-end encrypted upload. + case encryptionNotAllowed + /// The chat's seal could not be built, so the photo cannot be encrypted for it. + case sealUnavailable(Error) /// The photo could not be encoded within the policy's size ceiling. case encodingFailed(ChatMediaEncoder.Error) /// The server refused the stored bytes. @@ -60,13 +89,16 @@ enum ChatMediaUploadError: Error { /// Whether uploading the same photo again could succeed. var isRetryable: Bool { switch self { - case .noMatchingConstraint, .encodingFailed: + case .noMatchingConstraint, .encryptionNotAllowed, .encodingFailed: false + case .sealUnavailable(let error): + // A peer key that couldn't be fetched may arrive; one shared-core refuses never will. + error is PeerKeyUnavailable case .rejected(let reason): switch reason { case .moderation: false - case .unsupportedType, .mismatchedType, .tooLarge, .corrupt, .privacyMetadata, .unknown: + case .unsupportedType, .mismatchedType, .tooLarge, .corrupt, .privacyMetadata, .internal, .unknown, .unrecognized: true } case .failed: @@ -78,8 +110,10 @@ enum ChatMediaUploadError: Error { extension ChatMediaUploadError: ServerError { var reportingLevel: ErrorReportingLevel { switch self { - case .noMatchingConstraint, .encodingFailed: + case .noMatchingConstraint, .encryptionNotAllowed, .encodingFailed: .error + case .sealUnavailable(let error): + error is PeerKeyUnavailable ? .info : .error case .rejected: .info case .failed(let error): @@ -89,18 +123,34 @@ extension ChatMediaUploadError: ServerError { } /// Turns a staged photo into a finalized blob: downscaled to the upload policy's bounds, encoded -/// down the JPEG quality ladder, stored, and awaited until the server has finalized it. +/// down the JPEG quality ladder, stored, and awaited until the server has finalized it. In a chat +/// that encrypts, the bytes are encrypted for it before they leave the device. struct ChatMediaUploader { + /// The seal to encrypt a photo with, or nil when the chat takes plaintext photos. + typealias SealProvider = @MainActor () async throws -> ChatSeal? + let blob: any ChatMediaBlobStoring + /// Decides at upload time whether the photo is encrypted, the same decision text sends make. + var seal: SealProvider = { nil } + /// The wait before each automatic retry of a store that failed in transit; one entry per retry. var backoff: [Duration] = [.seconds(1), .seconds(2), .seconds(4)] - /// Returns the finalized blob for `image`, throwing `ChatMediaUploadError`. + /// Returns the finalized photo for `image`, throwing `ChatMediaUploadError`. /// /// `onPrepared` receives the uploaded pixel width and height once, before any bytes are stored. - func upload(_ image: UIImage, onPrepared: (Int, Int) -> Void) async throws -> BlobID { + func upload(_ image: UIImage, onPrepared: (Int, Int) -> Void) async throws -> UploadedPhoto { + let chatSeal: ChatSeal? + do { + chatSeal = try await seal() + } catch { + try Task.checkCancellation() + logger.info("No seal for chat photo", metadata: ["error": "\(error)"]) + throw ChatMediaUploadError.sealUnavailable(error) + } + let policy: UploadPolicy do { policy = try await blob.uploadPolicy() @@ -109,11 +159,25 @@ struct ChatMediaUploader { throw ChatMediaUploadError.failed(error) } - guard let constraint = policy.constraint(for: ChatMediaEncoder.mimeType) else { - logger.warning("Upload policy accepts no chat photo", metadata: ["policyVersion": "\(policy.version)"]) - throw ChatMediaUploadError.noMatchingConstraint + let bounds: UploadPolicy.ImageConstraints? + let maxSizeBytes: Int + if chatSeal != nil { + guard let encrypted = policy.encrypted else { + logger.warning("Upload policy allows no encrypted chat photo", metadata: ["policyVersion": "\(policy.version)"]) + throw ChatMediaUploadError.encryptionNotAllowed + } + bounds = encrypted.image + // The ceiling covers the sealed blob, so the image must leave room for nonce and tag. + maxSizeBytes = encrypted.maxSizeBytes - EncryptedBlobUpload.overhead + } else { + guard let constraint = policy.constraint(for: ChatMediaEncoder.mimeType) else { + logger.warning("Upload policy accepts no chat photo", metadata: ["policyVersion": "\(policy.version)"]) + throw ChatMediaUploadError.noMatchingConstraint + } + bounds = constraint.image + maxSizeBytes = constraint.maxSizeBytes } - guard let cgImage = image.cgImage else { + guard let cgImage = image.cgImage, maxSizeBytes > 0 else { throw ChatMediaUploadError.encodingFailed(.encodingFailed) } @@ -129,23 +193,37 @@ struct ChatMediaUploader { let target = ChatMediaDownscale.target( sourceWidth: isSideways ? cgImage.height : cgImage.width, sourceHeight: isSideways ? cgImage.width : cgImage.height, - maxWidth: constraint.image?.maxWidth ?? 0, - maxHeight: constraint.image?.maxHeight ?? 0, - maxPixels: constraint.image?.maxPixels ?? 0 + maxWidth: bounds?.maxWidth ?? 0, + maxHeight: bounds?.maxHeight ?? 0, + maxPixels: bounds?.maxPixels ?? 0 ) onPrepared(target.width, target.height) let data: Data do { - data = try await Self.encode(cgImage, orientation: orientation, target: target, maxSizeBytes: constraint.maxSizeBytes) + data = try await Self.encode(cgImage, orientation: orientation, target: target, maxSizeBytes: maxSizeBytes) } catch let error as ChatMediaEncoder.Error { throw ChatMediaUploadError.encodingFailed(error) } - let blobID = try await store(data) + let uploaded: UploadedPhoto + if let chatSeal { + let blurhash = await Self.blurhash(of: data) + let stored = try await store { try await blob.storeEncryptedBlob(data, seal: chatSeal) } + uploaded = .sealed(SealedPhoto( + blobID: stored.blobID, + mimeType: ChatMediaEncoder.mimeType, + sizeBytes: stored.plaintextSize, + width: target.width, + height: target.height, + blurhash: blurhash + )) + } else { + uploaded = .plain(try await store { try await blob.storeBlob(data, mimeType: ChatMediaEncoder.mimeType) }) + } do { - try await blob.awaitBlobFinalization(blobID: blobID) + try await blob.awaitBlobFinalization(blobID: uploaded.blobID) } catch ErrorBlob.rejected(let reason) { throw ChatMediaUploadError.rejected(reason) } catch { @@ -153,15 +231,15 @@ struct ChatMediaUploader { throw ChatMediaUploadError.failed(error) } - return blobID + return uploaded } /// Stores `data`, retrying through `backoff` while the failure is in transit. - private func store(_ data: Data) async throws -> BlobID { + private func store(_ attemptStore: () async throws -> Stored) async throws -> Stored { var attempt = 0 while true { do { - return try await blob.storeBlob(data, mimeType: ChatMediaEncoder.mimeType) + return try await attemptStore() } catch { try Task.checkCancellation() @@ -200,6 +278,21 @@ struct ChatMediaUploader { ) async throws -> Data { try ChatMediaEncoder().encode(image, orientation: orientation, target: target, maxSizeBytes: maxSizeBytes) } + + /// The BlurHash of the encoded photo, which a recipient of an encrypted photo draws until it + /// decrypts, since the server cannot derive one. Empty when the bytes don't decode, which the + /// contract allows. + @concurrent + nonisolated static func blurhash(of jpeg: Data) async -> String { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let thumbnail = CGImageSourceCreateThumbnailAtIndex(source, 0, [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: 64, + ] as CFDictionary) else { return "" } + let landscape = thumbnail.width >= thumbnail.height + return BlurHash.encode(thumbnail, componentsX: landscape ? 4 : 3, componentsY: landscape ? 3 : 4) ?? "" + } } private extension CGImagePropertyOrientation { diff --git a/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift b/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift index 754140835..374aa3a51 100644 --- a/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift +++ b/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift @@ -131,6 +131,8 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { /// Mints a signed download URL for a photo in this chat. The transcript's resolver caches what it /// returns and never asks for a photo drawn only from its BlurHash. var mintMediaURL: (BlobID) async throws -> URL? = { _ in nil } + /// Decrypts the chat's end-to-end encrypted photo blobs, or nil while its key is unknown. + var mediaBlobDecrypt: () async -> ChatMediaURLResolver.BlobDecrypt? = { nil } /// Fired when the user taps a photo the viewer may see. The owner opens the full-screen viewer. var onMediaTap: (ChatMediaViewerRequest) -> Void = { _ in } @@ -161,6 +163,7 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { screen.linkCardSource = linkCardSource screen.onMediaTap = onMediaTap context.coordinator.mintMediaURL = mintMediaURL + context.coordinator.mediaBlobDecrypt = mediaBlobDecrypt screen.mediaURLResolver = context.coordinator.mediaURLResolver screen.pendingMediaImage = { [conversationController] id in conversationController.pendingMediaImage(forMessageID: id) @@ -213,6 +216,7 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { screen.linkCardSource = linkCardSource screen.onMediaTap = onMediaTap context.coordinator.mintMediaURL = mintMediaURL + context.coordinator.mediaBlobDecrypt = mediaBlobDecrypt screen.onContactAction = onContactAction screen.onProfileTap = onProfileTap screen.onGroupInvite = onGroupInvite @@ -345,8 +349,8 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { model.returnToMenu() }, // Only a member can aim a reply, so the viewer may see what it quotes. - quoteThumbnailURL: { [resolver = coordinator.mediaURLResolver] kind in - await resolver.thumbnailURL(for: kind, canReact: true) + quoteThumbnailLocation: { [resolver = coordinator.mediaURLResolver] kind in + await resolver.thumbnailLocation(for: kind, canReact: true) } ) .environment(conversationController) @@ -437,10 +441,17 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { /// The one resolver the transcript and the composer's reply strip share, so a photo is minted /// once however many places draw it. Reads ``mintMediaURL`` at fetch time, so a chat created /// after this screen opened mints against its real id. - lazy var mediaURLResolver = ChatMediaURLResolver { [weak self] blobID in - guard let self else { return nil } - return try await self.mintMediaURL(blobID) - } + /// The latest ``ChatScreenRepresentable/mediaBlobDecrypt``, read by the resolver. + var mediaBlobDecrypt: () async -> ChatMediaURLResolver.BlobDecrypt? = { nil } + lazy var mediaURLResolver = ChatMediaURLResolver( + fetch: { [weak self] blobID in + guard let self else { return nil } + return try await self.mintMediaURL(blobID) + }, + decrypt: { [weak self] in + await self?.mediaBlobDecrypt() + } + ) } } diff --git a/Flipcash/Core/Screens/Conversation/ComposerChip.swift b/Flipcash/Core/Screens/Conversation/ComposerChip.swift index 763083366..be698189f 100644 --- a/Flipcash/Core/Screens/Conversation/ComposerChip.swift +++ b/Flipcash/Core/Screens/Conversation/ComposerChip.swift @@ -45,7 +45,7 @@ final class ComposerChip: Identifiable { /// The upload in flight or finished for this chip, which the send path awaits rather than /// starting its own. - @ObservationIgnored var uploadTask: Task? + @ObservationIgnored var uploadTask: Task? /// The uploader the last attempt went through, kept so a failed send can upload again after the /// composer has let go of the chip. @@ -65,15 +65,15 @@ final class ComposerChip: Identifiable { uploadTask = Task { do { - let blobID = try await uploader.upload(image) { width, height in + let photo = try await uploader.upload(image) { width, height in preparedWidth = width preparedHeight = height state = .uploading } if !Task.isCancelled { - state = .uploaded(blobID) + state = .uploaded(photo.blobID) } - return blobID + return photo } catch let error as ChatMediaUploadError { if !Task.isCancelled { state = .failed(error.isRetryable ? .retryable : .notRetryable) diff --git a/Flipcash/Core/Screens/Conversation/ComposerReplyStrip.swift b/Flipcash/Core/Screens/Conversation/ComposerReplyStrip.swift index f4766486e..2b0034e58 100644 --- a/Flipcash/Core/Screens/Conversation/ComposerReplyStrip.swift +++ b/Flipcash/Core/Screens/Conversation/ComposerReplyStrip.swift @@ -24,11 +24,11 @@ struct ComposerReplyStrip: View { let target: ComposerModel.ReplyTarget /// Where a quoted photo's thumbnail loads from; nil for anything else, or a photo with none to fetch. - var thumbnailURL: (ChatQuote.Kind) async -> URL? = { _ in nil } + var thumbnailLocation: (ChatQuote.Kind) async -> ChatMediaLocation? = { _ in nil } let onDismiss: () -> Void @Environment(\.barCardCollapsed) private var collapsed - @State private var resolvedThumbnail: URL? + @State private var resolvedThumbnail: ChatMediaLocation? /// Wider than the 4pt a blockquote rule usually takes, because the quote's corner radius is the /// bar's 14: the leading edge is straight for only `contentHeight - 14 * 2` of its run, and the @@ -159,12 +159,14 @@ struct ComposerReplyStrip: View { @ViewBuilder private var thumbnail: some View { switch target.kind { - case .media(let thumbnailBlobID?): + case .media(let thumbnailBlobID?, _): RoundedRectangle(cornerRadius: 6) .fill(Color.white.opacity(0.08)) .overlay { if let resolvedThumbnail { - KFImage(source: .network(ChatMediaImageSource.resource(blobID: thumbnailBlobID, url: resolvedThumbnail))) + KFImage(source: ChatMediaImageSource.source(blobID: thumbnailBlobID, location: resolvedThumbnail)) + .targetCache(ChatMediaImageSource.cache) + .onSuccess { ChatMediaImageSource.persist(.success($0)) } .resizable() .scaledToFill() } @@ -173,9 +175,9 @@ struct ComposerReplyStrip: View { .clipShape(.rect(cornerRadius: 6)) .accessibilityHidden(true) .task(id: thumbnailBlobID) { - resolvedThumbnail = await thumbnailURL(target.kind) + resolvedThumbnail = await thumbnailLocation(target.kind) } - case .media(nil), .text, .cash, .unavailable: + case .media(nil, _), .text, .cash, .unavailable: EmptyView() } } diff --git a/Flipcash/Core/Screens/Conversation/ConversationBottomBar.swift b/Flipcash/Core/Screens/Conversation/ConversationBottomBar.swift index ef3b64b3a..6a75e2bcb 100644 --- a/Flipcash/Core/Screens/Conversation/ConversationBottomBar.swift +++ b/Flipcash/Core/Screens/Conversation/ConversationBottomBar.swift @@ -260,7 +260,7 @@ struct ConversationBottomBar: View { /// Fired by the photo card's back chevron and escape gesture. var onPhotosBack: () -> Void = {} /// Where the reply strip's quoted photo loads its thumbnail from. - var quoteThumbnailURL: (ChatQuote.Kind) async -> URL? = { _ in nil } + var quoteThumbnailLocation: (ChatQuote.Kind) async -> ChatMediaLocation? = { _ in nil } /// The curve the bar narrows and widens on as the keyboard goes and comes. private static let widthSpring = Animation.spring(duration: 0.22, bounce: 0.14) @@ -448,7 +448,7 @@ struct ConversationBottomBar: View { // already drives this state in both directions, and wrapping the dismissal in a second // transaction gave the exit a curve the entry never had. AccessoryReveal(kind: .reply, item: barReply, collapsesInPlace: mentionCandidates != nil) { target in - ComposerReplyStrip(target: target, thumbnailURL: quoteThumbnailURL) { composer.endReplying() } + ComposerReplyStrip(target: target, thumbnailLocation: quoteThumbnailLocation) { composer.endReplying() } } // The quote narrows with the row below it, so the two keep one margin. .padding(.horizontal, compactExtraInset) diff --git a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift index 9d3ee13c3..dcbc59cec 100644 --- a/Flipcash/Core/Screens/Conversation/ConversationScreen.swift +++ b/Flipcash/Core/Screens/Conversation/ConversationScreen.swift @@ -542,22 +542,27 @@ struct ConversationScreen: View { onCameraCapture: stageCapturedPhoto, onPhotosAdded: stageAddedPhotos, mintMediaURL: mintMediaURL, + mediaBlobDecrypt: mediaBlobDecrypt, onMediaTap: openMediaViewer ) } - /// Whether the chat takes photos: any chat this device has a record of, except one it encrypts, whose - /// encryption does not cover media. False until the record loads, so the menu never offers a - /// photo the chat may refuse. + /// Whether the chat takes photos: any chat this device has a record of, encrypted or not. False + /// until the record loads, so the menu never offers a photo before the chat is known. private var acceptsMedia: Bool { - guard let conversationID, - let conversation = conversationController.conversation(withID: conversationID) else { return false } - return !E2eePolicy.shouldEncrypt(conversation) + ChatMediaGate.acceptsMedia(conversationID.flatMap(conversationController.conversation(withID:))) } - /// Uploads chat photos on behalf of the signed-in owner. + /// Uploads chat photos on behalf of the signed-in owner, encrypted for the chat when it encrypts. private var mediaUploader: ChatMediaUploader { - ChatMediaUploader(blob: SessionChatMediaBlobStore(session: session, flipClient: container.flipClient)) + let controller = conversationController + let conversationID = conversationID + var uploader = ChatMediaUploader(blob: SessionChatMediaBlobStore(session: session, flipClient: container.flipClient)) + uploader.seal = { + guard let conversationID else { return nil } + return try await controller.photoSeal(for: conversationID) + } + return uploader } /// Stages photos added from the photo card in the order they were selected, returning the chip @@ -595,6 +600,14 @@ struct ConversationScreen: View { } /// Mints a download URL for a photo in this chat, read through the chat's access context. + /// Decrypts this chat's end-to-end encrypted photos with the key its messages open with. + private var mediaBlobDecrypt: () async -> ChatMediaURLResolver.BlobDecrypt? { + { [controller = conversationController, conversationID] in + guard let conversationID else { return nil } + return await controller.mediaBlobDecrypt(for: conversationID) + } + } + private var mintMediaURL: (BlobID) async throws -> URL? { { [flipClient = container.flipClient, owner = session.ownerKeyPair, conversationID] blobID in guard let conversationID else { return nil } @@ -1161,7 +1174,7 @@ struct ConversationScreen: View { case .media(let attachments, let caption): ( ChatQuote.snippet(forText: ChatMediaStrings.quoteSnippet(caption: caption)), - .media(thumbnailBlobID: message.redacted ? nil : attachments.first?.blobID) + .media(thumbnailBlobID: message.redacted ? nil : attachments.first?.blobID, sealed: attachments.first?.sealed) ) case .deleted: (ChatQuote.deletedSnippet, .unavailable) diff --git a/Flipcash/Core/Screens/Main/Profile/DialogItem+ProfilePicture.swift b/Flipcash/Core/Screens/Main/Profile/DialogItem+ProfilePicture.swift index a63d3c362..026a649d8 100644 --- a/Flipcash/Core/Screens/Main/Profile/DialogItem+ProfilePicture.swift +++ b/Flipcash/Core/Screens/Main/Profile/DialogItem+ProfilePicture.swift @@ -58,7 +58,7 @@ extension DialogItem { // Privacy metadata is stripped from every upload, so a rejection for it // is a defect on our side rather than something a different photo fixes. - case .rejected(.privacyMetadata), .rejected(.unknown), .timedOut, + case .rejected(.privacyMetadata), .rejected(.internal), .rejected(.unknown), .rejected(.unrecognized), .timedOut, .uploadFailed, .notFound, .notUploaded, .unknown, .network: .error( title: "Couldn't Upload Your Photo", diff --git a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift index f84e13418..999d1c4c0 100644 --- a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift +++ b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift @@ -9,7 +9,8 @@ private let logger = Logger(label: "flipcash.blob-uploader") /// The blob RPCs an upload depends on. protocol BlobReserving: Sendable { - func initiateExternalUpload(mimeType: String, sizeBytes: Int, owner: KeyPair) async throws -> ReservedUpload + /// Reserves an upload of `sizeBytes`, end-to-end encrypted for the DM `encryptedFor` when set. + func initiateExternalUpload(mimeType: String, sizeBytes: Int, encryptedFor: ConversationID?, owner: KeyPair) async throws -> ReservedUpload func completeExternalUpload(blobID: BlobID, owner: KeyPair) async throws -> BlobState func blobState(blobID: BlobID, owner: KeyPair) async throws -> BlobState } @@ -49,6 +50,7 @@ final class BlobUploader: Sendable { let reserved = try await reserving.initiateExternalUpload( mimeType: mimeType, sizeBytes: data.count, + encryptedFor: nil, owner: owner ) @@ -62,12 +64,70 @@ final class BlobUploader: Sendable { switch try await reserving.completeExternalUpload(blobID: reserved.blobID, owner: owner) { case .rejected(let reason): + logger.info("Blob rejected on completion", metadata: [ + "blobId": "\(reserved.blobID)", + "reason": "\(reason)", + ]) throw ErrorBlob.rejected(reason) case .ready, .pending, .processing: return reserved.blobID } } + /// Stores `image` encrypted for the DM `conversationID` and returns its blob, leaving + /// finalization to the caller. + /// + /// `encrypt` seals the plaintext under the blob id the reservation assigns, which is part of + /// its aad, so the reservation declares the sealed size before the blob exists. + func storeEncrypted( + _ image: Data, + for conversationID: ConversationID, + owner: KeyPair, + encrypt: @Sendable (Data, BlobID) throws -> Data + ) async throws -> EncryptedBlobUpload { + // The server never reads these bytes, so stripping location and camera metadata is on us. + let plaintext = JPEGMetadata.stripped(image) + let sizeBytes = plaintext.count + EncryptedBlobUpload.overhead + + let reserved = try await reserving.initiateExternalUpload( + mimeType: Self.encryptedMimeType, + sizeBytes: sizeBytes, + encryptedFor: conversationID, + owner: owner + ) + + logger.info("Reserved encrypted blob upload", metadata: [ + "blobId": "\(reserved.blobID)", + "sizeBytes": "\(sizeBytes)", + ]) + + let blob = try encrypt(plaintext, reserved.blobID) + guard blob.count == sizeBytes else { + logger.error("Encrypted blob is not the size reserved", metadata: [ + "blobId": "\(reserved.blobID)", + "reserved": "\(sizeBytes)", + "actual": "\(blob.count)", + ]) + throw ErrorBlob.unknown + } + + try await store(blob, mimeType: Self.encryptedMimeType, to: reserved.target) + + switch try await reserving.completeExternalUpload(blobID: reserved.blobID, owner: owner) { + case .rejected(let reason): + logger.info("Blob rejected on completion", metadata: [ + "blobId": "\(reserved.blobID)", + "reason": "\(reason)", + ]) + throw ErrorBlob.rejected(reason) + case .ready, .pending, .processing: + return EncryptedBlobUpload(blobID: reserved.blobID, plaintextSize: plaintext.count) + } + } + + /// The MIME type every end-to-end encrypted upload declares; the server refuses any other. + static let encryptedMimeType = "application/octet-stream" + /// Polls until the blob is finalized. /// /// Returns immediately when it is already ready; a rejection is terminal. @@ -163,6 +223,18 @@ final class BlobUploader: Sendable { } } +/// An encrypted blob stored but not yet finalized, with the length of the plaintext it seals. +public struct EncryptedBlobUpload: Hashable, Sendable { + /// The blob the reservation assigned. + public let blobID: BlobID + /// The plaintext image's length after metadata stripping, the length the recipient checks. + public let plaintextSize: Int + + /// What encryption adds to an image: the 24-byte nonce ahead of the ciphertext and the 16-byte + /// tag after it. + public static let overhead = 40 +} + private extension Data { mutating func append(_ string: String) { append(Data(string.utf8)) diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift index f1a7bb29a..2e18231b1 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift @@ -16,6 +16,14 @@ extension FlipClient { try await blobUploader.store(data, mimeType: mimeType, owner: owner) } + /// Stores `image` end-to-end encrypted with `seal` for its DM and returns the blob, before the + /// server has finalized it. Pair with `awaitBlobFinalization(blobID:owner:)` as for `storeBlob`. + public func storeEncryptedBlob(_ image: Data, seal: ChatSeal, owner: KeyPair) async throws -> EncryptedBlobUpload { + try await blobUploader.storeEncrypted(image, for: seal.conversationID, owner: owner) { plaintext, blobID in + try seal.encryptBlob(plaintext, blobID: blobID) + } + } + /// Returns the upload constraints in force for `owner`. public func uploadPolicy(owner: KeyPair) async throws -> UploadPolicy { try await blobService.uploadPolicy(owner: owner) diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift index f4bf7094f..c3bed0792 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Chat.swift @@ -199,6 +199,15 @@ extension FlipClient { } } + /// Sends the finalized encrypted `photo` sealed with `seal`, with `caption` under it, and returns + /// the server's copy (decrypted). + @discardableResult + public func sendSealedMediaMessage(owner: KeyPair, conversationID: ConversationID, photo: SealedPhoto, caption: String?, repliedTo: MessageID?, seal: ChatSeal, clientMessageID: UUID) async throws -> ConversationMessage { + try await withCheckedThrowingContinuation { c in + chatMessagingService.sendSealedMediaMessage(owner: owner, conversationID: conversationID, photo: photo, caption: caption, repliedTo: repliedTo, seal: seal, clientMessageID: clientMessageID) { c.resume(with: $0) } + } + } + public func editMessage( owner: KeyPair, conversationID: ConversationID, diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/BlobService.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/BlobService.swift index e4b8dc606..9bba3e255 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/BlobService.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/BlobService.swift @@ -21,11 +21,8 @@ final class BlobService: Sendable { extension BlobService: BlobReserving { - func initiateExternalUpload(mimeType: String, sizeBytes: Int, owner: KeyPair) async throws -> ReservedUpload { - var request = Flipcash_Blob_V1_InitiateExternalUploadRequest() - request.mimeType = mimeType - request.sizeBytes = UInt64(sizeBytes) - request.auth = owner.authFor(message: request) + func initiateExternalUpload(mimeType: String, sizeBytes: Int, encryptedFor: ConversationID?, owner: KeyPair) async throws -> ReservedUpload { + let request = Self.initiateRequest(mimeType: mimeType, sizeBytes: sizeBytes, encryptedFor: encryptedFor, owner: owner) do { let response = try await service.initiateExternalUpload(request, options: .unaryDefault) @@ -114,6 +111,18 @@ extension BlobService: BlobReserving { /// A policy-driven denial echoes the version in force, which retires a /// stale cached policy. + /// The signed reservation request, naming the DM the bytes are encrypted for when set. + static func initiateRequest(mimeType: String, sizeBytes: Int, encryptedFor: ConversationID?, owner: KeyPair) -> Flipcash_Blob_V1_InitiateExternalUploadRequest { + var request = Flipcash_Blob_V1_InitiateExternalUploadRequest() + request.mimeType = mimeType + request.sizeBytes = UInt64(sizeBytes) + if let encryptedFor { + request.chat = encryptedFor.proto + } + request.auth = owner.authFor(message: request) + return request + } + private func observePolicyVersion(of response: Flipcash_Blob_V1_InitiateExternalUploadResponse) async { guard response.hasPolicyVersion else { return } await policyCache.observe(version: response.policyVersion.value) diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatMessagingService.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatMessagingService.swift index da11e901a..9d4aa7f01 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatMessagingService.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/Services/ChatMessagingService.swift @@ -198,6 +198,19 @@ final class ChatMessagingService: Sendable { send(content, owner: owner, conversationID: conversationID, seal: nil, clientMessageID: clientMessageID, completion: completion) } + /// Sends `photo`, its caption and reply inside the sealed plaintext, as `EncryptedContent`. + func sendSealedMediaMessage(owner: KeyPair, conversationID: ConversationID, photo: SealedPhoto, caption: String?, repliedTo: MessageID?, seal: ChatSeal, clientMessageID: UUID, completion: @Sendable @escaping (Result) -> Void) { + let content: Flipcash_Messaging_V1_Content + do { + content = try seal.seal(photo: photo, caption: caption, repliedTo: repliedTo) + } catch { + logger.error("Failed to encrypt photo message") + completion(.failure(.encryptionFailed)) + return + } + send(content, owner: owner, conversationID: conversationID, seal: seal, clientMessageID: clientMessageID, completion: completion) + } + private func send(_ content: Flipcash_Messaging_V1_Content, owner: KeyPair, conversationID: ConversationID, seal: ChatSeal?, clientMessageID: UUID, completion: @Sendable @escaping (Result) -> Void) { let request = Flipcash_Messaging_V1_SendMessageRequest.with { $0.chatID = conversationID.proto diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Blob.swift b/FlipcashCore/Sources/FlipcashCore/Models/Blob.swift index 048fcbd27..39dbcd649 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Blob.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Blob.swift @@ -25,7 +25,12 @@ public enum BlobRejectionReason: Sendable, Equatable { case tooLarge case corrupt case privacyMetadata + /// The server failed to process the blob. + case `internal` + /// The server sent no reason. case unknown + /// The server sent a reason this build doesn't know, by its raw proto value. + case unrecognized(Int) } /// A reserved upload: the blob it will become, and the request that stores its @@ -77,10 +82,27 @@ public struct UploadPolicy: Sendable, Equatable { /// Constraints ordered most specific first. public let constraints: [MimeConstraint] - init(version: String, ttl: Duration?, constraints: [MimeConstraint]) { + /// The constraints on an end-to-end encrypted upload, or nil when the caller may not make one. + public let encrypted: EncryptedConstraints? + + /// The bounds on an end-to-end encrypted upload, which the server checks by size alone. + public struct EncryptedConstraints: Sendable, Equatable { + /// The ceiling on the whole encrypted blob, nonce and tag included. + public let maxSizeBytes: Int + /// Advisory bounds to downscale an image to before encrypting it, or nil when there are none. + public let image: ImageConstraints? + + public init(maxSizeBytes: Int, image: ImageConstraints?) { + self.maxSizeBytes = maxSizeBytes + self.image = image + } + } + + init(version: String, ttl: Duration?, constraints: [MimeConstraint], encrypted: EncryptedConstraints? = nil) { self.version = version self.ttl = ttl self.constraints = constraints + self.encrypted = encrypted } /// Returns the constraint governing `mimeType` — the first match in policy @@ -119,8 +141,9 @@ extension BlobRejectionReason { case .tooLarge: self = .tooLarge case .corrupt: self = .corrupt case .privacyMetadata: self = .privacyMetadata - case .internal: self = .unknown - case .unknown, .UNRECOGNIZED: self = .unknown + case .internal: self = .internal + case .unknown: self = .unknown + case .UNRECOGNIZED(let value): self = .unrecognized(value) } } } @@ -144,7 +167,27 @@ extension UploadPolicy { self.init( version: proto.version.value, ttl: proto.hasTtl ? .seconds(proto.ttl.seconds) + .nanoseconds(proto.ttl.nanos) : nil, - constraints: proto.mimeTypeConstraints.map(MimeConstraint.init) + constraints: proto.mimeTypeConstraints.map(MimeConstraint.init), + encrypted: proto.hasEncrypted ? EncryptedConstraints(proto.encrypted) : nil + ) + } +} + +extension UploadPolicy.EncryptedConstraints { + init(_ proto: Flipcash_Blob_V1_EncryptedConstraints) { + self.init( + maxSizeBytes: Int(clamping: proto.maxSizeBytes), + image: proto.hasImage ? UploadPolicy.ImageConstraints(proto.image) : nil + ) + } +} + +extension UploadPolicy.ImageConstraints { + init(_ proto: Flipcash_Blob_V1_ImageConstraints) { + self.init( + maxWidth: Int(proto.maxWidth), + maxHeight: Int(proto.maxHeight), + maxPixels: Int(clamping: proto.maxPixels) ) } } @@ -154,11 +197,7 @@ extension UploadPolicy.MimeConstraint { let image: UploadPolicy.ImageConstraints? switch proto.kind { case .image(let bounds): - image = UploadPolicy.ImageConstraints( - maxWidth: Int(bounds.maxWidth), - maxHeight: Int(bounds.maxHeight), - maxPixels: Int(clamping: bounds.maxPixels) - ) + image = UploadPolicy.ImageConstraints(bounds) case nil: image = nil } diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMediaStrings.swift b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMediaStrings.swift index b2909853a..7ddc36651 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMediaStrings.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMediaStrings.swift @@ -11,6 +11,10 @@ import Foundation /// list, Spotlight, and notifications. The copy is the fixture's `strings` section, shared with Android. public enum ChatMediaStrings { + /// Drawn over an encrypted photo whose bytes failed to decrypt. Not in the shared fixture yet: + /// Android doesn't decrypt photos. + public static let undecryptable = "This photo can\u{2019}t be displayed" + /// A quote's snippet for a photo: its caption, or "Photo" when it has none. public static func quoteSnippet(caption: String?) -> String { nonEmpty(caption) ?? "Photo" diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMessage.swift b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMessage.swift index deb0d19a5..f4d8b8cd6 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMessage.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatMessage.swift @@ -348,14 +348,17 @@ public struct ChatMediaContent: Hashable, Sendable, Codable { public let caption: String? /// Whether the server withheld the photo's download URL; the bubble then only ever draws its BlurHash. public let isRedacted: Bool + /// What decrypting the blob needs when it is end-to-end encrypted, nil for a plaintext blob. + public let sealed: SealedBlob? - public init(blobID: BlobID?, width: Int, height: Int, blurhash: String?, caption: String?, isRedacted: Bool) { + public init(blobID: BlobID?, width: Int, height: Int, blurhash: String?, caption: String?, isRedacted: Bool, sealed: SealedBlob? = nil) { self.blobID = blobID self.width = width self.height = height self.blurhash = blurhash self.caption = caption self.isRedacted = isRedacted + self.sealed = sealed } } diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatPreviewMapping.swift b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatPreviewMapping.swift index 807d00b82..c8180173d 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatPreviewMapping.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatPreviewMapping.swift @@ -100,7 +100,8 @@ extension ChatItem { height: attachment?.height ?? 0, blurhash: attachment?.blurhash, caption: caption, - isRedacted: message.redacted + isRedacted: message.redacted, + sealed: attachment?.sealed )) case .deleted: continue // filtered out above; unreachable, kept for switch exhaustiveness diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatQuote.swift b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatQuote.swift index 2fce0bb69..b780f56dd 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatQuote.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Chat/ChatQuote.swift @@ -21,8 +21,9 @@ public struct ChatQuote: Hashable, Sendable, Codable { /// the currency's asset name, `nil` when the currency has no flag. case cash(token: String, flagImageName: String?) /// A photo. The snippet is its caption, or "Photo" when it has none; `thumbnailBlobID` is the - /// blob the panel draws beside it, `nil` when there is none to fetch. - case media(thumbnailBlobID: BlobID?) + /// blob the panel draws beside it, `nil` when there is none to fetch; `sealed` is what + /// decrypting it needs when it is end-to-end encrypted. + case media(thumbnailBlobID: BlobID?, sealed: SealedBlob?) /// The original is not in the local database, or it has been deleted. The panel renders /// the placeholder copy and the row is not tappable. case unavailable diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift index 9a975cd71..f28798686 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift @@ -10,7 +10,8 @@ import FlipcashAPI import SharedCore /// One DM's end-to-end encryption: the chat key both members derive, bound to the chat and both -/// public keys. Encrypts outgoing text and decrypts incoming `EncryptedContent`. +/// public keys. Encrypts outgoing text and photos, and decrypts incoming `EncryptedContent` and the +/// photo blobs it references. /// /// The cipher is shared-core's `ChatCipher` (scheme `X25519_XCHACHA20POLY1305`), so both apps agree /// byte for byte. Stateless after construction. @@ -26,6 +27,9 @@ public struct ChatSeal: @unchecked Sendable { private let chatID: KotlinByteArray private let selfUserID: UserID + /// The chat this seal encrypts for. + public let conversationID: ConversationID + /// Derives the chat key for `owner` and `peerPublicKey` in `conversationID`. /// /// Throws when shared-core rejects the peer key (not a valid point, low-order). @@ -47,11 +51,22 @@ public struct ChatSeal: @unchecked Sendable { self.peerPublicKey = peer self.chatID = chatID self.selfUserID = selfUserID + self.conversationID = conversationID } /// Encrypts `text`, as a reply to `repliedTo` when set, into the content to send in its place. public func seal(text: String, repliedTo: MessageID?) throws -> Flipcash_Messaging_V1_Content { - let plaintext = Flipcash_Messaging_V1_Content.plaintext(text: text, repliedTo: repliedTo) + try seal(.plaintext(text: text, repliedTo: repliedTo)) + } + + /// Encrypts a message carrying `photo`, with `caption` under it and as a reply to `repliedTo` + /// when set, into the content to send in its place. `photo` must already be uploaded through + /// ``encryptBlob(_:blobID:)`` and READY. + public func seal(photo: SealedPhoto, caption: String?, repliedTo: MessageID?) throws -> Flipcash_Messaging_V1_Content { + try seal(.plaintext(media: photo.media, caption: caption, repliedTo: repliedTo)) + } + + private func seal(_ plaintext: Flipcash_Messaging_V1_Content) throws -> Flipcash_Messaging_V1_Content { let payload = try cipher.encrypt( content: SharedBytes.shared.byteArray(data: try plaintext.serializedData()), chatKey: chatKey, @@ -68,6 +83,46 @@ public struct ChatSeal: @unchecked Sendable { } } + /// The blob to upload for `image`, the plaintext bytes of a photo this user sends as `blobID`: + /// a fresh 24-byte nonce followed by the ciphertext and its tag. + public func encryptBlob(_ image: Data, blobID: BlobID) throws -> Data { + let bytes = SharedBytes.shared + let blob = try cipher.encryptBlob( + image: bytes.byteArray(data: image), + chatKey: chatKey, + senderPk: ownPublicKey, + recipientPk: peerPublicKey, + chatId: chatID, + blobId: bytes.byteArray(data: blobID.data) + ) + return bytes.data(bytes: blob) + } + + /// The plaintext image in `blob`, the downloaded bytes of `blobID` that `sealed` describes. + /// + /// Throws ``BlobOpenFailure`` when the blob fails to authenticate or its plaintext is not the + /// length the sender declared. + public func decryptBlob(_ blob: Data, blobID: BlobID, sealed: SealedBlob) throws -> Data { + let bytes = SharedBytes.shared + let isFromSelf = sealed.senderID == selfUserID + let image: KotlinByteArray + do { + image = try cipher.decryptBlob( + blob: bytes.byteArray(data: blob), + chatKey: chatKey, + senderPk: isFromSelf ? ownPublicKey : peerPublicKey, + recipientPk: isFromSelf ? peerPublicKey : ownPublicKey, + chatId: chatID, + blobId: bytes.byteArray(data: blobID.data) + ) + } catch { + throw BlobOpenFailure.authentication + } + let data = bytes.data(bytes: image) + guard data.count == sealed.plaintextSize else { throw BlobOpenFailure.length } + return data + } + /// `message` with its content decrypted, or carrying why it could not be. A message that isn't /// `.encrypted` comes back unchanged. public func open(_ message: ConversationMessage) -> ConversationMessage { @@ -89,15 +144,72 @@ public struct ChatSeal: @unchecked Sendable { return message.opened(.failure(.authentication)) } - // Authenticated, so anything this client can't read is a newer client's content. + // Authenticated, so anything this client can't read, or a photo whose metadata breaks the + // contract, is a newer or misbehaving client's content. guard let content = try? Flipcash_Messaging_V1_Content(serializedBytes: bytes.data(bytes: plaintext)), - let readable = content.readableText else { + let senderID = message.senderID, + let readable = content.readable(senderID: senderID) else { return message.opened(.failure(.unsupported)) } return message.opened(.success(readable)) } } +/// Why a photo blob could not be decrypted; either way the photo draws as unsupported. +public enum BlobOpenFailure: Error, Hashable, Sendable { + /// The blob failed to authenticate under the chat key and its aad. + case authentication + /// The decrypted image is not the length the sender declared. + case length + /// The decrypted bytes don't decode as an image. + case undecodable +} + +/// A photo uploaded encrypted for a chat, with the plaintext metadata its sealed message carries. +public struct SealedPhoto: Hashable, Sendable { + /// The encrypted blob, READY before the message is sent. + public let blobID: BlobID + /// The plaintext image's MIME type, always an `image/` type. + public let mimeType: String + /// The plaintext image's length in bytes. + public let sizeBytes: Int + /// Pixel width of the plaintext image. + public let width: Int + /// Pixel height of the plaintext image. + public let height: Int + /// The plaintext image's BlurHash, at most 64 characters. + public let blurhash: String + + public init(blobID: BlobID, mimeType: String, sizeBytes: Int, width: Int, height: Int, blurhash: String) { + self.blobID = blobID + self.mimeType = mimeType + self.sizeBytes = sizeBytes + self.width = width + self.height = height + self.blurhash = blurhash + } + + /// The single-ORIGINAL `Media` the sealed message carries, with the sender-set metadata the + /// server never sees and no download URL. + var media: Flipcash_Blob_V1_Media { + .with { + $0.renditions = [.with { + $0.role = .original + $0.blobID = .with { $0.value = blobID.data } + $0.blob = .with { + $0.mimeType = mimeType + $0.sizeBytes = UInt64(sizeBytes) + $0.image = .with { + $0.width = UInt32(width) + $0.height = UInt32(height) + $0.blurhash = blurhash + } + } + }] + } + } +} + extension Flipcash_Messaging_V1_Content { /// The plaintext content for `text`, wrapped as a reply to `repliedTo` when set. @@ -112,15 +224,92 @@ extension Flipcash_Messaging_V1_Content { } } - /// The text and replied-to message of decrypted Text or Reply(Text) content; nil for any other. - fileprivate var readableText: (text: String, repliedTo: MessageID?)? { + /// The media content for `media`, with `caption` under it, wrapped as a reply to `repliedTo` + /// when set. A reply nests the media one level deeper on the wire. + static func plaintext(media: Flipcash_Blob_V1_Media, caption: String?, repliedTo: MessageID?) -> Self { + let body = Self.with { + $0.media = .with { + $0.items = [media] + if let caption { + $0.caption = .with { $0.text = caption } + } + } + } + guard let repliedTo else { return body } + return .with { + $0.reply = .with { + $0.repliedMessageID = repliedTo.proto + $0.content = [body] + } + } + } + + /// The content and replied-to message of decrypted Text, Media, or a Reply of either sent by + /// `senderID`; nil for any other type, or for media whose metadata breaks the contract. + fileprivate func readable(senderID: UserID) -> (content: ConversationMessage.Content, repliedTo: MessageID?)? { switch type { case .text(let text): - return (text.text, nil) + return (.text(text.text), nil) + case .media(let media): + return ConversationMessage.Content(sealed: media, senderID: senderID).map { ($0, nil) } case .reply(let reply): - guard reply.content.count == 1, case .text(let text) = reply.content[0].type else { return nil } - return (text.text, MessageID(reply.repliedMessageID)) - case .cash, .media, .system, .widget, .deleted, .encrypted, nil: + guard reply.content.count == 1 else { return nil } + let repliedTo = MessageID(reply.repliedMessageID) + switch reply.content[0].type { + case .text(let text): + return (.text(text.text), repliedTo) + case .media(let media): + return ConversationMessage.Content(sealed: media, senderID: senderID).map { ($0, repliedTo) } + case .cash, .system, .widget, .deleted, .encrypted, .reply, nil: + return nil + } + case .cash, .system, .widget, .deleted, .encrypted, nil: + return nil + } + } +} + +extension ConversationMessage.Content { + + /// The `.media` content of a decrypted `MediaContent` sent by `senderID`, or nil when it breaks + /// the encrypted-media contract: exactly one item with exactly one ORIGINAL rendition, a blob id, + /// and valid `BlobMetadata` describing an image. + /// + /// The dimensions are the sender's claim; the decoded image's own are authoritative once it loads. + init?(sealed proto: Flipcash_Messaging_V1_MediaContent, senderID: UserID) { + guard proto.items.count == 1, + proto.items[0].renditions.count == 1 else { return nil } + let rendition = proto.items[0].renditions[0] + guard rendition.role == .original, + rendition.hasBlobID, + rendition.blobID.value.count == 16, + rendition.hasBlob, + let image = Self.validImage(rendition.blob) else { return nil } + let caption = proto.caption.text + self = .media( + [MediaAttachment( + blobID: BlobID(data: rendition.blobID.value), + width: Int(image.width), + height: Int(image.height), + blurhash: image.blurhash.isEmpty ? nil : image.blurhash, + sealed: SealedBlob(senderID: senderID, plaintextSize: Int(rendition.blob.sizeBytes)) + )], + caption: caption.isEmpty ? nil : caption + ) + } + + /// `metadata`'s image description when it passes the `blob.v1.BlobMetadata` rules for an image. + private static func validImage(_ metadata: Flipcash_Blob_V1_BlobMetadata) -> Flipcash_Blob_V1_ImageMetadata? { + let mimeType = metadata.mimeType + guard (1...255).contains(mimeType.unicodeScalars.count), + mimeType.lowercased().hasPrefix("image/"), + metadata.sizeBytes >= 1, + metadata.sizeBytes <= UInt64(Int.max) else { return nil } + switch metadata.kind { + case .image(let image): + guard image.width >= 1, image.height >= 1, image.blurhash.unicodeScalars.count <= 64 else { return nil } + return image + case .encrypted, nil: return nil } } diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ConversationMessage.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ConversationMessage.swift index 08865d8f4..1bf6f51ab 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ConversationMessage.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ConversationMessage.swift @@ -51,7 +51,7 @@ public struct ConversationMessage: Identifiable, Hashable, Sendable { /// End-to-end-encrypted content not yet decrypted: either decryption failed (see /// ``ConversationMessage/decryptFailure``) or the peer's key hasn't been fetched yet. `scheme` /// is the wire `EncryptedContent.Scheme` raw value, kept as `Int` so this model doesn't depend - /// on the generated proto enum. A decrypted message carries `.text` instead. + /// on the generated proto enum. A decrypted message carries `.text` or `.media` instead. case encrypted(scheme: Int, nonce: Data, ciphertext: Data) /// A widget: structured content the sender's client drew as a card rather than text. case widget(Widget) @@ -220,15 +220,15 @@ extension ConversationMessage { if case .encrypted = content { decryptFailure == nil } else { false } } - /// A copy carrying the outcome of decrypting it: the plaintext and the message it replies to, - /// or the reason it failed. Everything else is kept, including the ciphertext. - public func opened(_ outcome: Result<(text: String, repliedTo: MessageID?), DecryptFailure>) -> ConversationMessage { + /// A copy carrying the outcome of decrypting it: the decrypted content and the message it + /// replies to, or the reason it failed. Everything else is kept, including the ciphertext. + public func opened(_ outcome: Result<(content: Content, repliedTo: MessageID?), DecryptFailure>) -> ConversationMessage { let content: Content let repliedTo: MessageID? let failure: DecryptFailure? switch outcome { case .success(let plaintext): - content = .text(plaintext.text) + content = plaintext.content repliedTo = plaintext.repliedTo failure = nil case .failure(let reason): diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/MediaAttachment.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/MediaAttachment.swift index 301e667a7..1c72a3bec 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/MediaAttachment.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/MediaAttachment.swift @@ -25,11 +25,30 @@ public struct MediaAttachment: Hashable, Sendable, Codable { /// The ORIGINAL's BlurHash preview, or `nil` when the server carried none. public let blurhash: String? - public init(blobID: BlobID?, width: Int, height: Int, blurhash: String?) { + /// How the blob decrypts when it is end-to-end encrypted for the chat; nil for a plaintext blob. + public let sealed: SealedBlob? + + public init(blobID: BlobID?, width: Int, height: Int, blurhash: String?, sealed: SealedBlob? = nil) { self.blobID = blobID self.width = width self.height = height self.blurhash = blurhash + self.sealed = sealed + } +} + +/// What an end-to-end encrypted photo's blob needs, besides the chat key, to be decrypted and +/// checked: who uploaded it, which orders the public keys in its aad, and the plaintext length the +/// decrypted bytes must match. +public struct SealedBlob: Hashable, Sendable, Codable { + /// The member who uploaded the blob, always the sender of the message that references it. + public let senderID: UserID + /// The plaintext image's length in bytes, from the sender's sealed metadata. + public let plaintextSize: Int + + public init(senderID: UserID, plaintextSize: Int) { + self.senderID = senderID + self.plaintextSize = plaintextSize } } diff --git a/FlipcashCore/Sources/FlipcashStore/Database.swift b/FlipcashCore/Sources/FlipcashStore/Database.swift index 2e97d2ca8..d194b4d9e 100644 --- a/FlipcashCore/Sources/FlipcashStore/Database.swift +++ b/FlipcashCore/Sources/FlipcashStore/Database.swift @@ -221,7 +221,7 @@ nonisolated open class Database: @unchecked Sendable { /// the notification service extension needs the same number to decide whether the store on disk /// is one it understands, and an extension cannot read the app's `Info.plist` — separate bundles. /// Both targets link this module, so they cannot disagree. - public static let schemaVersion = 46 + public static let schemaVersion = 47 /// Removes the store and the write-ahead log files beside it. /// diff --git a/FlipcashCore/Tests/FlipcashCoreTests/BlobRejectionReasonTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/BlobRejectionReasonTests.swift new file mode 100644 index 000000000..5068b45de --- /dev/null +++ b/FlipcashCore/Tests/FlipcashCoreTests/BlobRejectionReasonTests.swift @@ -0,0 +1,31 @@ +// +// BlobRejectionReasonTests.swift +// FlipcashCoreTests +// + +import Testing +import FlipcashAPI +@testable import FlipcashCore + +@Suite("Blob rejection reason") +struct BlobRejectionReasonTests { + + @Test("Keeps an internal rejection distinct from an unset reason") + func internal_isNotUnknown() { + #expect(BlobRejectionReason(.internal) == .internal) + #expect(BlobRejectionReason(.unknown) == .unknown) + } + + @Test("Keeps the raw value of a reason this build doesn't know") + func unrecognized_keepsRawValue() { + let reason = BlobRejectionReason(.UNRECOGNIZED(42)) + #expect(reason == .unrecognized(42)) + #expect("\(reason)" == "unrecognized(42)") + } + + @Test("A rejected status carries the mapped reason") + func rejectedState_carriesReason() { + let rejection = Flipcash_Blob_V1_RejectionMetadata.with { $0.reason = .internal } + #expect(BlobState(status: .rejected, rejection: rejection) == .rejected(.internal)) + } +} diff --git a/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift index 6fc90c087..8c52e4507 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift @@ -10,6 +10,70 @@ import Testing @Suite("Blob upload") struct BlobUploaderTests { + // MARK: - End-to-end encrypted - + + @Test("An encrypted upload reserves for its DM, as an opaque blob the size of the sealed bytes") + func encryptedUploadReservesForChat() async throws { + let transport = RecordingTransport() + let reserving = StubReserving(states: [.ready]) + let uploader = BlobUploader(reserving: reserving, transport: transport, pollInterval: .milliseconds(1), timeout: .seconds(5)) + let conversationID = ConversationID(data: Data(repeating: 7, count: 32)) + let sealed = Data(repeating: 0xCD, count: Self.fileBytes.count + EncryptedBlobUpload.overhead) + + let upload = try await uploader.storeEncrypted(Self.fileBytes, for: conversationID, owner: try Self.owner()) { plaintext, blobID in + #expect(blobID == StubReserving.blobID) + #expect(plaintext == Self.fileBytes) + return sealed + } + + #expect(upload == EncryptedBlobUpload(blobID: StubReserving.blobID, plaintextSize: Self.fileBytes.count)) + #expect(await reserving.encryptedFor == conversationID) + #expect(await reserving.declaredMimeType == "application/octet-stream") + #expect(await reserving.declaredSizeBytes == sealed.count) + let parsed = try await Self.parse(transport) + #expect(parsed.file == sealed) + } + + @Test("Metadata is stripped before encrypting, and the plaintext size is the stripped length") + func encryptedUploadStripsBeforeSealing() async throws { + let uploader = makeUploader(transport: RecordingTransport(), states: [.ready]) + let jpeg = Self.jpegCarryingComment() + let stripped = JPEGMetadata.stripped(jpeg) + #expect(stripped.count < jpeg.count) + + let upload = try await uploader.storeEncrypted(jpeg, for: ConversationID(data: Data(repeating: 7, count: 32)), owner: try Self.owner()) { plaintext, _ in + #expect(plaintext == stripped) + return Data(count: plaintext.count + EncryptedBlobUpload.overhead) + } + + #expect(upload.plaintextSize == stripped.count) + } + + @Test("A sealed blob of a size other than the one reserved is never uploaded") + func encryptedUploadRefusesWrongSize() async throws { + let transport = RecordingTransport() + let uploader = makeUploader(transport: transport, states: [.ready]) + + await #expect(throws: ErrorBlob.self) { + _ = try await uploader.storeEncrypted(Self.fileBytes, for: ConversationID(data: Data(repeating: 7, count: 32)), owner: try Self.owner()) { plaintext, _ in + plaintext + } + } + #expect(await transport.body == nil) + } + + @Test("The reservation names the chat only for an encrypted upload") + func reservationRequestNamesChat() throws { + let owner = try Self.owner() + let conversationID = ConversationID(data: Data(repeating: 7, count: 32)) + + let encrypted = BlobService.initiateRequest(mimeType: "application/octet-stream", sizeBytes: 10, encryptedFor: conversationID, owner: owner) + let plain = BlobService.initiateRequest(mimeType: "image/jpeg", sizeBytes: 10, encryptedFor: nil, owner: owner) + + #expect(encrypted.endToEndEncryptedFor == .chat(conversationID.proto)) + #expect(plain.endToEndEncryptedFor == nil) + } + // MARK: - Multipart body - @Test("Signed policy fields come first, the file last") @@ -305,6 +369,7 @@ struct BlobUploaderTests { let names: [String] let values: [String: String] let fileByteCount: Int + let file: Data } private static func parse(_ transport: RecordingTransport) async throws -> ParsedBody { @@ -320,6 +385,7 @@ struct BlobUploaderTests { var names: [String] = [] var values: [String: String] = [:] var fileByteCount = 0 + var file = Data() for segment in Self.segments(of: body, delimitedBy: "--\(boundary)\r\n") { guard let terminator = segment.range(of: Data("\r\n\r\n".utf8)) else { continue } @@ -334,12 +400,13 @@ struct BlobUploaderTests { if name == "file" { fileByteCount = payload.count + file = payload } else { values[name] = String(decoding: payload, as: UTF8.self) } } - return ParsedBody(names: names, values: values, fileByteCount: fileByteCount) + return ParsedBody(names: names, values: values, fileByteCount: fileByteCount, file: file) } private static func segments(of body: Data, delimitedBy delimiter: String) -> [Data] { @@ -409,15 +476,19 @@ private actor StubReserving: BlobReserving { private(set) var pollCount = 0 private(set) var reserveCount = 0 private(set) var declaredSizeBytes: Int? + private(set) var declaredMimeType: String? + private(set) var encryptedFor: ConversationID? init(states: [BlobState], completion: BlobState = .processing) { self.states = states self.completion = completion } - func initiateExternalUpload(mimeType: String, sizeBytes: Int, owner: KeyPair) async throws -> ReservedUpload { + func initiateExternalUpload(mimeType: String, sizeBytes: Int, encryptedFor: ConversationID?, owner: KeyPair) async throws -> ReservedUpload { reserveCount += 1 declaredSizeBytes = sizeBytes + declaredMimeType = mimeType + self.encryptedFor = encryptedFor return ReservedUpload( blobID: Self.blobID, diff --git a/FlipcashCore/Tests/FlipcashCoreTests/ChatPreviewDecryptionTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/ChatPreviewDecryptionTests.swift index bf2d53444..e0ec458c6 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/ChatPreviewDecryptionTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/ChatPreviewDecryptionTests.swift @@ -38,6 +38,25 @@ struct ChatPreviewDecryptionTests { #expect(contents(items) == [.text("hello")]) } + @Test("A decrypted photo previews as a photo that still knows it is encrypted") + func decryptedPhotoPreviewsAsPhoto() throws { + let blobID = BlobID(data: Data([7])) + let seal = SealedBlob(senderID: them, plaintextSize: 42) + let attachment = MediaAttachment(blobID: blobID, width: 4, height: 3, blurhash: "LKO2?U%2Tw=w", sealed: seal) + var message = encrypted(1, from: them) + message = message.opened(.success((content: .media([attachment], caption: "beach"), repliedTo: nil))) + + let items = ChatItem.preview(from: [message], selfUserID: me) + let content = try #require(contents(items).first) + guard case .media(let media) = content else { + Issue.record("Expected a photo, got \(content)") + return + } + #expect(media.blobID == blobID) + #expect(media.caption == "beach") + #expect(media.sealed == seal) + } + @Test("A message waiting on the peer's key is left out of the preview") func awaitingIsLeftOut() { let items = ChatItem.preview(from: [encrypted(1, from: them, text: "hi"), encrypted(2, from: them)], selfUserID: me) diff --git a/FlipcashCore/Tests/FlipcashCoreTests/ChatSealTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/ChatSealTests.swift index 8f2256b5c..a95ec7e2a 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/ChatSealTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/ChatSealTests.swift @@ -127,4 +127,174 @@ struct ChatSealTests { #expect(encrypted.scheme == .x25519Xchacha20Poly1305) #expect(encrypted.nonce.count == 24) } + + // MARK: - Photos - + + private let blobID = BlobID(data: Data(repeating: 0xA5, count: 16)) + + private func photo(sizeBytes: Int = 1234) -> SealedPhoto { + SealedPhoto(blobID: blobID, mimeType: "image/jpeg", sizeBytes: sizeBytes, width: 640, height: 480, blurhash: "LEHV6nWB2yk8") + } + + /// The decrypted `.media` attachment `content` carries. + private func attachment(_ content: ConversationMessage.Content) throws -> (MediaAttachment, String?) { + guard case .media(let attachments, let caption) = content, attachments.count == 1 else { + Issue.record("Expected one media attachment, got \(content)") + throw CancellationError() + } + return (attachments[0], caption) + } + + @Test("The peer decrypts a sealed photo to its blob and sender-set metadata") + func photoRoundTrip() throws { + let sealed = try seal(owner: alice, peer: bob, selfID: aliceID).seal(photo: photo(), caption: nil, repliedTo: nil) + let opened = try seal(owner: bob, peer: alice, selfID: bobID).open(message(sealed, from: aliceID)) + + let (media, caption) = try attachment(opened.content) + #expect(media == MediaAttachment( + blobID: blobID, width: 640, height: 480, blurhash: "LEHV6nWB2yk8", + sealed: SealedBlob(senderID: aliceID, plaintextSize: 1234) + )) + #expect(caption == nil) + #expect(opened.repliedTo == nil) + #expect(opened.decryptFailure == nil) + } + + @Test("A sealed photo carries its caption and the message it replies to inside the ciphertext") + func photoCaptionAndReply() throws { + let sealed = try seal(owner: alice, peer: bob, selfID: aliceID).seal(photo: photo(), caption: "sunset", repliedTo: MessageID(value: 9)) + guard case .encrypted = sealed.type else { + Issue.record("Expected encrypted content") + return + } + let opened = try seal(owner: bob, peer: alice, selfID: bobID).open(message(sealed, from: aliceID)) + + let (_, caption) = try attachment(opened.content) + #expect(caption == "sunset") + #expect(opened.repliedTo == MessageID(value: 9)) + } + + @Test("The sender's other device opens its own sealed photo") + func photoOwnMessage() throws { + let aliceSeal = try seal(owner: alice, peer: bob, selfID: aliceID) + let opened = aliceSeal.open(try message(aliceSeal.seal(photo: photo(), caption: nil, repliedTo: nil), from: aliceID)) + + let (media, _) = try attachment(opened.content) + #expect(media.sealed?.senderID == aliceID) + } + + @Test("Both members decrypt the blob the sender encrypted") + func blobRoundTrip() throws { + let image = Data((0..<4096).map { UInt8(truncatingIfNeeded: $0) }) + let sealedBlob = SealedBlob(senderID: aliceID, plaintextSize: image.count) + let blob = try seal(owner: alice, peer: bob, selfID: aliceID).encryptBlob(image, blobID: blobID) + + #expect(blob.count == image.count + 24 + 16) + #expect(try seal(owner: bob, peer: alice, selfID: bobID).decryptBlob(blob, blobID: blobID, sealed: sealedBlob) == image) + #expect(try seal(owner: alice, peer: bob, selfID: aliceID).decryptBlob(blob, blobID: blobID, sealed: sealedBlob) == image) + } + + @Test("A blob read under another blob id fails authentication") + func blobWrongID() throws { + let image = Data(repeating: 1, count: 100) + let blob = try seal(owner: alice, peer: bob, selfID: aliceID).encryptBlob(image, blobID: blobID) + let other = BlobID(data: Data(repeating: 0, count: 16)) + + #expect(throws: BlobOpenFailure.authentication) { + try seal(owner: bob, peer: alice, selfID: bobID).decryptBlob(blob, blobID: other, sealed: SealedBlob(senderID: aliceID, plaintextSize: 100)) + } + } + + @Test("A blob attributed to the wrong sender fails authentication") + func blobSwappedKeys() throws { + let image = Data(repeating: 1, count: 100) + let blob = try seal(owner: alice, peer: bob, selfID: aliceID).encryptBlob(image, blobID: blobID) + + // Bob reading it as his own upload puts the public keys in the aad the wrong way round. + #expect(throws: BlobOpenFailure.authentication) { + try seal(owner: bob, peer: alice, selfID: bobID).decryptBlob(blob, blobID: blobID, sealed: SealedBlob(senderID: bobID, plaintextSize: 100)) + } + } + + @Test("A blob whose plaintext is not the declared size is rejected") + func blobLengthMismatch() throws { + let image = Data(repeating: 1, count: 100) + let blob = try seal(owner: alice, peer: bob, selfID: aliceID).encryptBlob(image, blobID: blobID) + + #expect(throws: BlobOpenFailure.length) { + try seal(owner: bob, peer: alice, selfID: bobID).decryptBlob(blob, blobID: blobID, sealed: SealedBlob(senderID: aliceID, plaintextSize: 99)) + } + } + + /// The opened result of `media`, sealed by Alice by hand so it can break the contract. + private func openHandSealed(_ media: Flipcash_Messaging_V1_MediaContent) throws -> ConversationMessage { + let bytes = SharedBytes.shared + let plaintext = Flipcash_Messaging_V1_Content.with { $0.media = media } + let owner = SharedCore.KeyPair(publicKey: bytes.byteArray(data: alice.publicKey.data), privateKey: bytes.byteArray(data: alice.privateKey.data)) + let chatKey = try DefaultChatCipher.shared.chatKey(ownKeyPair: owner, peerPublicKey: bytes.byteArray(data: bob.publicKey.data), chatId: bytes.byteArray(data: chatID.data)) + let payload = try DefaultChatCipher.shared.encrypt( + content: bytes.byteArray(data: try plaintext.serializedData()), + chatKey: chatKey, + senderPk: bytes.byteArray(data: alice.publicKey.data), + recipientPk: bytes.byteArray(data: bob.publicKey.data), + chatId: bytes.byteArray(data: chatID.data) + ) + let content = Flipcash_Messaging_V1_Content.with { + $0.encrypted = .with { + $0.scheme = ChatSeal.scheme + $0.nonce = bytes.data(bytes: payload.nonce) + $0.ciphertext = bytes.data(bytes: payload.ciphertext) + } + } + return try seal(owner: bob, peer: alice, selfID: bobID).open(message(content, from: aliceID)) + } + + /// A valid sealed photo's `MediaContent`, for the validation cases to break one field of. + private var validMedia: Flipcash_Messaging_V1_MediaContent { + .with { $0.items = [photo().media] } + } + + @Test("Hand-sealed valid media opens, so the cases below fail for their one broken field") + func handSealedValid() throws { + #expect(try openHandSealed(validMedia).decryptFailure == nil) + } + + @Test( + "Media whose metadata breaks the contract is unsupported", + arguments: [ + "zero size", "non-image mime", "empty mime", "long mime", "zero width", "zero height", + "long blurhash", "no metadata", "no blob id", "short blob id", "not original", + "two renditions", "two items", "no items", "no image metadata", + ] + ) + func invalidMediaIsUnsupported(_ breakage: String) throws { + var media = validMedia + switch breakage { + case "zero size": media.items[0].renditions[0].blob.sizeBytes = 0 + case "non-image mime": media.items[0].renditions[0].blob.mimeType = "video/mp4" + case "empty mime": media.items[0].renditions[0].blob.mimeType = "" + case "long mime": media.items[0].renditions[0].blob.mimeType = "image/" + String(repeating: "x", count: 250) + case "zero width": media.items[0].renditions[0].blob.image.width = 0 + case "zero height": media.items[0].renditions[0].blob.image.height = 0 + case "long blurhash": media.items[0].renditions[0].blob.image.blurhash = String(repeating: "L", count: 65) + case "no metadata": media.items[0].renditions[0].clearBlob() + case "no blob id": media.items[0].renditions[0].clearBlobID() + case "short blob id": media.items[0].renditions[0].blobID.value = Data([1, 2, 3]) + case "not original": media.items[0].renditions[0].role = .unknown + case "two renditions": media.items[0].renditions.append(media.items[0].renditions[0]) + case "two items": media.items.append(media.items[0]) + case "no items": media.items = [] + case "no image metadata": media.items[0].renditions[0].blob.kind = nil + default: Issue.record("Unknown breakage \(breakage)") + } + + #expect(try openHandSealed(media).decryptFailure == .unsupported) + } + + @Test("A download URL inside sealed media is ignored") + func downloadURLIgnored() throws { + var media = validMedia + media.items[0].renditions[0].blob.downloadURL = .with { $0.url = "https://example.com/x" } + #expect(try openHandSealed(media).decryptFailure == nil) + } } diff --git a/FlipcashCore/Tests/FlipcashCoreTests/UploadPolicyTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/UploadPolicyTests.swift index 5c6644374..849f69b06 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/UploadPolicyTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/UploadPolicyTests.swift @@ -34,6 +34,22 @@ struct UploadPolicyTests { #expect(policy.constraints[2].image == nil) } + @Test("Maps the encrypted-upload constraints, and their absence to nil") + func mapsEncryptedConstraints() { + var proto = Flipcash_Blob_V1_UploadPolicy() + #expect(UploadPolicy(proto).encrypted == nil) + + proto.encrypted = .with { + $0.maxSizeBytes = 5_000_040 + $0.image = .with { $0.maxWidth = 2048; $0.maxHeight = 1536; $0.maxPixels = 3_000_000 } + } + + #expect(UploadPolicy(proto).encrypted == UploadPolicy.EncryptedConstraints( + maxSizeBytes: 5_000_040, + image: UploadPolicy.ImageConstraints(maxWidth: 2048, maxHeight: 1536, maxPixels: 3_000_000) + )) + } + @Test("An unset TTL maps to nil, not zero") func unsetTTLIsNil() { var proto = Self.proto(version: "v7", ttlSeconds: 300) diff --git a/FlipcashCoreVectors/Tests/FlipcashCoreVectorsTests/ChatCipherBlobVectorTests.swift b/FlipcashCoreVectors/Tests/FlipcashCoreVectorsTests/ChatCipherBlobVectorTests.swift new file mode 100644 index 000000000..06872e97b --- /dev/null +++ b/FlipcashCoreVectors/Tests/FlipcashCoreVectorsTests/ChatCipherBlobVectorTests.swift @@ -0,0 +1,121 @@ +import Foundation +import Testing +import SharedCore +@testable import FlipcashCore + +/// Photo-blob half of `test-vectors/chat_cipher.json`. The canonical copy lives in the orchestrator +/// repo; this one is synced. The vectors fix the nonce, which production draws at random, so only +/// the decrypt direction is reproducible: each blob must open to its image, and each reject must not. +@Suite struct ChatCipherBlobVectorTests { + + struct BlobVector: Decodable, Sendable, CustomTestStringConvertible { + let name: String + let chatId: String + let chatKey: String + let senderPublicKey: String + let recipientPublicKey: String + let blobId: String + let image: String? + let blob: String + + var testDescription: String { name } + } + + struct Member: Decodable, Sendable { + let edSeed: String + let edPublicKey: String + } + + struct ChatKeyVector: Decodable, Sendable { + let name: String + let chatId: String + let memberA: Member + let memberB: Member + let chatKey: String + } + + struct Rejects: Decodable { + let decryptBlob: [BlobVector] + } + + struct Fixture: Decodable { + let chatKey: [ChatKeyVector] + let blobs: [BlobVector] + let rejects: Rejects + } + + static let fixture: Fixture? = try? loadFixture() + static let blobs: [BlobVector] = fixture?.blobs ?? [] + static let rejects: [BlobVector] = fixture?.rejects.decryptBlob ?? [] + + private static func loadFixture() throws -> Fixture { + let url = try #require( + Bundle.module.url(forResource: "chat_cipher", withExtension: "json", subdirectory: "Fixtures") + ) + return try JSONDecoder().decode(Fixture.self, from: Data(contentsOf: url)) + } + + @Test func fixtureLoads() throws { + let fixture = try Self.loadFixture() + #expect(!fixture.blobs.isEmpty) + #expect(!fixture.rejects.decryptBlob.isEmpty) + } + + private func decrypt(_ vector: BlobVector) throws -> Data { + let bytes = SharedBytes.shared + let image = try DefaultChatCipher.shared.decryptBlob( + blob: bytes.byteArray(data: Data(hex: vector.blob)), + chatKey: bytes.byteArray(data: Data(hex: vector.chatKey)), + senderPk: bytes.byteArray(data: Data(hex: vector.senderPublicKey)), + recipientPk: bytes.byteArray(data: Data(hex: vector.recipientPublicKey)), + chatId: bytes.byteArray(data: Data(hex: vector.chatId)), + blobId: bytes.byteArray(data: Data(hex: vector.blobId)) + ) + return bytes.data(bytes: image) + } + + @Test(arguments: blobs) + func blobDecryptsToItsImage(_ vector: BlobVector) throws { + #expect(try decrypt(vector) == Data(hex: vector.image ?? "")) + } + + @Test(arguments: rejects) + func rejectedBlobFailsToDecrypt(_ vector: BlobVector) { + #expect(throws: (any Error).self) { try decrypt(vector) } + } + + /// `ChatSeal`, built from the members' seeds the way the app builds it, opens the same blobs: + /// the recipient through the peer's key and the sender's other device through its own. + @Test(arguments: blobs) + func chatSealOpensBlobForBothMembers(_ vector: BlobVector) throws { + let pair = try #require(Self.fixture?.chatKey.first { + $0.chatId == vector.chatId && $0.memberA.edPublicKey == vector.senderPublicKey + }) + let alice = try KeyPair(seed: Seed32(Data(hex: pair.memberA.edSeed))) + let bob = try KeyPair(seed: Seed32(Data(hex: pair.memberB.edSeed))) + let aliceID = UUID() + let conversationID = ConversationID(data: Data(hex: vector.chatId)) + let image = Data(hex: vector.image ?? "") + let sealed = SealedBlob(senderID: aliceID, plaintextSize: image.count) + let blobID = BlobID(data: Data(hex: vector.blobId)) + + let asBob = try ChatSeal(cipher: DefaultChatCipher.shared, owner: bob, peerPublicKey: alice.publicKey, conversationID: conversationID, selfUserID: UUID()) + let asAlice = try ChatSeal(cipher: DefaultChatCipher.shared, owner: alice, peerPublicKey: bob.publicKey, conversationID: conversationID, selfUserID: aliceID) + + #expect(try asBob.decryptBlob(Data(hex: vector.blob), blobID: blobID, sealed: sealed) == image) + #expect(try asAlice.decryptBlob(Data(hex: vector.blob), blobID: blobID, sealed: sealed) == image) + } +} + +private extension Data { + init(hex: String) { + var data = Data(capacity: hex.count / 2) + var index = hex.startIndex + while index < hex.endIndex { + let next = hex.index(index, offsetBy: 2) + data.append(UInt8(hex[index.. Data) throws -> ChatMediaCell { + let file = FileManager.default.temporaryDirectory.appendingPathComponent("chat-media-\(UUID().uuidString)") + try Data([1, 2, 3]).write(to: file) + let media = ChatMediaContent( + blobID: BlobID(data: Data(UUID().uuidString.utf8)), + width: 100, + height: 100, + blurhash: Self.blurhash, + caption: nil, + isRedacted: false, + sealed: SealedBlob(senderID: UUID(), plaintextSize: 3) + ) + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure( + with: ChatMessage(id: "1", content: .media(media), sender: .other, reactions: [], canReact: true), + maxWidth: 240, + localImage: nil, + remote: ChatMediaLocation(url: file, decrypt: decrypt) + ) + return cell + } + + private func waitForUnavailable(_ cell: ChatMediaCell) async { + for _ in 0..<200 where cell.unavailableLabel.isHidden { + try? await Task.sleep(for: .milliseconds(10)) + } + } + + @Test("An encrypted photo that fails to authenticate keeps its BlurHash, says it can't be shown, and takes no tap") + func undecryptablePhotoShowsUnavailable() async throws { + let cell = try encryptedCell { _ in throw BlobOpenFailure.authentication } + #expect(cell.unavailableLabel.isHidden) + + await waitForUnavailable(cell) + + #expect(!cell.unavailableLabel.isHidden) + #expect(!cell.imageTap.isEnabled) + #expect(cell.imageView.image != nil) + } + + @Test("An encrypted photo whose plaintext doesn't decode as an image says it can't be shown") + func undecodablePhotoShowsUnavailable() async throws { + let cell = try encryptedCell { $0 } + + await waitForUnavailable(cell) + + #expect(!cell.unavailableLabel.isHidden) + } + + @Test("Only a blob that will never open counts as undecryptable; a failed fetch stays retryable") + func undecryptableClassification() { + func providerError(_ underlying: any Error) -> KingfisherError { + .imageSettingError(reason: .dataProviderError(provider: RawImageDataProvider(data: Data(), cacheKey: "k"), error: underlying)) + } + #expect(ChatMediaImageSource.isUndecryptable(providerError(BlobOpenFailure.authentication))) + #expect(ChatMediaImageSource.isUndecryptable(providerError(BlobOpenFailure.length))) + #expect(ChatMediaImageSource.isUndecryptable(providerError(BlobOpenFailure.undecodable))) + #expect(!ChatMediaImageSource.isUndecryptable(providerError(URLError(.notConnectedToInternet)))) } } diff --git a/FlipcashTests/Chat/ChatMediaURLResolverTests.swift b/FlipcashTests/Chat/ChatMediaURLResolverTests.swift index 9b51bd4c3..bff8027e5 100644 --- a/FlipcashTests/Chat/ChatMediaURLResolverTests.swift +++ b/FlipcashTests/Chat/ChatMediaURLResolverTests.swift @@ -36,8 +36,8 @@ struct ChatMediaURLResolverTests { return url } - #expect(resolver.url(for: photo(isRedacted: true), canReact: true) { _ in } == nil) - #expect(resolver.url(for: photo(isRedacted: false), canReact: false) { _ in } == nil) + #expect(resolver.location(for: photo(isRedacted: true), canReact: true) { _ in }?.url == nil) + #expect(resolver.location(for: photo(isRedacted: false), canReact: false) { _ in }?.url == nil) // Give any stray fetch task a chance to run before asserting none did. await Task.yield() @@ -54,17 +54,17 @@ struct ChatMediaURLResolverTests { } let photo = photo(isRedacted: false) - #expect(resolver.url(for: photo, canReact: false) { _ in } == nil) + #expect(resolver.location(for: photo, canReact: false) { _ in }?.url == nil) let resolved = await withCheckedContinuation { continuation in - let immediate = resolver.url(for: photo, canReact: true) { continuation.resume(returning: $0) } + let immediate = resolver.location(for: photo, canReact: true) { continuation.resume(returning: $0.url) } #expect(immediate == nil) // A second dequeue while the first fetch is in flight does not start another. - #expect(resolver.url(for: photo, canReact: true) { _ in } == nil) + #expect(resolver.location(for: photo, canReact: true) { _ in }?.url == nil) } #expect(resolved == url) - #expect(resolver.url(for: photo, canReact: true) { _ in } == url) + #expect(resolver.location(for: photo, canReact: true) { _ in }?.url == url) #expect(spy.calls == [blobID]) } @@ -77,10 +77,10 @@ struct ChatMediaURLResolverTests { return url } - #expect(resolver.thumbnailURL(for: .media(thumbnailBlobID: nil), canReact: true) { _ in } == nil) - #expect(resolver.thumbnailURL(for: .media(thumbnailBlobID: blobID), canReact: false) { _ in } == nil) - #expect(resolver.thumbnailURL(for: .text, canReact: true) { _ in } == nil) - #expect(resolver.thumbnailURL(for: .unavailable, canReact: true) { _ in } == nil) + #expect(resolver.thumbnailLocation(for: .media(thumbnailBlobID: nil, sealed: nil), canReact: true) { _ in }?.url == nil) + #expect(resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: nil), canReact: false) { _ in }?.url == nil) + #expect(resolver.thumbnailLocation(for: .text, canReact: true) { _ in }?.url == nil) + #expect(resolver.thumbnailLocation(for: .unavailable, canReact: true) { _ in }?.url == nil) await Task.yield() #expect(spy.calls.isEmpty) @@ -96,14 +96,14 @@ struct ChatMediaURLResolverTests { } let resolved = await withCheckedContinuation { continuation in - let immediate = resolver.thumbnailURL(for: .media(thumbnailBlobID: blobID), canReact: true) { - continuation.resume(returning: $0) + let immediate = resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: nil), canReact: true) { + continuation.resume(returning: $0.url) } #expect(immediate == nil) } #expect(resolved == url) - #expect(resolver.url(for: photo(isRedacted: false), canReact: true) { _ in } == url) + #expect(resolver.location(for: photo(isRedacted: false), canReact: true) { _ in }?.url == url) #expect(spy.calls == [blobID]) } @@ -116,11 +116,11 @@ struct ChatMediaURLResolverTests { return url } - #expect(await resolver.thumbnailURL(for: .media(thumbnailBlobID: nil), canReact: true) == nil) + #expect(await resolver.thumbnailLocation(for: .media(thumbnailBlobID: nil, sealed: nil), canReact: true)?.url == nil) #expect(spy.calls.isEmpty) - #expect(await resolver.thumbnailURL(for: .media(thumbnailBlobID: blobID), canReact: true) == url) - #expect(await resolver.thumbnailURL(for: .media(thumbnailBlobID: blobID), canReact: true) == url) + #expect(await resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: nil), canReact: true)?.url == url) + #expect(await resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: nil), canReact: true)?.url == url) #expect(spy.calls == [blobID]) } @@ -132,13 +132,54 @@ struct ChatMediaURLResolverTests { let (row, quote) = await withCheckedContinuation { continuation in var row: URL? - let immediate = resolver.url(for: photo, canReact: true) { row = $0 } + let immediate = resolver.location(for: photo, canReact: true) { row = $0.url } #expect(immediate == nil) - _ = resolver.thumbnailURL(for: .media(thumbnailBlobID: blobID), canReact: true) { quote in - continuation.resume(returning: (row, quote)) + _ = resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: nil), canReact: true) { quote in + continuation.resume(returning: (row, quote.url)) } } #expect(row == url) #expect(quote == url) } + + @Test("An encrypted photo waits on the chat's decryption, fetches it once, and decrypts with its blob") + func encryptedPhotoDecrypts() async throws { + let url = url + let sealed = SealedBlob(senderID: UserID(), plaintextSize: 3) + let decryptCalls = FetchSpy() + let resolver = ChatMediaURLResolver( + fetch: { _ in url }, + decrypt: { + decryptCalls.calls.append(BlobID(data: Data())) + return { blob, blobID, sealed in Data(blob.reversed()) + blobID.data + Data([UInt8(sealed.plaintextSize)]) } + } + ) + let sealedPhoto = ChatMediaContent(blobID: blobID, width: 1, height: 1, blurhash: nil, caption: nil, isRedacted: false, sealed: sealed) + + let location = await withCheckedContinuation { continuation in + let immediate = resolver.location(for: sealedPhoto, canReact: true) { continuation.resume(returning: $0) } + #expect(immediate == nil) + } + #expect(location.url == url) + let decrypt = try #require(location.decrypt) + #expect(try decrypt(Data([1, 2])) == Data([2, 1, 1, 3])) + + let cached = try #require(resolver.location(for: sealedPhoto, canReact: true) { _ in }) + #expect(cached.decrypt != nil) + #expect(await resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: sealed), canReact: true)?.decrypt != nil) + #expect(decryptCalls.calls.count == 1) + #expect(resolver.location(for: photo(isRedacted: false), canReact: true) { _ in }?.decrypt == nil) + } + + @Test("An encrypted photo stays on its BlurHash while the chat's key is unknown") + func encryptedPhotoWithoutKey() async { + let url = url + let resolver = ChatMediaURLResolver(fetch: { _ in url }, decrypt: { nil }) + let sealed = SealedBlob(senderID: UserID(), plaintextSize: 3) + let photo = ChatMediaContent(blobID: blobID, width: 1, height: 1, blurhash: nil, caption: nil, isRedacted: false, sealed: sealed) + + let location = await resolver.thumbnailLocation(for: .media(thumbnailBlobID: blobID, sealed: sealed), canReact: true) + #expect(location == nil) + #expect(resolver.location(for: photo, canReact: true) { _ in } == nil) + } } diff --git a/FlipcashTests/Chat/ChatMediaUploaderTests.swift b/FlipcashTests/Chat/ChatMediaUploaderTests.swift index 27e9722ea..e82923ee5 100644 --- a/FlipcashTests/Chat/ChatMediaUploaderTests.swift +++ b/FlipcashTests/Chat/ChatMediaUploaderTests.swift @@ -7,6 +7,7 @@ import Testing import UIKit +import SharedCore @testable import FlipcashCore @testable import Flipcash @@ -46,7 +47,7 @@ struct ChatMediaUploaderTests { let blobID = try await ChatMediaUploader(blob: blob).upload(Self.image(width: 3000, height: 2000)) { width, height in prepared = (width, height, blob.storedData.count) - } + }.blobID #expect(prepared?.width == 2048) #expect(prepared?.height == 1365) @@ -117,9 +118,9 @@ struct ChatMediaUploaderTests { let blob = MockChatMediaBlobStore() blob.storeResults = [.failure(ErrorBlob.network(URLError(.timedOut)))] - let blobID = try await ChatMediaUploader(blob: blob, backoff: [.zero]).upload(Self.image(width: 40, height: 30)) { _, _ in } + let photo = try await ChatMediaUploader(blob: blob, backoff: [.zero]).upload(Self.image(width: 40, height: 30)) { _, _ in } - #expect(blobID == MockChatMediaBlobStore.blobID) + #expect(photo == .plain(MockChatMediaBlobStore.blobID)) #expect(blob.storeAttempts == 2) } @@ -146,6 +147,91 @@ struct ChatMediaUploaderTests { #expect(blob.storeAttempts == 0) } + // MARK: - Encrypted - + + private static func seal() throws -> ChatSeal { + try ChatSeal( + cipher: DefaultChatCipher.shared, + owner: KeyPair.generate()!, + peerPublicKey: KeyPair.generate()!.publicKey, + conversationID: ConversationID(data: Data(repeating: 0x07, count: 32)), + selfUserID: UUID() + ) + } + + @Test("In a chat that encrypts, the photo is sealed for it within the encrypted ceiling, with its blurhash") + func encryptedChatSealsPhoto() async throws { + let policy = UploadPolicy(version: "v1", ttl: nil, constraints: [ + .init(pattern: "image/*", maxSizeBytes: 5_000_000, image: .init(maxWidth: 4096, maxHeight: 4096, maxPixels: 0)), + ], encrypted: .init(maxSizeBytes: 2_000_000, image: .init(maxWidth: 1000, maxHeight: 1000, maxPixels: 0))) + let blob = MockChatMediaBlobStore(policy: policy) + let seal = try Self.seal() + var uploader = ChatMediaUploader(blob: blob) + uploader.seal = { seal } + + let photo = try await uploader.upload(Self.image(width: 3000, height: 2000)) { _, _ in } + + guard case .sealed(let sealed) = photo else { + Issue.record("Expected a sealed photo") + return + } + #expect(blob.storedData.isEmpty) + #expect(blob.encryptedStores.count == 1) + #expect(blob.encryptedStores.first?.conversationID == seal.conversationID) + #expect(sealed.width == 1000) + #expect((666...667).contains(sealed.height)) + #expect(sealed.mimeType == "image/jpeg") + #expect(sealed.sizeBytes == blob.encryptedStores.first?.image.count) + #expect(!sealed.blurhash.isEmpty && sealed.blurhash.count <= 64) + #expect(blob.finalizedBlobIDs == [sealed.blobID]) + } + + @Test("A chat that encrypts under a policy with no encrypted constraints uploads nothing, for good") + func encryptedChatWithoutPolicyUploadsNothing() async throws { + let blob = MockChatMediaBlobStore() + let seal = try Self.seal() + var uploader = ChatMediaUploader(blob: blob) + uploader.seal = { seal } + + let error = await Self.uploadError(uploader) + + #expect(error?.isRetryable == false) + #expect(blob.storeAttempts == 0) + } + + @Test("A missing peer key uploads nothing and stays retryable; it never falls back to plaintext") + func missingPeerKeyUploadsNothing() async { + let blob = MockChatMediaBlobStore() + var uploader = ChatMediaUploader(blob: blob) + uploader.seal = { throw PeerKeyUnavailable(userID: UUID()) } + + let error = await Self.uploadError(uploader) + + #expect(error?.isRetryable == true) + #expect(blob.storeAttempts == 0) + } + + // MARK: - Gate - + + @Test("Camera and Photos show in an encrypted DM, a plaintext chat, and not before the chat loads") + func gateShowsPhotosInEncryptedDMs() { + let encrypted = Conversation( + id: ConversationID(data: Data(repeating: 0x01, count: 32)), + members: [ConversationMember(userID: UUID(), displayName: "A"), ConversationMember(userID: UUID(), displayName: "B")], + lastMessage: nil, + lastActivity: Date(timeIntervalSince1970: 0), + type: .contactDm, + useE2Ee: true + ) + #expect(E2eePolicy.shouldEncrypt(encrypted)) + #expect(ChatMediaGate.acceptsMedia(encrypted)) + + var plain = encrypted + plain.useE2Ee = false + #expect(ChatMediaGate.acceptsMedia(plain)) + #expect(!ChatMediaGate.acceptsMedia(nil)) + } + // MARK: - Chip - @Test("A staged chip uploads, with its dimensions set before the bytes are stored") @@ -158,7 +244,7 @@ struct ChatMediaUploaderTests { #expect(chip.state == .preparing) - let blobID = try await #require(chip.uploadTask).value + let blobID = try await #require(chip.uploadTask).value.blobID #expect(dimensionsAtStore?.0 == 20) #expect(dimensionsAtStore?.1 == 15) @@ -189,7 +275,7 @@ struct ChatMediaUploaderTests { blob.finalization = .success(()) chip.startUpload(using: uploader) - let blobID = try await #require(chip.uploadTask).value + let blobID = try await #require(chip.uploadTask).value.blobID #expect(chip.state == .uploaded(blobID)) } @@ -243,4 +329,22 @@ struct ChatMediaUploaderTests { #expect(ChatMediaUploadError.failed(ErrorBlob.quotaExceeded).reportingLevel == .info) #expect(ChatMediaUploadError.failed(ErrorBlob.unknown).reportingLevel == .error) } + + @Test("A photo goes out only under the encryption choice it was uploaded for") + func photoSendMatchesSeal() throws { + let seal = try Self.seal() + let chat = seal.conversationID + let other = ConversationID(data: Data(repeating: 0x08, count: 32)) + let blobID = BlobID(data: Data([1])) + let sealed = SealedPhoto(blobID: blobID, mimeType: "image/jpeg", sizeBytes: 10, width: 1, height: 1, blurhash: "00") + + #expect(EncryptedChatClient.photoSend(.plain(blobID), seal: nil, conversationID: chat) == .plain(blobID)) + #expect(EncryptedChatClient.photoSend(.sealed(sealed), seal: seal, conversationID: chat) == .sealed(sealed, seal)) + // The chat started encrypting after the photo uploaded in plaintext. + #expect(EncryptedChatClient.photoSend(.plain(blobID), seal: seal, conversationID: chat) == .mismatch) + // The chat stopped encrypting after the photo uploaded sealed. + #expect(EncryptedChatClient.photoSend(.sealed(sealed), seal: nil, conversationID: chat) == .mismatch) + // A seal for a different chat. + #expect(EncryptedChatClient.photoSend(.sealed(sealed), seal: seal, conversationID: other) == .mismatch) + } } diff --git a/FlipcashTests/Chat/ChatMediaViewerTests.swift b/FlipcashTests/Chat/ChatMediaViewerTests.swift index 53da3ca17..bc9cbcca7 100644 --- a/FlipcashTests/Chat/ChatMediaViewerTests.swift +++ b/FlipcashTests/Chat/ChatMediaViewerTests.swift @@ -38,7 +38,7 @@ struct ChatMediaViewerTests { localImage: UIImage? = nil, remoteURL: URL? = ChatMediaViewerTests.remoteURL ) -> ChatMediaViewerRequest? { - ChatMediaViewerRequest(message: message, localImage: localImage, remoteURL: remoteURL, placeholder: nil) { nil } + ChatMediaViewerRequest(message: message, localImage: localImage, remote: remoteURL.map { ChatMediaLocation(url: $0) }, placeholder: nil) { nil } } // MARK: - Request @@ -47,7 +47,7 @@ struct ChatMediaViewerTests { func viewablePhotoOpens() throws { let request = try #require(request(message(media()))) #expect(request.blobID == Self.blobID) - #expect(request.remoteURL == Self.remoteURL) + #expect(request.remote?.url == Self.remoteURL) } @Test("A redacted photo, or one seen while previewing a group, never opens") @@ -79,7 +79,7 @@ struct ChatMediaViewerTests { func zoomsFromSource() throws { let source = UIView() let request = try #require(ChatMediaViewerRequest( - message: message(media()), localImage: Self.localImage, remoteURL: nil, placeholder: nil + message: message(media()), localImage: Self.localImage, remote: nil, placeholder: nil ) { source }) let viewer = ChatMediaViewerController(request: request) { _ in } @@ -118,7 +118,7 @@ struct ChatMediaViewerTests { func shareWaitsForPhoto() throws { let placeholder = UIImage(systemName: "photo")! let request = try #require(ChatMediaViewerRequest( - message: message(media()), localImage: nil, remoteURL: URL(string: "https://example.invalid/never.jpg")!, placeholder: placeholder + message: message(media()), localImage: nil, remote: ChatMediaLocation(url: URL(string: "https://example.invalid/never.jpg")!), placeholder: placeholder ) { nil }) var shared: UIImage? let viewer = ChatMediaViewerController(request: request) { shared = $0 } diff --git a/FlipcashTests/Chat/ChatQuoteBubbleTests.swift b/FlipcashTests/Chat/ChatQuoteBubbleTests.swift index 3d4037b4a..8785bb956 100644 --- a/FlipcashTests/Chat/ChatQuoteBubbleTests.swift +++ b/FlipcashTests/Chat/ChatQuoteBubbleTests.swift @@ -142,8 +142,8 @@ struct ChatQuoteBubbleTests { @Test("A quoted photo draws its thumbnail; a redacted one draws none") func mediaQuote_drawsThumbnailOnlyWhenFetchable() { - let photo = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: BlobID(data: Data([1])))) - let redacted = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: nil)) + let photo = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: BlobID(data: Data([1])), sealed: nil)) + let redacted = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: nil, sealed: nil)) #expect(laidOutCell(quote: photo).bubbleView.quotePanel.thumbnailView.isHidden == false) #expect(laidOutCell(quote: redacted).bubbleView.quotePanel.thumbnailView.isHidden == true) @@ -151,7 +151,7 @@ struct ChatQuoteBubbleTests { @Test("A recycled panel drops the photo it quoted before") func reuse_dropsThumbnail() { - let photo = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: BlobID(data: Data([1])))) + let photo = ChatQuote(stableID: "7", authorName: "Ada", snippet: "Photo", kind: .media(thumbnailBlobID: BlobID(data: Data([1])), sealed: nil)) let cell = laidOutCell(quote: photo) cell.configure(with: ChatMessage(id: "2", content: .text("hi"), sender: .me, quote: quote), maxWidth: Self.maxWidth) diff --git a/FlipcashTests/Chat/ConversationMediaSendTests.swift b/FlipcashTests/Chat/ConversationMediaSendTests.swift index b6a298de3..47c9ad38b 100644 --- a/FlipcashTests/Chat/ConversationMediaSendTests.swift +++ b/FlipcashTests/Chat/ConversationMediaSendTests.swift @@ -45,14 +45,14 @@ struct ConversationMediaSendTests { private func uploadedChip(_ name: String) -> ComposerChip { let chip = ComposerChip(image: testImage()) let blobID = blobID(name) - chip.uploadTask = Task { blobID } + chip.uploadTask = Task { .plain(blobID) } return chip } /// A chip whose upload ends only when the test opens `gate`. private func gatedChip(_ gate: UploadGate, image: UIImage? = nil) -> ComposerChip { let chip = ComposerChip(image: image ?? testImage()) - chip.uploadTask = Task { try await gate.wait() } + chip.uploadTask = Task { .plain(try await gate.wait()) } return chip } diff --git a/FlipcashTests/Database/Database+ConversationsTests.swift b/FlipcashTests/Database/Database+ConversationsTests.swift index f614bfbf0..76593f653 100644 --- a/FlipcashTests/Database/Database+ConversationsTests.swift +++ b/FlipcashTests/Database/Database+ConversationsTests.swift @@ -501,6 +501,27 @@ struct DatabaseConversationsTests { #expect(try database.getConversationMessages(conversationID: id) == [captioned, bare]) } + @Test("A decrypted photo reloads with what it needs to decrypt its blob again") + func sealedMediaRoundTrip() throws { + let (database, url) = try Database.makeTemp() + defer { Database.removeTemp(at: url) } + let id = ConversationID.test(1) + let attachment = MediaAttachment( + blobID: BlobID(data: Data(repeating: 9, count: 16)), width: 300, height: 400, blurhash: "LEHV6nWB2yk8", + sealed: SealedBlob(senderID: otherID, plaintextSize: 4321) + ) + let message = ConversationMessage( + id: MessageID(value: 1), senderID: otherID, + content: .media([attachment], caption: "secret"), + date: Date(timeIntervalSince1970: 10), unreadSeq: 1, eventSequence: 3, + sealed: ConversationMessage.Sealed(scheme: 1, nonce: Data([1]), ciphertext: Data([2])) + ) + + try database.upsertConversationMessages([message], conversationID: id) + + #expect(try database.getConversationMessages(conversationID: id) == [message]) + } + @Test("A redacted media message reloads still redacted, so its bytes stay unfetched after a relaunch") func redactedMediaRoundTrip() throws { let (database, url) = try Database.makeTemp() diff --git a/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift b/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift index e21dcef49..92a8464b5 100644 --- a/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift +++ b/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift @@ -25,6 +25,8 @@ final class MockChatMediaBlobStore: ChatMediaBlobStoring { private(set) var storedData: [Data] = [] private(set) var storedMimeTypes: [String] = [] private(set) var finalizedBlobIDs: [BlobID] = [] + /// The plaintext images `storeEncryptedBlob` was handed, with the chat each was sealed for. + private(set) var encryptedStores: [(image: Data, conversationID: ConversationID)] = [] init(policy: UploadPolicy = UploadPolicy(version: "v1", ttl: nil, constraints: [ .init(pattern: "image/*", maxSizeBytes: 5_000_000, image: .init(maxWidth: 2048, maxHeight: 2048, maxPixels: 0)), @@ -46,6 +48,15 @@ final class MockChatMediaBlobStore: ChatMediaBlobStoring { return blobID } + func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload { + storeAttempts += 1 + onStore?() + let result = storeResults.isEmpty ? .success(Self.blobID) : storeResults.removeFirst() + let blobID = try result.get() + encryptedStores.append((image, seal.conversationID)) + return EncryptedBlobUpload(blobID: blobID, plaintextSize: image.count) + } + func awaitBlobFinalization(blobID: BlobID) async throws { try finalization.get() finalizedBlobIDs.append(blobID) diff --git a/FlipcashTests/TestSupport/MockConversations.swift b/FlipcashTests/TestSupport/MockConversations.swift index 30f1de532..5d4980151 100644 --- a/FlipcashTests/TestSupport/MockConversations.swift +++ b/FlipcashTests/TestSupport/MockConversations.swift @@ -23,6 +23,8 @@ final class MockConversations: ConversationFetching, ConversationMembership, Con let caption: String? let repliedTo: MessageID? let clientMessageID: UUID + /// The sealed metadata the photo went out with, nil for a plaintext photo. + var sealed: SealedPhoto? = nil } struct TypingCall: Sendable { let conversationID: ConversationID; let state: TypingState } /// A scripted `GetDelta` batch: one `onBatch` call with these messages + checkpoint. @@ -411,9 +413,23 @@ final class MockConversations: ConversationFetching, ConversationMembership, Con ) } - func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, blobID: BlobID, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage { + func openingSeal(owner: KeyPair, conversationID: ConversationID) async -> ChatSeal? { + nil + } + + func photoSeal(owner: KeyPair, conversationID: ConversationID) async throws -> ChatSeal? { + nil + } + + func sendMediaMessage(owner: KeyPair, conversationID: ConversationID, photo: UploadedPhoto, caption: String?, repliedTo: MessageID?, clientMessageID: UUID) async throws -> ConversationMessage { + let blobID = photo.blobID + let sealed: SealedPhoto? + switch photo { + case .plain: sealed = nil + case .sealed(let photo): sealed = photo + } let (count, error) = lock.withLock { - _sentMedia.append(SentMedia(conversationID: conversationID, blobID: blobID, caption: caption, repliedTo: repliedTo, clientMessageID: clientMessageID)) + _sentMedia.append(SentMedia(conversationID: conversationID, blobID: blobID, caption: caption, repliedTo: repliedTo, clientMessageID: clientMessageID, sealed: sealed)) return (_sentMedia.count, _mediaSendError) } if let error { throw error } diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatBubbleView.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatBubbleView.swift index 82406c59b..265dfc244 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatBubbleView.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatBubbleView.swift @@ -148,8 +148,8 @@ public final class ChatBubbleView: UIView { /// Whether this bubble is currently flashing. var isFlashingAttention: Bool { background.isFlashingAttention } - /// Fills the bubble; `quoteThumbnailURL` is where a quoted photo's thumbnail loads from. - public func configure(with message: ChatMessage, quoteThumbnailURL: URL? = nil) { + /// Fills the bubble; `quoteThumbnail` is where a quoted photo's thumbnail loads from. + public func configure(with message: ChatMessage, quoteThumbnail: ChatMediaLocation? = nil) { label.attributedText = Self.displayText(for: message) editedLabel.isHidden = !Self.showsEditedMarker(for: message) || message.rendersAsLargeEmoji isBare = message.rendersAsLargeEmoji @@ -182,7 +182,7 @@ public final class ChatBubbleView: UIView { // unsatisfiable, and UIKit resolves that by breaking one at random. if let quote = message.quote { quotePanel.isHidden = false - quotePanel.configure(with: quote, thumbnailURL: quoteThumbnailURL) + quotePanel.configure(with: quote, thumbnail: quoteThumbnail) NSLayoutConstraint.deactivate(quoteCollapse) quoteTrailing.isActive = true labelTopToBubble.isActive = false diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatLinkMessageCell.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatLinkMessageCell.swift index 88d18f5f8..e25f43e95 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatLinkMessageCell.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatLinkMessageCell.swift @@ -97,17 +97,17 @@ public final class ChatLinkMessageCell: ChatColumnCell { /// - Parameters: /// - maxWidth: the widest the bubble may grow before its text wraps. - /// - quoteThumbnailURL: where a quoted photo's thumbnail loads from, or nil until it resolves. + /// - quoteThumbnail: where a quoted photo's thumbnail loads from, or nil until it resolves. public func configure( with message: ChatMessage, maxWidth: CGFloat, authorImageData: Data? = nil, - quoteThumbnailURL: URL? = nil + quoteThumbnail: ChatMediaLocation? = nil ) { bubbleMaxWidthConstraint.constant = maxWidth bubbleCardWidthConstraint.constant = maxWidth bubbleCardWidthConstraint.isActive = Self.cardFillsWidth(message.linkPreview?.card) - bubble.configure(with: message, quoteThumbnailURL: quoteThumbnailURL) + bubble.configure(with: message, quoteThumbnail: quoteThumbnail) reactionRowWidthConstraint.constant = maxWidth reactionRow.layoutWidth = maxWidth reactionRow.hugsTrailingEdge = message.sender == .me diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift index 17d39cf6c..e4f301146 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift @@ -30,6 +30,8 @@ public final class ChatMediaCell: ChatColumnCell { private let stack = UIStackView() private let imageBubble = BubbleBackgroundView() let imageView = UIImageView() + /// Shown over the BlurHash when an encrypted photo's bytes fail to decrypt or check out. + let unavailableLabel = UILabel() let captionBubble = BubbleBackgroundView() let captionLabel = UILabel() let reactionRow = ReactionPillRowView() @@ -65,6 +67,14 @@ public final class ChatMediaCell: ChatColumnCell { imageBubble.addSubview(imageView) imageBubble.isAccessibilityElement = true imageBubble.accessibilityLabel = "Photo" + unavailableLabel.text = ChatMediaStrings.undecryptable + unavailableLabel.font = .default(size: 14, weight: .medium) + unavailableLabel.textColor = .white + unavailableLabel.textAlignment = .center + unavailableLabel.numberOfLines = 0 + unavailableLabel.isHidden = true + unavailableLabel.translatesAutoresizingMaskIntoConstraints = false + imageBubble.addSubview(unavailableLabel) imageTap.addTarget(self, action: #selector(imageTapped)) imageBubble.addGestureRecognizer(imageTap) @@ -103,6 +113,10 @@ public final class ChatMediaCell: ChatColumnCell { imageView.trailingAnchor.constraint(equalTo: imageBubble.trailingAnchor), imageView.bottomAnchor.constraint(equalTo: imageBubble.bottomAnchor), + unavailableLabel.centerYAnchor.constraint(equalTo: imageBubble.centerYAnchor), + unavailableLabel.leadingAnchor.constraint(equalTo: imageBubble.leadingAnchor, constant: Self.captionInset), + unavailableLabel.trailingAnchor.constraint(equalTo: imageBubble.trailingAnchor, constant: -Self.captionInset), + captionLabel.topAnchor.constraint(equalTo: captionBubble.topAnchor, constant: Self.captionPadding), captionLabel.bottomAnchor.constraint(equalTo: captionBubble.bottomAnchor, constant: -Self.captionPadding), captionLabel.leadingAnchor.constraint(equalTo: captionBubble.leadingAnchor, constant: Self.captionInset), @@ -124,6 +138,7 @@ public final class ChatMediaCell: ChatColumnCell { super.prepareForReuse() imageView.kf.cancelDownloadTask() imageView.image = nil + showUnavailable(false) drawnRowID = nil reactionRow.prepareForReuse() } @@ -137,12 +152,12 @@ public final class ChatMediaCell: ChatColumnCell { /// - Parameters: /// - maxWidth: the transcript's widest bubble, which the photo always spans. /// - localImage: the picked image of a pending send this device staged, or nil. - /// - remoteURL: the photo's resolved download URL, or nil until it is resolved. + /// - remote: where the photo downloads from, or nil until it is resolved. public func configure( with message: ChatMessage, maxWidth: CGFloat, localImage: UIImage?, - remoteURL: URL?, + remote: ChatMediaLocation?, authorImageData: Data? = nil ) { guard case .media(let media) = message.content else { return } @@ -161,7 +176,7 @@ public final class ChatMediaCell: ChatColumnCell { blobID: media.blobID, blurhash: media.blurhash, localImage: blurhashOnly ? nil : localImage, - remoteURL: blurhashOnly ? nil : remoteURL, + remote: blurhashOnly ? nil : remote, size: CGSize(width: size.width, height: size.height) ) imageTap.isEnabled = !blurhashOnly && !message.isFailed @@ -197,9 +212,10 @@ public final class ChatMediaCell: ChatColumnCell { updateColumn(for: message, authorImageData: authorImageData) } - private func drawImage(for rowID: String, blobID: BlobID?, blurhash: String?, localImage: UIImage?, remoteURL: URL?, size: CGSize) { + private func drawImage(for rowID: String, blobID: BlobID?, blurhash: String?, localImage: UIImage?, remote: ChatMediaLocation?, size: CGSize) { let isSameRow = drawnRowID == rowID drawnRowID = rowID + if !isSameRow { showUnavailable(false) } if let localImage { imageView.kf.cancelDownloadTask() @@ -208,7 +224,7 @@ public final class ChatMediaCell: ChatColumnCell { } let preview = BlurHashCache.shared.image(for: blurhash) - guard let remoteURL else { + guard let remote else { imageView.kf.cancelDownloadTask() imageView.image = preview return @@ -217,15 +233,28 @@ public final class ChatMediaCell: ChatColumnCell { // Whatever this row already shows — its local image, or the photo itself on a reconfigure — // stays under the load, so only a BlurHash is ever faded over. let placeholder = isSameRow ? (imageView.image ?? preview) : preview + let processor = DownsamplingImageProcessor(size: size) imageView.kf.setImage( - with: ChatMediaImageSource.resource(blobID: blobID, url: remoteURL), + with: ChatMediaImageSource.source(blobID: blobID, location: remote), placeholder: placeholder, - options: [ - .processor(DownsamplingImageProcessor(size: size)), + options: ChatMediaImageSource.options(processor: processor) + [ .scaleFactor(traitCollection.displayScale), .transition(.fade(Self.fadeDuration)), ] - ) + ) { [weak self] result in + ChatMediaImageSource.persist(result, processor: processor) + if case .failure(let error) = result, ChatMediaImageSource.isUndecryptable(error) { + self?.showUnavailable(true) + } + } + } + + /// Marks the photo as one that can't be shown: an encrypted blob that failed to authenticate or + /// wasn't the length its sender declared. It keeps its BlurHash and takes no tap. + private func showUnavailable(_ unavailable: Bool) { + unavailableLabel.isHidden = !unavailable + imageBubble.accessibilityLabel = unavailable ? ChatMediaStrings.undecryptable : "Photo" + if unavailable { imageTap.isEnabled = false } } @objc private func imageTapped() { diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaImageSource.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaImageSource.swift index b18a3e5fc..32dfa42ce 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaImageSource.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaImageSource.swift @@ -6,15 +6,117 @@ // import Foundation +import UIKit import FlipcashCore import Kingfisher +/// Where one chat photo downloads from, and how its bytes become an image. +public struct ChatMediaLocation: Sendable { + /// The photo's signed download URL. + public let url: URL + /// Turns the downloaded blob into the image's plaintext bytes; nil for a plaintext blob. + public let decrypt: (@Sendable (Data) throws -> Data)? + + public init(url: URL, decrypt: (@Sendable (Data) throws -> Data)? = nil) { + self.url = url + self.decrypt = decrypt + } +} + /// Where a chat photo's bytes load from, for every view that draws one. public enum ChatMediaImageSource { - /// The photo at `url`, cached under its blob rather than the URL. The URL is signed and minted - /// per fetch, so keying on it would re-download the same photo every session. - public static func resource(blobID: BlobID?, url: URL) -> KF.ImageResource { - KF.ImageResource(downloadURL: url, cacheKey: blobID.map { "chat-media-\($0.description)" } ?? url.absoluteString) + /// The photo at `location`, cached under its blob rather than the URL. The URL is signed and + /// minted per fetch, so keying on it would re-download the same photo every session. An + /// encrypted blob is decrypted before Kingfisher decodes it. + public static func source(blobID: BlobID?, location: ChatMediaLocation) -> Source { + let cacheKey = blobID.map { "chat-media-\($0.description)" } ?? location.url.absoluteString + guard let decrypt = location.decrypt else { + return .network(KF.ImageResource(downloadURL: location.url, cacheKey: cacheKey)) + } + return .provider(DecryptingProvider(cacheKey: cacheKey, url: location.url, decrypt: decrypt)) + } + + /// Whether a failed load was an encrypted blob that will never decrypt, rather than a fetch + /// that may succeed later. + public static func isUndecryptable(_ error: KingfisherError) -> Bool { + switch error { + case .imageSettingError(reason: .dataProviderError(_, let underlying)): + underlying is BlobOpenFailure + default: + false + } + } + + /// The cache chat photos read and write. A blob's bytes never change, so unlike the app's other + /// remote images they persist to disk and never expire from memory. + public static let cache: ImageCache = { + let cache = ImageCache(name: "chat-media") + cache.memoryStorage.config.expiration = .never + cache.diskStorage.config.sizeLimit = 500 * 1024 * 1024 + cache.diskStorage.config.expiration = .days(30) + return cache + }() + + /// The options every chat photo load passes, `processor` included when it draws a downsampled copy. + public static func options(processor: ImageProcessor? = nil) -> KingfisherOptionsInfo { + var options: KingfisherOptionsInfo = [.targetCache(cache)] + if let processor { options.append(.processor(processor)) } + return options + } + + /// Writes a freshly downloaded photo to disk under `resource`'s key. + /// + /// The app-wide `.cacheMemoryOnly` default stops Kingfisher writing it there itself, and the + /// signed URL changes per fetch, so without this every relaunch or memory trim re-downloads it. + public static func persist( + _ result: Result, + processor: ImageProcessor? = nil + ) { + guard case .success(let value) = result else { return } + persist(value.image, cacheType: value.cacheType, forKey: value.source.cacheKey, processor: processor) + } + + static func persist( + _ image: UIImage, + cacheType: CacheType, + forKey key: String, + processor: ImageProcessor?, + completion: (@Sendable () -> Void)? = nil + ) { + guard cacheType == .none else { return } + cache.store( + image, + forKey: key, + processorIdentifier: processor?.identifier ?? DefaultImageProcessor.default.identifier, + cacheSerializer: FormatIndicatedCacheSerializer.jpeg, + toDisk: true + ) { _ in completion?() } + } +} + +/// Downloads an end-to-end encrypted chat blob and hands Kingfisher its decrypted bytes. +private struct DecryptingProvider: ImageDataProvider { + let cacheKey: String + let url: URL + let decrypt: @Sendable (Data) throws -> Data + + var contentURL: URL? { url } + + func data(handler: @escaping @Sendable (Result) -> Void) { + Task { + do { + let (blob, response) = try await URLSession.shared.data(from: url) + if let status = (response as? HTTPURLResponse)?.statusCode, !(200..<300).contains(status) { + throw URLError(.badServerResponse) + } + let image = try decrypt(blob) + // The decoded image is authoritative; bytes that don't decode render as unsupported. + guard UIImage(data: image) != nil else { throw BlobOpenFailure.undecodable } + handler(.success(image)) + } catch { + handler(.failure(error)) + } + } } } diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaURLResolver.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaURLResolver.swift index 0815161d8..09d93989b 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaURLResolver.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaURLResolver.swift @@ -17,55 +17,67 @@ public final class ChatMediaURLResolver { /// Mints a download URL for a blob, or returns nil when the server has none to give. public typealias Fetch = @MainActor (BlobID) async throws -> URL? + /// Decrypts an end-to-end encrypted blob into its image bytes, throwing when it fails to + /// authenticate or is not the declared length. + public typealias BlobDecrypt = @Sendable (Data, BlobID, SealedBlob) throws -> Data + /// Builds the conversation's blob decryption, or returns nil while it can't be (its key is not + /// known yet). + public typealias FetchDecrypt = @MainActor () async -> BlobDecrypt? private let fetch: Fetch + private let fetchDecrypt: FetchDecrypt private var cache: [BlobID: URL] = [:] /// Everyone waiting on a blob's fetch, called with its URL, or nil when the fetch failed. private var waiters: [BlobID: [(URL?) -> Void]] = [:] + private var decrypt: BlobDecrypt? + /// Everyone waiting on the decryption, or nil when no fetch of it is in flight. + private var decryptWaiters: [(BlobDecrypt?) -> Void]? - public init(fetch: @escaping Fetch) { + public init(fetch: @escaping Fetch, decrypt: @escaping FetchDecrypt = { nil }) { self.fetch = fetch + self.fetchDecrypt = decrypt } - /// The URL to draw `media` from, or nil for now. + /// Where to draw `media` from, or nil for now. /// /// A BlurHash-only row returns nil without fetching. A cache miss starts at most one fetch per /// blob and calls `onResolved` when it lands so the caller can redraw the row; a failed fetch - /// leaves the row on its BlurHash, and the next call tries again. - public func url( + /// leaves the row on its BlurHash, and the next call tries again. An encrypted photo also waits + /// on the chat's decryption. + public func location( for media: ChatMediaContent, canReact: Bool, - onResolved: @escaping (URL) -> Void - ) -> URL? { + onResolved: @escaping (ChatMediaLocation) -> Void + ) -> ChatMediaLocation? { guard !ChatMediaCell.isBlurhashOnly(media, canReact: canReact), let blobID = media.blobID else { return nil } - return resolve(blobID) { $0.map(onResolved) } + return resolve(blobID, sealed: media.sealed) { $0.map(onResolved) } } - /// The URL to draw a quoted photo's thumbnail from, or nil for now, sharing the photo row's cache. + /// Where to draw a quoted photo's thumbnail from, or nil for now, sharing the photo row's cache. /// /// Nil without fetching for anything but a photo whose bytes the viewer may see: a quote of text, - /// cash, or an unavailable original, a redacted photo (``ChatQuote/Kind/media(thumbnailBlobID:)`` + /// cash, or an unavailable original, a redacted photo (``ChatQuote/Kind/media(thumbnailBlobID:sealed:)`` /// carries no blob), or any quote while the viewer previews a group they have not joined. - public func thumbnailURL( + public func thumbnailLocation( for kind: ChatQuote.Kind, canReact: Bool, - onResolved: @escaping (URL) -> Void - ) -> URL? { - guard let blobID = Self.thumbnailBlobID(for: kind, canReact: canReact) else { return nil } - return resolve(blobID) { $0.map(onResolved) } + onResolved: @escaping (ChatMediaLocation) -> Void + ) -> ChatMediaLocation? { + guard let (blobID, sealed) = Self.thumbnailBlob(for: kind, canReact: canReact) else { return nil } + return resolve(blobID, sealed: sealed) { $0.map(onResolved) } } - /// The URL to draw a quoted photo's thumbnail from, waiting for the fetch; nil when there is none + /// Where to draw a quoted photo's thumbnail from, waiting for the fetch; nil when there is none /// to draw or the fetch failed. Follows the same rules as the callback form. - public func thumbnailURL(for kind: ChatQuote.Kind, canReact: Bool) async -> URL? { - guard let blobID = Self.thumbnailBlobID(for: kind, canReact: canReact) else { return nil } + public func thumbnailLocation(for kind: ChatQuote.Kind, canReact: Bool) async -> ChatMediaLocation? { + guard let (blobID, sealed) = Self.thumbnailBlob(for: kind, canReact: canReact) else { return nil } return await withCheckedContinuation { continuation in var resumed = false - let immediate = resolve(blobID) { url in + let immediate = resolve(blobID, sealed: sealed) { location in guard !resumed else { return } resumed = true - continuation.resume(returning: url) + continuation.resume(returning: location) } if let immediate, !resumed { resumed = true @@ -74,21 +86,49 @@ public final class ChatMediaURLResolver { } } - private static func thumbnailBlobID(for kind: ChatQuote.Kind, canReact: Bool) -> BlobID? { + private static func thumbnailBlob(for kind: ChatQuote.Kind, canReact: Bool) -> (BlobID, SealedBlob?)? { switch kind { - case .media(let thumbnailBlobID): - canReact ? thumbnailBlobID : nil + case .media(let thumbnailBlobID, let sealed): + guard canReact, let thumbnailBlobID else { return nil } + return (thumbnailBlobID, sealed) case .text, .cash, .unavailable: - nil + return nil } } - /// The cached URL for `blobID`, or nil after queueing `completion` on the one fetch for it. - private func resolve(_ blobID: BlobID, completion: @escaping (URL?) -> Void) -> URL? { - if let cached = cache[blobID] { return cached } + /// The location for `blobID` when everything it needs is cached, or nil after queueing + /// `completion` on the fetches it still waits on. + private func resolve(_ blobID: BlobID, sealed: SealedBlob?, completion: @escaping (ChatMediaLocation?) -> Void) -> ChatMediaLocation? { + if let url = cache[blobID] { + guard let sealed else { return ChatMediaLocation(url: url) } + if let decrypt { return Self.location(url: url, blobID: blobID, sealed: sealed, decrypt: decrypt) } + awaitDecrypt { decrypt in + completion(decrypt.map { Self.location(url: url, blobID: blobID, sealed: sealed, decrypt: $0) }) + } + return nil + } + fetchURL(blobID) { [weak self] url in + guard let url else { return completion(nil) } + guard let sealed else { return completion(ChatMediaLocation(url: url)) } + if let decrypt = self?.decrypt { + return completion(Self.location(url: url, blobID: blobID, sealed: sealed, decrypt: decrypt)) + } + self?.awaitDecrypt { decrypt in + completion(decrypt.map { Self.location(url: url, blobID: blobID, sealed: sealed, decrypt: $0) }) + } + } + return nil + } + + private static func location(url: URL, blobID: BlobID, sealed: SealedBlob, decrypt: @escaping BlobDecrypt) -> ChatMediaLocation { + ChatMediaLocation(url: url) { try decrypt($0, blobID, sealed) } + } + + /// Queues `completion` on the one fetch of `blobID`'s URL, starting it if none is in flight. + private func fetchURL(_ blobID: BlobID, completion: @escaping (URL?) -> Void) { let isFirst = waiters[blobID] == nil waiters[blobID, default: []].append(completion) - guard isFirst else { return nil } + guard isFirst else { return } Task { // Best-effort: the row keeps its BlurHash, and the next dequeue retries. let resolved = try? await fetch(blobID) @@ -96,6 +136,23 @@ public final class ChatMediaURLResolver { let completions = waiters.removeValue(forKey: blobID) ?? [] for completion in completions { completion(resolved) } } - return nil + } + + /// Queues `completion` on the one fetch of the chat's decryption, starting it if none is in flight. + private func awaitDecrypt(_ completion: @escaping (BlobDecrypt?) -> Void) { + guard decryptWaiters == nil else { + decryptWaiters?.append(completion) + return + } + decryptWaiters = [completion] + Task { + // A chat whose key is not known yet leaves its photos on their BlurHash; the next + // dequeue tries again. + let resolved = await fetchDecrypt() + if let resolved { decrypt = resolved } + let completions = decryptWaiters ?? [] + decryptWaiters = nil + for completion in completions { completion(resolved) } + } } } diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaViewerController.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaViewerController.swift index 07ccf6f8a..c78363446 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaViewerController.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaViewerController.swift @@ -20,8 +20,8 @@ public struct ChatMediaViewerRequest { public let blobID: BlobID? /// The staged image of a send this device has not confirmed yet, drawn instead of downloading. public let localImage: UIImage? - /// The photo's resolved download URL, or nil for a pending send. - public let remoteURL: URL? + /// Where the photo downloads from, or nil for a pending send. + public let remote: ChatMediaLocation? /// Whatever the row drew when it was tapped, shown while the full photo loads. public let placeholder: UIImage? /// The on-screen view the photo zooms out of and back into, looked up on each use because the @@ -34,18 +34,18 @@ public struct ChatMediaViewerRequest { public init?( message: ChatMessage, localImage: UIImage?, - remoteURL: URL?, + remote: ChatMediaLocation?, placeholder: UIImage?, sourceView: @escaping () -> UIView? ) { guard case .media(let media) = message.content, !ChatMediaCell.isBlurhashOnly(media, canReact: message.canReact), !message.isFailed, - localImage != nil || remoteURL != nil else { return nil } + localImage != nil || remote != nil else { return nil } self.messageID = message.id self.blobID = media.blobID self.localImage = localImage - self.remoteURL = remoteURL + self.remote = remote self.placeholder = placeholder self.sourceView = sourceView } @@ -159,11 +159,13 @@ public final class ChatMediaViewerController: UIViewController, UIScrollViewDele } imageView.image = request.placeholder loadedImage = nil - guard let remoteURL = request.remoteURL else { return } + guard let remote = request.remote else { return } imageView.kf.setImage( - with: ChatMediaImageSource.resource(blobID: request.blobID, url: remoteURL), - placeholder: request.placeholder + with: ChatMediaImageSource.source(blobID: request.blobID, location: remote), + placeholder: request.placeholder, + options: ChatMediaImageSource.options() ) { [weak self] result in + ChatMediaImageSource.persist(result) switch result { case .success(let value): self?.loadedImage = value.image diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMessageCell.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMessageCell.swift index 3ac34f097..d02896b06 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMessageCell.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMessageCell.swift @@ -66,14 +66,14 @@ public final class ChatMessageCell: ChatColumnCell { /// - Parameters: /// - maxWidth: the widest the bubble may grow before its text wraps, in points. The owner /// derives it from the collection view's width. - /// - quoteThumbnailURL: where a quoted photo's thumbnail loads from, or nil until it resolves. + /// - quoteThumbnail: where a quoted photo's thumbnail loads from, or nil until it resolves. public func configure( with message: ChatMessage, maxWidth: CGFloat, authorImageData: Data? = nil, - quoteThumbnailURL: URL? = nil + quoteThumbnail: ChatMediaLocation? = nil ) { - bubble.configure(with: message, quoteThumbnailURL: quoteThumbnailURL) + bubble.configure(with: message, quoteThumbnail: quoteThumbnail) maxWidthConstraint.constant = maxWidth reactionRowWidthConstraint.constant = maxWidth reactionRow.layoutWidth = maxWidth diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatQuotePanelView.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatQuotePanelView.swift index 87235c503..17bda467e 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatQuotePanelView.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatQuotePanelView.swift @@ -171,8 +171,8 @@ final class ChatQuotePanelView: UIView { accessibilityIdentifier = "chat-quote-panel" } - /// - Parameter thumbnailURL: where a quoted photo's thumbnail loads from, or nil until it resolves. - func configure(with quote: ChatQuote, thumbnailURL: URL? = nil) { + /// - Parameter thumbnail: where a quoted photo's thumbnail loads from, or nil until it resolves. + func configure(with quote: ChatQuote, thumbnail: ChatMediaLocation? = nil) { targetStableID = quote.stableID // The author's own colour, derived from their user id — the same colour the composer's strip // draws them in, and the same one Android does. An original with no known author falls back @@ -196,17 +196,17 @@ final class ChatQuotePanelView: UIView { // A payment's amount is the whole of what was said, so it is read rather than glanced // at — a step brighter than the preview grey a quoted sentence gets. snippetLabel.textColor = UIColor.white.withAlphaComponent(0.75) - showThumbnail(blobID: nil, url: nil) - case .media(let thumbnailBlobID): + showThumbnail(blobID: nil, location: nil) + case .media(let thumbnailBlobID, _): flagView.isHidden = true tokenLabel.isHidden = true snippetLabel.textColor = Self.snippetColor - showThumbnail(blobID: thumbnailBlobID, url: thumbnailURL) + showThumbnail(blobID: thumbnailBlobID, location: thumbnail) case .text, .unavailable: flagView.isHidden = true tokenLabel.isHidden = true snippetLabel.textColor = Self.snippetColor - showThumbnail(blobID: nil, url: nil) + showThumbnail(blobID: nil, location: nil) } let spoken = switch quote.kind { case .cash(let token, _): "\(quote.snippet) \(token)" @@ -230,14 +230,14 @@ final class ChatQuotePanelView: UIView { flagView.image = nil flagView.isHidden = true tokenLabel.isHidden = true - showThumbnail(blobID: nil, url: nil) + showThumbnail(blobID: nil, location: nil) isUserInteractionEnabled = false accessibilityLabel = nil } - /// Shows the thumbnail slot for a photo with a blob, loading it once `url` resolves; hides and - /// empties it otherwise. A redacted photo has no blob, so it never reaches the network from here. - private func showThumbnail(blobID: BlobID?, url: URL?) { + /// Shows the thumbnail slot for a photo with a blob, loading it once `location` resolves; hides + /// and empties it otherwise. A redacted photo has no blob, so it never reaches the network from here. + private func showThumbnail(blobID: BlobID?, location: ChatMediaLocation?) { guard let blobID else { thumbnailView.kf.cancelDownloadTask() thumbnailView.image = nil @@ -249,19 +249,21 @@ final class ChatQuotePanelView: UIView { thumbnailView.isHidden = false textTrailingToEdge.isActive = false NSLayoutConstraint.activate(thumbnailConstraints + [textTrailingToThumbnail]) - guard let url else { + guard let location else { thumbnailView.kf.cancelDownloadTask() thumbnailView.image = nil return } let side = Self.thumbnailSide + let processor = DownsamplingImageProcessor(size: CGSize(width: side, height: side)) thumbnailView.kf.setImage( - with: ChatMediaImageSource.resource(blobID: blobID, url: url), - options: [ - .processor(DownsamplingImageProcessor(size: CGSize(width: side, height: side))), + with: ChatMediaImageSource.source(blobID: blobID, location: location), + options: ChatMediaImageSource.options(processor: processor) + [ .scaleFactor(traitCollection.displayScale), ] - ) + ) { result in + ChatMediaImageSource.persist(result, processor: processor) + } } @objc private func handleTap() { diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift index 010afb4e3..2b87c5462 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift @@ -657,7 +657,7 @@ public final class ChatViewController: UICollectionViewController { with: message, maxWidth: maxWidth, authorImageData: authorImageData, - quoteThumbnailURL: quoteThumbnailURL(for: message) + quoteThumbnail: quoteThumbnail(for: message) ) cell.onRetry = { [weak self] id in self?.onRetry?(id) } cell.onOpenURL = { [weak self] url in self?.onOpenURL?(url) } @@ -672,7 +672,7 @@ public final class ChatViewController: UICollectionViewController { with: message, maxWidth: maxWidth, authorImageData: authorImageData, - quoteThumbnailURL: quoteThumbnailURL(for: message) + quoteThumbnail: quoteThumbnail(for: message) ) cell.onRetry = { [weak self] id in self?.onRetry?(id) } cell.onQuoteTap = { [weak self] id in self?.onQuoteTap?(id) } @@ -696,7 +696,7 @@ public final class ChatViewController: UICollectionViewController { cell.onReactionAdd = { [weak self] in self?.onReactionAdd?(message.messageID) } case let cell as ChatMediaCell: guard case .media(let media) = message.content else { return } - let remoteURL = mediaURLResolver?.url(for: media, canReact: message.canReact) { [weak self] _ in + let remote = mediaURLResolver?.location(for: media, canReact: message.canReact) { [weak self] _ in self?.reconfigureVisibleMessage(id: message.id) } let localImage = pendingMediaImage?(message.id) @@ -704,14 +704,14 @@ public final class ChatViewController: UICollectionViewController { with: message, maxWidth: maxWidth, localImage: localImage, - remoteURL: remoteURL, + remote: remote, authorImageData: authorImageData ) cell.onImageTap = { [weak self, weak cell] in guard let self, let request = ChatMediaViewerRequest( message: message, localImage: localImage, - remoteURL: remoteURL, + remote: remote, placeholder: cell?.imageView.image, sourceView: { [weak self] in self?.mediaSourceView(forMessageID: message.id) } ) else { return } @@ -728,9 +728,9 @@ public final class ChatViewController: UICollectionViewController { /// Where a reply's quoted photo draws its thumbnail from, or nil when it has none or is not yet /// resolved; a resolution redraws the reply. - private func quoteThumbnailURL(for message: ChatMessage) -> URL? { + private func quoteThumbnail(for message: ChatMessage) -> ChatMediaLocation? { guard let quote = message.quote else { return nil } - return mediaURLResolver?.thumbnailURL(for: quote.kind, canReact: message.canReact) { [weak self] _ in + return mediaURLResolver?.thumbnailLocation(for: quote.kind, canReact: message.canReact) { [weak self] _ in self?.reconfigureVisibleMessage(id: message.id) } } diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/LinkableBubbleView.swift b/FlipcashUI/Sources/FlipcashUI/Chat/LinkableBubbleView.swift index 4d21ed0ea..705746424 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/LinkableBubbleView.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/LinkableBubbleView.swift @@ -248,8 +248,8 @@ public final class LinkableBubbleView: UIView { cardView.prepareForReuse() } - /// Fills the bubble; `quoteThumbnailURL` is where a quoted photo's thumbnail loads from. - public func configure(with message: ChatMessage, quoteThumbnailURL: URL? = nil) { + /// Fills the bubble; `quoteThumbnail` is where a quoted photo's thumbnail loads from. + public func configure(with message: ChatMessage, quoteThumbnail: ChatMediaLocation? = nil) { // Shares the plain bubble's text builder so a link message gets the same body styling, the // same tombstone copy, and the same "Edited" reservation, with the link spans laid over it // from the preview the mapper already detected. @@ -296,7 +296,7 @@ public final class LinkableBubbleView: UIView { // and UIKit resolves that by breaking one at random. if let quote = message.quote { quotePanel.isHidden = false - quotePanel.configure(with: quote, thumbnailURL: quoteThumbnailURL) + quotePanel.configure(with: quote, thumbnail: quoteThumbnail) NSLayoutConstraint.deactivate(quoteCollapse + [textTopToBubble, cardTopToBubble]) quoteTrailing.isActive = true textTopToQuote.isActive = !bare diff --git a/FlipcashUI/Sources/FlipcashUI/Images/BlurHash.swift b/FlipcashUI/Sources/FlipcashUI/Images/BlurHash.swift index d0371e6df..d8982d591 100644 --- a/FlipcashUI/Sources/FlipcashUI/Images/BlurHash.swift +++ b/FlipcashUI/Sources/FlipcashUI/Images/BlurHash.swift @@ -182,6 +182,107 @@ nonisolated public enum BlurHash { return UIImage(cgImage: cgImage) } + // MARK: - Encode - + + /// Encodes `image` into a hash with `componentsX` × `componentsY` components (each 1…9), or nil + /// when the image cannot be drawn. + /// + /// The image is sampled at a small fixed size first: a hash keeps only the lowest frequencies, so + /// more pixels change nothing but the cost. Squashing it to a square is harmless for the same + /// reason; the decoder stretches back to whatever aspect it is drawn at. + public static func encode(_ image: CGImage, componentsX: Int, componentsY: Int) -> String? { + guard (1...9).contains(componentsX), (1...9).contains(componentsY) else { return nil } + let side = 32 + let bytesPerRow = side * 4 + var pixels = [UInt8](repeating: 0, count: side * bytesPerRow) + let drawn = pixels.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext( + data: buffer.baseAddress, + width: side, + height: side, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) else { return false } + context.interpolationQuality = .medium + context.draw(image, in: CGRect(x: 0, y: 0, width: side, height: side)) + return true + } + guard drawn else { return nil } + return encode(rgb: pixels, width: side, height: side, bytesPerRow: bytesPerRow, componentsX: componentsX, componentsY: componentsY) + } + + /// Encodes 8-bit sRGB pixels laid out as RGBx rows, the reference implementation's algorithm. + static func encode(rgb pixels: [UInt8], width: Int, height: Int, bytesPerRow: Int, componentsX: Int, componentsY: Int) -> String { + var linear = [SIMD3](repeating: .zero, count: width * height) + for y in 0..] = [] + for j in 0...zero + for y in 0.., _ maxValue: Float) -> Int { + func quantise(_ component: Float) -> Int { + let scaled = component / maxValue + let root = scaled < 0 ? -sqrtf(-scaled) : sqrtf(scaled) + return max(0, min(18, Int(floorf(root * 9 + 9.5)))) + } + return quantise(value.x) * 19 * 19 + quantise(value.y) * 19 + quantise(value.z) + } + + private static func encode83(_ value: Int, length: Int) -> String { + var result = "" + for i in 1...length { + var divisor = 1 + for _ in 0..<(length - i) { divisor *= 83 } + result.append(alphabet[(value / divisor) % 83]) + } + return result + } + private static let alphabet = Array( "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~" ) From f26ae2d9ea89303cda9e0874662819c9c652c340 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Sat, 3 Oct 2026 15:00:46 -0400 Subject: [PATCH 2/2] feat(chat): show photo send progress and survive backgrounding Outgoing photo bubbles show a thin progress bar in the corner: the upload's byte fraction, then a sliding segment while the server processes the blob, which has no progress signal. A send no longer fails when the app is backgrounded mid-send. The READY poll retries through a dropped stream, its deadline only counts time the app is awake, and the send holds a background task assertion. A retry after the bytes are stored polls again instead of uploading again. Pending photos are written to disk at send, so a relaunch restores them: one whose bytes were stored resumes, one that never reached the server comes back failed with retry, and one that already sent is dropped. --- .../2026-10-03-pending-photo-persistence.md | 70 +++++ .../Controllers/ConversationController.swift | 187 +++++++++++- .../Conversation/ChatMediaUploader.swift | 169 ++++++++--- .../ChatScreenRepresentable.swift | 3 + .../Screens/Conversation/ComposerChip.swift | 71 ++++- .../Conversation/ComposerChipStrip.swift | 16 +- .../Conversation/PendingMediaStore.swift | 143 +++++++++ .../Session/BackgroundStoreWriteGuard.swift | 37 ++- .../Core/Session/SessionAuthenticator.swift | 11 + .../FlipcashCore/Blob/BlobUploader.swift | 65 ++-- .../FlipcashCore/Blob/BlobUploading.swift | 51 +++- .../Clients/Flip API/FlipClient+Blob.swift | 20 +- .../Models/Conversation/ChatSeal.swift | 2 +- .../FlipcashCoreTests/BlobUploaderTests.swift | 110 ++++++- FlipcashTests/Chat/ChatMediaCellTests.swift | 91 ++++++ .../Chat/ChatMediaUploaderTests.swift | 45 ++- .../Chat/ChatPhotoSendProgressTests.swift | 113 +++++++ .../Chat/ComposerChipStripTests.swift | 6 +- .../Chat/ConversationMediaSendTests.swift | 31 +- .../Chat/PendingMediaPersistenceTests.swift | 281 ++++++++++++++++++ .../TestSupport/MockChatMediaBlobStore.swift | 8 +- .../FlipcashUI/Chat/ChatMediaCell.swift | 16 + .../Chat/ChatPhotoProgressOverlay.swift | 200 +++++++++++++ .../Chat/ChatPhotoSendProgress.swift | 88 ++++++ .../Chat/ChatScreenViewController.swift | 6 + .../FlipcashUI/Chat/ChatViewController.swift | 5 + 26 files changed, 1724 insertions(+), 121 deletions(-) create mode 100644 .claude/plans/2026-10-03-pending-photo-persistence.md create mode 100644 Flipcash/Core/Screens/Conversation/PendingMediaStore.swift create mode 100644 FlipcashTests/Chat/ChatPhotoSendProgressTests.swift create mode 100644 FlipcashTests/Chat/PendingMediaPersistenceTests.swift create mode 100644 FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoProgressOverlay.swift create mode 100644 FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoSendProgress.swift diff --git a/.claude/plans/2026-10-03-pending-photo-persistence.md b/.claude/plans/2026-10-03-pending-photo-persistence.md new file mode 100644 index 000000000..21af559fb --- /dev/null +++ b/.claude/plans/2026-10-03-pending-photo-persistence.md @@ -0,0 +1,70 @@ +# Persist pending chat photo sends + +## Problem + +A pending photo send lives only in memory. That's `ConversationController.pendingMediaChips` (`ComposerChip`, holding a `UIImage`) plus the in-memory `ConversationStore.pendingByConversation` row. If iOS kills the app in the background, the photo and its bubble are both lost. + +On-device evidence, 2026-10-03 15:39–15:45: +- blob `51993f1f…` was reserved, +- the app was backgrounded 3 s later, +- the next log line is a cold launch at 15:45:07, +- the chat showed no photo and no failed bubble. + +## What Android does (text only — Android has no media send) + +- Pending rows are stored in the same Room table as messages, with status `SENDING`/`SENT`/`FAILED`. +- At login, every `SENDING` row is marked `FAILED`, and the user taps retry. +- A retry reuses the same client id, and the server dedupes on `client_message_id`. + +## Design + +### Storage +- Use a JSON manifest plus one JPEG file per photo, following `ChatDraftStore`'s precedent and reasoning. +- `FlipcashStore` (SQLite) is the wrong home because its versioning is destructive, and an outbox can't be re-fetched. +- Location: Application Support, scoped to the owner: + - manifest `flipcash--pending-media.json`, + - photos in `flipcash--pending-media/.jpg`. +- Leave the files at the default protection class, `completeUntilFirstUserAuthentication`. + +### Entry +Each entry holds: +- `clientMessageID` +- `conversationID` +- `createdAt` +- the image file name +- `caption?` and `replyTo?`, exactly as `ChatMediaSendPlan` assigned them to this message +- `stored: UploadedPhoto?` + +`stored` is set the moment `store` returns. `SealedPhoto` is metadata only (blob id, MIME type, size, dimensions, blurhash), so it's safe to persist, and it needs `Codable`. + +### Encode once +- Encode the JPEG before the upload starts and write it to disk then. A kill mid-upload must not lose the photo. +- The upload then sends those same bytes. The retry path never re-encodes a re-decoded image. + +### Lifecycle +- Write the entry when `sendMedia` runs. Attached-but-unsent composer chips are not persisted. +- Update `stored` when the bytes land. +- Delete the entry and its file at the same point `dropPending` runs: after `upsertConversationMessages` succeeds, never before. +- Also delete when the user discards a failed row, and when the failure can't be retried (`.rejected`). + +### Launch +In `ConversationController.start()`, after hydrate: +- Load the manifest. +- Sweep orphans: files without an entry, and entries without a file. +- Re-insert each entry as a pending row, anchored by its `createdAt` rather than `newestMessageID`, and rebuild its chip from the file. +- `stored == nil`: show the row as failed with retry, as Android does. Retry uploads the file. +- `stored != nil`: resume the READY poll automatically, then send. + +### Dedupe +- The risk is a kill after `sendMediaMessage` succeeds but before the entry is deleted. +- Before re-inserting an entry that has `stored`, look for a self-authored media message with that blob id in the loaded messages. If one exists, the photo already sent, so drop the entry. +- An entry with no blob id can't exist on the server, so it's always safe to re-insert. + +## Tests +- The store round-trips its entries and sweeps orphans. +- Reconciliation drops an entry whose blob id is in the loaded messages and keeps one whose blob id isn't. +- At launch, `stored == nil` comes back as failed and `stored != nil` resumes. + +## Known gaps +- An end-to-end encrypted photo sits as a plaintext JPEG in the app's sandbox until it sends. +- `pendingMatch`'s date window: a resumed send's row keeps its old `createdAt`, so the stream echo may not match it. The send RPC's own confirm still removes the row. diff --git a/Flipcash/Core/Controllers/ConversationController.swift b/Flipcash/Core/Controllers/ConversationController.swift index bb06755e2..e3d320973 100644 --- a/Flipcash/Core/Controllers/ConversationController.swift +++ b/Flipcash/Core/Controllers/ConversationController.swift @@ -10,6 +10,7 @@ import os import SwiftUI import FlipcashCore import FlipcashStore +import FlipcashUI nonisolated private let logger = Logger(label: "flipcash.conversation-controller") @@ -315,9 +316,21 @@ final class ConversationController { /// Keeps the words of a send that failed — see ``FailedSendDrafts``. @ObservationIgnored private var failedSends: FailedSendDrafts? + /// The session's pending photo sends, wired by `SessionContainer` like ``chatDrafts``; a controller + /// without one keeps photo sends in memory only. + @ObservationIgnored var pendingMedia: PendingMediaStore? + + /// Builds the uploader a photo send restored from disk finishes through, wired with + /// ``pendingMedia``. + @ObservationIgnored var restoredPhotoUploader: ((ConversationID) -> ChatMediaUploader)? + /// The chip behind each photo send not yet confirmed, keyed by its client id: its local image /// draws the pending bubble, and a failed send uploads or posts again through it. @ObservationIgnored private var pendingMediaChips: [UUID: ComposerChip] = [:] + /// The local image of each photo this session sent, newest last and capped, so its bubble keeps + /// drawing it after confirm instead of falling back to the BlurHash while the server copy loads. + @ObservationIgnored private var sentMediaImages: [(clientMessageID: UUID, image: UIImage)] = [] + private static let sentMediaImageLimit = 20 /// The receive-side analytics concern (cumulative counters + received events), /// owned by its own unit. Exposed so `SessionContainer` can wire its rate lookup. @ObservationIgnored let receipts: ConversationReceiptReporter @@ -333,6 +346,8 @@ final class ConversationController { /// Moves new senders to the front of the mention picker's held suggestions. Set by the session /// after init. @ObservationIgnored var mentionSearch: ServerMentionSearch? + /// Keeps a photo send running for the time iOS allows after the app leaves the screen. + private let backgroundTasks: any BackgroundTaskAsserting init( fetching: any ConversationFetching, @@ -350,9 +365,11 @@ final class ConversationController { typingStoppedLinger: Duration = ConversationTyping.defaultStoppedLinger, typingExpiryClock: TypingExpiryClock = .continuous, reconcileTiming: FeedReconcileTiming = .launch, - receipts: ConversationReceiptReporter? = nil + receipts: ConversationReceiptReporter? = nil, + backgroundTasks: any BackgroundTaskAsserting = UIApplicationBackgroundTasks(name: "chat.photo.send") ) { self.reconcileTiming = reconcileTiming + self.backgroundTasks = backgroundTasks self.fetching = fetching self.membership = membership self.viewerSettings = viewerSettings @@ -456,6 +473,7 @@ final class ConversationController { } startTask = Task { await hydrateFromDatabase(loading: cache, unlessApplied: true) + restorePendingMedia() await openStream() await loadFeed() } @@ -2099,6 +2117,10 @@ final class ConversationController { to conversationID: ConversationID, repliedTo: MessageID? = nil ) async -> Bool { + let assertion = BackgroundTimeAssertion(assertions: backgroundTasks) + assertion.begin() + defer { assertion.end() } + let sends = ChatMediaSendPlan.mediaMessages(chips: chips, caption: caption, replyTo: repliedTo).map { message in (message: message, clientMessageID: insertPendingMedia(message, into: conversationID)) } @@ -2117,15 +2139,50 @@ final class ConversationController { return deliveredAll } - /// The local image drawing the pending photo bubble whose transcript id is `messageID`, or `nil` - /// once the send is confirmed or when the row is not a photo this device is sending. + /// The local image drawing the photo bubble whose transcript id is `messageID` — pending, or sent + /// from this device this session — or `nil` when the row is not one of those. func pendingMediaImage(forMessageID messageID: String) -> UIImage? { - UUID(uuidString: messageID).flatMap { pendingMediaChips[$0]?.image } + guard let id = UUID(uuidString: messageID) else { return nil } + return pendingMediaChips[id]?.image ?? sentMediaImages.last { $0.clientMessageID == id }?.image + } + + private func keepSentImage(_ image: UIImage, clientMessageID: UUID) { + sentMediaImages.append((clientMessageID, image)) + if sentMediaImages.count > Self.sentMediaImageLimit { + sentMediaImages.removeFirst(sentMediaImages.count - Self.sentMediaImageLimit) + } + } + + /// The send progress of the pending photo bubble whose transcript id is `messageID`, or `nil` + /// once the send is confirmed or when the row is not a photo this device is sending. + func pendingMediaProgress(forMessageID messageID: String) -> ChatPhotoSendProgress? { + UUID(uuidString: messageID).flatMap { pendingMediaChips[$0]?.progress } } private func insertPendingMedia(_ message: ChatMediaSendPlan.MediaMessage, into conversationID: ConversationID) -> UUID { let clientMessageID = UUID() let chip = message.chip + let createdAt = Date.now + let pending = pendingMediaRow( + chip: chip, caption: message.caption, repliedTo: message.replyTo, + clientMessageID: clientMessageID, date: createdAt, status: .sending + ) + let anchor = (try? database.newestMessageID(conversationID: conversationID)).flatMap { $0 }?.value ?? 0 + store.insertPending(pending, anchoredTo: anchor, into: conversationID) + receiptSettle.hold(clientMessageID.uuidString) + pendingMediaChips[clientMessageID] = chip + keepOnDisk(chip, clientMessageID: clientMessageID, conversationID: conversationID, createdAt: createdAt, caption: message.caption, repliedTo: message.replyTo) + return clientMessageID + } + + private func pendingMediaRow( + chip: ComposerChip, + caption: String?, + repliedTo: MessageID?, + clientMessageID: UUID, + date: Date, + status: SendStatus + ) -> ConversationMessage { // A chip still preparing has no upload size yet; the source's own pixels carry the same // aspect ratio, which is all the bubble's clamp reads. let pixels = CGSize(width: chip.image.size.width * chip.image.scale, height: chip.image.size.height * chip.image.scale) @@ -2135,23 +2192,111 @@ final class ConversationController { height: chip.preparedHeight ?? Int(pixels.height.rounded()), blurhash: nil ) - let pending = ConversationMessage( + return ConversationMessage( id: .unassigned, senderID: selfUserID, - content: .media([attachment], caption: message.caption), - date: .now, + content: .media([attachment], caption: caption), + date: date, unreadSeq: 0, - repliedTo: message.replyTo, - status: .sending, + repliedTo: repliedTo, + status: status, clientMessageID: clientMessageID ) - let anchor = (try? database.newestMessageID(conversationID: conversationID)).flatMap { $0 }?.value ?? 0 - store.insertPending(pending, anchoredTo: anchor, into: conversationID) - receiptSettle.hold(clientMessageID.uuidString) - pendingMediaChips[clientMessageID] = chip - return clientMessageID } + /// Records the send in ``pendingMedia`` and keeps its JPEG and stored photo current there as the + /// chip produces them. + private func keepOnDisk( + _ chip: ComposerChip, + clientMessageID: UUID, + conversationID: ConversationID, + createdAt: Date, + caption: String?, + repliedTo: MessageID? + ) { + guard let pendingMedia else { return } + pendingMedia.add(.init(clientMessageID: clientMessageID, conversationID: conversationID, createdAt: createdAt, caption: caption, replyTo: repliedTo)) + chip.persist( + onEncoded: { [weak pendingMedia] data in pendingMedia?.writeImage(data, for: clientMessageID) }, + onStored: { [weak pendingMedia] photo in pendingMedia?.setStored(photo, for: clientMessageID) } + ) + } + + /// Puts the photo sends a killed app left behind back in their transcripts: a send whose bytes + /// never landed as a failed row to retry, one whose bytes had landed resumed. + func restorePendingMedia() { + guard let pendingMedia, let uploaderFor = restoredPhotoUploader else { return } + pendingMedia.sweepOrphans() + + var resuming: [(clientMessageID: UUID, chip: ComposerChip, entry: PendingMediaStore.Entry)] = [] + for entry in pendingMedia.entries { + let conversationID = entry.chatID + let recent = (try? database.messagesWindow(conversationID: conversationID, limit: Self.restoreWindow)) ?? [] + guard let jpeg = pendingMedia.imageData(for: entry), let image = UIImage(data: jpeg) else { + pendingMedia.remove(clientMessageID: entry.clientMessageID) + continue + } + if let stored = entry.stored, alreadySent(stored.blobID, in: recent) { + pendingMedia.remove(clientMessageID: entry.clientMessageID) + continue + } + + let chip = ComposerChip(image: image) + chip.restore(encoded: jpeg, stored: entry.stored, uploader: uploaderFor(conversationID)) + let status: SendStatus = entry.stored == nil ? .failed : .sending + let row = pendingMediaRow( + chip: chip, caption: entry.caption, repliedTo: entry.replyTo, + clientMessageID: entry.clientMessageID, date: entry.createdAt, status: status + ) + let anchor = recent.filter { $0.date <= entry.createdAt }.map(\.id.value).max() ?? 0 + store.insertPending(row, anchoredTo: anchor, into: conversationID) + receiptSettle.hold(entry.clientMessageID.uuidString) + pendingMediaChips[entry.clientMessageID] = chip + chip.persist( + onEncoded: { _ in }, + onStored: { [weak pendingMedia] photo in pendingMedia?.setStored(photo, for: entry.clientMessageID) } + ) + if entry.stored != nil { + resuming.append((entry.clientMessageID, chip, entry)) + } + } + + guard !resuming.isEmpty else { return } + Task { [weak self] in + for send in resuming { + guard let self, let uploader = self.restoredPhotoUploader?(send.entry.chatID) else { return } + let assertion = BackgroundTimeAssertion(assertions: self.backgroundTasks) + assertion.begin() + send.chip.startUpload(using: uploader) + _ = await self.deliverMedia( + clientMessageID: send.clientMessageID, + chip: send.chip, + caption: send.entry.caption, + repliedTo: send.entry.replyTo, + to: send.entry.chatID + ) + assertion.end() + } + } + } + + /// Whether `recent` holds a message this user sent that references `blobID`, so its photo already + /// reached the server. + private func alreadySent(_ blobID: BlobID, in recent: [ConversationMessage]) -> Bool { + recent.contains { message in + guard message.senderID == selfUserID else { return false } + switch message.content { + case .media(let attachments, _): + return attachments.contains { $0.blobID == blobID } + case .text, .cash, .deleted, .encrypted, .widget: + return false + } + } + } + + /// How many of a chat's newest messages ``restorePendingMedia()`` searches for an already-sent photo. + private static let restoreWindow = 100 + /// Re-sends a failed photo: posts again when its blob is finalized, uploads again first when the /// upload is what failed, and leaves a photo the server refused for good as it is. private func retryMedia(clientMessageID: UUID, caption: String?, repliedTo: MessageID?, in conversationID: ConversationID) async { @@ -2170,6 +2315,9 @@ final class ConversationController { return } store.markPending(clientMessageID: clientMessageID, status: .sending, in: conversationID) + let assertion = BackgroundTimeAssertion(assertions: backgroundTasks) + assertion.begin() + defer { assertion.end() } _ = await deliverMedia(clientMessageID: clientMessageID, chip: chip, caption: caption, repliedTo: repliedTo, to: conversationID) } @@ -2183,6 +2331,7 @@ final class ConversationController { ) async -> Bool { // Read at await time: a retry replaces the chip's task. guard let upload = chip.uploadTask else { + chip.progress.fail() store.markPending(clientMessageID: clientMessageID, status: .failed, in: conversationID) logger.error("Photo send has no upload to await", metadata: ["conversationID": "\(conversationID)"]) return false @@ -2191,7 +2340,11 @@ final class ConversationController { do { photo = try await upload.value } catch { + chip.progress.fail() store.markPending(clientMessageID: clientMessageID, status: .failed, in: conversationID) + if let error = error as? ChatMediaUploadError, error.isTerminal { + pendingMedia?.remove(clientMessageID: clientMessageID) + } logger.error("Failed to upload conversation photo", metadata: [ "conversationID": "\(conversationID)", "error": "\(error)", @@ -2201,6 +2354,7 @@ final class ConversationController { } let chatType = conversation(withID: conversationID)?.type + chip.progress.beginSending() do { let message = try await messaging.sendMediaMessage( owner: owner, @@ -2210,12 +2364,16 @@ final class ConversationController { repliedTo: repliedTo, clientMessageID: clientMessageID ) + // Before the chip is dropped, so the bubble still holding it fades its overlay out. + chip.progress.finish() var confirmed = message confirmed.clientMessageID = clientMessageID let ok = persist(operation: "send-media-message") { try database.upsertConversationMessages([confirmed], conversationID: conversationID) } if ok { + keepSentImage(chip.image, clientMessageID: clientMessageID) store.dropPending(clientMessageID: clientMessageID, confirmedAt: message.id, in: conversationID) pendingMediaChips[clientMessageID] = nil + pendingMedia?.remove(clientMessageID: clientMessageID) } else { scheduleGapCatchUp(conversationID) } @@ -2225,6 +2383,7 @@ final class ConversationController { Analytics.sentMessage(chatType: chatType) return true } catch { + chip.progress.fail() store.markPending(clientMessageID: clientMessageID, status: .failed, in: conversationID) logger.error("Failed to send conversation photo", metadata: [ "conversationID": "\(conversationID)", diff --git a/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift b/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift index c2c6beb8d..f0770f1db 100644 --- a/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift +++ b/Flipcash/Core/Screens/Conversation/ChatMediaUploader.swift @@ -20,12 +20,13 @@ protocol ChatMediaBlobStoring { /// Returns the upload constraints in force for the owner. func uploadPolicy() async throws -> UploadPolicy - /// Stores `data` and returns its blob, before the server has finalized it. - func storeBlob(_ data: Data, mimeType: String) async throws -> BlobID + /// Stores `data` and returns its blob, before the server has finalized it. `onProgress` hears + /// the bytes going out, from any thread. + func storeBlob(_ data: Data, mimeType: String, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> BlobID /// Stores `image` end-to-end encrypted with `seal` for its DM and returns the blob, before the - /// server has finalized it. - func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload + /// server has finalized it. `onProgress` hears the sealed bytes going out, from any thread. + func storeEncryptedBlob(_ image: Data, seal: ChatSeal, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> EncryptedBlobUpload /// Returns once the blob is servable, throwing `ErrorBlob.rejected` when it is refused and /// `ErrorBlob.timedOut` when it is still processing. @@ -42,12 +43,12 @@ struct SessionChatMediaBlobStore: ChatMediaBlobStoring { try await flipClient.uploadPolicy(owner: session.ownerKeyPair) } - func storeBlob(_ data: Data, mimeType: String) async throws -> BlobID { - try await flipClient.storeBlob(data, mimeType: mimeType, owner: session.ownerKeyPair) + func storeBlob(_ data: Data, mimeType: String, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> BlobID { + try await flipClient.storeBlob(data, mimeType: mimeType, owner: session.ownerKeyPair, onProgress: onProgress) } - func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload { - try await flipClient.storeEncryptedBlob(image, seal: seal, owner: session.ownerKeyPair) + func storeEncryptedBlob(_ image: Data, seal: ChatSeal, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> EncryptedBlobUpload { + try await flipClient.storeEncryptedBlob(image, seal: seal, owner: session.ownerKeyPair, onProgress: onProgress) } func awaitBlobFinalization(blobID: BlobID) async throws { @@ -56,7 +57,7 @@ struct SessionChatMediaBlobStore: ChatMediaBlobStoring { } /// A chat photo uploaded and finalized, in the form its message references it. -enum UploadedPhoto: Hashable, Sendable { +enum UploadedPhoto: Hashable, Sendable, Codable { /// A plaintext blob, whose metadata the server derives. case plain(BlobID) /// A blob end-to-end encrypted for the chat, with the metadata its sealed message carries. @@ -86,6 +87,17 @@ enum ChatMediaUploadError: Error { /// The upload failed for a reason other than the bytes themselves, after any automatic retries. case failed(Error) + /// Whether the send can never succeed, so nothing about it is worth keeping: the server refused + /// the bytes, or the photo can't be uploaded again. + var isTerminal: Bool { + switch self { + case .rejected: + true + case .noMatchingConstraint, .encryptionNotAllowed, .sealUnavailable, .encodingFailed, .failed: + !isRetryable + } + } + /// Whether uploading the same photo again could succeed. var isRetryable: Bool { switch self { @@ -141,7 +153,73 @@ struct ChatMediaUploader { /// Returns the finalized photo for `image`, throwing `ChatMediaUploadError`. /// /// `onPrepared` receives the uploaded pixel width and height once, before any bytes are stored. - func upload(_ image: UIImage, onPrepared: (Int, Int) -> Void) async throws -> UploadedPhoto { + /// `onEncoded` receives the JPEG that will be stored, before it is, so the caller can keep the + /// exact bytes. `onStored` receives the photo once its bytes are stored, before the server has + /// finalized it, so a failed wait can resume through ``finalize(_:progress:)`` instead of + /// storing again. `progress`, when given, follows each store attempt's bytes and then the + /// server's processing. + func upload( + _ image: UIImage, + progress: ChatPhotoSendProgress? = nil, + onStored: (UploadedPhoto) -> Void = { _ in }, + onPrepared: (Int, Int) -> Void, + onEncoded: (Data) -> Void = { _ in } + ) async throws -> UploadedPhoto { + let (chatSeal, bounds, maxSizeBytes) = try await resolveConstraints() + guard let cgImage = image.cgImage, maxSizeBytes > 0 else { + throw ChatMediaUploadError.encodingFailed(.encodingFailed) + } + + let orientation = CGImagePropertyOrientation(image.imageOrientation) + let isSideways: Bool + switch orientation { + case .left, .leftMirrored, .right, .rightMirrored: + isSideways = true + case .up, .upMirrored, .down, .downMirrored: + isSideways = false + } + + let target = ChatMediaDownscale.target( + sourceWidth: isSideways ? cgImage.height : cgImage.width, + sourceHeight: isSideways ? cgImage.width : cgImage.height, + maxWidth: bounds?.maxWidth ?? 0, + maxHeight: bounds?.maxHeight ?? 0, + maxPixels: bounds?.maxPixels ?? 0 + ) + onPrepared(target.width, target.height) + + let data: Data + do { + data = try await Self.encode(cgImage, orientation: orientation, target: target, maxSizeBytes: maxSizeBytes) + } catch let error as ChatMediaEncoder.Error { + throw ChatMediaUploadError.encodingFailed(error) + } + onEncoded(data) + + return try await storeAndFinalize(data, width: target.width, height: target.height, chatSeal: chatSeal, progress: progress, onStored: onStored) + } + + /// Returns the finalized photo for `jpeg`, bytes an earlier ``upload(_:progress:onStored:onPrepared:onEncoded:)`` + /// produced, stored as they are rather than encoded again; throws `ChatMediaUploadError`. + func upload( + jpeg: Data, + progress: ChatPhotoSendProgress? = nil, + onStored: (UploadedPhoto) -> Void = { _ in }, + onPrepared: (Int, Int) -> Void + ) async throws -> UploadedPhoto { + let (chatSeal, _, _) = try await resolveConstraints() + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], + let width = properties[kCGImagePropertyPixelWidth] as? Int, + let height = properties[kCGImagePropertyPixelHeight] as? Int else { + throw ChatMediaUploadError.encodingFailed(.encodingFailed) + } + onPrepared(width, height) + return try await storeAndFinalize(jpeg, width: width, height: height, chatSeal: chatSeal, progress: progress, onStored: onStored) + } + + /// The seal for this photo and the policy's bounds and size ceiling for the form it goes out in. + private func resolveConstraints() async throws -> (ChatSeal?, UploadPolicy.ImageConstraints?, Int) { let chatSeal: ChatSeal? do { chatSeal = try await seal() @@ -159,71 +237,62 @@ struct ChatMediaUploader { throw ChatMediaUploadError.failed(error) } - let bounds: UploadPolicy.ImageConstraints? - let maxSizeBytes: Int if chatSeal != nil { guard let encrypted = policy.encrypted else { logger.warning("Upload policy allows no encrypted chat photo", metadata: ["policyVersion": "\(policy.version)"]) throw ChatMediaUploadError.encryptionNotAllowed } - bounds = encrypted.image // The ceiling covers the sealed blob, so the image must leave room for nonce and tag. - maxSizeBytes = encrypted.maxSizeBytes - EncryptedBlobUpload.overhead + return (chatSeal, encrypted.image, encrypted.maxSizeBytes - EncryptedBlobUpload.overhead) } else { guard let constraint = policy.constraint(for: ChatMediaEncoder.mimeType) else { logger.warning("Upload policy accepts no chat photo", metadata: ["policyVersion": "\(policy.version)"]) throw ChatMediaUploadError.noMatchingConstraint } - bounds = constraint.image - maxSizeBytes = constraint.maxSizeBytes - } - guard let cgImage = image.cgImage, maxSizeBytes > 0 else { - throw ChatMediaUploadError.encodingFailed(.encodingFailed) - } - - let orientation = CGImagePropertyOrientation(image.imageOrientation) - let isSideways: Bool - switch orientation { - case .left, .leftMirrored, .right, .rightMirrored: - isSideways = true - case .up, .upMirrored, .down, .downMirrored: - isSideways = false + return (nil, constraint.image, constraint.maxSizeBytes) } + } - let target = ChatMediaDownscale.target( - sourceWidth: isSideways ? cgImage.height : cgImage.width, - sourceHeight: isSideways ? cgImage.width : cgImage.height, - maxWidth: bounds?.maxWidth ?? 0, - maxHeight: bounds?.maxHeight ?? 0, - maxPixels: bounds?.maxPixels ?? 0 - ) - onPrepared(target.width, target.height) - - let data: Data - do { - data = try await Self.encode(cgImage, orientation: orientation, target: target, maxSizeBytes: maxSizeBytes) - } catch let error as ChatMediaEncoder.Error { - throw ChatMediaUploadError.encodingFailed(error) + private func storeAndFinalize( + _ data: Data, + width: Int, + height: Int, + chatSeal: ChatSeal?, + progress: ChatPhotoSendProgress?, + onStored: (UploadedPhoto) -> Void + ) async throws -> UploadedPhoto { + // Byte counts arrive on URLSession's delegate queue. + let onBytes: @Sendable (BlobUploadProgress) -> Void = { [weak progress] bytes in + Task { @MainActor in progress?.didUpload(bytes) } } let uploaded: UploadedPhoto if let chatSeal { let blurhash = await Self.blurhash(of: data) - let stored = try await store { try await blob.storeEncryptedBlob(data, seal: chatSeal) } + let stored = try await store(progress: progress) { try await blob.storeEncryptedBlob(data, seal: chatSeal, onProgress: onBytes) } uploaded = .sealed(SealedPhoto( blobID: stored.blobID, mimeType: ChatMediaEncoder.mimeType, sizeBytes: stored.plaintextSize, - width: target.width, - height: target.height, + width: width, + height: height, blurhash: blurhash )) } else { - uploaded = .plain(try await store { try await blob.storeBlob(data, mimeType: ChatMediaEncoder.mimeType) }) + uploaded = .plain(try await store(progress: progress) { try await blob.storeBlob(data, mimeType: ChatMediaEncoder.mimeType, onProgress: onBytes) }) } + onStored(uploaded) + return try await finalize(uploaded, progress: progress) + } + + /// Returns `photo` once the server has finalized its stored bytes, throwing + /// `ChatMediaUploadError`. + func finalize(_ photo: UploadedPhoto, progress: ChatPhotoSendProgress? = nil) async throws -> UploadedPhoto { + progress?.beginProcessing() + do { - try await blob.awaitBlobFinalization(blobID: uploaded.blobID) + try await blob.awaitBlobFinalization(blobID: photo.blobID) } catch ErrorBlob.rejected(let reason) { throw ChatMediaUploadError.rejected(reason) } catch { @@ -231,13 +300,15 @@ struct ChatMediaUploader { throw ChatMediaUploadError.failed(error) } - return uploaded + return photo } /// Stores `data`, retrying through `backoff` while the failure is in transit. - private func store(_ attemptStore: () async throws -> Stored) async throws -> Stored { + private func store(progress: ChatPhotoSendProgress?, _ attemptStore: () async throws -> Stored) async throws -> Stored { var attempt = 0 while true { + // A retried store sends every byte again. + progress?.beginAttempt() do { return try await attemptStore() } catch { diff --git a/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift b/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift index 374aa3a51..30ee0d332 100644 --- a/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift +++ b/Flipcash/Core/Screens/Conversation/ChatScreenRepresentable.swift @@ -168,6 +168,9 @@ struct ChatScreenRepresentable: UIViewControllerRepresentable { screen.pendingMediaImage = { [conversationController] id in conversationController.pendingMediaImage(forMessageID: id) } + screen.pendingMediaProgress = { [conversationController] id in + conversationController.pendingMediaProgress(forMessageID: id) + } screen.onContactAction = onContactAction screen.onProfileTap = onProfileTap screen.onGroupInvite = onGroupInvite diff --git a/Flipcash/Core/Screens/Conversation/ComposerChip.swift b/Flipcash/Core/Screens/Conversation/ComposerChip.swift index be698189f..16cc726a3 100644 --- a/Flipcash/Core/Screens/Conversation/ComposerChip.swift +++ b/Flipcash/Core/Screens/Conversation/ComposerChip.swift @@ -8,6 +8,7 @@ import UIKit import Observation import FlipcashCore +import FlipcashUI /// Whether a failed chip can be uploaded again, or the server refused its bytes for good. enum ChatMediaChipFailure: Equatable { @@ -37,6 +38,9 @@ final class ComposerChip: Identifiable { let preview: UIImage? var state: State = .preparing + /// How far this photo's send has got, which its transcript bubble draws. + let progress = ChatPhotoSendProgress() + /// Pixel width of the image as it will be uploaded, known before the upload starts. var preparedWidth: Int? @@ -51,32 +55,93 @@ final class ComposerChip: Identifiable { /// composer has let go of the chip. @ObservationIgnored private var uploader: ChatMediaUploader? + /// The photo whose bytes storage already holds, so a retry waits on the server again instead of + /// storing a second copy. Nil until a store succeeds, and again once the server rejects it. + @ObservationIgnored private var stored: UploadedPhoto? + + /// The JPEG the upload encoded, kept so a retry stores the same bytes instead of encoding again. + @ObservationIgnored private var encoded: Data? + + /// Hears the encoded JPEG as soon as it exists; see ``persist(onEncoded:onStored:)``. + @ObservationIgnored private var onEncoded: ((Data) -> Void)? + + /// Hears the stored photo as soon as its bytes land; see ``persist(onEncoded:onStored:)``. + @ObservationIgnored private var onStored: ((UploadedPhoto) -> Void)? + init(image: UIImage, preview: UIImage? = nil) { self.image = image self.preview = preview } + /// Hands the encoded JPEG and the stored photo to the callbacks, now if the chip already has + /// them and otherwise when they arrive, so a send can keep them on disk. + func persist(onEncoded: @escaping (Data) -> Void, onStored: @escaping (UploadedPhoto) -> Void) { + self.onEncoded = onEncoded + self.onStored = onStored + if let encoded { onEncoded(encoded) } + if let stored { onStored(stored) } + } + + /// Rebuilds a chip for a send read back from disk: `encoded` is the JPEG that was held and + /// `stored` the photo whose bytes had landed, if they had. With nothing stored the chip starts + /// failed and retryable; otherwise ``startUpload(using:)`` resumes by waiting on the server. + func restore(encoded: Data, stored: UploadedPhoto?, uploader: ChatMediaUploader) { + self.encoded = encoded + self.stored = stored + self.uploader = uploader + if stored == nil { + state = .failed(.retryable) + progress.fail() + } + } + /// Uploads the image through `uploader`, replacing any earlier attempt; calling it again after /// a retryable failure is how the chip is retried. func startUpload(using uploader: ChatMediaUploader) { self.uploader = uploader uploadTask?.cancel() state = .preparing + progress.beginAttempt() uploadTask = Task { do { - let photo = try await uploader.upload(image) { width, height in - preparedWidth = width - preparedHeight = height + let photo: UploadedPhoto + if let stored { state = .uploading + photo = try await uploader.finalize(stored, progress: progress) + } else if let encoded { + photo = try await uploader.upload(jpeg: encoded, progress: progress, onStored: { photo in + stored = photo + onStored?(photo) + }, onPrepared: { width, height in + preparedWidth = width + preparedHeight = height + state = .uploading + }) + } else { + photo = try await uploader.upload(image, progress: progress, onStored: { photo in + stored = photo + onStored?(photo) + }, onPrepared: { width, height in + preparedWidth = width + preparedHeight = height + state = .uploading + }, onEncoded: { data in + encoded = data + onEncoded?(data) + }) } if !Task.isCancelled { state = .uploaded(photo.blobID) } return photo } catch let error as ChatMediaUploadError { + if case .rejected = error { + stored = nil + } if !Task.isCancelled { state = .failed(error.isRetryable ? .retryable : .notRetryable) + progress.fail() } throw error } diff --git a/Flipcash/Core/Screens/Conversation/ComposerChipStrip.swift b/Flipcash/Core/Screens/Conversation/ComposerChipStrip.swift index af495806c..470bacdeb 100644 --- a/Flipcash/Core/Screens/Conversation/ComposerChipStrip.swift +++ b/Flipcash/Core/Screens/Conversation/ComposerChipStrip.swift @@ -10,20 +10,19 @@ import FlipcashUI /// What a staged chip draws over its thumbnail. enum ComposerChipBadge: Equatable { - /// The photo is still being prepared or uploaded. - case progress /// The upload failed and can be tried again. case retry /// The server refused the photo; the only way forward is to remove it. case error - /// The photo is uploaded, so the thumbnail stands alone. + /// The thumbnail stands alone: uploaded, or uploading quietly until the send, whose bubble + /// shows the progress. case none /// Returns the badge for a chip in `state`. init(_ state: ComposerChip.State) { switch state { - case .preparing, .uploading: self = .progress - case .uploaded: self = .none + case .preparing, .uploading, .uploaded: + self = .none case .failed(.retryable): self = .retry case .failed(.notRetryable): self = .error } @@ -159,13 +158,6 @@ private struct ComposerChipView: View { @ViewBuilder private var badge: some View { switch ComposerChipBadge(chip.state) { - case .progress: - ZStack { - Color.black.opacity(0.35) - ProgressView() - .tint(.white) - } - .accessibilityLabel("Uploading photo") case .retry: Button(action: onRetry) { ZStack { diff --git a/Flipcash/Core/Screens/Conversation/PendingMediaStore.swift b/Flipcash/Core/Screens/Conversation/PendingMediaStore.swift new file mode 100644 index 000000000..d7fe787e8 --- /dev/null +++ b/Flipcash/Core/Screens/Conversation/PendingMediaStore.swift @@ -0,0 +1,143 @@ +// +// PendingMediaStore.swift +// Flipcash +// +// Copyright © 2026 Code Inc. All rights reserved. +// + +import Foundation +import FlipcashCore + +private let logger = Logger(label: "flipcash.pending-media-store") + +/// The photo sends that have left the composer but not yet been confirmed, kept on disk so a send +/// survives iOS killing the app while it is backgrounded. +/// +/// A JSON manifest plus one JPEG per photo, owner-scoped in Application Support and outside +/// `FlipcashStore` for the reasons `ChatDraftStore` gives: that store is rebuilt from the server on +/// a version bump, and an unsent photo cannot be re-fetched. +@MainActor +final class PendingMediaStore { + + /// One photo send, as `ChatMediaSendPlan` assigned it. + struct Entry: Codable, Equatable, Sendable { + var clientMessageID: UUID + /// The raw bytes of the chat's `ConversationID`. + var conversationID: Data + var createdAt: Date + var fileName: String + var caption: String? + var replyTo: MessageID? + /// The photo once its bytes are stored; nil while the encoded JPEG is the only thing held. + var stored: UploadedPhoto? + + init(clientMessageID: UUID, conversationID: ConversationID, createdAt: Date, caption: String?, replyTo: MessageID?, stored: UploadedPhoto? = nil) { + self.clientMessageID = clientMessageID + self.conversationID = conversationID.data + self.createdAt = createdAt + self.fileName = "\(clientMessageID.uuidString).jpg" + self.caption = caption + self.replyTo = replyTo + self.stored = stored + } + + /// The chat the photo is being sent to. + var chatID: ConversationID { ConversationID(data: conversationID) } + } + + private struct File: Codable { + var version: Int + var entries: [Entry] + } + + private static let version = 1 + + private let manifestURL: URL + private let photosDirectory: URL + private var held: [Entry] + + /// Loads the owner's pending sends. A manifest that cannot be read is treated as empty. + init(directory: URL, owner: PublicKey) { + let prefix = "flipcash-\(owner.base58)-pending-media" + self.manifestURL = directory.appendingPathComponent("\(prefix).json") + self.photosDirectory = directory.appendingPathComponent(prefix, isDirectory: true) + guard + let data = try? Data(contentsOf: manifestURL), + let file = try? JSONDecoder().decode(File.self, from: data), + file.version == Self.version + else { + self.held = [] + return + } + self.held = file.entries + } + + /// Every send still held, oldest first. + var entries: [Entry] { held } + + /// Starts holding `entry`, replacing one with the same client id. + func add(_ entry: Entry) { + held.removeAll { $0.clientMessageID == entry.clientMessageID } + held.append(entry) + writeManifest() + } + + /// Writes the encoded JPEG for a held send, so the photo survives a kill from here on. + func writeImage(_ data: Data, for clientMessageID: UUID) { + guard let entry = held.first(where: { $0.clientMessageID == clientMessageID }) else { return } + do { + try FileManager.default.createDirectory(at: photosDirectory, withIntermediateDirectories: true) + try data.write(to: photosDirectory.appendingPathComponent(entry.fileName), options: .atomic) + } catch { + ErrorReporting.captureError(error, reason: "Failed to write pending chat photo") + } + } + + /// The JPEG held for `entry`, or nil when its file is gone. + func imageData(for entry: Entry) -> Data? { + try? Data(contentsOf: photosDirectory.appendingPathComponent(entry.fileName)) + } + + /// Records that the send's bytes are stored, so a relaunch resumes instead of uploading again. + func setStored(_ photo: UploadedPhoto, for clientMessageID: UUID) { + guard let index = held.firstIndex(where: { $0.clientMessageID == clientMessageID }) else { return } + held[index].stored = photo + writeManifest() + } + + /// Forgets a send and deletes its photo — confirmed, discarded, or refused for good. + func remove(clientMessageID: UUID) { + guard let index = held.firstIndex(where: { $0.clientMessageID == clientMessageID }) else { return } + let entry = held.remove(at: index) + try? FileManager.default.removeItem(at: photosDirectory.appendingPathComponent(entry.fileName)) + writeManifest() + } + + /// Drops entries whose photo file is missing and deletes photo files no entry refers to. + func sweepOrphans() { + let before = held.count + held.removeAll { imageData(for: $0) == nil } + if held.count != before { writeManifest() } + + let known = Set(held.map(\.fileName)) + let files = (try? FileManager.default.contentsOfDirectory(atPath: photosDirectory.path)) ?? [] + for name in files where !known.contains(name) { + try? FileManager.default.removeItem(at: photosDirectory.appendingPathComponent(name)) + } + if before != held.count || files.count != known.count { + logger.info("Swept orphaned pending photos", metadata: [ + "droppedEntries": "\(before - held.count)", + "files": "\(files.count)", + ]) + } + } + + private func writeManifest() { + do { + let data = try JSONEncoder().encode(File(version: Self.version, entries: held)) + try data.write(to: manifestURL, options: .atomic) + } catch { + ErrorReporting.captureError(error, reason: "Failed to write pending chat photo manifest") + } + } +} diff --git a/Flipcash/Core/Session/BackgroundStoreWriteGuard.swift b/Flipcash/Core/Session/BackgroundStoreWriteGuard.swift index 2a53f1b13..00173bc6a 100644 --- a/Flipcash/Core/Session/BackgroundStoreWriteGuard.swift +++ b/Flipcash/Core/Session/BackgroundStoreWriteGuard.swift @@ -117,10 +117,13 @@ nonisolated protocol BackgroundTaskAsserting: Sendable { /// ``BackgroundTaskAsserting`` over `UIApplication.shared`, hopping to the main thread it requires. nonisolated struct UIApplicationBackgroundTasks: BackgroundTaskAsserting { + /// The name iOS reports the task under in its diagnostics. + var name = "database.write" + func begin(expiration: @escaping @Sendable () -> Void) -> UIBackgroundTaskIdentifier { let begin = { MainActor.assumeIsolated { - UIApplication.shared.beginBackgroundTask(withName: "database.write", expirationHandler: expiration) + UIApplication.shared.beginBackgroundTask(withName: name, expirationHandler: expiration) } } return Thread.isMainThread ? begin() : DispatchQueue.main.sync(execute: begin) @@ -134,3 +137,35 @@ nonisolated struct UIApplicationBackgroundTasks: BackgroundTaskAsserting { } } } + +/// One background-task assertion, held from ``begin()`` until ``end()`` or until iOS expires it. +nonisolated final class BackgroundTimeAssertion: @unchecked Sendable { + + private let assertions: any BackgroundTaskAsserting + + /// Guarded by `lock`. + private let lock = NSLock() + private var taskID: UIBackgroundTaskIdentifier = .invalid + + init(assertions: any BackgroundTaskAsserting) { + self.assertions = assertions + } + + /// Asks iOS to keep the app running after it leaves the screen; a no-op when none is granted. + func begin() { + // Expiry only ends the assertion: the work under it carries on until iOS suspends the app. + let acquired = assertions.begin { [weak self] in self?.end() } + lock.withLock { taskID = acquired } + } + + /// Releases the assertion; safe to call more than once. + func end() { + let held = lock.withLock { + defer { taskID = .invalid } + return taskID + } + if held != .invalid { + assertions.end(held) + } + } +} diff --git a/Flipcash/Core/Session/SessionAuthenticator.swift b/Flipcash/Core/Session/SessionAuthenticator.swift index b637b4ea2..1d39d6a57 100644 --- a/Flipcash/Core/Session/SessionAuthenticator.swift +++ b/Flipcash/Core/Session/SessionAuthenticator.swift @@ -701,6 +701,17 @@ final class SessionContainer { // puts its text back — none of which the controller can do before it has the store. conversationController.chatDrafts = chatDrafts conversationController.chatArchive = chatArchive + // Same owner-scoped placement as the drafts; a photo whose send a kill interrupted comes back + // through `start()`, which needs the uploader to finish it. + conversationController.pendingMedia = PendingMediaStore( + directory: .applicationSupportDirectory, + owner: owner.publicKey + ) + conversationController.restoredPhotoUploader = { [weak conversationController, session, flipClient] conversationID in + var uploader = ChatMediaUploader(blob: SessionChatMediaBlobStore(session: session, flipClient: flipClient)) + uploader.seal = { try await conversationController?.photoSeal(for: conversationID) } + return uploader + } let recentReactions = RecentReactionsStore(owner: owner.publicKey) self.recentReactions = recentReactions conversationController.reactions.recents = recentReactions diff --git a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift index 999d1c4c0..9ad6431a7 100644 --- a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift +++ b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploader.swift @@ -41,7 +41,14 @@ final class BlobUploader: Sendable { /// /// Separate from `awaitFinalization` so a caller whose poll times out can /// resume the same blob instead of uploading a second copy. - func store(_ data: Data, mimeType: String, owner: KeyPair) async throws -> BlobID { + /// + /// `onProgress` hears the bytes going out to storage, from any thread. + func store( + _ data: Data, + mimeType: String, + owner: KeyPair, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void = { _ in } + ) async throws -> BlobID { // Sanitize before reserving: the reservation signs the byte count, and // storage refuses an image carrying personal metadata. Doing it here // rather than at the call site is what makes it true of every upload. @@ -60,7 +67,7 @@ final class BlobUploader: Sendable { "sizeBytes": "\(data.count)", ]) - try await store(data, mimeType: mimeType, to: reserved.target) + try await store(data, mimeType: mimeType, to: reserved.target, onProgress: onProgress) switch try await reserving.completeExternalUpload(blobID: reserved.blobID, owner: owner) { case .rejected(let reason): @@ -78,11 +85,13 @@ final class BlobUploader: Sendable { /// finalization to the caller. /// /// `encrypt` seals the plaintext under the blob id the reservation assigns, which is part of - /// its aad, so the reservation declares the sealed size before the blob exists. + /// its aad, so the reservation declares the sealed size before the blob exists. `onProgress` + /// hears the sealed bytes going out to storage, from any thread. func storeEncrypted( _ image: Data, for conversationID: ConversationID, owner: KeyPair, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void = { _ in }, encrypt: @Sendable (Data, BlobID) throws -> Data ) async throws -> EncryptedBlobUpload { // The server never reads these bytes, so stripping location and camera metadata is on us. @@ -111,7 +120,7 @@ final class BlobUploader: Sendable { throw ErrorBlob.unknown } - try await store(blob, mimeType: Self.encryptedMimeType, to: reserved.target) + try await store(blob, mimeType: Self.encryptedMimeType, to: reserved.target, onProgress: onProgress) switch try await reserving.completeExternalUpload(blobID: reserved.blobID, owner: owner) { case .rejected(let reason): @@ -130,27 +139,39 @@ final class BlobUploader: Sendable { /// Polls until the blob is finalized. /// - /// Returns immediately when it is already ready; a rejection is terminal. + /// Returns immediately when it is already ready; a rejection is terminal. A poll lost in transit + /// is retried, since iOS drops the connection when it suspends the app mid-wait. The `timeout` + /// budget is counted in polls rather than wall time, so time spent suspended doesn't use it up. func awaitFinalization(blobID: BlobID, owner: KeyPair) async throws { - let deadline = ContinuousClock.now.advanced(by: timeout) + let maxPolls = max(1, Int((timeout / pollInterval).rounded(.up))) + var polls = 0 while true { try Task.checkCancellation() - - switch try await reserving.blobState(blobID: blobID, owner: owner) { - case .ready: - return - case .rejected(let reason): - logger.info("Blob rejected", metadata: [ + polls += 1 + + do { + switch try await reserving.blobState(blobID: blobID, owner: owner) { + case .ready: + return + case .rejected(let reason): + logger.info("Blob rejected", metadata: [ + "blobId": "\(blobID)", + "reason": "\(reason)", + ]) + throw ErrorBlob.rejected(reason) + case .pending, .processing: + break + } + } catch ErrorBlob.network(let error) { + try Task.checkCancellation() + logger.info("Blob poll lost in transit", metadata: [ "blobId": "\(blobID)", - "reason": "\(reason)", + "error": "\(error)", ]) - throw ErrorBlob.rejected(reason) - case .pending, .processing: - break } - guard ContinuousClock.now < deadline else { + guard polls < maxPolls else { logger.info("Blob finalization timed out", metadata: ["blobId": "\(blobID)"]) throw ErrorBlob.timedOut } @@ -161,7 +182,12 @@ final class BlobUploader: Sendable { // MARK: - Upload - - private func store(_ data: Data, mimeType: String, to target: UploadTarget) async throws { + private func store( + _ data: Data, + mimeType: String, + to target: UploadTarget, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void + ) async throws { let boundary = "Boundary-\(UUID().uuidString)" let status: Int @@ -180,7 +206,8 @@ final class BlobUploader: Sendable { file: data, mimeType: mimeType, boundary: boundary - ) + ), + onProgress: onProgress ) } catch is CancellationError { throw CancellationError() diff --git a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploading.swift b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploading.swift index 1e5904562..c2e901673 100644 --- a/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploading.swift +++ b/FlipcashCore/Sources/FlipcashCore/Blob/BlobUploading.swift @@ -14,10 +14,30 @@ protocol BlobUploading: Sendable { url: URL, contentType: String, headers: [String: String], - body: Data + body: Data, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void ) async throws -> (status: Int, body: Data) } +/// How much of an upload's request body has gone out. +public struct BlobUploadProgress: Sendable, Equatable { + /// Bytes of the request body sent so far. + public let sentBytes: Int64 + /// The request body's full length. + public let totalBytes: Int64 + + public init(sentBytes: Int64, totalBytes: Int64) { + self.sentBytes = sentBytes + self.totalBytes = totalBytes + } + + /// The share of the body sent, from 0 to 1, or nil when the length is unknown. + public var fraction: Double? { + guard totalBytes > 0 else { return nil } + return min(max(Double(sentBytes) / Double(totalBytes), 0), 1) + } +} + /// The production `BlobUploading`, over a shared `URLSession`. struct URLSessionBlobUploader: BlobUploading { @@ -31,7 +51,8 @@ struct URLSessionBlobUploader: BlobUploading { url: URL, contentType: String, headers: [String: String], - body: Data + body: Data, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void ) async throws -> (status: Int, body: Data) { var request = URLRequest(url: url) @@ -42,9 +63,33 @@ struct URLSessionBlobUploader: BlobUploading { request.setValue(value, forHTTPHeaderField: field) } - let (data, response) = try await session.upload(for: request, from: body) + // The body's own length, since `totalBytesExpectedToSend` can be unknown (-1). + let delegate = UploadProgressDelegate(totalBytes: Int64(body.count), onProgress: onProgress) + let (data, response) = try await session.upload(for: request, from: body, delegate: delegate) let status = (response as? HTTPURLResponse)?.statusCode ?? 0 return (status, data) } } + +/// Forwards one upload task's sent-byte counts. +private final class UploadProgressDelegate: NSObject, URLSessionTaskDelegate, Sendable { + + private let totalBytes: Int64 + private let onProgress: @Sendable (BlobUploadProgress) -> Void + + init(totalBytes: Int64, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) { + self.totalBytes = totalBytes + self.onProgress = onProgress + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + didSendBodyData bytesSent: Int64, + totalBytesSent: Int64, + totalBytesExpectedToSend: Int64 + ) { + onProgress(BlobUploadProgress(sentBytes: totalBytesSent, totalBytes: totalBytes)) + } +} diff --git a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift index 2e18231b1..8ccb1c3df 100644 --- a/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift +++ b/FlipcashCore/Sources/FlipcashCore/Clients/Flip API/FlipClient+Blob.swift @@ -11,15 +11,25 @@ extension FlipClient { /// /// Pair with `awaitBlobFinalization(blobID:owner:)`: holding the blob from /// the moment the bytes land is what lets a timed-out wait resume rather - /// than upload a second copy. - public func storeBlob(_ data: Data, mimeType: String, owner: KeyPair) async throws -> BlobID { - try await blobUploader.store(data, mimeType: mimeType, owner: owner) + /// than upload a second copy. `onProgress` hears the bytes going out, from any thread. + public func storeBlob( + _ data: Data, + mimeType: String, + owner: KeyPair, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void = { _ in } + ) async throws -> BlobID { + try await blobUploader.store(data, mimeType: mimeType, owner: owner, onProgress: onProgress) } /// Stores `image` end-to-end encrypted with `seal` for its DM and returns the blob, before the /// server has finalized it. Pair with `awaitBlobFinalization(blobID:owner:)` as for `storeBlob`. - public func storeEncryptedBlob(_ image: Data, seal: ChatSeal, owner: KeyPair) async throws -> EncryptedBlobUpload { - try await blobUploader.storeEncrypted(image, for: seal.conversationID, owner: owner) { plaintext, blobID in + public func storeEncryptedBlob( + _ image: Data, + seal: ChatSeal, + owner: KeyPair, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void = { _ in } + ) async throws -> EncryptedBlobUpload { + try await blobUploader.storeEncrypted(image, for: seal.conversationID, owner: owner, onProgress: onProgress) { plaintext, blobID in try seal.encryptBlob(plaintext, blobID: blobID) } } diff --git a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift index f28798686..6a164a8a4 100644 --- a/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift +++ b/FlipcashCore/Sources/FlipcashCore/Models/Conversation/ChatSeal.swift @@ -166,7 +166,7 @@ public enum BlobOpenFailure: Error, Hashable, Sendable { } /// A photo uploaded encrypted for a chat, with the plaintext metadata its sealed message carries. -public struct SealedPhoto: Hashable, Sendable { +public struct SealedPhoto: Hashable, Sendable, Codable { /// The encrypted blob, READY before the message is sent. public let blobID: BlobID /// The plaintext image's MIME type, always an `image/` type. diff --git a/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift b/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift index 8c52e4507..a69223731 100644 --- a/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift +++ b/FlipcashCore/Tests/FlipcashCoreTests/BlobUploaderTests.swift @@ -4,6 +4,7 @@ // import Foundation +import Synchronization import Testing @testable import FlipcashCore @@ -62,6 +63,55 @@ struct BlobUploaderTests { #expect(await transport.body == nil) } + // MARK: - Progress - + + @Test("Forwards the transport's sent-byte counts for a plaintext upload") + func store_forwardsProgress() async throws { + let uploader = makeUploader(transport: RecordingTransport(), states: [.ready]) + let seen = Mutex<[BlobUploadProgress]>([]) + + _ = try await uploader.store(Self.fileBytes, mimeType: "image/jpeg", owner: try Self.owner()) { progress in + seen.withLock { $0.append(progress) } + } + + let fractions = seen.withLock { $0 }.compactMap(\.fraction) + #expect(fractions.count == 2) + #expect(fractions.first.map { $0 > 0 && $0 < 1 } == true) + #expect(fractions.last == 1.0) + } + + @Test("Forwards the sealed bytes' progress for an encrypted upload") + func storeEncrypted_forwardsProgress() async throws { + let uploader = makeUploader(transport: RecordingTransport(), states: [.ready]) + let seen = Mutex<[BlobUploadProgress]>([]) + + _ = try await uploader.storeEncrypted( + Self.fileBytes, + for: ConversationID(data: Data(repeating: 7, count: 32)), + owner: try Self.owner(), + onProgress: { progress in seen.withLock { $0.append(progress) } } + ) { plaintext, _ in + Data(count: plaintext.count + EncryptedBlobUpload.overhead) + } + + #expect(seen.withLock { $0 }.last?.fraction == 1.0) + } + + @Test("Maps sent bytes to a fraction of the body, clamped to 0...1") + func progress_fraction() { + #expect(BlobUploadProgress(sentBytes: 0, totalBytes: 200).fraction == 0) + #expect(BlobUploadProgress(sentBytes: 50, totalBytes: 200).fraction == 0.25) + #expect(BlobUploadProgress(sentBytes: 200, totalBytes: 200).fraction == 1) + #expect(BlobUploadProgress(sentBytes: 300, totalBytes: 200).fraction == 1) + #expect(BlobUploadProgress(sentBytes: -1, totalBytes: 200).fraction == 0) + } + + @Test("Reports no fraction when the body's length is unknown") + func progress_unknownLength() { + #expect(BlobUploadProgress(sentBytes: 10, totalBytes: 0).fraction == nil) + #expect(BlobUploadProgress(sentBytes: 10, totalBytes: -1).fraction == nil) + } + @Test("The reservation names the chat only for an encrypted upload") func reservationRequestNamesChat() throws { let owner = try Self.owner() @@ -325,6 +375,44 @@ struct BlobUploaderTests { #expect(await reserving.reserveCount == 0) } + /// iOS drops the connection when it suspends the app mid-wait; one lost poll is not a failed + /// upload. + @Test("A poll lost in transit is retried rather than failing the upload") + func lostPollIsRetried() async throws { + let transport = RecordingTransport() + let reserving = StubReserving(states: [.processing, .ready], lostPolls: 2) + let uploader = BlobUploader( + reserving: reserving, + transport: transport, + pollInterval: .milliseconds(1), + timeout: .seconds(5) + ) + + try await uploader.awaitFinalization(blobID: StubReserving.blobID, owner: try Self.owner()) + + #expect(await reserving.pollCount == 4) + #expect(await transport.body == nil) + } + + @Test("Polls lost in transit still count toward the deadline") + func lostPollsHonourTheDeadline() async throws { + let reserving = StubReserving(states: [], lostPolls: .max) + let uploader = BlobUploader( + reserving: reserving, + transport: RecordingTransport(), + pollInterval: .milliseconds(1), + timeout: .milliseconds(5) + ) + + await #expect { + try await uploader.awaitFinalization(blobID: StubReserving.blobID, owner: try Self.owner()) + } throws: { error in + guard case ErrorBlob.timedOut = error else { return false } + return true + } + #expect(await reserving.pollCount == 5) + } + // MARK: - Fixtures - private static let fileBytes = Data(repeating: 0xAB, count: 4096) @@ -453,7 +541,18 @@ private actor RecordingTransport: BlobUploading { self.failure = error } - func post(url: URL, contentType: String, headers: [String: String], body: Data) async throws -> (status: Int, body: Data) { + func post( + url: URL, + contentType: String, + headers: [String: String], + body: Data, + onProgress: @escaping @Sendable (BlobUploadProgress) -> Void + ) async throws -> (status: Int, body: Data) { + // Reports the body going out in two halves, as URLSession does in chunks. + let total = Int64(body.count) + onProgress(BlobUploadProgress(sentBytes: total / 2, totalBytes: total)) + onProgress(BlobUploadProgress(sentBytes: total, totalBytes: total)) + self.url = url self.contentType = contentType self.headers = headers @@ -473,15 +572,18 @@ private actor StubReserving: BlobReserving { private var states: [BlobState] private let completion: BlobState + private var lostPolls: Int private(set) var pollCount = 0 private(set) var reserveCount = 0 private(set) var declaredSizeBytes: Int? private(set) var declaredMimeType: String? private(set) var encryptedFor: ConversationID? - init(states: [BlobState], completion: BlobState = .processing) { + /// `lostPolls` is how many polls fail in transit before the stub starts answering. + init(states: [BlobState], completion: BlobState = .processing, lostPolls: Int = 0) { self.states = states self.completion = completion + self.lostPolls = lostPolls } func initiateExternalUpload(mimeType: String, sizeBytes: Int, encryptedFor: ConversationID?, owner: KeyPair) async throws -> ReservedUpload { @@ -512,6 +614,10 @@ private actor StubReserving: BlobReserving { func blobState(blobID: BlobID, owner: KeyPair) async throws -> BlobState { pollCount += 1 + if lostPolls > 0 { + lostPolls -= 1 + throw ErrorBlob.network(URLError(.networkConnectionLost)) + } return states.isEmpty ? .processing : states.removeFirst() } } diff --git a/FlipcashTests/Chat/ChatMediaCellTests.swift b/FlipcashTests/Chat/ChatMediaCellTests.swift index c3b6e1fb4..0cd036db9 100644 --- a/FlipcashTests/Chat/ChatMediaCellTests.swift +++ b/FlipcashTests/Chat/ChatMediaCellTests.swift @@ -46,6 +46,97 @@ struct ChatMediaCellTests { return cell } + private func outgoing(id: String = "1", receipt: ChatReceipt? = nil) -> ChatMessage { + ChatMessage(id: id, content: .media(media()), sender: .me, receipt: receipt) + } + + // MARK: - Send progress - + + @Test("A row with no send progress draws no overlay") + func noProgressNoOverlay() { + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: nil, remote: nil) + #expect(!cell.progressOverlay.isShowing) + #expect(cell.progressOverlay.isHidden) + } + + @Test("An uploading photo shows its share of bytes sent; processing shows a full, indeterminate bar") + func overlayFollowsProgress() async { + let progress = ChatPhotoSendProgress() + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + #expect(cell.progressOverlay.isShowing) + #expect(cell.progressOverlay.fraction == 0) + #expect(!cell.progressOverlay.isIndeterminate) + + progress.didUpload(BlobUploadProgress(sentBytes: 40, totalBytes: 100)) + await waitFor { cell.progressOverlay.fraction == 0.4 } + #expect(cell.progressOverlay.fraction == 0.4) + + progress.beginProcessing() + await waitFor { cell.progressOverlay.isIndeterminate } + #expect(cell.progressOverlay.fraction == 1) + #expect(cell.progressOverlay.isIndeterminate) + #expect(cell.progressOverlay.isShowing) + } + + @Test("The overlay hides once the photo is sent") + func overlayHidesWhenSent() async { + let progress = ChatPhotoSendProgress() + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + progress.beginSending() + + progress.finish() + await waitFor { !cell.progressOverlay.isShowing } + #expect(!cell.progressOverlay.isShowing) + } + + @Test("The confirmed row, reconfigured without progress, hides the overlay") + func overlayHidesWhenConfirmedRowLosesProgress() { + let progress = ChatPhotoSendProgress() + progress.beginSending() + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + #expect(cell.progressOverlay.isShowing) + + cell.configure(with: outgoing(receipt: .delivered), maxWidth: 240, localImage: nil, progress: nil, remote: nil) + #expect(!cell.progressOverlay.isShowing) + } + + @Test("A failed row hides the overlay for the failed receipt, and a retry brings it back") + func failedRowHidesOverlay() async { + let progress = ChatPhotoSendProgress() + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + + progress.fail() + cell.configure(with: outgoing(receipt: .failed("Not delivered")), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + #expect(!cell.progressOverlay.isShowing) + #expect(cell.progressOverlay.isHidden) + + progress.beginAttempt() + cell.configure(with: outgoing(), maxWidth: 240, localImage: nil, progress: progress, remote: nil) + #expect(cell.progressOverlay.isShowing) + } + + @Test("A recycled cell drops the previous row's progress") + func recycledCellIgnoresOldProgress() async { + let old = ChatPhotoSendProgress() + let cell = ChatMediaCell(frame: CGRect(x: 0, y: 0, width: 320, height: 480)) + cell.configure(with: outgoing(id: "old"), maxWidth: 240, localImage: nil, progress: old, remote: nil) + cell.prepareForReuse() + cell.configure(with: outgoing(id: "new", receipt: .delivered), maxWidth: 240, localImage: nil, progress: nil, remote: nil) + + old.didUpload(BlobUploadProgress(sentBytes: 50, totalBytes: 100)) + for _ in 0..<5 { await Task.yield() } + #expect(!cell.progressOverlay.isShowing) + } + + private func waitFor(_ condition: () -> Bool) async { + for _ in 0..<50 where !condition() { await Task.yield() } + } + @Test("A row with no image yet draws its BlurHash") func drawsBlurhashBeforeImageLoads() { let cell = configuredCell(media()) diff --git a/FlipcashTests/Chat/ChatMediaUploaderTests.swift b/FlipcashTests/Chat/ChatMediaUploaderTests.swift index e82923ee5..ddff56d7c 100644 --- a/FlipcashTests/Chat/ChatMediaUploaderTests.swift +++ b/FlipcashTests/Chat/ChatMediaUploaderTests.swift @@ -263,8 +263,10 @@ struct ChatMediaUploaderTests { #expect(chip.state == .failed(.notRetryable)) } - @Test("Retrying a failed chip uploads it again") - func retryingFailedChipUploadsAgain() async throws { + /// The bytes were stored before the wait failed, which is how a send interrupted by the app + /// going to the background fails; storing them again would leave an orphan blob. + @Test("Retrying a chip whose wait for the server failed waits again without storing again") + func retryingFailedChipResumesStoredBlob() async throws { let blob = MockChatMediaBlobStore() blob.finalization = .failure(ErrorBlob.timedOut) let uploader = ChatMediaUploader(blob: blob) @@ -278,9 +280,46 @@ struct ChatMediaUploaderTests { let blobID = try await #require(chip.uploadTask).value.blobID #expect(chip.state == .uploaded(blobID)) + #expect(blob.storeAttempts == 1) + #expect(blob.finalizedBlobIDs == [blobID]) + } + + @Test("A chip's progress follows its bytes, then waits on the server's processing once stored") + func chipProgressReachesProcessing() async throws { + let blob = MockChatMediaBlobStore() + blob.progressReports = [ + BlobUploadProgress(sentBytes: 50, totalBytes: 100), + BlobUploadProgress(sentBytes: 100, totalBytes: 100), + ] + let composer = ComposerModel() + let chip = try #require(composer.stageChip(image: Self.image(width: 40, height: 30), uploader: ChatMediaUploader(blob: blob))) + #expect(chip.progress.phase == .preparing) + + _ = try await #require(chip.uploadTask).value + + #expect(chip.progress.phase == .processing) + #expect(chip.progress.showsOverlay) + } + + @Test("A chip whose upload fails hands its progress to the failed state; retrying starts it over") + func chipProgressFailsAndRetries() async throws { + let blob = MockChatMediaBlobStore() + blob.finalization = .failure(ErrorBlob.timedOut) + let uploader = ChatMediaUploader(blob: blob) + let composer = ComposerModel() + let chip = try #require(composer.stageChip(image: Self.image(width: 40, height: 30), uploader: uploader)) + _ = await chip.uploadTask?.result + #expect(chip.progress.phase == .failed) + #expect(!chip.progress.showsOverlay) + + blob.finalization = .success(()) + chip.startUpload(using: uploader) + #expect(chip.progress.phase == .preparing) + _ = try await #require(chip.uploadTask).value + #expect(chip.progress.phase == .processing) } - @Test("Removing a chip cancels its upload") + @Test("Removing a chip cancels its upload") func removingChipCancelsUpload() throws { let composer = ComposerModel() let chip = try #require(composer.stageChip(image: Self.image(width: 40, height: 30), uploader: ChatMediaUploader(blob: MockChatMediaBlobStore()))) diff --git a/FlipcashTests/Chat/ChatPhotoSendProgressTests.swift b/FlipcashTests/Chat/ChatPhotoSendProgressTests.swift new file mode 100644 index 000000000..4281815e5 --- /dev/null +++ b/FlipcashTests/Chat/ChatPhotoSendProgressTests.swift @@ -0,0 +1,113 @@ +// +// ChatPhotoSendProgressTests.swift +// FlipcashTests +// +// Copyright © 2026 Code Inc. All rights reserved. +// + +import Testing +import FlipcashCore +@testable import FlipcashUI + +@Suite("Chat photo send progress") +@MainActor +struct ChatPhotoSendProgressTests { + + private func bytes(_ sent: Int64, of total: Int64) -> BlobUploadProgress { + BlobUploadProgress(sentBytes: sent, totalBytes: total) + } + + @Test("Bytes sent map to the share of the body, clamped, and nil for an unknown length") + func bytesToFraction() { + #expect(bytes(0, of: 200).fraction == 0) + #expect(bytes(50, of: 200).fraction == 0.25) + #expect(bytes(200, of: 200).fraction == 1) + #expect(bytes(300, of: 200).fraction == 1) + #expect(bytes(10, of: 0).fraction == nil) + #expect(bytes(10, of: -1).fraction == nil) + } + + @Test("A send runs preparing, uploading, processing, sending, sent") + func happyPath() { + let progress = ChatPhotoSendProgress() + #expect(progress.phase == .preparing) + #expect(progress.showsOverlay) + + progress.didUpload(bytes(25, of: 100)) + #expect(progress.phase == .uploading(fraction: 0.25)) + + progress.didUpload(bytes(100, of: 100)) + #expect(progress.phase == .uploading(fraction: 1)) + + progress.beginProcessing() + #expect(progress.phase == .processing) + #expect(progress.showsOverlay) + + progress.beginSending() + #expect(progress.phase == .sending) + #expect(progress.showsOverlay) + + progress.finish() + #expect(progress.phase == .sent) + #expect(!progress.showsOverlay) + } + + @Test("The bar never runs backwards within an attempt") + func monotonicWithinAttempt() { + let progress = ChatPhotoSendProgress() + progress.didUpload(bytes(60, of: 100)) + progress.didUpload(bytes(40, of: 100)) + #expect(progress.phase == .uploading(fraction: 0.6)) + } + + @Test("A byte count that lands after the upload moved on is ignored") + func lateBytesIgnored() { + let progress = ChatPhotoSendProgress() + progress.beginProcessing() + progress.didUpload(bytes(50, of: 100)) + #expect(progress.phase == .processing) + + progress.fail() + progress.didUpload(bytes(50, of: 100)) + #expect(progress.phase == .failed) + } + + @Test("A byte count with no known length leaves the phase alone") + func unknownLengthIgnored() { + let progress = ChatPhotoSendProgress() + progress.didUpload(bytes(50, of: 0)) + #expect(progress.phase == .preparing) + } + + @Test("A failure hides the overlay; a new attempt brings it back from the start", arguments: [ + ChatPhotoSendProgress.Phase.preparing, .uploading(fraction: 0.5), .processing, .sending, + ]) + func failureThenRetry(from phase: ChatPhotoSendProgress.Phase) { + let progress = ChatPhotoSendProgress() + switch phase { + case .preparing: break + case .uploading(let fraction): progress.didUpload(bytes(Int64(fraction * 100), of: 100)) + case .processing: progress.beginProcessing() + case .sending: progress.beginSending() + case .sent, .failed: Issue.record("not a starting phase") + } + #expect(progress.phase == phase) + + progress.fail() + #expect(progress.phase == .failed) + #expect(!progress.showsOverlay) + + progress.beginAttempt() + #expect(progress.phase == .preparing) + #expect(progress.showsOverlay) + } + + @Test("A retried store starts the bar over") + func retriedStoreResets() { + let progress = ChatPhotoSendProgress() + progress.didUpload(bytes(80, of: 100)) + progress.beginAttempt() + progress.didUpload(bytes(10, of: 100)) + #expect(progress.phase == .uploading(fraction: 0.1)) + } +} diff --git a/FlipcashTests/Chat/ComposerChipStripTests.swift b/FlipcashTests/Chat/ComposerChipStripTests.swift index 0b74624bf..724dcf2a3 100644 --- a/FlipcashTests/Chat/ComposerChipStripTests.swift +++ b/FlipcashTests/Chat/ComposerChipStripTests.swift @@ -14,12 +14,12 @@ import FlipcashCore @Suite("Composer chip strip") struct ComposerChipStripTests { - @Test("A chip still preparing or uploading shows the progress ring", arguments: [ + @Test("A chip still preparing or uploading shows only its thumbnail; the sent bubble shows progress", arguments: [ ComposerChip.State.preparing, .uploading, ]) - func inFlightShowsProgress(state: ComposerChip.State) { - #expect(ComposerChipBadge(state) == .progress) + func inFlightShowsNothing(state: ComposerChip.State) { + #expect(ComposerChipBadge(state) == .none) } @Test("An uploaded chip shows only its thumbnail") diff --git a/FlipcashTests/Chat/ConversationMediaSendTests.swift b/FlipcashTests/Chat/ConversationMediaSendTests.swift index 47c9ad38b..ee5507031 100644 --- a/FlipcashTests/Chat/ConversationMediaSendTests.swift +++ b/FlipcashTests/Chat/ConversationMediaSendTests.swift @@ -158,6 +158,7 @@ struct ConversationMediaSendTests { #expect(pending.count == 1) #expect(pending.first?.status == .failed) #expect(controller.pendingMediaImage(forMessageID: try #require(pending.first?.stableID)) === rejected.image) + #expect(rejected.progress.phase == .failed) } @Test("A photo the server refuses to post leaves a failed bubble") @@ -166,9 +167,11 @@ struct ConversationMediaSendTests { mock.mediaSendError = ErrorSendMessage.transportFailure let controller = makeController(mock) - #expect(!(await controller.sendMedia([uploadedChip("a")], caption: "hi", to: conversationID))) + let chip = uploadedChip("a") + #expect(!(await controller.sendMedia([chip], caption: "hi", to: conversationID))) #expect(pendingMedia(controller).map(\.status) == [.failed]) + #expect(chip.progress.phase == .failed) } // MARK: - Pending bubble @@ -210,8 +213,8 @@ struct ConversationMediaSendTests { _ = await send.value } - @Test("The local image is served for a pending photo and dropped once it is confirmed") - func localImageLivesUntilConfirmed() async throws { + @Test("The local image is served for a pending photo and kept once it is confirmed") + func localImageOutlivesConfirm() async throws { let controller = makeController(MockConversations()) let gate = UploadGate() let chip = gatedChip(gate) @@ -223,7 +226,25 @@ struct ConversationMediaSendTests { gate.open(.success(blobID("a"))) _ = await send.value - #expect(controller.pendingMediaImage(forMessageID: messageID) == nil) + // The sent row keeps the client id, so its bubble keeps drawing the photo instead of the BlurHash. + #expect(controller.pendingMediaImage(forMessageID: messageID) === chip.image) + } + + @Test("The send progress is served for a pending photo, ends sent, and is dropped once confirmed") + func progressLivesUntilConfirmed() async throws { + let controller = makeController(MockConversations()) + let gate = UploadGate() + let chip = gatedChip(gate) + + let send = Task { await controller.sendMedia([chip], caption: nil, to: conversationID) } + await yield { pendingMedia(controller).count == 1 } + let messageID = try #require(pendingMedia(controller).first?.stableID) + #expect(controller.pendingMediaProgress(forMessageID: messageID) === chip.progress) + + gate.open(.success(blobID("a"))) + #expect(await send.value) + #expect(chip.progress.phase == .sent) + #expect(controller.pendingMediaProgress(forMessageID: messageID) == nil) } // MARK: - Retry @@ -260,6 +281,7 @@ struct ConversationMediaSendTests { #expect(!(await controller.sendMedia([chip], caption: nil, to: conversationID))) #expect(chip.state == .failed(.retryable)) + #expect(chip.progress.phase == .failed) let clientMessageID = try #require(pendingMedia(controller).first?.clientMessageID) await controller.retry(clientMessageID: clientMessageID, in: conversationID) @@ -267,6 +289,7 @@ struct ConversationMediaSendTests { #expect(blob.storeAttempts == 2) #expect(mock.sentMedia.map(\.blobID) == [MockChatMediaBlobStore.blobID]) #expect(pendingMedia(controller).isEmpty) + #expect(chip.progress.phase == .sent, "the retried upload brings the overlay back and ends it sent") } @Test("A photo refused by moderation is not retried") diff --git a/FlipcashTests/Chat/PendingMediaPersistenceTests.swift b/FlipcashTests/Chat/PendingMediaPersistenceTests.swift new file mode 100644 index 000000000..239092c73 --- /dev/null +++ b/FlipcashTests/Chat/PendingMediaPersistenceTests.swift @@ -0,0 +1,281 @@ +// +// PendingMediaPersistenceTests.swift +// FlipcashTests +// +// Copyright © 2026 Code Inc. All rights reserved. +// + +import Foundation +import Testing +import UIKit +import FlipcashCore +import FlipcashStore +@testable import Flipcash + +@MainActor +@Suite("Pending photo persistence") +struct PendingMediaPersistenceTests { + + private let owner = try! PublicKey(Data(repeating: 7, count: 32)) + private let conversationID = ConversationID.test(1) + private let selfUserID = UUID() + + private func makeDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("pending-media-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url + } + + private func jpeg() -> Data { + let format = UIGraphicsImageRendererFormat() + format.scale = 1 + return UIGraphicsImageRenderer(size: CGSize(width: 8, height: 6), format: format).jpegData(withCompressionQuality: 0.8) { context in + UIColor.red.setFill() + context.fill(CGRect(x: 0, y: 0, width: 8, height: 6)) + } + } + + private func blobID(_ name: String) -> BlobID { BlobID(data: Data(name.utf8)) } + + private func entry(stored: UploadedPhoto? = nil, caption: String? = nil, createdAt: Date = .now) -> PendingMediaStore.Entry { + .init(clientMessageID: UUID(), conversationID: conversationID, createdAt: createdAt, caption: caption, replyTo: MessageID(value: 9), stored: stored) + } + + private func makeController( + _ mock: MockConversations, + database: Database, + store: PendingMediaStore, + blob: MockChatMediaBlobStore + ) -> ConversationController { + let controller = ConversationController( + fetching: mock, membership: mock, viewerSettings: mock, messaging: mock, streaming: mock, + contactNaming: MockDMContactNaming(), + database: database, + owner: .generate()!, selfUserID: selfUserID, + typingHeartbeatInterval: .seconds(3), incomingTypingExpiry: .seconds(10) + ) + controller.pendingMedia = store + controller.restoredPhotoUploader = { _ in ChatMediaUploader(blob: blob) } + return controller + } + + private func pendingRows(_ controller: ConversationController) -> [ConversationMessage] { + controller.messages(for: conversationID).filter { $0.clientMessageID != nil && $0.id == .unassigned } + } + + private func yield(until condition: () -> Bool) async { + for _ in 0..<500 where !condition() { + await Task.yield() + } + } + + // MARK: - Store + + @Test("Entries, their photo and their stored state survive a new store over the same directory") + func roundTrips() throws { + let directory = try makeDirectory() + let store = PendingMediaStore(directory: directory, owner: owner) + let plain = entry(stored: .plain(blobID("a")), caption: "hi") + let sealed = entry(stored: .sealed(SealedPhoto(blobID: blobID("b"), mimeType: "image/jpeg", sizeBytes: 10, width: 8, height: 6, blurhash: "LEHV6n"))) + let bare = entry() + for item in [plain, sealed, bare] { + store.add(item) + store.writeImage(jpeg(), for: item.clientMessageID) + } + store.setStored(.plain(blobID("c")), for: bare.clientMessageID) + + let reloaded = PendingMediaStore(directory: directory, owner: owner) + + #expect(reloaded.entries.map(\.clientMessageID) == [plain, sealed, bare].map(\.clientMessageID)) + #expect(reloaded.entries[0].caption == "hi") + #expect(reloaded.entries[0].replyTo == MessageID(value: 9)) + #expect(reloaded.entries[0].chatID == conversationID) + #expect(reloaded.entries[1].stored == sealed.stored) + #expect(reloaded.entries[2].stored == .plain(blobID("c"))) + #expect(reloaded.imageData(for: reloaded.entries[0]) == store.imageData(for: plain)) + } + + @Test("Another owner's store does not see the entries") + func ownerScoped() throws { + let directory = try makeDirectory() + let store = PendingMediaStore(directory: directory, owner: owner) + store.add(entry()) + + let other = PendingMediaStore(directory: directory, owner: try PublicKey(Data(repeating: 8, count: 32))) + + #expect(other.entries.isEmpty) + } + + @Test("Removing an entry deletes its photo") + func removeDeletesFile() throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let item = entry() + store.add(item) + store.writeImage(jpeg(), for: item.clientMessageID) + + store.remove(clientMessageID: item.clientMessageID) + + #expect(store.entries.isEmpty) + #expect(store.imageData(for: item) == nil) + } + + @Test("Sweeping drops an entry with no photo and a photo with no entry") + func sweepsOrphans() throws { + let directory = try makeDirectory() + let store = PendingMediaStore(directory: directory, owner: owner) + let kept = entry() + let noFile = entry() + store.add(kept) + store.writeImage(jpeg(), for: kept.clientMessageID) + store.add(noFile) + let orphan = directory + .appendingPathComponent("flipcash-\(owner.base58)-pending-media") + .appendingPathComponent("\(UUID().uuidString).jpg") + try jpeg().write(to: orphan) + + store.sweepOrphans() + + #expect(store.entries.map(\.clientMessageID) == [kept.clientMessageID]) + #expect(!FileManager.default.fileExists(atPath: orphan.path)) + #expect(PendingMediaStore(directory: directory, owner: owner).entries.count == 1) + } + + @Test("A manifest that cannot be read is an empty store") + func unreadableManifestIsEmpty() throws { + let directory = try makeDirectory() + try Data("not json".utf8).write(to: directory.appendingPathComponent("flipcash-\(owner.base58)-pending-media.json")) + + #expect(PendingMediaStore(directory: directory, owner: owner).entries.isEmpty) + } + + // MARK: - Restore + + @Test("A send whose bytes never landed comes back as a failed, retryable row") + func unstoredEntryRestoresFailed() async throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let item = entry(caption: "hi") + store.add(item) + let bytes = jpeg() + store.writeImage(bytes, for: item.clientMessageID) + let mock = MockConversations() + let blob = MockChatMediaBlobStore() + let controller = makeController(mock, database: try Database.makeTemp().database, store: store, blob: blob) + + controller.restorePendingMedia() + + let rows = pendingRows(controller) + #expect(rows.map(\.status) == [.failed]) + #expect(rows.first?.clientMessageID == item.clientMessageID) + if case .media(_, let caption) = try #require(rows.first).content { + #expect(caption == "hi") + } else { + Issue.record("expected a media row") + } + #expect(rows.first?.repliedTo == MessageID(value: 9)) + #expect(controller.pendingMediaImage(forMessageID: try #require(rows.first?.stableID)) != nil) + #expect(blob.storeAttempts == 0) + + // Retrying uploads the held bytes as they are. + await controller.retry(clientMessageID: item.clientMessageID, in: conversationID) + + #expect(blob.storedData == [bytes]) + #expect(mock.sentMedia.count == 1) + #expect(mock.sentMedia.first?.clientMessageID == item.clientMessageID) + #expect(pendingRows(controller).isEmpty) + #expect(store.entries.isEmpty, "confirmed sends leave the store") + } + + @Test("A send whose bytes had landed resumes and posts without storing again") + func storedEntryResumes() async throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let item = entry(stored: .plain(blobID("landed"))) + store.add(item) + store.writeImage(jpeg(), for: item.clientMessageID) + let mock = MockConversations() + let blob = MockChatMediaBlobStore() + let controller = makeController(mock, database: try Database.makeTemp().database, store: store, blob: blob) + + controller.restorePendingMedia() + #expect(pendingRows(controller).map(\.status) == [.sending]) + await yield { !mock.sentMedia.isEmpty && store.entries.isEmpty } + + #expect(mock.sentMedia.map(\.blobID) == [blobID("landed")]) + #expect(blob.storeAttempts == 0) + #expect(blob.finalizedBlobIDs == [blobID("landed")]) + #expect(store.entries.isEmpty) + #expect(pendingRows(controller).isEmpty) + } + + @Test("An entry whose blob is already in a message this user sent is dropped; one whose blob is not is kept") + func reconcilesAgainstLoadedMessages() async throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let sent = entry(stored: .plain(blobID("sent"))) + let unsent = entry(stored: .plain(blobID("unsent"))) + for item in [sent, unsent] { + store.add(item) + store.writeImage(jpeg(), for: item.clientMessageID) + } + let database = try Database.makeTemp().database + let message = ConversationMessage( + id: MessageID(value: 5), + senderID: selfUserID, + content: .media([MediaAttachment(blobID: blobID("sent"), width: 8, height: 6, blurhash: nil)], caption: nil), + date: .now, + unreadSeq: 0, + repliedTo: nil, + status: .sent, + clientMessageID: nil + ) + try database.upsertConversationMessages([message], conversationID: conversationID) + let mock = MockConversations() + mock.mediaSendError = ErrorSendMessage.transportFailure + let controller = makeController(mock, database: database, store: store, blob: MockChatMediaBlobStore()) + + controller.restorePendingMedia() + await yield { pendingRows(controller).first?.status == .failed } + + #expect(store.entries.map(\.clientMessageID) == [unsent.clientMessageID]) + #expect(pendingRows(controller).map(\.clientMessageID) == [unsent.clientMessageID]) + #expect(mock.sentMedia.map(\.blobID) == [blobID("unsent")], "the already-sent photo is never posted again") + } + + @Test("A photo the server refuses for good leaves the store") + func rejectedUploadLeavesStore() async throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let item = entry(stored: .plain(blobID("bad"))) + store.add(item) + store.writeImage(jpeg(), for: item.clientMessageID) + let blob = MockChatMediaBlobStore() + blob.finalization = .failure(ErrorBlob.rejected(.moderation)) + let controller = makeController(MockConversations(), database: try Database.makeTemp().database, store: store, blob: blob) + + controller.restorePendingMedia() + await yield { store.entries.isEmpty } + + #expect(store.entries.isEmpty) + #expect(pendingRows(controller).map(\.status) == [.failed]) + } + + @Test("A photo sent through the controller is held until its post is confirmed") + func sendKeepsEntryUntilConfirmed() async throws { + let store = PendingMediaStore(directory: try makeDirectory(), owner: owner) + let mock = MockConversations() + mock.mediaSendError = ErrorSendMessage.transportFailure + let blob = MockChatMediaBlobStore() + let controller = makeController(mock, database: try Database.makeTemp().database, store: store, blob: blob) + let chip = ComposerChip(image: UIImage(data: jpeg())!) + chip.startUpload(using: ChatMediaUploader(blob: blob)) + + #expect(!(await controller.sendMedia([chip], caption: "hi", to: conversationID))) + + let held = try #require(store.entries.first) + #expect(held.caption == "hi") + #expect(held.stored == .plain(MockChatMediaBlobStore.blobID)) + #expect(store.imageData(for: held) == blob.storedData.first) + + mock.mediaSendError = nil + await controller.retry(clientMessageID: held.clientMessageID, in: conversationID) + + #expect(store.entries.isEmpty) + } +} diff --git a/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift b/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift index 92a8464b5..16f2deac8 100644 --- a/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift +++ b/FlipcashTests/TestSupport/MockChatMediaBlobStore.swift @@ -20,6 +20,8 @@ final class MockChatMediaBlobStore: ChatMediaBlobStoring { var storeResults: [Result] = [] var finalization: Result = .success(()) var onStore: (() -> Void)? + /// Reported through each store call's `onProgress` before it ends. + var progressReports: [BlobUploadProgress] = [] private(set) var storeAttempts = 0 private(set) var storedData: [Data] = [] @@ -38,8 +40,9 @@ final class MockChatMediaBlobStore: ChatMediaBlobStoring { policy } - func storeBlob(_ data: Data, mimeType: String) async throws -> BlobID { + func storeBlob(_ data: Data, mimeType: String, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> BlobID { storeAttempts += 1 + progressReports.forEach(onProgress) onStore?() let result = storeResults.isEmpty ? .success(Self.blobID) : storeResults.removeFirst() let blobID = try result.get() @@ -48,8 +51,9 @@ final class MockChatMediaBlobStore: ChatMediaBlobStoring { return blobID } - func storeEncryptedBlob(_ image: Data, seal: ChatSeal) async throws -> EncryptedBlobUpload { + func storeEncryptedBlob(_ image: Data, seal: ChatSeal, onProgress: @escaping @Sendable (BlobUploadProgress) -> Void) async throws -> EncryptedBlobUpload { storeAttempts += 1 + progressReports.forEach(onProgress) onStore?() let result = storeResults.isEmpty ? .success(Self.blobID) : storeResults.removeFirst() let blobID = try result.get() diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift index e4f301146..2a35e76e4 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatMediaCell.swift @@ -26,12 +26,16 @@ public final class ChatMediaCell: ChatColumnCell { private static let captionInset: CGFloat = 12 private static let captionPadding: CGFloat = 9 private static let fadeDuration: TimeInterval = 0.25 + /// The progress capsule's inset from the photo's bottom-right corner. + private static let progressInset: CGFloat = 10 private let stack = UIStackView() private let imageBubble = BubbleBackgroundView() let imageView = UIImageView() /// Shown over the BlurHash when an encrypted photo's bytes fail to decrypt or check out. let unavailableLabel = UILabel() + /// Shows how far an outgoing photo's send has got, until it is sent or fails. + let progressOverlay = ChatPhotoProgressOverlay() let captionBubble = BubbleBackgroundView() let captionLabel = UILabel() let reactionRow = ReactionPillRowView() @@ -75,6 +79,8 @@ public final class ChatMediaCell: ChatColumnCell { unavailableLabel.isHidden = true unavailableLabel.translatesAutoresizingMaskIntoConstraints = false imageBubble.addSubview(unavailableLabel) + progressOverlay.translatesAutoresizingMaskIntoConstraints = false + imageBubble.addSubview(progressOverlay) imageTap.addTarget(self, action: #selector(imageTapped)) imageBubble.addGestureRecognizer(imageTap) @@ -113,6 +119,11 @@ public final class ChatMediaCell: ChatColumnCell { imageView.trailingAnchor.constraint(equalTo: imageBubble.trailingAnchor), imageView.bottomAnchor.constraint(equalTo: imageBubble.bottomAnchor), + progressOverlay.widthAnchor.constraint(equalToConstant: ChatPhotoProgressOverlay.size.width), + progressOverlay.heightAnchor.constraint(equalToConstant: ChatPhotoProgressOverlay.size.height), + progressOverlay.trailingAnchor.constraint(equalTo: imageBubble.trailingAnchor, constant: -Self.progressInset), + progressOverlay.bottomAnchor.constraint(equalTo: imageBubble.bottomAnchor, constant: -Self.progressInset), + unavailableLabel.centerYAnchor.constraint(equalTo: imageBubble.centerYAnchor), unavailableLabel.leadingAnchor.constraint(equalTo: imageBubble.leadingAnchor, constant: Self.captionInset), unavailableLabel.trailingAnchor.constraint(equalTo: imageBubble.trailingAnchor, constant: -Self.captionInset), @@ -139,6 +150,7 @@ public final class ChatMediaCell: ChatColumnCell { imageView.kf.cancelDownloadTask() imageView.image = nil showUnavailable(false) + progressOverlay.bind(nil, suppressed: false, animated: false) drawnRowID = nil reactionRow.prepareForReuse() } @@ -152,11 +164,13 @@ public final class ChatMediaCell: ChatColumnCell { /// - Parameters: /// - maxWidth: the transcript's widest bubble, which the photo always spans. /// - localImage: the picked image of a pending send this device staged, or nil. + /// - progress: the send progress of a pending photo this device staged, or nil. /// - remote: where the photo downloads from, or nil until it is resolved. public func configure( with message: ChatMessage, maxWidth: CGFloat, localImage: UIImage?, + progress: ChatPhotoSendProgress? = nil, remote: ChatMediaLocation?, authorImageData: Data? = nil ) { @@ -171,6 +185,8 @@ public final class ChatMediaCell: ChatColumnCell { ) imageWidthConstraint.constant = size.width imageHeightConstraint.constant = size.height + // The same row losing its progress is a confirmed send, which fades; a new row starts clean. + progressOverlay.bind(progress, suppressed: message.isFailed, animated: drawnRowID == message.id) drawImage( for: message.id, blobID: media.blobID, diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoProgressOverlay.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoProgressOverlay.swift new file mode 100644 index 000000000..48d397816 --- /dev/null +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoProgressOverlay.swift @@ -0,0 +1,200 @@ +// +// ChatPhotoProgressOverlay.swift +// FlipcashUI +// +// Copyright © 2026 Code Inc. All rights reserved. +// + +#if canImport(UIKit) +import UIKit +import SwiftUI +import Observation + +/// A thin bar in the corner of an outgoing photo, after iMessage's send bar, that follows its +/// ``ChatPhotoSendProgress``. +/// +/// The bar fills with the bytes sent. Once they are stored, the server's processing and the message +/// send have no measurable progress, so a blue segment slides along the track instead — held still +/// in the middle under Reduce Motion. The bar fades out when the message is sent, and is gone at once on a failure, +/// where the row's failed receipt and tap-to-retry take over. +final class ChatPhotoProgressOverlay: UIView { + + static let size = CGSize(width: 64, height: 5) + private static let fadeDuration: TimeInterval = 0.25 + private static let fillDuration: TimeInterval = 0.2 + private static let segmentShare: CGFloat = 0.35 + private static let slideDuration: CFTimeInterval = 1.1 + private static let slideKey = "slide" + + private let track = UIView() + private let fill = UIView() + private let segment = UIView() + + private var progress: ChatPhotoSendProgress? + /// Set while the row is failed, which hides the bar whatever the progress says. + private var suppressed = false + /// Bumped on every bind, so an observation armed for an earlier row is dropped. + private var generation = 0 + + /// The share of the bar drawn filled. + private(set) var fraction: CGFloat = 0 + /// Whether the bar is showing, or fading in to show. + private(set) var isShowing = false + /// Whether the bar is in its indeterminate, post-upload state. + private(set) var isIndeterminate = false + + override init(frame: CGRect) { + super.init(frame: frame) + isUserInteractionEnabled = false + isAccessibilityElement = false + alpha = 0 + isHidden = true + + // The shadow keeps the bar legible over a white region of the photo. + layer.shadowColor = UIColor.black.cgColor + layer.shadowOpacity = 0.3 + layer.shadowRadius = 2 + layer.shadowOffset = .zero + + track.backgroundColor = UIColor(Color.backgroundMain).withAlphaComponent(0.35) + track.layer.cornerRadius = Self.size.height / 2 + track.clipsToBounds = true + addSubview(track) + + fill.backgroundColor = .systemBlue + track.addSubview(fill) + + segment.backgroundColor = .systemBlue + segment.layer.cornerRadius = Self.size.height / 2 + segment.isHidden = true + track.addSubview(segment) + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } + + override var intrinsicContentSize: CGSize { Self.size } + + override func layoutSubviews() { + super.layoutSubviews() + track.frame = bounds + layer.shadowPath = UIBezierPath(roundedRect: bounds, cornerRadius: bounds.height / 2).cgPath + fill.frame = CGRect(x: 0, y: 0, width: track.bounds.width * fraction, height: track.bounds.height) + let width = track.bounds.width * Self.segmentShare + segment.bounds = CGRect(x: 0, y: 0, width: width, height: track.bounds.height) + segment.center = CGPoint(x: track.bounds.midX, y: track.bounds.midY) + // A slide armed before the track had a width never started. + if isIndeterminate, segment.layer.animation(forKey: Self.slideKey) == nil { startSliding() } + } + + /// Follows `progress`, or shows nothing when it is nil. `animated` is false for a row the + /// overlay was not already drawing, so a recycled cell never fades another row's state. + func bind(_ progress: ChatPhotoSendProgress?, suppressed: Bool, animated: Bool) { + generation += 1 + self.progress = progress + self.suppressed = suppressed + render(animated: animated) + observe(generation: generation) + } + + private func observe(generation armed: Int) { + guard let progress else { return } + withObservationTracking { + _ = progress.phase + } onChange: { [weak self] in + // Fires before the new value is readable; render once it has landed. + Task { @MainActor [weak self] in + guard let self, self.generation == armed else { return } + self.render(animated: true) + self.observe(generation: armed) + } + } + } + + private func render(animated: Bool) { + let phase = suppressed ? nil : progress?.phase + let animated = animated && window != nil + + let target: (show: Bool, fraction: CGFloat, indeterminate: Bool) + switch phase { + case .preparing: + target = (true, 0, false) + case .uploading(let value): + target = (true, CGFloat(value), false) + case .processing, .sending: + target = (true, 1, true) + case .sent: + target = (false, 1, false) + case .failed, nil: + target = (false, fraction, false) + } + + if target.show { + setFraction(target.fraction, animated: animated) + setIndeterminate(target.indeterminate) + } + // A failure hands straight to the failed receipt. Anything else leaving fades: a send, or a + // row whose progress went away because its send confirmed. + setShowing(target.show, animated: animated && !suppressed && phase != .failed) + } + + private func setFraction(_ value: CGFloat, animated: Bool) { + guard value != fraction else { return } + fraction = value + guard animated, !UIAccessibility.isReduceMotionEnabled, !isHidden else { + setNeedsLayout() + return + } + UIView.animate(withDuration: Self.fillDuration, delay: 0, options: [.beginFromCurrentState, .curveEaseOut]) { + self.layoutIfNeeded() + } + setNeedsLayout() + } + + private func setIndeterminate(_ indeterminate: Bool) { + guard indeterminate != isIndeterminate else { return } + isIndeterminate = indeterminate + fill.isHidden = indeterminate + segment.isHidden = !indeterminate + if indeterminate { startSliding() } else { stopSliding() } + } + + private func startSliding() { + guard !UIAccessibility.isReduceMotionEnabled, track.bounds.width > 0 else { return } + let half = segment.bounds.width / 2 + let slide = CABasicAnimation(keyPath: "position.x") + slide.fromValue = -half + slide.toValue = track.bounds.width + half + slide.duration = Self.slideDuration + slide.timingFunction = CAMediaTimingFunction(name: .easeInEaseOut) + slide.repeatCount = .infinity + slide.isRemovedOnCompletion = false + segment.layer.add(slide, forKey: Self.slideKey) + } + + private func stopSliding() { + segment.layer.removeAnimation(forKey: Self.slideKey) + } + + private func setShowing(_ show: Bool, animated: Bool) { + guard show != isShowing else { return } + isShowing = show + if show { + isHidden = false + layoutIfNeeded() + } + let apply = { self.alpha = show ? 1 : 0 } + guard animated else { + apply() + isHidden = !show + if !show { setIndeterminate(false) } + return + } + UIView.animate(withDuration: Self.fadeDuration, delay: 0, options: [.beginFromCurrentState], animations: apply) { _ in + guard !self.isShowing else { return } + self.isHidden = true + self.setIndeterminate(false) + } + } +} +#endif diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoSendProgress.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoSendProgress.swift new file mode 100644 index 000000000..a8ff05d43 --- /dev/null +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatPhotoSendProgress.swift @@ -0,0 +1,88 @@ +// +// ChatPhotoSendProgress.swift +// FlipcashUI +// +// Copyright © 2026 Code Inc. All rights reserved. +// + +import Foundation +import Observation +import FlipcashCore + +/// How far one outgoing photo has got, from encoding through the message that carries it. +/// +/// One per staged photo, so each bubble in a multi-photo send draws its own progress. +@MainActor +@Observable +public final class ChatPhotoSendProgress { + + public enum Phase: Equatable, Sendable { + /// Encoding, encrypting, or reserving the upload; no bytes have gone out yet. + case preparing + /// Sending bytes to storage, with the share of them sent. + case uploading(fraction: Double) + /// Stored, while the server checks the bytes before they can be served. + case processing + /// Posting the message that references the stored photo. + case sending + /// The message is confirmed. + case sent + /// The upload or the message failed; the transcript's retry affordance takes over. + case failed + } + + public private(set) var phase: Phase = .preparing + + public init() {} + + /// Whether the bubble draws a progress overlay for this phase. + public var showsOverlay: Bool { + switch phase { + case .preparing, .uploading, .processing, .sending: + true + case .sent, .failed: + false + } + } + + /// Starts an upload attempt over, including after a failure or a retried store. + public func beginAttempt() { + phase = .preparing + } + + /// Records bytes sent to storage. + /// + /// Byte counts arrive asynchronously from the network, so one landing after the upload has + /// moved on is ignored, and the bar never runs backwards within an attempt. + public func didUpload(_ progress: BlobUploadProgress) { + guard let fraction = progress.fraction else { return } + switch phase { + case .preparing: + phase = .uploading(fraction: fraction) + case .uploading(let current): + if fraction > current { phase = .uploading(fraction: fraction) } + case .processing, .sending, .sent, .failed: + break + } + } + + /// Records that the bytes are stored and the server is finalizing them. + public func beginProcessing() { + phase = .processing + } + + /// Records that the message referencing the photo is being posted. + public func beginSending() { + phase = .sending + } + + /// Records that the message is confirmed. + public func finish() { + phase = .sent + } + + /// Records that the upload or the message failed. + public func fail() { + phase = .failed + } +} diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatScreenViewController.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatScreenViewController.swift index 6e44d29f5..ad5c7a0c1 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatScreenViewController.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatScreenViewController.swift @@ -230,6 +230,12 @@ public final class ChatScreenViewController: UIViewController { set { transcript.pendingMediaImage = newValue } } + /// A pending photo's send progress — see ``ChatViewController/pendingMediaProgress``. + public var pendingMediaProgress: ((String) -> ChatPhotoSendProgress?)? { + get { transcript.pendingMediaProgress } + set { transcript.pendingMediaProgress = newValue } + } + public var onContactAction: (() -> Void)? { get { transcript.onContactAction } set { transcript.onContactAction = newValue } diff --git a/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift b/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift index 2b87c5462..fb2a9ddfd 100644 --- a/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift +++ b/FlipcashUI/Sources/FlipcashUI/Chat/ChatViewController.swift @@ -129,6 +129,10 @@ public final class ChatViewController: UICollectionViewController { /// it until the send confirms and the download takes over. public var pendingMediaImage: ((String) -> UIImage?)? + /// The send progress of a photo this device is still sending, by the pending row's id; the row + /// draws it over the photo until the send confirms or fails. + public var pendingMediaProgress: ((String) -> ChatPhotoSendProgress?)? + /// The widest a bubble may grow, as a share of the collection view's width. private static let maxBubbleWidthFraction: CGFloat = 0.78 @@ -704,6 +708,7 @@ public final class ChatViewController: UICollectionViewController { with: message, maxWidth: maxWidth, localImage: localImage, + progress: pendingMediaProgress?(message.id), remote: remote, authorImageData: authorImageData )