From cca6d0b60fc8b09ab8d6456e8300bfdfd71564c1 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Mon, 5 Oct 2026 16:10:13 -0400 Subject: [PATCH 1/4] feat(chat): send, store and open photo messages The transport and storage half of chat photos, ported from code-ios-app#959. Nothing in the app calls it yet; the UI follows in a stacked PR. - Encoding: photos are downscaled and encoded as JPEG down a quality ladder of 0.9, 0.8, 0.7, 0.6 until they fit the blob policy, with EXIF orientation baked in and a BlurHash for the placeholder. The chat_media.json vectors are the iOS file, byte for byte. - Encrypted chats: the blob is sealed with the shared-core ChatCipher, uploaded with the `chat` field, and sent as EncryptedContent. A photo that can't be opened loads as ChatPhotoUnavailable. - Sending: one message per photo, each retried on its own. Finalization polls GetBlobs every 2s, at most 30 times, counted only while the app is in the foreground. - Storage: pending uploads live in Room (schema 43) with the JPEG in filesDir, so a send resumes after the app is killed. - Loading: a Coil fetcher keyed `chat-media-`, registered in FlipcashApp. - Fixes found on the way: captions were never sent, because `caption` inside the builder's `apply` resolved to the builder's own field; and a sent message disappeared when the stream delivered its stored copy before the send returned, because clearing the pending row deleted by its client id. --- apps/flipcash/app/build.gradle.kts | 1 + .../kotlin/com/flipcash/app/FlipcashApp.kt | 7 + apps/flipcash/shared/blob/build.gradle.kts | 3 + .../app/blob/BlobStorageCoordinator.kt | 81 +- .../com/flipcash/app/blob/BlurHashEncoder.kt | 127 ++ .../flipcash/app/blob/ChatMediaConstraints.kt | 29 + .../flipcash/app/blob/ChatMediaDownscale.kt | 40 + .../com/flipcash/app/blob/ChatMediaEncoder.kt | 102 ++ .../com/flipcash/app/blob/ChatMediaLimits.kt | 75 ++ .../com/flipcash/app/blob/ChatMediaRetry.kt | 41 + .../app/blob/BlobStorageCoordinatorTest.kt | 44 + .../flipcash/app/blob/BlurHashEncoderTest.kt | 77 ++ .../app/blob/ChatMediaEncodingTest.kt | 101 ++ .../flipcash/app/blob/ChatMediaRetryTest.kt | 63 + .../flipcash/app/blob/ChatMediaVectorTest.kt | 86 ++ .../blob/src/test/resources/chat_media.json | 639 ++++++++++ apps/flipcash/shared/chat/build.gradle.kts | 1 + .../flipcash/shared/chat/ChatCoordinator.kt | 4 +- .../flipcash/shared/chat/MessageCapability.kt | 17 +- .../flipcash/shared/chat/inject/ChatModule.kt | 8 + .../chat/internal/OutgoingEncryption.kt | 55 +- .../chat/internal/RealChatCoordinator.kt | 7 +- .../internal/delegates/MediaSendDelegate.kt | 463 ++++++++ .../internal/delegates/MessagingDelegate.kt | 6 +- .../shared/chat/media/ChatMediaSendPlan.kt | 36 + .../shared/chat/media/ChatMediaSending.kt | 76 ++ .../shared/chat/media/ChatMediaText.kt | 27 + .../shared/chat/media/ChatMediaUploads.kt | 374 ++++++ .../shared/chat/media/ChatPhotoFetcher.kt | 121 ++ .../shared/chat/media/ChatPhotoLoader.kt | 119 ++ .../shared/chat/media/SentPhotoPreviews.kt | 47 + .../shared/chat/MessageCapabilityTest.kt | 48 + .../chat/MessagingSendEncryptionTest.kt | 70 ++ .../chat/media/ChatMediaSendPlanTest.kt | 24 + .../shared/chat/media/ChatMediaUploadsTest.kt | 325 +++++ .../shared/chat/media/ChatMediaVectorTest.kt | 90 ++ .../shared/chat/media/ChatPhotoLoaderTest.kt | 112 ++ .../chat/media/MediaSendDelegateTest.kt | 364 ++++++ .../chat/src/test/resources/chat_media.json | 639 ++++++++++ .../43.json | 1043 +++++++++++++++++ .../app/persistence/FlipcashDatabase.kt | 9 +- .../app/persistence/dao/ChatMessageDao.kt | 62 +- .../app/persistence/dao/PendingMediaDao.kt | 39 + .../entities/PendingMediaEntity.kt | 63 + .../persistence/dao/PendingMediaDaoTest.kt | 175 +++ .../sources/ChatMessageDataSource.kt | 36 +- .../sources/PendingMediaDataSource.kt | 97 ++ .../sources/mapper/chat/ChatEntityMapper.kt | 9 +- services/flipcash/build.gradle.kts | 1 + .../com/flipcash/services/BlobUploader.kt | 12 +- .../com/flipcash/services/ForegroundGate.kt | 31 + .../services/chat/ChatContentCrypto.kt | 96 +- .../com/flipcash/services/chat/SealedMedia.kt | 92 ++ .../controllers/BlobStorageController.kt | 163 ++- .../services/inject/FlipcashModule.kt | 7 + .../internal/extensions/JpegMetadata.kt | 2 +- .../internal/network/HttpBlobUploader.kt | 37 +- .../internal/network/api/BlobStorageApi.kt | 6 +- .../network/extensions/LocalToProtobuf.kt | 46 +- .../network/extensions/ProtobufToLocal.kt | 16 +- .../network/services/BlobStorageService.kt | 4 +- .../InternalBlobStorageRepository.kt | 4 +- .../services/models/blob/UploadPolicy.kt | 11 + .../repository/BlobStorageRepository.kt | 4 + .../services/chat/ChatContentCryptoTest.kt | 171 ++- .../BlobStorageControllerChatMediaTest.kt | 304 +++++ .../controllers/BlobStorageControllerTest.kt | 4 +- .../extensions/MediaContentMappingTest.kt | 145 +++ .../flipcash/services/chat/FakeChatCipher.kt | 10 +- 69 files changed, 7192 insertions(+), 56 deletions(-) create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlurHashEncoder.kt create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaConstraints.kt create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaDownscale.kt create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaEncoder.kt create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaLimits.kt create mode 100644 apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaRetry.kt create mode 100644 apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlurHashEncoderTest.kt create mode 100644 apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaEncodingTest.kt create mode 100644 apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaRetryTest.kt create mode 100644 apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaVectorTest.kt create mode 100644 apps/flipcash/shared/blob/src/test/resources/chat_media.json create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlan.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSending.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaText.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaUploads.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoFetcher.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoader.kt create mode 100644 apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/SentPhotoPreviews.kt create mode 100644 apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlanTest.kt create mode 100644 apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaUploadsTest.kt create mode 100644 apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaVectorTest.kt create mode 100644 apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoaderTest.kt create mode 100644 apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt create mode 100644 apps/flipcash/shared/chat/src/test/resources/chat_media.json create mode 100644 apps/flipcash/shared/persistence/db/schemas/com.flipcash.app.persistence.FlipcashDatabase/43.json create mode 100644 apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/PendingMediaDao.kt create mode 100644 apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/entities/PendingMediaEntity.kt create mode 100644 apps/flipcash/shared/persistence/db/src/test/kotlin/com/flipcash/app/persistence/dao/PendingMediaDaoTest.kt create mode 100644 apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/PendingMediaDataSource.kt create mode 100644 services/flipcash/src/main/kotlin/com/flipcash/services/ForegroundGate.kt create mode 100644 services/flipcash/src/main/kotlin/com/flipcash/services/chat/SealedMedia.kt create mode 100644 services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerChatMediaTest.kt create mode 100644 services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/extensions/MediaContentMappingTest.kt diff --git a/apps/flipcash/app/build.gradle.kts b/apps/flipcash/app/build.gradle.kts index 357ccdf8c0..c03a3aeb28 100644 --- a/apps/flipcash/app/build.gradle.kts +++ b/apps/flipcash/app/build.gradle.kts @@ -232,6 +232,7 @@ dependencies { implementation(project(":apps:flipcash:shared:contacts")) implementation(project(":apps:flipcash:shared:common-ui")) implementation(project(":apps:flipcash:shared:notifications")) + implementation(project(":apps:flipcash:shared:chat")) implementation(project(":apps:flipcash:shared:onramp:coinbase")) implementation(project(":apps:flipcash:shared:onramp:deeplinks")) implementation(libs.phantom.connect) { diff --git a/apps/flipcash/app/src/main/kotlin/com/flipcash/app/FlipcashApp.kt b/apps/flipcash/app/src/main/kotlin/com/flipcash/app/FlipcashApp.kt index 7d4db85466..649e548a0e 100644 --- a/apps/flipcash/app/src/main/kotlin/com/flipcash/app/FlipcashApp.kt +++ b/apps/flipcash/app/src/main/kotlin/com/flipcash/app/FlipcashApp.kt @@ -14,6 +14,8 @@ import coil3.request.crossfade import com.flipcash.app.auth.AuthManager import okio.Path.Companion.toOkioPath import com.flipcash.app.core.android.ActivityProvider +import com.flipcash.shared.chat.media.ChatPhotoFetcher +import com.flipcash.shared.chat.media.ChatPhotoKeyer import com.flipcash.app.currency.PreferredCurrencyController import com.flipcash.app.bills.share.TipCodePreviewCache import com.getcode.opencode.repositories.EventRepository @@ -51,6 +53,9 @@ class FlipcashApp : Application(), Configuration.Provider, SingletonImageLoader. @Inject lateinit var preferredCurrencyController: Lazy + @Inject + lateinit var chatPhotoFetcher: Lazy + @Inject lateinit var workerFactory: Lazy @@ -109,6 +114,8 @@ class FlipcashApp : Application(), Configuration.Provider, SingletonImageLoader. // would flash the BlurHash again). The default strategy respects HTTP cache headers and // would revalidate/re-fetch the ephemeral, expiring download URLs; blobs are static. .components { + add(ChatPhotoKeyer()) + add(chatPhotoFetcher.get()) add(OkHttpNetworkFetcherFactory(cacheStrategy = { ImmutableBlobCacheStrategy })) } .build() diff --git a/apps/flipcash/shared/blob/build.gradle.kts b/apps/flipcash/shared/blob/build.gradle.kts index 213b86c547..e58f0cecea 100644 --- a/apps/flipcash/shared/blob/build.gradle.kts +++ b/apps/flipcash/shared/blob/build.gradle.kts @@ -19,4 +19,7 @@ dependencies { testImplementation(kotlin("test")) testImplementation(libs.bundles.unit.testing) testImplementation(libs.robolectric) + // The BlurHash round-trip test decodes with the real decoder; common-ui is compose-versioned. + testImplementation(platform(libs.compose.bom)) + testImplementation(project(":apps:flipcash:shared:common-ui")) } diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlobStorageCoordinator.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlobStorageCoordinator.kt index dd35938d2e..d7e6de88e4 100644 --- a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlobStorageCoordinator.kt +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlobStorageCoordinator.kt @@ -11,12 +11,15 @@ import androidx.datastore.preferences.preferencesDataStoreFile import com.flipcash.libs.coroutines.DispatcherProvider import com.flipcash.services.controllers.BlobStorageController import com.flipcash.services.models.InitiateExternalUploadError +import com.flipcash.services.models.blob.EncryptedConstraints import com.flipcash.services.models.blob.MimeTypeConstraints import com.flipcash.services.models.blob.UploadPolicy import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.ChatId import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.map @@ -75,12 +78,75 @@ class BlobStorageCoordinator @Inject constructor( * poll are all handled inside the controller. A policy-driven rejection invalidates the cached * policy (the server echoes a newer policy version on such denials). */ - suspend fun upload(bytes: ByteArray, mimeType: String): Result { - val result = blobStorageController.upload(bytes, mimeType) + suspend fun upload( + bytes: ByteArray, + mimeType: String, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { + val result = blobStorageController.upload(bytes, mimeType, onProgress) result.exceptionOrNull()?.let { refreshIfPolicyChanged(it) } return result } + /** + * Stores an encoded chat photo and returns its READY [BlobId], retrying what a retry can fix. + * + * [jpeg] is the encoder's output. With [sealing] it is encrypted for that chat and uploaded as + * opaque bytes; without, it goes up as plain `image/jpeg` for the server to moderate. Each + * attempt reserves afresh, so a retry gets a new blob id and re-seals under it. + * + * Up to [ChatMediaRetry.BACKOFFS].size retries, 1 s, 2 s, then 4 s apart, for failures in + * transit or while finalizing; a refusal that repeating can't change (see + * [ChatMediaRetry.isRetryable]) is returned at once. [onProgress] restarts from zero on a retry. + */ + suspend fun storeChatMedia( + jpeg: ByteArray, + sealing: ChatMediaSealing? = null, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result = withStoreRetries { _ -> + if (sealing != null) { + blobStorageController.uploadSealed(jpeg, sealing.chatId, sealing.seal, onProgress) + } else { + blobStorageController.uploadChatMedia(jpeg, ChatMediaEncoder.UPLOAD_MIME_TYPE, onProgress) + } + } + + /** + * [storeChatMedia] up to the point the bytes are in storage, without waiting for finalization. + * Retries only what happens before that, so a returned id means "stored": a caller that then + * fails to see it finalize re-polls it with [awaitChatMediaReady] instead of uploading again. + * [onAttempt] runs before each attempt, first included, so progress can restart with a retry. + */ + suspend fun storeChatMediaUnfinalized( + jpeg: ByteArray, + sealing: ChatMediaSealing? = null, + onAttempt: (() -> Unit)? = null, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result = withStoreRetries { _ -> + onAttempt?.invoke() + if (sealing != null) { + blobStorageController.storeSealed(jpeg, sealing.chatId, sealing.seal, onProgress) + } else { + blobStorageController.storeChatMedia(jpeg, ChatMediaEncoder.UPLOAD_MIME_TYPE, onProgress) + } + } + + /** Waits for a blob [storeChatMediaUnfinalized] stored to finalize. */ + suspend fun awaitChatMediaReady(blobId: BlobId): Result = + blobStorageController.awaitChatMediaReady(blobId) + + private suspend fun withStoreRetries(attemptStore: suspend (attempt: Int) -> Result): Result { + var attempt = 0 + while (true) { + val result = attemptStore(attempt) + val failure = result.exceptionOrNull() ?: return result + + refreshIfPolicyChanged(failure) + if (!ChatMediaRetry.isRetryable(failure) || attempt >= ChatMediaRetry.BACKOFFS.size) return result + delay(ChatMediaRetry.BACKOFFS[attempt++]) + } + } + suspend fun reset() { dataStore.edit { it.remove(KEY_UPLOAD_POLICY) } } @@ -128,14 +194,16 @@ class BlobStorageCoordinator @Inject constructor( private fun now(): Long = System.currentTimeMillis() companion object { - private val KEY_UPLOAD_POLICY = stringPreferencesKey("cached_upload_policy") + // v2: entries cached before the policy carried `encrypted` would read back as "encrypted + // uploads not allowed" for the rest of their ttl; a new key makes them refetch instead. + private val KEY_UPLOAD_POLICY = stringPreferencesKey("cached_upload_policy_v2") } } /** * On-disk form. [UploadPolicy.ttl] is a [kotlin.time.Duration] (not kotlinx-serializable) so it is * stored as milliseconds; [fetchedAtMillis] stamps when it was cached so freshness can be checked - * against the ttl; [MimeTypeConstraints] is already `@Serializable` and stored as-is. + * against the ttl; [MimeTypeConstraints] and [EncryptedConstraints] are already `@Serializable` and stored as-is. */ @kotlinx.serialization.Serializable private data class CachedUploadPolicy( @@ -143,11 +211,15 @@ private data class CachedUploadPolicy( val ttlMillis: Long, val fetchedAtMillis: Long, val mimeTypeConstraints: List, + // Absent in entries cached before encrypted uploads existed, which read back as "not allowed" + // until the next refresh. + val encrypted: EncryptedConstraints? = null, ) { fun toDomain(): UploadPolicy = UploadPolicy( version = version, ttl = ttlMillis.milliseconds, mimeTypeConstraints = mimeTypeConstraints, + encrypted = encrypted, ) companion object { @@ -156,6 +228,7 @@ private data class CachedUploadPolicy( ttlMillis = policy.ttl.inWholeMilliseconds, fetchedAtMillis = fetchedAtMillis, mimeTypeConstraints = policy.mimeTypeConstraints, + encrypted = policy.encrypted, ) } } diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlurHashEncoder.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlurHashEncoder.kt new file mode 100644 index 0000000000..796bc66c17 --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlurHashEncoder.kt @@ -0,0 +1,127 @@ +package com.flipcash.app.blob + +import kotlin.math.PI +import kotlin.math.abs +import kotlin.math.cos +import kotlin.math.floor +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow +import kotlin.math.sign + +/** + * Encoder for [BlurHash](https://blurha.sh) strings, the counterpart of the decoder in + * `shared/common-ui`. A sealed photo can't be previewed from the server, so its hash is computed + * here from the plaintext and travels inside the encrypted message. + * + * Works on an ARGB [IntArray] so the maths runs on the JVM; [fromThumbnail] is the only part that + * needs a [android.graphics.Bitmap]. + */ +object BlurHashEncoder { + + private const val THUMBNAIL_EDGE = 64 + + /** + * Hash of [pixels] (row-major ARGB, alpha ignored) using [componentsX] × [componentsY] DCT + * components. Returns "" when the input is unusable; a missing preview must never fail a send. + */ + fun encode(pixels: IntArray, width: Int, height: Int, componentsX: Int, componentsY: Int): String { + if (width <= 0 || height <= 0 || pixels.size < width * height) return "" + if (componentsX !in 1..9 || componentsY !in 1..9) return "" + + val factors = Array(componentsX * componentsY) { index -> + factor(pixels, width, height, index % componentsX, index / componentsX) + } + val dc = factors[0] + val ac = factors.drop(1) + + val hash = StringBuilder() + hash.append(encode83((componentsX - 1) + (componentsY - 1) * 9, 1)) + + val maxValue: Float + if (ac.isNotEmpty()) { + val actualMax = ac.maxOf { channels -> channels.maxOf { abs(it) } } + val quantisedMax = floor(actualMax * 166f - 0.5f).toInt().coerceIn(0, 82) + maxValue = (quantisedMax + 1) / 166f + hash.append(encode83(quantisedMax, 1)) + } else { + maxValue = 1f + hash.append(encode83(0, 1)) + } + + hash.append(encode83(encodeDc(dc), 4)) + for (factor in ac) hash.append(encode83(encodeAc(factor, maxValue), 2)) + return hash.toString() + } + + /** + * Hash of an image of [width] × [height] from a [THUMBNAIL_EDGE]-px thumbnail of [source], + * 4×3 components for landscape (and square) and 3×4 for portrait. "" on any failure. + */ + fun fromThumbnail(source: android.graphics.Bitmap, width: Int, height: Int): String = + runCatching { + val scale = min(1f, THUMBNAIL_EDGE.toFloat() / max(source.width, source.height)) + val tw = max((source.width * scale).toInt(), 1) + val th = max((source.height * scale).toInt(), 1) + val thumb = android.graphics.Bitmap.createScaledBitmap(source, tw, th, true) + val pixels = IntArray(tw * th) + thumb.getPixels(pixels, 0, tw, 0, 0, tw, th) + if (thumb !== source) thumb.recycle() + val landscape = width >= height + encode(pixels, tw, th, if (landscape) 4 else 3, if (landscape) 3 else 4) + }.getOrDefault("") + + private fun factor(pixels: IntArray, width: Int, height: Int, i: Int, j: Int): FloatArray { + var r = 0f + var g = 0f + var b = 0f + val normalisation = if (i == 0 && j == 0) 1f else 2f + for (y in 0 until height) { + val basisY = cos(PI * j * y / height).toFloat() + for (x in 0 until width) { + val basis = normalisation * cos(PI * i * x / width).toFloat() * basisY + val pixel = pixels[y * width + x] + r += basis * srgbToLinear(pixel shr 16 and 255) + g += basis * srgbToLinear(pixel shr 8 and 255) + b += basis * srgbToLinear(pixel and 255) + } + } + val scale = 1f / (width * height) + return floatArrayOf(r * scale, g * scale, b * scale) + } + + private fun encodeDc(value: FloatArray): Int = + (linearToSrgb(value[0]) shl 16) or (linearToSrgb(value[1]) shl 8) or linearToSrgb(value[2]) + + private fun encodeAc(value: FloatArray, maxValue: Float): Int { + fun quantise(v: Float): Int = + floor(signPow(v / maxValue, 0.5f) * 9f + 9.5f).toInt().coerceIn(0, 18) + return quantise(value[0]) * 19 * 19 + quantise(value[1]) * 19 + quantise(value[2]) + } + + private fun signPow(value: Float, exp: Float): Float = abs(value).pow(exp) * sign(value) + + private fun srgbToLinear(value: Int): Float { + val v = value / 255f + return if (v <= 0.04045f) v / 12.92f else ((v + 0.055f) / 1.055f).pow(2.4f) + } + + private fun linearToSrgb(value: Float): Int { + val v = value.coerceIn(0f, 1f) + val srgb = if (v <= 0.0031308f) v * 12.92f else 1.055f * v.pow(1f / 2.4f) - 0.055f + return (srgb * 255f + 0.5f).toInt() + } + + private fun encode83(value: Int, length: Int): String { + val out = CharArray(length) + var remaining = value + for (i in length - 1 downTo 0) { + out[i] = CHARS[remaining % 83] + remaining /= 83 + } + return String(out) + } + + private const val CHARS = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#\$%*+,-.:;=?@[]^_{|}~" +} diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaConstraints.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaConstraints.kt new file mode 100644 index 0000000000..d903d3fb09 --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaConstraints.kt @@ -0,0 +1,29 @@ +package com.flipcash.app.blob + +/** Picks which upload-policy entry governs a MIME type. */ +object ChatMediaConstraints { + + /** + * Index of the first entry in [patterns], in policy order, that matches [mimeType]: the + * catch-all pattern matches anything, a type wildcard matches on the type alone, and anything + * else is an exact `type/subtype`. + * Null when [mimeType] has no `/` or nothing matches. + * + * Policy order is authoritative — a later, more specific entry never overrides an earlier + * catch-all. Must agree with iOS (`test-vectors/chat_media.json`, `constraintSelection`). + */ + fun firstMatchIndex(patterns: List, mimeType: String): Int? { + val slash = mimeType.indexOf('/') + if (slash < 0) return null + val type = mimeType.substring(0, slash) + + val index = patterns.indexOfFirst { pattern -> + when { + pattern == "*/*" -> true + pattern.endsWith("/*") -> pattern.dropLast(2).equals(type, ignoreCase = true) + else -> pattern.equals(mimeType, ignoreCase = true) + } + } + return index.takeIf { it >= 0 } + } +} diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaDownscale.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaDownscale.kt new file mode 100644 index 0000000000..55164b4b25 --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaDownscale.kt @@ -0,0 +1,40 @@ +package com.flipcash.app.blob + +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +data class PixelSize(val width: Int, val height: Int) + +/** + * The pixel size a photo is scaled to before encoding. A bound of 0 means unbounded, and an image + * is never scaled up. + */ +object ChatMediaDownscale { + + /** + * [width] and [height] are display-space — after the EXIF rotation has been applied. + * + * The scale is computed in doubles and truncated, which can land a pixel or two over + * [maxPixels], so the longer side is then walked down until the area fits. Must agree with iOS + * (`test-vectors/chat_media.json`, `downscale`) to the pixel. + */ + fun target(width: Int, height: Int, maxWidth: Int, maxHeight: Int, maxPixels: Long): PixelSize { + if (width <= 0 || height <= 0) return PixelSize(max(width, 1), max(height, 1)) + + var scale = 1.0 + if (maxWidth > 0) scale = min(scale, maxWidth.toDouble() / width) + if (maxHeight > 0) scale = min(scale, maxHeight.toDouble() / height) + if (maxPixels > 0) scale = min(scale, sqrt(maxPixels.toDouble() / (width.toDouble() * height))) + + var w = max((width * scale).toInt(), 1) + var h = max((height * scale).toInt(), 1) + + if (maxPixels > 0) { + while (w.toLong() * h > maxPixels && (w > 1 || h > 1)) { + if (w >= h) w-- else h-- + } + } + return PixelSize(w, h) + } +} diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaEncoder.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaEncoder.kt new file mode 100644 index 0000000000..c012807922 --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaEncoder.kt @@ -0,0 +1,102 @@ +package com.flipcash.app.blob + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.ImageDecoder +import android.net.Uri +import com.flipcash.libs.coroutines.DispatcherProvider +import com.flipcash.services.internal.extensions.withoutJpegMetadata +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Turns a picked or captured photo into the JPEG a chat message uploads: decoded with its EXIF + * rotation applied, scaled under the policy's bounds, flattened onto white, and compressed at the + * highest quality that fits the byte cap. + * + * The result is always `image/jpeg`. There is no fallback to another type and, for a sealed + * upload, none to plaintext — a photo that can't be made to fit is refused. + */ +@Singleton +class ChatMediaEncoder @Inject constructor( + @param:ApplicationContext private val context: Context, + private val dispatchers: DispatcherProvider, +) { + /** [bytes] are metadata-free; [width] × [height] are display-space pixels. */ + class Encoded(val bytes: ByteArray, val width: Int, val height: Int) + + suspend fun encode(uri: Uri, limits: ChatMediaLimits): Result = withContext(dispatchers.IO) { + val source = decode(uri, limits) + .getOrElse { return@withContext Result.failure(ChatMediaEncodingException.EncodingFailed(it)) } + try { + JpegLadder.select(cap = limits.maxBytes) { quality -> source.compressJpeg(quality) } + .map { Encoded(it, source.width, source.height) } + } finally { + source.recycle() + } + } + + /** + * [ImageDecoder] applies the EXIF orientation and reports `info.size` in that display space, + * so the target is computed from it as-is and [Bitmap.getWidth] is what ships as the width. + */ + private fun decode(uri: Uri, limits: ChatMediaLimits): Result = runCatching { + val decoded = ImageDecoder.decodeBitmap(ImageDecoder.createSource(context.contentResolver, uri)) { decoder, info, _ -> + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + val target = ChatMediaDownscale.target( + width = info.size.width, + height = info.size.height, + maxWidth = limits.maxWidth, + maxHeight = limits.maxHeight, + maxPixels = limits.maxPixels, + ) + decoder.setTargetSize(target.width, target.height) + } + // JPEG has no alpha; compositing on white keeps a transparent PNG from going black. + val flattened = Bitmap.createBitmap(decoded.width, decoded.height, Bitmap.Config.ARGB_8888) + Canvas(flattened).apply { + drawColor(Color.WHITE) + drawBitmap(decoded, 0f, 0f, null) + } + decoded.recycle() + flattened + } + + private fun Bitmap.compressJpeg(quality: Int): ByteArray? { + val out = ByteArrayOutputStream() + return if (compress(Bitmap.CompressFormat.JPEG, quality, out)) out.toByteArray() else null + } + + companion object { + const val UPLOAD_MIME_TYPE = "image/jpeg" + } +} + +/** The quality ladder, and the rule for choosing a rung. Pure so the choice is JVM-testable. */ +internal object JpegLadder { + val QUALITIES = listOf(90, 80, 70, 60) + + /** + * Walks [QUALITIES] from the top and returns the first encode whose size, measured after the + * metadata strip the upload applies, is at most [cap] (<= 0 means no cap). + * [TooLarge][ChatMediaEncodingException.TooLarge] when something encoded but nothing fit; + * [EncodingFailed][ChatMediaEncodingException.EncodingFailed] when nothing encoded at all. + */ + fun select(cap: Long, encode: (quality: Int) -> ByteArray?): Result { + var encodedAny = false + for (quality in QUALITIES) { + val bytes = encode(quality)?.withoutJpegMetadata() ?: continue + encodedAny = true + if (cap <= 0 || bytes.size <= cap) return Result.success(bytes) + } + return Result.failure( + if (encodedAny) ChatMediaEncodingException.TooLarge() + else ChatMediaEncodingException.EncodingFailed() + ) + } +} diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaLimits.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaLimits.kt new file mode 100644 index 0000000000..f0907a6d5d --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaLimits.kt @@ -0,0 +1,75 @@ +package com.flipcash.app.blob + +import com.flipcash.services.controllers.BlobStorageController +import com.flipcash.services.models.blob.UploadPolicy + +/** + * What the server will accept for one chat photo upload, resolved from the cached + * [UploadPolicy]. A bound of 0 means unbounded. + * + * [maxBytes] is a cap on the *encoded JPEG*, already net of any framing the upload adds, so the + * encoder compares against it directly. + */ +data class ChatMediaLimits( + val maxBytes: Long, + val maxWidth: Int, + val maxHeight: Int, + val maxPixels: Long, +) { + companion object { + /** + * Limits for a photo sealed for a chat: the policy's encrypted image bounds, and its size + * ceiling less [BlobStorageController.SEAL_OVERHEAD] — the ceiling applies to the sealed + * bytes, the encoder measures the plaintext. + */ + fun sealed(policy: UploadPolicy): Result { + val encrypted = policy.encrypted + ?: return Result.failure(ChatMediaEncodingException.EncryptionNotAllowed()) + return Result.success( + ChatMediaLimits( + maxBytes = encrypted.maxSizeBytes - BlobStorageController.SEAL_OVERHEAD, + maxWidth = encrypted.image?.maxWidth ?: 0, + maxHeight = encrypted.image?.maxHeight ?: 0, + maxPixels = encrypted.image?.maxPixels ?: 0, + ) + ) + } + + /** The entry the policy names for [ChatMediaEncoder.UPLOAD_MIME_TYPE], in policy order. */ + fun plain(policy: UploadPolicy): Result { + val index = ChatMediaConstraints.firstMatchIndex( + patterns = policy.mimeTypeConstraints.map { it.mimeTypePattern }, + mimeType = ChatMediaEncoder.UPLOAD_MIME_TYPE, + ) ?: return Result.failure(ChatMediaEncodingException.NoMatchingConstraint()) + + val constraint = policy.mimeTypeConstraints[index] + return Result.success( + ChatMediaLimits( + maxBytes = constraint.maxSizeBytes, + maxWidth = constraint.image?.maxWidth ?: 0, + maxHeight = constraint.image?.maxHeight ?: 0, + maxPixels = constraint.image?.maxPixels ?: 0, + ) + ) + } + } +} + +/** Why a photo could not be turned into an uploadable JPEG. None of these are worth retrying. */ +sealed class ChatMediaEncodingException(message: String) : Exception(message) { + /** The policy has no entry for `image/jpeg`. */ + class NoMatchingConstraint : ChatMediaEncodingException("Policy has no entry for image/jpeg") + + /** The policy carries no end-to-end-encrypted constraints, so a sealed upload isn't allowed. */ + class EncryptionNotAllowed : ChatMediaEncodingException("Policy does not allow encrypted uploads") + + /** Even the lowest-quality encode is over the byte cap. */ + class TooLarge : ChatMediaEncodingException("Photo is too large at every quality") + + /** The source couldn't be decoded, or no quality produced bytes. */ + class EncodingFailed(cause: Throwable? = null) : ChatMediaEncodingException("Photo could not be encoded") { + init { + cause?.let { initCause(it) } + } + } +} diff --git a/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaRetry.kt b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaRetry.kt new file mode 100644 index 0000000000..ce76002d9f --- /dev/null +++ b/apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/ChatMediaRetry.kt @@ -0,0 +1,41 @@ +package com.flipcash.app.blob + +import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.CompleteExternalUploadError +import com.flipcash.services.models.GetBlobsError +import com.flipcash.services.models.InitiateExternalUploadError +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.RejectionReason +import kotlin.time.Duration.Companion.seconds + +/** Seals a photo for [chatId]; [seal] gets the reserved blob id, which the cipher binds to. */ +class ChatMediaSealing( + val chatId: ChatId, + val seal: suspend (blobId: BlobId) -> ByteArray, +) + +/** Which failed chat-photo stores are worth another attempt. */ +object ChatMediaRetry { + /** Waits before the first, second and third retry. */ + val BACKOFFS = listOf(1.seconds, 2.seconds, 4.seconds) + + /** + * False for a refusal that repeating can't change: the server denied it, said the type or size + * is wrong, or the user is out of quota; or moderation rejected the photo. Everything else — + * transport errors, a non-2xx from storage, finalization that timed out or was rejected for + * another reason, and failures nobody classified — is retried. + */ + fun isRetryable(cause: Throwable): Boolean = when (cause) { + is InitiateExternalUploadError.Denied, + is InitiateExternalUploadError.UnsupportedType, + is InitiateExternalUploadError.TooLarge, + is InitiateExternalUploadError.QuotaExceeded, + is CompleteExternalUploadError.NotFound, + is CompleteExternalUploadError.NotUploaded, + is GetBlobsError.Denied, + is ChatMediaEncodingException -> false + is BlobRejectedException -> cause.rejection.reason != RejectionReason.MODERATION + else -> true + } +} diff --git a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlobStorageCoordinatorTest.kt b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlobStorageCoordinatorTest.kt index 8d1810eb87..88d1dbd11c 100644 --- a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlobStorageCoordinatorTest.kt +++ b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlobStorageCoordinatorTest.kt @@ -5,6 +5,7 @@ import androidx.test.core.app.ApplicationProvider import com.flipcash.libs.coroutines.DispatcherProvider import com.flipcash.services.controllers.BlobStorageController import com.flipcash.services.models.InitiateExternalUploadError +import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.blob.MimeTypeConstraints import com.flipcash.services.models.blob.UploadPolicy import io.mockk.coEvery @@ -99,4 +100,47 @@ class BlobStorageCoordinatorTest { coVerify(exactly = 1) { controller.getUploadPolicy() } } + + @Test + fun `an unfinalized store retries transport failures on the backoff and announces each attempt`() = runTest { + val id = BlobId(ByteArray(16) { 1 }) + coEvery { controller.storeChatMedia(any(), any(), any()) } returnsMany listOf( + Result.failure(RuntimeException("offline")), + Result.failure(RuntimeException("offline")), + Result.success(id), + ) + var attempts = 0 + + val result = newCoordinator().storeChatMediaUnfinalized(byteArrayOf(1), onAttempt = { attempts++ }) + + assertEquals(id, result.getOrNull()) + assertEquals(3, attempts) + assertEquals(3_000L, testScheduler.currentTime) + coVerify(exactly = 0) { controller.awaitChatMediaReady(any()) } + } + + @Test + fun `an unfinalized store gives up after the third retry`() = runTest { + coEvery { controller.storeChatMedia(any(), any(), any()) } returns Result.failure(RuntimeException("offline")) + + val result = newCoordinator().storeChatMediaUnfinalized(byteArrayOf(1)) + + assertEquals(true, result.isFailure) + coVerify(exactly = 4) { controller.storeChatMedia(any(), any(), any()) } + assertEquals(7_000L, testScheduler.currentTime) + } + + @Test + fun `an unfinalized store does not retry a refusal`() = runTest { + coEvery { controller.getUploadPolicy() } returns Result.success(policy("v1", 1.hours)) + coEvery { controller.storeChatMedia(any(), any(), any()) } returns + Result.failure(InitiateExternalUploadError.TooLarge(policyVersion = "v1")) + val coordinator = newCoordinator() + coordinator.reset() + coordinator.preloadPolicy() + + coordinator.storeChatMediaUnfinalized(byteArrayOf(1)) + + coVerify(exactly = 1) { controller.storeChatMedia(any(), any(), any()) } + } } diff --git a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlurHashEncoderTest.kt b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlurHashEncoderTest.kt new file mode 100644 index 0000000000..7d91d6b3da --- /dev/null +++ b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/BlurHashEncoderTest.kt @@ -0,0 +1,77 @@ +package com.flipcash.app.blob + +import android.graphics.Bitmap +import com.flipcash.shared.common.ui.BlurHash +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import kotlin.math.abs +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +@RunWith(RobolectricTestRunner::class) +class BlurHashEncoderTest { + + private fun solid(rgb: Int, w: Int = 8, h: Int = 8) = IntArray(w * h) { (0xFF shl 24) or rgb } + + @Test + fun `hash length follows the component count`() { + assertEquals(4 + 2 * 4 * 3, BlurHashEncoder.encode(solid(0x336699), 8, 8, 4, 3).length) + assertEquals(4 + 2 * 3 * 4, BlurHashEncoder.encode(solid(0x336699), 8, 8, 3, 4).length) + } + + @Test + fun `a solid colour round-trips through the average colour`() { + for (rgb in listOf(0x000000, 0xFFFFFF, 0x336699, 0xE8552B)) { + val hash = BlurHashEncoder.encode(solid(rgb), 8, 8, 4, 3) + + val average = assertNotNull(BlurHash.averageColor(hash)) + for (shift in listOf(16, 8, 0)) { + assertTrue( + abs((average shr shift and 255) - (rgb shr shift and 255)) <= 1, + "channel at $shift of ${rgb.toString(16)} came back ${average.toString(16)}", + ) + } + } + } + + @Test + fun `a gradient round-trips through the decoder`() { + val w = 32 + val h = 24 + val pixels = IntArray(w * h) { i -> + val x = i % w + val r = x * 255 / (w - 1) + (0xFF shl 24) or (r shl 16) or ((255 - r) shl 8) or 0x40 + } + + val hash = BlurHashEncoder.encode(pixels, w, h, 4, 3) + val decoded = assertNotNull(BlurHash.decode(hash, w, h)) + + val out = IntArray(w * h).also { decoded.getPixels(it, 0, w, 0, 0, w, h) } + val meanError = pixels.indices.sumOf { i -> + listOf(16, 8, 0).sumOf { s -> abs((pixels[i] shr s and 255) - (out[i] shr s and 255)) } + } / (pixels.size * 3.0) + // 12 DCT components of a horizontal ramp: soft, but nowhere near a different image. + assertTrue(meanError < 20, "mean channel error $meanError") + } + + @Test + fun `unusable input yields an empty hash`() { + assertEquals("", BlurHashEncoder.encode(IntArray(0), 0, 0, 4, 3)) + assertEquals("", BlurHashEncoder.encode(IntArray(4), 4, 4, 4, 3)) + assertEquals("", BlurHashEncoder.encode(solid(0), 8, 8, 10, 3)) + } + + @Test + fun `thumbnail hash is 4x3 for landscape and 3x4 for portrait`() { + val landscape = Bitmap.createBitmap(200, 100, Bitmap.Config.ARGB_8888) + val portrait = Bitmap.createBitmap(100, 200, Bitmap.Config.ARGB_8888) + + assertEquals(4 + 2 * 12, BlurHashEncoder.fromThumbnail(landscape, 200, 100).length) + assertEquals("L".single(), BlurHashEncoder.fromThumbnail(landscape, 200, 100)[0]) + // size flag = (cx-1) + (cy-1)*9: 4x3 -> 21 -> 'L', 3x4 -> 29 -> 'T' + assertEquals('T', BlurHashEncoder.fromThumbnail(portrait, 100, 200)[0]) + } +} diff --git a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaEncodingTest.kt b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaEncodingTest.kt new file mode 100644 index 0000000000..7d0a86134a --- /dev/null +++ b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaEncodingTest.kt @@ -0,0 +1,101 @@ +package com.flipcash.app.blob + +import com.flipcash.services.models.blob.ImageConstraints +import com.flipcash.services.models.blob.MimeTypeConstraints +import com.flipcash.services.models.blob.UploadPolicy +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.time.Duration.Companion.hours + +class ChatMediaEncodingTest { + + private fun policy(vararg constraints: MimeTypeConstraints) = + UploadPolicy(version = "v1", ttl = 1.hours, mimeTypeConstraints = constraints.toList()) + + @Test + fun `plain limits come from the first entry matching image-jpeg`() { + val limits = ChatMediaLimits.plain( + policy( + MimeTypeConstraints("image/png", 1, null), + MimeTypeConstraints("image/*", 5_000, ImageConstraints(2048, 1024, 3_000_000)), + MimeTypeConstraints("*/*", 9, null), + ) + ).getOrThrow() + + assertEquals(ChatMediaLimits(5_000, 2048, 1024, 3_000_000), limits) + } + + @Test + fun `plain limits without image bounds are unbounded`() { + val limits = ChatMediaLimits.plain(policy(MimeTypeConstraints("image/jpeg", 700, null))).getOrThrow() + + assertEquals(ChatMediaLimits(700, 0, 0, 0), limits) + } + + @Test + fun `plain limits fail when no entry matches`() { + val result = ChatMediaLimits.plain(policy(MimeTypeConstraints("video/*", 1, null))) + + assertIs(result.exceptionOrNull()) + } + + @Test + fun `ladder returns the first quality that fits`() { + val sizes = mapOf(90 to 900, 80 to 700, 70 to 500, 60 to 300) + val tried = mutableListOf() + + val result = JpegLadder.select(cap = 700) { q -> tried += q; ByteArray(sizes.getValue(q)) } + + assertEquals(700, result.getOrThrow().size) + assertEquals(listOf(90, 80), tried) + } + + @Test + fun `ladder is TooLarge when nothing fits`() { + val result = JpegLadder.select(cap = 100) { ByteArray(200) } + + assertIs(result.exceptionOrNull()) + } + + @Test + fun `ladder is EncodingFailed when nothing encodes`() { + val result = JpegLadder.select(cap = 100) { null } + + assertIs(result.exceptionOrNull()) + } + + @Test + fun `ladder skips a failed rung and keeps going`() { + val result = JpegLadder.select(cap = 100) { q -> if (q == 90) null else ByteArray(50) } + + assertEquals(50, result.getOrThrow().size) + } + + @Test + fun `ladder measures the size after the metadata strip`() { + // APP1 (EXIF) segment of 22 bytes on a minimal JPEG; stripped it shrinks under the cap. + val exif = byteArrayOf(0xFF.toByte(), 0xE1.toByte(), 0x00, 0x14) + ByteArray(18) + val jpeg = byteArrayOf(0xFF.toByte(), 0xD8.toByte()) + exif + + byteArrayOf(0xFF.toByte(), 0xDA.toByte(), 0x00, 0x02, 0xFF.toByte(), 0xD9.toByte()) + + val result = JpegLadder.select(cap = (jpeg.size - exif.size).toLong()) { jpeg } + + assertContentEquals( + byteArrayOf(0xFF.toByte(), 0xD8.toByte(), 0xFF.toByte(), 0xDA.toByte(), 0x00, 0x02, 0xFF.toByte(), 0xD9.toByte()), + result.getOrThrow(), + ) + } + + @Test + fun `a non-positive cap is unbounded`() { + assertEquals(10, JpegLadder.select(cap = 0) { ByteArray(10) }.getOrThrow().size) + } + + @Test + fun `non-slash mime type never matches`() { + assertNull(ChatMediaConstraints.firstMatchIndex(listOf("*/*"), "jpeg")) + } +} diff --git a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaRetryTest.kt b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaRetryTest.kt new file mode 100644 index 0000000000..9ae10e9fcd --- /dev/null +++ b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaRetryTest.kt @@ -0,0 +1,63 @@ +package com.flipcash.app.blob + +import com.flipcash.services.models.BlobNotReadyException +import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.CompleteExternalUploadError +import com.flipcash.services.models.GetBlobsError +import com.flipcash.services.models.InitiateExternalUploadError +import com.flipcash.services.models.ModerationResult +import com.flipcash.services.models.chat.BlobRejection +import com.flipcash.services.models.chat.RejectionReason +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.seconds + +class ChatMediaRetryTest { + + private fun rejected(reason: RejectionReason) = + BlobRejectedException(BlobRejection(reason, ModerationResult.FlaggedCategory.NONE)) + + @Test + fun `backoff is one, two, then four seconds`() { + assertEquals(listOf(1.seconds, 2.seconds, 4.seconds), ChatMediaRetry.BACKOFFS) + } + + @Test + fun `refusals that repeating cannot change are not retried`() { + listOf( + InitiateExternalUploadError.Denied(), + InitiateExternalUploadError.UnsupportedType(), + InitiateExternalUploadError.TooLarge(), + InitiateExternalUploadError.QuotaExceeded(), + CompleteExternalUploadError.NotFound(), + CompleteExternalUploadError.NotUploaded(), + GetBlobsError.Denied(), + ).forEach { assertFalse(ChatMediaRetry.isRetryable(it), it::class.simpleName) } + } + + @Test + fun `transport, timeout and unclassified failures are retried`() { + listOf( + java.io.IOException("offline"), + BlobNotReadyException(), + RuntimeException("unknown"), + InitiateExternalUploadError.Other(RuntimeException("x")), + ).forEach { assertTrue(ChatMediaRetry.isRetryable(it), it::class.simpleName) } + } + + @Test + fun `moderation rejection is final, other rejections are retried`() { + assertFalse(ChatMediaRetry.isRetryable(rejected(RejectionReason.MODERATION))) + RejectionReason.entries.filter { it != RejectionReason.MODERATION }.forEach { + assertTrue(ChatMediaRetry.isRetryable(rejected(it)), it.name) + } + } + + @Test + fun `encoding failures are final`() { + assertFalse(ChatMediaRetry.isRetryable(ChatMediaEncodingException.TooLarge())) + assertFalse(ChatMediaRetry.isRetryable(ChatMediaEncodingException.EncodingFailed())) + } +} diff --git a/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaVectorTest.kt b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaVectorTest.kt new file mode 100644 index 0000000000..6b62074218 --- /dev/null +++ b/apps/flipcash/shared/blob/src/test/kotlin/com/flipcash/app/blob/ChatMediaVectorTest.kt @@ -0,0 +1,86 @@ +package com.flipcash.app.blob + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.double +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * `test-vectors/chat_media.json` (git blob 54f976f2737e37811af0d9bba5a849e7ba3f7ade). The canonical + * copy lives in the orchestrator repo and is byte-identical to iOS's; this one is synced. A failure + * here is a real regression or a cross-platform decision to make in the canonical fixture, never a + * local edit. + * + * Covers dimensions and policy choice, not encoded bytes, which differ by platform. + */ +class ChatMediaVectorTest { + + private val root: JsonObject = Json.parseToJsonElement( + javaClass.classLoader!!.getResourceAsStream("chat_media.json")!! + .bufferedReader().use { it.readText() } + ).jsonObject + + private fun cases(key: String): JsonArray = root.getValue(key).jsonArray + + @Test + fun `downscale matches the vectors`() { + val cases = cases("downscale") + // A fixture that failed to load reads as empty, and a loop over nothing passes. + assertTrue(cases.isNotEmpty(), "chat_media.json has no downscale vectors") + + for (case in cases.map { it.jsonObject }) { + val name = case.getValue("name").jsonPrimitive.content + val expected = case.getValue("expected").jsonObject + val actual = ChatMediaDownscale.target( + width = case.getValue("sourceWidth").jsonPrimitive.int, + height = case.getValue("sourceHeight").jsonPrimitive.int, + maxWidth = case.getValue("maxWidth").jsonPrimitive.int, + maxHeight = case.getValue("maxHeight").jsonPrimitive.int, + maxPixels = case.getValue("maxPixels").jsonPrimitive.long, + ) + assertEquals( + PixelSize(expected.getValue("width").jsonPrimitive.int, expected.getValue("height").jsonPrimitive.int), + actual, + "downscale: $name", + ) + } + } + + @Test + fun `constraint selection matches the vectors`() { + val cases = cases("constraintSelection") + assertTrue(cases.isNotEmpty(), "chat_media.json has no constraintSelection vectors") + + for (case in cases.map { it.jsonObject }) { + val name = case.getValue("name").jsonPrimitive.content + val expected = case.getValue("expectedIndex") + val actual = ChatMediaConstraints.firstMatchIndex( + patterns = case.getValue("patterns").jsonArray.map { it.jsonPrimitive.content }, + mimeType = case.getValue("mimeType").jsonPrimitive.content, + ) + if (expected is JsonNull) { + assertNull(actual, "constraintSelection: $name") + } else { + assertEquals(expected.jsonPrimitive.int, actual, "constraintSelection: $name") + } + } + } + + @Test + fun `quality ladder and upload mime type match the vectors`() { + val ladder = root.getValue("jpegQualityLadder").jsonArray.map { (it.jsonPrimitive.double * 100).let(Math::round).toInt() } + assertEquals(ladder, JpegLadder.QUALITIES) + assertEquals(root.getValue("uploadMimeType").jsonPrimitive.contentOrNull, ChatMediaEncoder.UPLOAD_MIME_TYPE) + } +} diff --git a/apps/flipcash/shared/blob/src/test/resources/chat_media.json b/apps/flipcash/shared/blob/src/test/resources/chat_media.json new file mode 100644 index 0000000000..54f976f273 --- /dev/null +++ b/apps/flipcash/shared/blob/src/test/resources/chat_media.json @@ -0,0 +1,639 @@ +{ + "algorithm": "chat-media-photos", + "note": "Behavior fixture for docs/superpowers/specs/2026-09-28-chat-media-photos-design.md. Heights are points or dp as floats; compare with a 0.001 tolerance.", + "maxAttachments": 10, + "minAspect": 0.5, + "maxAspect": 2.0, + "jpegQualityLadder": [ + 0.9, + 0.8, + 0.7, + 0.6 + ], + "uploadMimeType": "image/jpeg", + "fanOut": [ + { + "name": "text-only", + "chips": [], + "text": "hello", + "replyTo": null, + "messages": [ + { + "kind": "text", + "chip": null, + "caption": null, + "text": "hello", + "replyTo": null + } + ], + "note": "No chips: a text message, exactly as today." + }, + { + "name": "text-only-reply", + "chips": [], + "text": "hello", + "replyTo": "m1", + "messages": [ + { + "kind": "text", + "chip": null, + "caption": null, + "text": "hello", + "replyTo": "m1" + } + ], + "note": "No chips, with a reply: a text reply, exactly as today." + }, + { + "name": "empty", + "chips": [], + "text": "", + "replyTo": null, + "messages": [], + "note": "Nothing to send. Send is disabled, so this pins that the builder agrees." + }, + { + "name": "one-photo", + "chips": [ + "a" + ], + "text": "", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "One chip, no text: one media message with no caption." + }, + { + "name": "one-photo-caption", + "chips": [ + "a" + ], + "text": "look", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": "look", + "text": null, + "replyTo": null + } + ], + "note": "One chip with text: the text is the caption." + }, + { + "name": "three-photos-caption", + "chips": [ + "a", + "b", + "c" + ], + "text": "from today", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": "from today", + "text": null, + "replyTo": null + } + ], + "note": "The caption rides on the last message, so it lands under the last image." + }, + { + "name": "three-photos-reply", + "chips": [ + "a", + "b", + "c" + ], + "text": "", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": "m1" + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "The reply attaches to the first message only." + }, + { + "name": "three-photos-caption-reply", + "chips": [ + "a", + "b", + "c" + ], + "text": "these?", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": "m1" + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": "these?", + "text": null, + "replyTo": null + } + ], + "note": "Reply on the first, caption on the last." + }, + { + "name": "one-photo-caption-reply", + "chips": [ + "a" + ], + "text": "this one", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": "this one", + "text": null, + "replyTo": "m1" + } + ], + "note": "With one chip, the same message carries both reply and caption." + }, + { + "name": "ten-photos", + "chips": [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j" + ], + "text": "", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "d", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "e", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "f", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "g", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "h", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "i", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "j", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "The composer maximum. Order is chip order." + } + ], + "bubble": [ + { + "name": "square", + "imageWidth": 1000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 240.0, + "cropped": false + }, + "note": "1:1 fills the width, same height." + }, + { + "name": "landscape-4-3", + "imageWidth": 4032, + "imageHeight": 3024, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 180.0, + "cropped": false + }, + "note": "Landscape phone photo." + }, + { + "name": "portrait-3-4", + "imageWidth": 3024, + "imageHeight": 4032, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 320.0, + "cropped": false + }, + "note": "Portrait phone photo." + }, + { + "name": "wide-at-limit", + "imageWidth": 2000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 120.0, + "cropped": false + }, + "note": "Exactly 1:2 is not cropped." + }, + { + "name": "panorama", + "imageWidth": 8000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 120.0, + "cropped": true + }, + "note": "Wider than 1:2 clamps to 1:2 and crops." + }, + { + "name": "tall-at-limit", + "imageWidth": 1000, + "imageHeight": 2000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 480.0, + "cropped": false + }, + "note": "Exactly 2:1 is not cropped." + }, + { + "name": "screenshot", + "imageWidth": 1179, + "imageHeight": 2556, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 480.0, + "cropped": true + }, + "note": "An iPhone screenshot is taller than 2:1 and crops." + }, + { + "name": "tiny-upscaled", + "imageWidth": 40, + "imageHeight": 30, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 180.0, + "cropped": false + }, + "note": "Small images still take the full width. The bubble never shrinks to the image." + }, + { + "name": "missing-metadata", + "imageWidth": 0, + "imageHeight": 0, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 240.0, + "cropped": false + }, + "note": "No dimensions (should not happen after READY): a square placeholder." + }, + { + "name": "android-dp", + "imageWidth": 4032, + "imageHeight": 3024, + "maxWidth": 264.5, + "expected": { + "width": 264.5, + "height": 198.375, + "cropped": false + }, + "note": "Fractional widths are fine; compare with 0.001 tolerance." + } + ], + "downscale": [ + { + "name": "within-bounds", + "sourceWidth": 1200, + "sourceHeight": 900, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 1200, + "height": 900 + }, + "note": "Already fits: unchanged." + }, + { + "name": "edge-bound-landscape", + "sourceWidth": 4032, + "sourceHeight": 3024, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 2048, + "height": 1536 + }, + "note": "Long edge binds." + }, + { + "name": "edge-bound-portrait", + "sourceWidth": 3024, + "sourceHeight": 4032, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 1536, + "height": 2048 + }, + "note": "Long edge binds, portrait." + }, + { + "name": "asymmetric-bounds", + "sourceWidth": 4000, + "sourceHeight": 4000, + "maxWidth": 3000, + "maxHeight": 1000, + "maxPixels": 0, + "expected": { + "width": 1000, + "height": 1000 + }, + "note": "The tighter of width and height binds." + }, + { + "name": "pixels-bind-first", + "sourceWidth": 4032, + "sourceHeight": 3024, + "maxWidth": 4096, + "maxHeight": 4096, + "maxPixels": 4000000, + "expected": { + "width": 2309, + "height": 1732 + }, + "note": "Edges would allow the original; max_pixels binds instead." + }, + { + "name": "pixels-and-edges", + "sourceWidth": 8000, + "sourceHeight": 6000, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 2000000, + "expected": { + "width": 1632, + "height": 1224 + }, + "note": "max_pixels is tighter than the edge limit here." + }, + { + "name": "unbounded", + "sourceWidth": 5000, + "sourceHeight": 3000, + "maxWidth": 0, + "maxHeight": 0, + "maxPixels": 0, + "expected": { + "width": 5000, + "height": 3000 + }, + "note": "No constraints: unchanged." + }, + { + "name": "never-upscale", + "sourceWidth": 300, + "sourceHeight": 200, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 16000000, + "expected": { + "width": 300, + "height": 200 + }, + "note": "Small source: unchanged." + }, + { + "name": "panorama-min-edge", + "sourceWidth": 20000, + "sourceHeight": 10, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 2048, + "height": 1 + }, + "note": "An edge never goes below 1." + }, + { + "name": "odd-pixels", + "sourceWidth": 3001, + "sourceHeight": 2999, + "maxWidth": 0, + "maxHeight": 0, + "maxPixels": 1000000, + "expected": { + "width": 1000, + "height": 999 + }, + "note": "Floor rounding still fits max_pixels." + } + ], + "constraintSelection": [ + { + "name": "exact-first", + "patterns": [ + "image/jpeg", + "image/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 0, + "note": "Exact match wins when listed first." + }, + { + "name": "wildcard-image", + "patterns": [ + "image/png", + "image/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 1, + "note": "image/png does not match; image/* does." + }, + { + "name": "catch-all", + "patterns": [ + "video/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 1, + "note": "Only */* matches." + }, + { + "name": "none", + "patterns": [ + "video/*", + "application/pdf" + ], + "mimeType": "image/jpeg", + "expectedIndex": null, + "note": "No entry: do not upload." + }, + { + "name": "policy-order-wins", + "patterns": [ + "*/*", + "image/jpeg" + ], + "mimeType": "image/jpeg", + "expectedIndex": 0, + "note": "Policy order is authoritative, even if a later entry is more specific." + } + ], + "strings": [ + { + "name": "no-caption", + "caption": null, + "expected": { + "snippet": "Photo", + "preview": "📷 Photo" + }, + "note": "No caption: the word Photo." + }, + { + "name": "caption", + "caption": "from today", + "expected": { + "snippet": "from today", + "preview": "📷 from today" + }, + "note": "The caption replaces the word Photo." + }, + { + "name": "emoji-caption", + "caption": "🎉 done", + "expected": { + "snippet": "🎉 done", + "preview": "📷 🎉 done" + }, + "note": "Captions are used as written." + } + ] +} diff --git a/apps/flipcash/shared/chat/build.gradle.kts b/apps/flipcash/shared/chat/build.gradle.kts index 8a52e3379d..3764ae2786 100644 --- a/apps/flipcash/shared/chat/build.gradle.kts +++ b/apps/flipcash/shared/chat/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { implementation(libs.androidx.paging.runtime) + implementation(project(":apps:flipcash:shared:blob")) implementation(project(":apps:flipcash:shared:persistence:sources")) implementation(project(":apps:flipcash:shared:persistence:db")) implementation(project(":apps:flipcash:shared:contacts")) diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/ChatCoordinator.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/ChatCoordinator.kt index db72ee3645..326e5d2fa8 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/ChatCoordinator.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/ChatCoordinator.kt @@ -21,6 +21,7 @@ import com.flipcash.services.models.chat.ReactionSummary import com.flipcash.services.models.chat.StartChatParameters import com.flipcash.services.models.chat.TypingState import com.flipcash.services.repository.ReactorsPage +import com.flipcash.shared.chat.media.ChatMediaSending import com.flipcash.shared.chat.reactions.ReactionError import com.flipcash.shared.chat.reactions.ReactionPill import com.flipcash.shared.chat.reactions.SelfReaction @@ -559,7 +560,8 @@ interface ChatCoordinator : DmChatResolver, MessagingOperations, GroupOperations, - ReactionOperations { + ReactionOperations, + ChatMediaSending { /** Full observable snapshot of chat state (feed, typing, reactions, active chat). */ val state: StateFlow diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/MessageCapability.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/MessageCapability.kt index f8bd247111..23115ce8fa 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/MessageCapability.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/MessageCapability.kt @@ -99,6 +99,8 @@ data class MessagePolicy( * | Own text, confirmed, past both windows | Copy, Reply | * | Own text, unconfirmed (`eventSequence == 0`) | none | * | Another participant's text | Copy, Reply, Report | + * | Own photo, confirmed, inside the delete window | Reply, Delete | + * | Another participant's photo | Reply, Report | * | Own cash or tip message | Reply | * | Another participant's cash or tip message | Reply, Report | * | A tombstone | none | @@ -183,11 +185,12 @@ private fun resolveForParticipant( return if (canSpeak) setOf(MessageCapability.Reply) else emptySet() } - val hasText = contents.any { it is MessageContent.Text || it is MessageContent.Reply } + // A photo, bare or quoted in a reply, is never copied or edited: its caption rides with the + // image, and an edit would have to re-seal or replace the picture. + val isPhoto = contents.any { it.isPhoto() } + val hasText = !isPhoto && contents.any { it is MessageContent.Text || it is MessageContent.Reply } return buildSet { - // Media carries no text, and this change edits text only. Not covered by the shared table; - // revisit when media messages actually ship. if (hasText) add(MessageCapability.Copy) if (canSpeak) add(MessageCapability.Reply) if (message.isFromSelf) { @@ -199,6 +202,12 @@ private fun resolveForParticipant( }.withinWindows(message.timestamp, policy, now) } +private fun MessageContent.isPhoto(): Boolean = when (this) { + is MessageContent.Media -> true + is MessageContent.Reply -> content.any { it is MessageContent.Media } + else -> false +} + /** * Drops the capabilities of a message sent at [sentAt] whose window has since closed. * @@ -248,5 +257,7 @@ fun canReact(message: ChatMessage, canSpeak: Boolean = true): Boolean { if (contents.any { it is MessageContent.Deleted }) return false if (message.eventSequence == 0L) return false if (contents.all { it is MessageContent.System }) return false + // A redacted photo is hidden from everyone; there is nothing left to react to. + if (message.redacted) return false return canSpeak } diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/inject/ChatModule.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/inject/ChatModule.kt index c3eddf96fe..d3d1c3c6b7 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/inject/ChatModule.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/inject/ChatModule.kt @@ -12,6 +12,8 @@ import com.flipcash.shared.chat.internal.OutgoingEncryption import com.flipcash.shared.chat.internal.RealChatArchiveStore import com.flipcash.shared.chat.internal.RealChatCoordinator import com.flipcash.shared.chat.internal.RealChatDraftStore +import com.flipcash.shared.chat.internal.delegates.MediaSendDelegate +import com.flipcash.shared.chat.media.ChatMediaSending import com.getcode.opencode.providers.SessionListener import dagger.Binds import dagger.Module @@ -42,6 +44,12 @@ abstract class ChatModule { impl: RealChatArchiveStore ): ChatArchiveStore + @Binds + @Singleton + abstract fun bindChatMediaSending( + impl: MediaSendDelegate + ): ChatMediaSending + @Binds internal abstract fun bindOutgoingEncryption( impl: DmOutgoingEncryption diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/OutgoingEncryption.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/OutgoingEncryption.kt index ad81279141..c6494445f7 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/OutgoingEncryption.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/OutgoingEncryption.kt @@ -1,11 +1,15 @@ package com.flipcash.shared.chat.internal +import com.flipcash.app.blob.ChatMediaSealing import com.flipcash.app.persistence.sources.ChatMemberDataSource import com.flipcash.app.persistence.sources.ChatMetadataDataSource import com.flipcash.services.chat.ChatContentCrypto +import com.flipcash.services.chat.carriesMedia +import com.flipcash.services.chat.checkBlobSealing import com.flipcash.services.chat.E2eePolicy import com.flipcash.services.chat.MessageEncryption import com.flipcash.services.controllers.ChatController +import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.chat.ChatId import com.flipcash.services.models.chat.ChatMessage import com.flipcash.services.models.chat.ChatMetadata @@ -32,23 +36,47 @@ interface OutgoingEncryption { * * Fails when the chat can't be read, since a chat that should be encrypted would otherwise go * out in plaintext, and when sealing fails. Either way the send is failed and can be retried. + * + * [blobSealedFor] is the chat a photo in [content] was sealed for, or null if it was uploaded + * plain. A photo whose sealing doesn't match the chat's, see [checkBlobSealing], fails the send. */ suspend fun prepare( chatId: ChatId, content: List, wasEncrypted: Boolean = false, + blobSealedFor: ChatId? = null, ): Result + /** + * How photos bound for [chatId] are uploaded: a [BlobSealer] if the chat encrypts, null if they + * go up plain. Fails when the chat can't be read, since guessing "plain" for a chat that + * encrypts would hand the photo to the server. + * + * [BlobSealer.chatId] is what [prepare] expects as `blobSealedFor` when the message is posted. + */ + suspend fun blobSealer(chatId: ChatId): Result + /** Sends everything in plaintext. What a delegate built without one -- a unit test -- is given. */ object None : OutgoingEncryption { override suspend fun prepare( chatId: ChatId, content: List, wasEncrypted: Boolean, + blobSealedFor: ChatId?, ): Result = Result.success(Outgoing(plaintext = content, wire = content, isSealed = false)) + + override suspend fun blobSealer(chatId: ChatId): Result = Result.success(null) } } +/** Seals a photo's bytes for [chatId] under the blob id the upload reserved. */ +class BlobSealer( + val chatId: ChatId, + private val sealWith: suspend (blobId: BlobId, plaintext: ByteArray) -> ByteArray, +) { + fun sealing(plaintext: ByteArray) = ChatMediaSealing(chatId) { blobId -> sealWith(blobId, plaintext) } +} + data class Outgoing( val plaintext: List, val wire: List, @@ -79,10 +107,15 @@ internal class DmOutgoingEncryption @Inject constructor( chatId: ChatId, content: List, wasEncrypted: Boolean, + blobSealedFor: ChatId?, ): Result { val chat = chat(chatId) ?: return Result.failure(IllegalStateException("Can't tell whether $chatId encrypts")) - if (!wasEncrypted && !policy.shouldEncrypt(chat)) { + val seals = wasEncrypted || policy.shouldEncrypt(chat) + if (content.any { it.carriesMedia() }) { + checkBlobSealing(blobSealedFor, chatId, seals).onFailure { return Result.failure(it) } + } + if (!seals) { return Result.success(Outgoing(plaintext = content, wire = content, isSealed = false)) } @@ -90,8 +123,8 @@ internal class DmOutgoingEncryption @Inject constructor( ?: return Result.failure(IllegalStateException("No account to encrypt from")) val peerId = chat.members.firstOrNull { it.userId != selfId }?.userId ?: return Result.failure(IllegalStateException("No peer in $chatId to encrypt to")) - // A DM message is one Text, or one Reply around Text. Anything else can't be sealed yet, - // and sending it in plaintext would break the chat's promise. + // A DM message is one Text, one photo, or one Reply around either. Anything else can't be + // sealed, and sending it in plaintext would break the chat's promise. val single = content.singleOrNull() ?: return Result.failure(IllegalArgumentException("Can't encrypt ${content.size} content items")) @@ -99,6 +132,22 @@ internal class DmOutgoingEncryption @Inject constructor( .map { sealed -> Outgoing(plaintext = content, wire = listOf(sealed), isSealed = true) } } + override suspend fun blobSealer(chatId: ChatId): Result { + val chat = chat(chatId) + ?: return Result.failure(IllegalStateException("Can't tell whether $chatId encrypts")) + if (!policy.shouldEncrypt(chat)) return Result.success(null) + + val selfId = userManager.accountId + ?: return Result.failure(IllegalStateException("No account to encrypt from")) + val peerId = chat.members.firstOrNull { it.userId != selfId }?.userId + ?: return Result.failure(IllegalStateException("No peer in $chatId to encrypt to")) + return Result.success( + BlobSealer(chatId) { blobId, plaintext -> + crypto.sealBlob(chatId, peerId, blobId.bytes, plaintext).getOrThrow() + } + ) + } + private suspend fun chat(chatId: ChatId): ChatMetadata? { metadataDataSource.observeById(chatId).first()?.let { entity -> val members = memberDataSource.getMembersForChat(chatId) diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/RealChatCoordinator.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/RealChatCoordinator.kt index d296780214..7716837351 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/RealChatCoordinator.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/RealChatCoordinator.kt @@ -26,6 +26,7 @@ import com.flipcash.shared.chat.internal.delegates.FeedSyncDelegate import com.flipcash.shared.chat.internal.delegates.GroupFeedDelegate import com.flipcash.shared.chat.internal.delegates.DmChatResolverDelegate import com.flipcash.shared.chat.internal.delegates.MessagingDelegate +import com.flipcash.shared.chat.media.ChatMediaSending import com.flipcash.shared.chat.internal.delegates.ReactionsDelegate import com.getcode.opencode.model.accounts.AccountCluster import com.getcode.opencode.providers.SessionListener @@ -99,6 +100,7 @@ class RealChatCoordinator @Inject constructor( private val networkObserver: NetworkConnectivityListener, private val dispatchers: DispatcherProvider, private val archiveStore: ChatArchiveStore = ChatArchiveStore.None, + private val mediaSender: ChatMediaSending = ChatMediaSending.None, ) : ChatCoordinator, SessionListener, DefaultLifecycleObserver, @@ -107,7 +109,8 @@ class RealChatCoordinator @Inject constructor( DmChatResolver by dmChatResolverDelegate, MessagingOperations by messagingDelegate, GroupOperations by groupFeedDelegate, - ReactionOperations by reactionsDelegate { + ReactionOperations by reactionsDelegate, + ChatMediaSending by mediaSender { companion object { private const val TAG = "ChatCoordinator" @@ -153,6 +156,8 @@ class RealChatCoordinator @Inject constructor( // was killed mid-flight has to be marked failed first or it is destroyed instead of // becoming retryable. See [MessagingDelegate.recoverInterruptedSends]. messagingDelegate.recoverInterruptedSends() + // After the sweep, which leaves a stored photo sending; this resumes it. + scope.launch { mediaSender.reconcilePendingMedia() } feedDelegate.observeFeedFromDb() // The list the Chats tab opens on comes from this read, so let it finish before the sync // below starts competing with it for CPU (a cold-launch trace had the sync's gRPC work diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt new file mode 100644 index 0000000000..e9431a1c96 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt @@ -0,0 +1,463 @@ +package com.flipcash.shared.chat.internal.delegates + +import com.flipcash.app.persistence.sources.ChatMessageDataSource +import com.flipcash.app.persistence.sources.ChatMetadataDataSource +import com.flipcash.app.persistence.sources.PendingMediaDataSource +import com.flipcash.app.persistence.sources.PendingMediaRecord +import com.flipcash.app.persistence.sources.mapper.chat.ChatEntityMapper +import com.flipcash.libs.coroutines.DispatcherProvider +import com.flipcash.services.controllers.ChatMessagingController +import com.flipcash.services.models.SendMessageError +import com.flipcash.services.models.chat.BlobMetadata +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.ChatMessage +import com.flipcash.services.models.chat.ClientMessageId +import com.flipcash.services.models.chat.ImageMetadata +import com.flipcash.services.models.chat.MediaItem +import com.flipcash.services.models.chat.MediaItemRendition +import com.flipcash.services.models.chat.MessageContent +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.user.UserManager +import com.flipcash.shared.chat.internal.OutgoingEncryption +import com.flipcash.shared.chat.media.ChatMediaSending +import com.flipcash.shared.chat.media.ChatMediaSendPlan +import com.flipcash.shared.chat.media.ChatMediaUploadState +import com.flipcash.shared.chat.media.ChatMediaUploads +import com.flipcash.shared.chat.media.MediaSendProgress +import com.flipcash.shared.chat.media.UploadedPhoto +import com.getcode.opencode.model.core.RandomId +import com.getcode.utils.TraceType +import com.getcode.utils.hexEncodedString +import com.getcode.utils.trace +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import com.flipcash.shared.chat.media.ChatPhoto +import com.flipcash.shared.chat.media.SentPhotoPreviews +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.time.Clock + +/** + * Sends photo messages: a pending row per photo up front, then a post per row, in chip order, as + * each upload settles. See [ChatMediaSending]. + * + * What is queued is in `pending_media`, so a photo survives the process. The photo's own file and + * state are [ChatMediaUploads]'; this decides what happens to the message around them. + */ +@Singleton +class MediaSendDelegate internal constructor( + private val messaging: MessagingDelegate, + private val messagingController: ChatMessagingController, + private val messageDataSource: ChatMessageDataSource, + private val metadataDataSource: ChatMetadataDataSource, + private val pendingMedia: PendingMediaDataSource, + private val uploads: ChatMediaUploads, + private val outgoing: OutgoingEncryption, + private val userManager: UserManager, + private val scope: CoroutineScope, + private val now: () -> Long = { Clock.System.now().toEpochMilliseconds() }, +) : ChatMediaSending { + + @Inject + internal constructor( + messaging: MessagingDelegate, + messagingController: ChatMessagingController, + messageDataSource: ChatMessageDataSource, + metadataDataSource: ChatMetadataDataSource, + pendingMedia: PendingMediaDataSource, + uploads: ChatMediaUploads, + outgoing: OutgoingEncryption, + userManager: UserManager, + dispatchers: DispatcherProvider, + ) : this( + messaging, messagingController, messageDataSource, metadataDataSource, pendingMedia, uploads, + outgoing, userManager, CoroutineScope(SupervisorJob() + dispatchers.IO), + ) + + private val mapper = ChatEntityMapper() + + /** What the post of a row did past the upload, which the upload's own state doesn't say. */ + private sealed interface Stage { + data object Sending : Stage + data object Sent : Stage + data class Failed(val retryable: Boolean) : Stage + } + + private val stages = MutableStateFlow>(emptyMap()) + + /** Client id hex to chip id, for every row this process is tracking. */ + private val chips = MutableStateFlow>(emptyMap()) + + private val inFlight = HashSet() + + override suspend fun sendMedia( + chatId: ChatId, + chipIds: List, + text: String, + replyToMessageId: Long?, + ): Result> { + val plan = ChatMediaSendPlan.build(chipIds, text, replyToMessageId) + if (plan.isEmpty()) return Result.success(emptyList()) + if (chipIds.isEmpty()) { + val message = plan.single() as ChatMediaSendPlan.Message.Text + return messaging.sendMessage(chatId, message.text, message.replyTo).map { emptyList() } + } + if (chipIds.distinct().size != chipIds.size || chipIds.any { !uploads.has(it) }) { + return Result.failure(IllegalArgumentException("Unknown or repeated photo")) + } + val senderId = userManager.accountId + ?: return Result.failure(IllegalStateException("Cannot send a photo without an account")) + + val createdAt = now() + val queued = plan.filterIsInstance().map { message -> + val (width, height) = uploads.pixelSize(message.chip) ?: (1 to 1) + Queued( + clientMessageId = ClientMessageId(RandomId.toByteArray()), + message = message, + width = width, + height = height, + ) + } + + // The entries first: a refresh that lands between the two writes would otherwise delete a + // row nothing yet says is a photo in flight. + pendingMedia.insert( + queued.map { q -> + PendingMediaRecord( + clientIdHex = hex(q.clientMessageId), + chatIdHex = mapper.chatIdHex(chatId), + fileName = fileName(q.message.chip), + caption = q.message.caption, + replyToMessageId = q.message.replyTo, + storedBlobIdHex = null, + sealedForHex = null, + width = q.width, + height = q.height, + blurhash = null, + sizeBytes = null, + createdAt = createdAt, + ) + }, + ) + queued.forEachIndexed { index, q -> + val local = localContent(q.message.caption, q.message.replyTo, q.width, q.height, fileUri(q.message.chip)) + SentPhotoPreviews.handOff(q.message.chip, hex(q.clientMessageId)) + messageDataSource.insertPending( + chatId = chatId, + content = local, + senderId = senderId, + clientMessageId = q.clientMessageId, + ordinal = index, + ) + } + chips.update { it + queued.associate { q -> hex(q.clientMessageId) to q.message.chip } } + val hexes = queued.map { hex(it.clientMessageId) } + synchronized(inFlight) { inFlight.addAll(hexes) } + + scope.launch { + for (q in queued) { + val record = pendingMedia.get(hex(q.clientMessageId)) ?: continue + post(chatId, record) + } + } + return Result.success(hexes) + } + + override suspend fun retryMedia(chatId: ChatId, pendingClientIdHex: String): Result { + val record = pendingMedia.get(pendingClientIdHex) + ?: return Result.failure(IllegalStateException("Nothing left to retry for $pendingClientIdHex")) + val chip = chipId(record) + if (!uploads.has(chip)) track(record) + if (!synchronized(inFlight) { inFlight.add(pendingClientIdHex) }) return Result.success(Unit) + + messageDataSource.retryPending(chatId, pendingClientIdHex) + stages.update { it - pendingClientIdHex } + chips.update { it + (pendingClientIdHex to chip) } + if (uploads.current(chip) is ChatMediaUploadState.Failed) uploads.retry(chip) + scope.launch { post(chatId, record) } + return Result.success(Unit) + } + + override fun observeMediaSendProgress(): Flow> = + combine(chips, stages, uploads.allStates) { chips, stages, states -> + chips.mapNotNull { (hex, chip) -> + val progress = when (val stage = stages[hex]) { + Stage.Sending -> MediaSendProgress.Sending + Stage.Sent -> MediaSendProgress.Sent + is Stage.Failed -> MediaSendProgress.Failed(stage.retryable) + null -> when (val state = states[chip]) { + is ChatMediaUploadState.Preparing -> MediaSendProgress.Preparing + is ChatMediaUploadState.Uploading -> MediaSendProgress.Uploading(state.fraction) + is ChatMediaUploadState.Processing -> MediaSendProgress.Processing + is ChatMediaUploadState.Uploaded -> MediaSendProgress.Sending + is ChatMediaUploadState.Failed -> MediaSendProgress.Failed(state.retryable) + null -> return@mapNotNull null + } + } + hex to progress + }.toMap() + }.distinctUntilChanged() + + override suspend fun reconcilePendingMedia() { + val selfId = userManager.accountId ?: return + val records = pendingMedia.getAll().sortedBy { it.createdAt } + if (records.isEmpty()) return + + val recentByChat = HashMap>() + val resume = ArrayList() + for (record in records) { + val chatId = mapper.chatIdFromHex(record.chatIdHex) + val stored = record.storedBlobIdHex + if (stored != null) { + val recent = recentByChat.getOrPut(record.chatIdHex) { + messageDataSource.getRecentSentBy(chatId, selfId, RECENT_WINDOW) + } + val arrived = recent.firstOrNull { message -> message.carries(stored) } + if (arrived != null) { + // The send got through before the process died; the stream already delivered it. + messageDataSource.confirmPending(chatId, mapper.clientMessageIdFromHex(record.clientIdHex), arrived, replaceContent = true) + discard(record) + continue + } + track(record) + resume += record + } else if (uploads.hasFile(record.fileName)) { + track(record) + } else { + // Nothing to upload again: the file went with a cleared cache. + discard(record) + } + } + + scope.launch { + for (record in resume) { + val chatId = mapper.chatIdFromHex(record.chatIdHex) + // Stored rows survive the startup sweep as SENDING; a row that was failed anyway + // goes back to it, since nothing here waits on the viewer. + retryMedia(chatId, record.clientIdHex) + } + } + } + + // region posting + + private class Queued( + val clientMessageId: ClientMessageId, + val message: ChatMediaSendPlan.Message.Media, + val width: Int, + val height: Int, + ) + + /** Waits for the photo of [record], then posts its message and settles the row. */ + private suspend fun post(chatId: ChatId, record: PendingMediaRecord) { + val hex = record.clientIdHex + val clientMessageId = mapper.clientMessageIdFromHex(hex) + try { + val chip = chipId(record) + var recorded = false + val settled = uploads.state(chip) + .onEach { state -> + if (recorded) return@onEach + val photo = (state as? ChatMediaUploadState.Processing)?.photo + ?: (state as? ChatMediaUploadState.Uploaded)?.photo + ?: (state as? ChatMediaUploadState.Failed)?.stored + if (photo != null) { + recorded = true + recordStored(hex, photo) + } + } + .first { it == null || it is ChatMediaUploadState.Uploaded || it is ChatMediaUploadState.Failed } + + when (settled) { + null -> fail(chatId, clientMessageId, hex, retryable = false, discard = record) + is ChatMediaUploadState.Failed -> + fail(chatId, clientMessageId, hex, settled.retryable, discard = record.takeUnless { settled.retryable }) + is ChatMediaUploadState.Uploaded -> postMessage(chatId, record, clientMessageId, settled.photo) + else -> Unit + } + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + trace(tag = TAG, message = "Photo send failed in $chatId", type = TraceType.Error, error = e) + fail(chatId, clientMessageId, hex, retryable = true, discard = null) + } finally { + synchronized(inFlight) { inFlight.remove(hex) } + } + } + + private suspend fun postMessage( + chatId: ChatId, + record: PendingMediaRecord, + clientMessageId: ClientMessageId, + photo: UploadedPhoto, + ) { + val hex = record.clientIdHex + stages.update { it + (hex to Stage.Sending) } + + val content = sentContent(record, photo) + val prepared = outgoing.prepare(chatId, content, blobSealedFor = photo.sealedFor).getOrElse { cause -> + trace(tag = TAG, message = "Couldn't prepare photo in $chatId", type = TraceType.Error, error = cause) + fail(chatId, clientMessageId, hex, retryable = true, discard = null) + return + } + + messagingController.sendMessage(chatId, prepared.wire, clientMessageId) + .map(prepared::echo) + .onSuccess { serverMessage -> + messageDataSource.confirmPending(chatId, clientMessageId, serverMessage, replaceContent = true) + messaging.advanceReadPointer(chatId, serverMessage.messageId) + metadataDataSource.updateLastMessageId(chatId, serverMessage.messageId) + metadataDataSource.updateLastActivity(chatId, serverMessage.timestamp.toEpochMilliseconds()) + discard(record, keepAs = ChatPhoto.cacheKeyOf(photo.blobId)) + stages.update { it + (hex to Stage.Sent) } + } + .onFailure { cause -> + // A chat that stopped taking ciphertext won't take this blob on a retry either. + val terminal = cause is SendMessageError.EncryptionNotAllowed + fail(chatId, clientMessageId, hex, retryable = !terminal, discard = record.takeIf { terminal }) + } + } + + private suspend fun fail( + chatId: ChatId, + clientMessageId: ClientMessageId, + hex: String, + retryable: Boolean, + discard: PendingMediaRecord?, + ) { + messageDataSource.failPending(chatId, clientMessageId) + stages.update { it + (hex to Stage.Failed(retryable)) } + // A terminal failure has nothing a retry could resend, so the queue entry and the file go. + if (discard != null) discard(discard) + } + + private suspend fun recordStored(hex: String, photo: UploadedPhoto) { + pendingMedia.markStored( + clientIdHex = hex, + blobIdHex = hex(photo.blobId.bytes), + sealedForHex = photo.sealedFor?.let { hex(it.bytes) }, + sizeBytes = photo.sizeBytes, + blurhash = photo.blurhash, + ) + } + + private suspend fun discard(record: PendingMediaRecord, keepAs: String? = null) { + pendingMedia.delete(record.clientIdHex) + uploads.remove(chipId(record), keepAs) + } + + // endregion + + // region content + + private fun track(record: PendingMediaRecord) { + val chip = chipId(record) + uploads.restore( + id = chip, + chatId = mapper.chatIdFromHex(record.chatIdHex), + fileName = record.fileName, + width = record.width, + height = record.height, + blurhash = record.blurhash, + stored = record.storedBlobIdHex?.let { blob -> + UploadedPhoto( + blobId = BlobId(unhex(blob)), + width = record.width, + height = record.height, + blurhash = record.blurhash, + sizeBytes = record.sizeBytes ?: 0, + sealedFor = record.sealedForHex?.let { ChatId(unhex(it)) }, + ) + }, + ) + chips.update { it + (record.clientIdHex to chip) } + } + + /** + * The optimistic row's content: one ORIGINAL rendition with an empty blob id, since none is + * known yet, and the local file as its URL for the bubble to draw. + */ + private fun localContent(caption: String?, replyTo: Long?, width: Int, height: Int, uri: String): List = + wrap( + replyTo, + MessageContent.Media( + items = listOf( + MediaItem( + listOf( + MediaItemRendition( + role = MediaItemRendition.Role.ORIGINAL, + blobId = BlobId(ByteArray(0)), + blob = BlobMetadata( + mimeType = JPEG, + sizeBytes = 0, + downloadUrl = uri, + image = ImageMetadata(width, height, ""), + ), + ), + ), + ), + ), + caption = caption?.let { MessageContent.Text(it) }, + ), + ) + + /** + * What goes to the server. A sealed photo carries the full metadata its contract needs; a plain + * one only its id, the server filling in the rest from the bytes. + */ + private fun sentContent(record: PendingMediaRecord, photo: UploadedPhoto): List { + val sealed = photo.sealedFor != null + val blob = if (sealed) { + BlobMetadata( + mimeType = JPEG, + sizeBytes = photo.sizeBytes, + downloadUrl = "", + image = ImageMetadata(photo.width, photo.height, photo.blurhash.orEmpty()), + ) + } else null + return wrap( + record.replyToMessageId, + MessageContent.Media( + items = listOf( + MediaItem(listOf(MediaItemRendition(MediaItemRendition.Role.ORIGINAL, photo.blobId, blob))), + ), + caption = record.caption?.let { MessageContent.Text(it) }, + ), + ) + } + + private fun wrap(replyTo: Long?, media: MessageContent.Media): List = + listOf(replyTo?.let { MessageContent.Reply(it, listOf(media)) } ?: media) + + private fun ChatMessage.carries(blobIdHex: String): Boolean = content.any { c -> + val media = (c as? MessageContent.Reply)?.content?.filterIsInstance() + ?: listOfNotNull(c as? MessageContent.Media) + media.any { m -> m.items.any { item -> item.renditions.any { hex(it.blobId.bytes) == blobIdHex } } } + } + + private fun fileName(chip: String) = "$chip.jpg" + private fun chipId(record: PendingMediaRecord) = record.fileName.removeSuffix(".jpg") + private fun fileUri(chip: String) = "file://" + uploads.file(fileName(chip)).absolutePath + private fun hex(id: ClientMessageId) = hex(id.bytes) + private fun hex(bytes: ByteArray) = bytes.toList().hexEncodedString() + private fun unhex(hex: String) = ByteArray(hex.length / 2) { hex.substring(it * 2, it * 2 + 2).toInt(16).toByte() } + + // endregion + + private companion object { + const val TAG = "MediaSendDelegate" + const val JPEG = "image/jpeg" + + /** How far back a launch looks for the message a queued photo may already have become. */ + const val RECENT_WINDOW = 50 + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MessagingDelegate.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MessagingDelegate.kt index 3a5f4077f5..78f67c295a 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MessagingDelegate.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MessagingDelegate.kt @@ -43,6 +43,7 @@ import com.flipcash.shared.chat.PendingMutation import com.flipcash.shared.chat.UnreadBoundary import com.flipcash.shared.chat.internal.ChatStateHolder import com.flipcash.shared.chat.internal.OutgoingEncryption +import com.flipcash.shared.chat.media.ChatMediaText import com.flipcash.shared.chat.replacingText import com.flipcash.services.user.UserManager import com.flipcash.shared.chat.MessageLinkPrefetch @@ -319,10 +320,11 @@ class MessagingDelegate @Inject constructor( return opened.takeIf { it.encryption is MessageEncryption.Decrypted } } - /** Text and replies with text are what DMs encrypt; anything else keeps the server's body. */ + /** Text, photos and replies to either are what DMs encrypt; anything else keeps the server's body. */ private fun MessageContent.pushText(): String? = when (this) { is MessageContent.Text -> text - is MessageContent.Reply -> (content.singleOrNull() as? MessageContent.Text)?.text + is MessageContent.Media -> ChatMediaText.pushText(caption?.text) + is MessageContent.Reply -> content.singleOrNull()?.pushText() else -> null } diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlan.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlan.kt new file mode 100644 index 0000000000..d76f1651d4 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlan.kt @@ -0,0 +1,36 @@ +package com.flipcash.shared.chat.media + +/** + * What a send with attachments turns into, decided before anything is uploaded or stored. + * + * The rules are iOS's (code-payments/code-ios-app#959) and the vectors in `chat_media.json` pin + * them: with no chips the text is an ordinary message; with chips there is one media message per + * chip, in chip order, the caption on the last so it lands under the last image, the reply on the + * first. Text is never a message of its own once a chip exists. + */ +object ChatMediaSendPlan { + + /** The composer's ceiling on staged photos. */ + const val MAX_ATTACHMENTS = 10 + + sealed interface Message { + data class Text(val text: String, val replyTo: Long?) : Message + + /** [chip] is whatever id the caller staged the photo under. */ + data class Media(val chip: String, val caption: String?, val replyTo: Long?) : Message + } + + fun build(chips: List, text: String, replyTo: Long?): List { + val body = text.trim() + if (chips.isEmpty()) { + return if (body.isEmpty()) emptyList() else listOf(Message.Text(body, replyTo)) + } + return chips.mapIndexed { index, chip -> + Message.Media( + chip = chip, + caption = body.takeIf { index == chips.lastIndex && it.isNotEmpty() }, + replyTo = replyTo.takeIf { index == 0 }, + ) + } + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSending.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSending.kt new file mode 100644 index 0000000000..9f4b957328 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaSending.kt @@ -0,0 +1,76 @@ +package com.flipcash.shared.chat.media + +import com.flipcash.services.models.chat.ChatId +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flowOf + +/** Where one photo message is on its way out, for the overlay on its bubble. */ +sealed interface MediaSendProgress { + data object Preparing : MediaSendProgress + + /** [fraction] is 0..1 and only grows within an attempt. */ + data class Uploading(val fraction: Float) : MediaSendProgress + + data object Processing : MediaSendProgress + data object Sending : MediaSendProgress + data object Sent : MediaSendProgress + + /** [retryable] is whether `retryMedia` can help; a moderation rejection can't be retried. */ + data class Failed(val retryable: Boolean) : MediaSendProgress +} + +/** + * Sending photos. Everything after [sendMedia] returns happens in the app's scope, so leaving the + * chat doesn't stop a photo that is still going up. + */ +interface ChatMediaSending { + + /** + * Sends the staged photos [chipIds] (see [ChatMediaUploads.stage]) and [text] to [chatId], + * following [ChatMediaSendPlan]: with no chips, [text] is an ordinary message; with chips, one + * photo message per chip, the caption on the last and the reply on the first. + * + * Returns once every photo message is on the transcript as a sending row, with the client id + * hex of each in chip order (empty for a text-only send, which is sent before this returns). + * The rows are posted in chip order as their uploads settle; one that fails leaves the others + * going. Follow them with [observeMediaSendProgress]. + */ + suspend fun sendMedia( + chatId: ChatId, + chipIds: List, + text: String, + replyToMessageId: Long?, + ): Result> + + /** + * Retries a failed photo message by its pending client id: uploads the stored JPEG again if the + * photo never reached storage, polls it if it did, or posts it again if only the post failed. + * Fails if there is nothing left to retry, as after a moderation rejection. + */ + suspend fun retryMedia(chatId: ChatId, pendingClientIdHex: String): Result + + /** Progress of the photo messages sent or resumed in this process, by client id hex. */ + fun observeMediaSendProgress(): Flow> + + /** + * Picks up what a previous process left queued: stored photos are polled and posted, the rest + * stay failed with a retry, and entries whose message already reached the chat are dropped. + */ + suspend fun reconcilePendingMedia() + + /** Does nothing. The default for a coordinator built without photo sending, as in unit tests. */ + object None : ChatMediaSending { + override suspend fun sendMedia( + chatId: ChatId, + chipIds: List, + text: String, + replyToMessageId: Long?, + ): Result> = Result.failure(UnsupportedOperationException("Photos are not available")) + + override suspend fun retryMedia(chatId: ChatId, pendingClientIdHex: String): Result = + Result.failure(UnsupportedOperationException("Photos are not available")) + + override fun observeMediaSendProgress(): Flow> = flowOf(emptyMap()) + override suspend fun reconcilePendingMedia() = Unit + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaText.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaText.kt new file mode 100644 index 0000000000..7c5e28e597 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaText.kt @@ -0,0 +1,27 @@ +package com.flipcash.shared.chat.media + +/** + * The words a photo message goes by where there is no room to show it: a reply's citation, the + * conversation list, a notification. Pure so the vectors in `chat_media.json` can pin them; the + * localized word for "Photo" is the caller's to pass. + */ +object ChatMediaText { + + /** Leads every one-line preview of a photo. An emoji, so not a string resource. */ + const val PREVIEW_PREFIX = "📷 " + + /** The caption as written, or [photoLabel] when there is none. */ + fun snippet(caption: String?, photoLabel: String): String = + caption?.takeIf { it.isNotBlank() } ?: photoLabel + + /** [snippet] behind [PREVIEW_PREFIX]: what the conversation list and a quote's one-liner show. */ + fun preview(caption: String?, photoLabel: String): String = + PREVIEW_PREFIX + snippet(caption, photoLabel) + + /** + * A push for a photo: the caption, else the preview of the bare photo. English, because a push + * body is built where no resources are at hand; the server's own body is localized if it ships one. + */ + fun pushText(caption: String?): String = + caption?.takeIf { it.isNotBlank() } ?: preview(null, "Photo") +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaUploads.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaUploads.kt new file mode 100644 index 0000000000..51b1993c85 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatMediaUploads.kt @@ -0,0 +1,374 @@ +package com.flipcash.shared.chat.media + +import android.content.Context +import android.graphics.BitmapFactory +import android.net.Uri +import com.flipcash.app.blob.BlobStorageCoordinator +import com.flipcash.app.blob.BlurHashEncoder +import com.flipcash.app.blob.ChatMediaEncoder +import com.flipcash.app.blob.ChatMediaLimits +import com.flipcash.app.blob.ChatMediaRetry +import com.flipcash.libs.coroutines.DispatcherProvider +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.ChatId +import com.flipcash.shared.chat.internal.OutgoingEncryption +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.io.File +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton + +/** A photo that is in storage: what a media message needs to reference it. */ +data class UploadedPhoto( + val blobId: BlobId, + val width: Int, + val height: Int, + /** Only computed for a sealed photo, whose preview the server can't derive. */ + val blurhash: String?, + /** Length of the stripped JPEG, before any sealing overhead. */ + val sizeBytes: Long, + /** The chat the bytes were sealed for, or null if they went up plain. */ + val sealedFor: ChatId?, +) + +/** Where one staged photo is, from picking it to having it ready to post. */ +sealed interface ChatMediaUploadState { + /** Being encoded, or waiting for a store attempt to move bytes; every attempt starts here. */ + data class Preparing(val sourceWidth: Int?, val sourceHeight: Int?) : ChatMediaUploadState + + /** Bytes are going up. [fraction] only grows within an attempt. */ + data class Uploading(val fraction: Float) : ChatMediaUploadState + + /** Stored; waiting on the server to moderate and finalize it. */ + data class Processing(val photo: UploadedPhoto) : ChatMediaUploadState + + data class Uploaded(val photo: UploadedPhoto) : ChatMediaUploadState + + /** + * [retryable] is whether [ChatMediaUploads.retry] can help. [stored] is set when the bytes did + * reach storage, so a retry polls rather than uploads. + */ + data class Failed( + val cause: Throwable, + val retryable: Boolean, + val stored: UploadedPhoto? = null, + ) : ChatMediaUploadState +} + +/** + * Uploads chat photos as they are staged, before the viewer sends them, so sending is mostly + * posting. App-scoped: a staged photo keeps uploading when the chat screen is left. + * + * Each photo is encoded once and written to `filesDir/pending-media/.jpg`; every later attempt + * resends those bytes. In a chat that encrypts the photo is sealed and a failure to do so fails the + * chip, never falls back to plain bytes. + * + * The id [stage] returns is the chip id: it keys [state], [remove] and [retry], and is what + * `ChatCoordinator.sendMedia` takes. + */ +@Singleton +class ChatMediaUploads internal constructor( + private val directory: () -> File, + private val encoder: ChatMediaEncoder, + private val blobs: BlobStorageCoordinator, + private val outgoing: OutgoingEncryption, + private val sourceSize: (Uri) -> Pair?, + private val blurhashOf: (ByteArray) -> String?, + private val scope: CoroutineScope, + private val newId: () -> String = { UUID.randomUUID().toString() }, + private val keepImage: (cacheKey: String, file: File) -> Unit = { _, _ -> }, +) { + @Inject + internal constructor( + @ApplicationContext context: Context, + encoder: ChatMediaEncoder, + blobs: BlobStorageCoordinator, + outgoing: OutgoingEncryption, + dispatchers: DispatcherProvider, + ) : this( + directory = { File(context.filesDir, DIRECTORY) }, + encoder = encoder, + blobs = blobs, + outgoing = outgoing, + sourceSize = { uri -> context.sourceSize(uri) }, + blurhashOf = ::blurhashOfJpeg, + scope = CoroutineScope(SupervisorJob() + dispatchers.IO), + keepImage = { key, file -> context.keepInImageCache(key, file) }, + ) + + private class Entry( + val id: String, + val chatId: ChatId, + val uri: Uri?, + val fileName: String, + /** Completes true once [uri] has been written, for a camera shot staged at the shutter. */ + val ready: Deferred? = null, + ) { + @Volatile var job: Job? = null + @Volatile var source: Pair? = null + @Volatile var encoded: Encoded? = null + } + + private class Encoded(val width: Int, val height: Int, val blurhash: String?) + + private val entries = HashMap() + private val states = MutableStateFlow>(emptyMap()) + + /** + * Starts preparing and uploading [uri] for [chatId]. Returns the chip id. + * + * With [ready], the chip exists at once but reading [uri] waits for it: the camera stages its + * shot at the shutter, before the file is written. Completing it false leaves the chip + * preparing, for the caller to [remove]. + */ + fun stage(chatId: ChatId, uri: Uri, ready: Deferred? = null): String { + val id = newId() + val entry = Entry(id, chatId, uri, "$id.jpg", ready) + synchronized(entries) { entries[id] = entry } + set(id, ChatMediaUploadState.Preparing(null, null)) + launch(entry) + return id + } + + /** + * Re-creates the chip of a message queued by an earlier process, from the record that survived + * it: [stored] set resumes at polling ([retry] to go), unset resumes at uploading the file. + * A no-op if [id] is already known. + */ + fun restore( + id: String, + chatId: ChatId, + fileName: String, + width: Int, + height: Int, + blurhash: String?, + stored: UploadedPhoto?, + ) { + val entry = synchronized(entries) { + if (id in entries) return + Entry(id, chatId, null, fileName).also { + it.encoded = Encoded(width, height, blurhash) + entries[id] = it + } + } + // Failed either way: with [stored] set, a retry polls it, and nothing runs until asked to. + set(entry.id, ChatMediaUploadState.Failed(IllegalStateException("Interrupted"), retryable = true, stored = stored)) + } + + /** The file of a photo staged as [fileName]. */ + fun file(fileName: String): File = File(directory(), fileName) + + fun hasFile(fileName: String): Boolean = file(fileName).isFile + + fun has(id: String): Boolean = synchronized(entries) { id in entries } + + /** The chip's state, null once it is removed. */ + fun state(id: String): Flow = states.map { it[id] }.distinctUntilChanged() + + /** Every chip's state at once, for a view of many. */ + val allStates: Flow> get() = states + + fun current(id: String): ChatMediaUploadState? = states.value[id] + + /** The photo's pixel size: the encoded size once known, the source's before that. */ + fun pixelSize(id: String): Pair? { + val entry = synchronized(entries) { entries[id] } ?: return null + return entry.encoded?.let { it.width to it.height } ?: entry.source + } + + /** Suspends until the chip is uploaded or failed; null if it is removed first. */ + suspend fun awaitSettled(id: String): ChatMediaUploadState? = + state(id).first { it == null || it is ChatMediaUploadState.Uploaded || it is ChatMediaUploadState.Failed } + + /** + * Cancels the chip's work and deletes its file. With [keepAs], the file is first copied into the + * image disk cache under that key, so the sent message draws the photo straight from it rather + * than fetching what was just uploaded. + */ + fun remove(id: String, keepAs: String? = null) { + val entry = synchronized(entries) { entries.remove(id) } + if (entry != null) { + entry.job?.cancel() + states.update { it - id } + } + if (keepAs == null) SentPhotoPreviews.drop(id) + // An entry not tracked in this process (a queue entry dropped at launch): the file is still ours. + val file = entry?.let(::file) ?: file("$id.jpg") + scope.launch { + if (keepAs != null && file.isFile) runCatching { keepImage(keepAs, file) } + runCatching { file.delete() } + } + } + + /** + * Resumes a failed chip: a stored photo is polled again, one that never reached storage is + * uploaded again from the file written the first time. + */ + fun retry(id: String) { + val entry = synchronized(entries) { entries[id] } ?: return + if (entry.job?.isActive == true) return + if (states.value[id] is ChatMediaUploadState.Uploaded) return + val stored = when (val s = states.value[id]) { + is ChatMediaUploadState.Failed -> s.stored + is ChatMediaUploadState.Processing -> s.photo + else -> null + } + set(id, stored?.let { ChatMediaUploadState.Processing(it) } ?: ChatMediaUploadState.Preparing(entry.source?.first, entry.source?.second)) + launch(entry) + } + + private fun launch(entry: Entry) { + entry.job = scope.launch { run(entry) } + } + + private fun file(entry: Entry) = File(directory(), entry.fileName) + + /** Writes the chip's state unless it was removed meanwhile, so a late write can't resurrect it. */ + private fun set(id: String, state: ChatMediaUploadState) { + if (has(id)) states.update { it + (id to state) } + } + + private suspend fun run(entry: Entry) { + try { + if (entry.ready?.await() == false) return + val stored = (states.value[entry.id] as? ChatMediaUploadState.Processing)?.photo + ?: store(entry) + ?: return + set(entry.id, ChatMediaUploadState.Processing(stored)) + blobs.awaitChatMediaReady(stored.blobId).fold( + onSuccess = { set(entry.id, ChatMediaUploadState.Uploaded(stored)) }, + onFailure = { fail(entry, it, stored) }, + ) + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + fail(entry, e, null) + } + } + + /** Encodes (once), then stores. Null if it failed, with the chip already marked. */ + private suspend fun store(entry: Entry): UploadedPhoto? { + if (entry.uri != null && entry.source == null) { + entry.source = sourceSize(entry.uri) + } + set(entry.id, ChatMediaUploadState.Preparing(entry.source?.first, entry.source?.second)) + + val sealer = outgoing.blobSealer(entry.chatId).getOrElse { return fail(entry, it) } + + if (entry.encoded == null) { + val uri = entry.uri ?: return fail(entry, IllegalStateException("Nothing to encode"), retryable = false) + val policy = blobs.policy.first() + ?: blobs.preloadPolicy().getOrElse { return fail(entry, it) } + val limits = (if (sealer != null) ChatMediaLimits.sealed(policy) else ChatMediaLimits.plain(policy)) + .getOrElse { return fail(entry, it) } + val encoded = encoder.encode(uri, limits).getOrElse { return fail(entry, it) } + + runCatching { + val file = file(entry) + file.parentFile?.mkdirs() + file.writeBytes(encoded.bytes) + }.onFailure { return fail(entry, it) } + entry.encoded = Encoded( + width = encoded.width, + height = encoded.height, + blurhash = if (sealer != null) blurhashOf(encoded.bytes) else null, + ) + } + + val encoded = entry.encoded!! + val bytes = runCatching { file(entry).readBytes() }.getOrElse { return fail(entry, it, retryable = false) } + SentPhotoPreviews.stage(entry.id, bytes) + + var storing = true + val blobId = blobs.storeChatMediaUnfinalized( + jpeg = bytes, + sealing = sealer?.sealing(bytes), + onAttempt = { set(entry.id, ChatMediaUploadState.Preparing(entry.source?.first, entry.source?.second)) }, + onProgress = { sent, total -> if (storing) progress(entry.id, sent, total) }, + ).also { storing = false } + .getOrElse { return fail(entry, it) } + + return UploadedPhoto( + blobId = blobId, + width = encoded.width, + height = encoded.height, + blurhash = encoded.blurhash, + sizeBytes = bytes.size.toLong(), + sealedFor = sealer?.chatId, + ) + } + + /** Monotonic within an attempt; an update with no usable length says nothing about progress. */ + private fun progress(id: String, sent: Long, total: Long) { + if (total <= 0 || sent < 0) return + val fraction = (sent.toDouble() / total).toFloat().coerceIn(0f, 1f) + states.update { all -> + when (val current = all[id]) { + is ChatMediaUploadState.Preparing -> all + (id to ChatMediaUploadState.Uploading(fraction)) + is ChatMediaUploadState.Uploading -> + if (fraction > current.fraction) all + (id to ChatMediaUploadState.Uploading(fraction)) else all + else -> all + } + } + } + + private fun fail(entry: Entry, cause: Throwable, stored: UploadedPhoto? = null, retryable: Boolean? = null): Nothing? { + // A chip removed while its work was in flight stays removed. + if (has(entry.id)) { + states.update { + it + (entry.id to ChatMediaUploadState.Failed(cause, retryable ?: ChatMediaRetry.isRetryable(cause), stored)) + } + } + return null + } + + companion object { + const val DIRECTORY = "pending-media" + + /** The photo's file in [filesDir], where `sendMedia`'s pending row points the UI. */ + fun fileFor(filesDir: File, fileName: String) = File(File(filesDir, DIRECTORY), fileName) + } +} + +private fun Context.sourceSize(uri: Uri): Pair? = runCatching { + contentResolver.openInputStream(uri)?.use { stream -> + val options = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeStream(stream, null, options) + if (options.outWidth > 0 && options.outHeight > 0) options.outWidth to options.outHeight else null + } +}.getOrNull() + +private fun blurhashOfJpeg(bytes: ByteArray): String? = runCatching { + val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return null + try { + BlurHashEncoder.fromThumbnail(bitmap, bitmap.width, bitmap.height).takeIf { it.isNotEmpty() } + } finally { + bitmap.recycle() + } +}.getOrNull() + +/** Copies [file] into Coil's disk cache under [key], where [ChatPhotoFetcher] looks first. */ +private fun Context.keepInImageCache(key: String, file: File) { + val cache = coil3.SingletonImageLoader.get(this).diskCache ?: return + val editor = cache.openEditor(key) ?: return + try { + cache.fileSystem.write(editor.data) { write(file.readBytes()) } + editor.commit() + } catch (t: Throwable) { + editor.abort() + throw t + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoFetcher.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoFetcher.kt new file mode 100644 index 0000000000..3ba43a7da0 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoFetcher.kt @@ -0,0 +1,121 @@ +package com.flipcash.shared.chat.media + +import coil3.ImageLoader +import coil3.decode.DataSource +import coil3.decode.ImageSource +import coil3.disk.DiskCache +import coil3.fetch.FetchResult +import coil3.fetch.Fetcher +import coil3.fetch.SourceFetchResult +import coil3.key.Keyer +import coil3.request.Options +import com.flipcash.services.chat.BlobOpenFailure +import com.getcode.utils.TraceType +import com.getcode.utils.trace +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import okio.Buffer +import okio.buffer +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Keys a [ChatPhoto] by its blob id, so the memory cache never keys on a signed URL that rotates + * on every mint. + */ +class ChatPhotoKeyer : Keyer { + override fun key(data: ChatPhoto, options: Options): String = data.cacheKey +} + +/** + * Loads a [ChatPhoto]: the disk cache by blob id first, else a download from a resolved (and, if + * need be, re-minted) URL, opened when the message is sealed, then cached. + * + * Failures surface as [ChatPhotoUnavailable] so a UI can draw "This photo can't be displayed" + * for any of them; a [ChatPhotoUnavailable.Open] carrying [BlobOpenFailure.KeyPending] may succeed + * on a later attempt. Decode failures of opened bytes surface as Coil's own decode error and + * are, to the UI, the same as [BlobOpenFailure.Undecodable]. + */ +class ChatPhotoFetcher( + private val photo: ChatPhoto, + private val loader: ChatPhotoLoader, + private val http: OkHttpClient, + private val imageLoader: ImageLoader, +) : Fetcher { + + override suspend fun fetch(): FetchResult = try { + load() + } catch (e: ChatPhotoUnavailable) { + trace(tag = TAG, message = "Chat photo ${photo.cacheKey} unavailable: ${e.message}", type = TraceType.Log) + throw e + } + + private suspend fun load(): FetchResult { + if (photo.redacted) throw ChatPhotoUnavailable.Redacted() + val cache = imageLoader.diskCache + val key = photo.cacheKey + cache?.openSnapshot(key)?.let { return it.asResult() } + + var url = loader.resolveUrl(photo) ?: throw ChatPhotoUnavailable.NotFound() + var bytes = download(url) + if (bytes == null) { + // The signed URL can die before its stated expiry; one re-mint is worth trying. + url = loader.resolveUrl(photo, failedUrl = url) ?: throw ChatPhotoUnavailable.NotFound() + bytes = download(url) ?: throw ChatPhotoUnavailable.Download(null) + } + val opened = loader.open(photo, bytes) + + if (cache != null) { + cache.write(key, opened)?.let { return it.asResult() } + } + val buffer = Buffer().write(opened) + return SourceFetchResult( + source = ImageSource(buffer, imageLoader.diskCacheFileSystem()), + mimeType = "image/jpeg", + dataSource = DataSource.NETWORK, + ) + } + + private suspend fun download(url: String): ByteArray? = withContext(Dispatchers.IO) { + runCatching { + http.newCall(Request.Builder().url(url).build()).execute().use { response -> + if (response.isSuccessful) response.body?.bytes() else null + } + }.getOrNull() + } + + private fun DiskCache.write(key: String, bytes: ByteArray): DiskCache.Snapshot? { + val editor = openEditor(key) ?: return openSnapshot(key) + return try { + fileSystem.sink(editor.data).buffer().use { it.write(bytes) } + editor.commitAndOpenSnapshot() + } catch (t: Throwable) { + runCatching { editor.abort() } + null + } + } + + private fun DiskCache.Snapshot.asResult() = SourceFetchResult( + source = ImageSource(file = data, fileSystem = imageLoader.diskCacheFileSystem(), diskCacheKey = photo.cacheKey, closeable = this), + mimeType = "image/jpeg", + dataSource = DataSource.DISK, + ) + + private fun ImageLoader.diskCacheFileSystem() = diskCache?.fileSystem ?: okio.FileSystem.SYSTEM + + private companion object { + const val TAG = "ChatPhotoFetcher" + } + + @Singleton + class Factory @Inject constructor( + private val loader: ChatPhotoLoader, + ) : Fetcher.Factory { + private val http by lazy { OkHttpClient() } + + override fun create(data: ChatPhoto, options: Options, imageLoader: ImageLoader): Fetcher = + ChatPhotoFetcher(data, loader, http, imageLoader) + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoader.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoader.kt new file mode 100644 index 0000000000..ed3b9deae4 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoader.kt @@ -0,0 +1,119 @@ +package com.flipcash.shared.chat.media + +import com.flipcash.app.persistence.sources.ChatMemberDataSource +import com.flipcash.services.chat.BlobOpenFailure +import com.flipcash.services.chat.ChatContentCrypto +import com.flipcash.services.chat.OpenedBlob +import com.flipcash.services.controllers.BlobStorageController +import com.flipcash.services.models.chat.BlobAccessContext +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.MediaItemRendition +import com.flipcash.services.user.UserManager +import com.getcode.opencode.model.core.ID +import com.getcode.utils.hexEncodedString +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.time.Clock + +/** + * What an image loader needs to draw a chat photo: the rendition to fetch and what it takes to open. + * + * The cache key is the blob id, never the signed URL, which rotates on every mint. + * + * @property sealed whether the message is end-to-end encrypted, so the bytes must be opened with + * [senderId]'s key before they decode. + * @property redacted a redacted message's photo is never fetched. + */ +data class ChatPhoto( + val chatId: ChatId, + val rendition: MediaItemRendition, + val senderId: ID?, + val sealed: Boolean, + val redacted: Boolean = false, +) { + val cacheKey: String get() = cacheKeyOf(rendition) + + companion object { + fun cacheKeyOf(rendition: MediaItemRendition) = cacheKeyOf(rendition.blobId) + fun cacheKeyOf(blobId: BlobId) = "chat-media-${blobId.bytes.toList().hexEncodedString()}" + } +} + +/** Why a photo can't be drawn; the UI shows "This photo can't be displayed" for every one. */ +sealed class ChatPhotoUnavailable(message: String, cause: Throwable? = null) : Exception(message, cause) { + class Redacted : ChatPhotoUnavailable("Redacted message") + + /** No URL could be resolved for the blob. */ + class NotFound : ChatPhotoUnavailable("No download URL for the photo") + + /** The bytes could not be downloaded. */ + class Download(cause: Throwable?) : ChatPhotoUnavailable("Photo download failed", cause) + + /** The sealed bytes did not open or decode; [reason] says why. [BlobOpenFailure.KeyPending] can succeed later. */ + class Open(val reason: BlobOpenFailure) : ChatPhotoUnavailable("Photo did not open: $reason") +} + +/** + * The network-free parts of loading a chat photo, so the Coil fetcher around them stays thin: + * URL resolution and opening sealed bytes. + */ +@Singleton +class ChatPhotoLoader internal constructor( + private val blobStorage: BlobStorageController, + private val crypto: ChatContentCrypto, + private val userManager: UserManager, + private val members: ChatMemberDataSource, + private val now: () -> kotlin.time.Instant, +) { + @Inject + constructor( + blobStorage: BlobStorageController, + crypto: ChatContentCrypto, + userManager: UserManager, + members: ChatMemberDataSource, + ) : this(blobStorage, crypto, userManager, members, { Clock.System.now() }) + + /** + * A URL to download [photo] from: the stored one while it is usable, else a freshly minted one. + * [failedUrl] is one that just failed, which forces a mint even if it looks valid, since + * metadata stored before expiry was modelled carries no expiry. Null if none can be had. + */ + suspend fun resolveUrl(photo: ChatPhoto, failedUrl: String? = null): String? { + if (photo.redacted) return null + val stored = photo.rendition.blob?.downloadUrl?.takeIf { it.isNotBlank() } + val usable = stored != null && stored != failedUrl && !photo.rendition.isDownloadUrlExpired(now()) + if (usable) return stored + return blobStorage.refreshMetadata(listOf(photo.rendition.blobId), BlobAccessContext.Chat(photo.chatId)) + .getOrNull() + ?.get(photo.rendition.cacheKey) + ?.downloadUrl + ?.takeIf { it.isNotBlank() } + } + + /** + * The decodable bytes for [downloaded]: as they are for a plain photo, opened for a sealed one. + * Throws [ChatPhotoUnavailable.Open] when they don't open. + */ + suspend fun open(photo: ChatPhoto, downloaded: ByteArray): ByteArray { + if (!photo.sealed) return downloaded + val selfId = userManager.accountId ?: throw ChatPhotoUnavailable.Open(BlobOpenFailure.KeyPending) + val peerId = members.getMembersForChat(photo.chatId).firstOrNull { it.userId != selfId }?.userId + ?: throw ChatPhotoUnavailable.Open(BlobOpenFailure.KeyPending) + val expected = photo.rendition.blob?.sizeBytes ?: throw ChatPhotoUnavailable.Open(BlobOpenFailure.Length) + return when ( + val opened = crypto.openBlob( + chatId = photo.chatId, + selfId = selfId, + peerId = peerId, + senderId = photo.senderId, + blobId = photo.rendition.blobId.bytes, + expectedSize = expected, + sealed = downloaded, + ) + ) { + is OpenedBlob.Plaintext -> opened.bytes + is OpenedBlob.Failed -> throw ChatPhotoUnavailable.Open(opened.reason) + } + } +} diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/SentPhotoPreviews.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/SentPhotoPreviews.kt new file mode 100644 index 0000000000..23828801e4 --- /dev/null +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/media/SentPhotoPreviews.kt @@ -0,0 +1,47 @@ +package com.flipcash.shared.chat.media + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.LruCache + +/** + * Decoded copies of photos this process staged, so a sent photo's bubble has its picture on the + * first frame it is drawn and keeps it while the row moves from the staged file to the stored blob. + * + * Staged photos are kept by chip id; [handOff] moves one to its message's pending client id, which + * the row keeps after the server copy replaces it. Nothing here outlives the process. + */ +object SentPhotoPreviews { + private const val MAX_ENTRIES = 12 + private const val MAX_EDGE = 1080 + + private val cache = LruCache(MAX_ENTRIES) + + /** The preview for a message's pending client id, if this process staged it. */ + fun forMessage(pendingClientIdHex: String): Bitmap? = cache.get(message(pendingClientIdHex)) + + internal fun stage(chipId: String, jpeg: ByteArray) { + decode(jpeg)?.let { cache.put(chip(chipId), it) } + } + + internal fun handOff(chipId: String, pendingClientIdHex: String) { + val bitmap = cache.remove(chip(chipId)) ?: return + cache.put(message(pendingClientIdHex), bitmap) + } + + internal fun drop(chipId: String) { + cache.remove(chip(chipId)) + } + + private fun chip(id: String) = "chip:$id" + private fun message(hex: String) = "msg:$hex" + + private fun decode(jpeg: ByteArray): Bitmap? = runCatching { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(jpeg, 0, jpeg.size, bounds) + var sample = 1 + while (maxOf(bounds.outWidth, bounds.outHeight) / (sample * 2) >= MAX_EDGE) sample *= 2 + BitmapFactory.decodeByteArray(jpeg, 0, jpeg.size, BitmapFactory.Options().apply { inSampleSize = sample }) + ?.also { it.prepareToDraw() } + }.getOrNull() +} diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessageCapabilityTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessageCapabilityTest.kt index 4e2cf7be75..327aef1015 100644 --- a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessageCapabilityTest.kt +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessageCapabilityTest.kt @@ -500,4 +500,52 @@ class MessageCapabilityTest { fun `empty content is not reactable`() { assertEquals(false, canReact(message(emptyList()))) } + + private fun photo(isFromSelf: Boolean = true, eventSequence: Long = 4, redacted: Boolean = false, reply: Boolean = false): ChatMessage { + val media = MessageContent.Media(items = emptyList(), caption = MessageContent.Text("hi")) + return message( + listOf(if (reply) MessageContent.Reply(repliedMessageId = 2, content = listOf(media)) else media), + isFromSelf, + eventSequence, + ).copy(redacted = redacted) + } + + @Test + fun `own confirmed photo can be replied to and deleted but not copied or edited`() { + assertEquals( + setOf(MessageCapability.Reply, MessageCapability.Delete), + resolveCapabilities(photo(), now = sentAt), + ) + assertEquals( + setOf(MessageCapability.Reply, MessageCapability.Delete), + resolveCapabilities(photo(reply = true), now = sentAt), + ) + } + + @Test + fun `own photo past the delete window can only be replied to`() { + assertEquals( + setOf(MessageCapability.Reply), + resolveCapabilities(photo(), now = sentAt + 49.hours), + ) + } + + @Test + fun `another participant's photo can be replied to and reported`() { + assertEquals( + setOf(MessageCapability.Reply, MessageCapability.Report), + resolveCapabilities(photo(isFromSelf = false), now = sentAt), + ) + } + + @Test + fun `an unconfirmed photo has no actions`() { + assertEquals(emptySet(), resolveCapabilities(photo(eventSequence = 0), now = sentAt)) + } + + @Test + fun `a redacted photo takes no reactions`() { + assertEquals(true, canReact(photo(isFromSelf = false))) + assertEquals(false, canReact(photo(isFromSelf = false, redacted = true))) + } } diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessagingSendEncryptionTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessagingSendEncryptionTest.kt index f8435de1d9..3c0a859384 100644 --- a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessagingSendEncryptionTest.kt +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/MessagingSendEncryptionTest.kt @@ -16,7 +16,12 @@ import com.flipcash.services.controllers.ChatMessagingController import com.flipcash.services.models.GetChatError import com.flipcash.services.models.SendMessageError import com.flipcash.services.models.UserProfile +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobMetadata import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.ImageMetadata +import com.flipcash.services.models.chat.MediaItem +import com.flipcash.services.models.chat.MediaItemRendition import com.flipcash.services.models.chat.ChatMember import com.flipcash.services.models.chat.ChatMessage import com.flipcash.services.models.chat.ChatMetadata @@ -177,6 +182,71 @@ class MessagingSendEncryptionTest { ) } + private val photo = MessageContent.Media( + items = listOf( + MediaItem( + listOf( + MediaItemRendition( + role = MediaItemRendition.Role.ORIGINAL, + blobId = BlobId(ByteArray(16) { it.toByte() }), + blob = BlobMetadata( + mimeType = "image/jpeg", + sizeBytes = 10, + downloadUrl = "", + image = ImageMetadata(width = 4, height = 3, blurhash = ""), + ), + ) + ) + ) + ), + caption = null, + ) + + private fun outgoing() = DmOutgoingEncryption( + policy = E2eePolicy(), + crypto = ChatContentCrypto(FakeChatCipher, keys), + userManager = userManager, + chatController = chatController, + metadataDataSource = metadata, + memberDataSource = members, + ) + + @Test + fun `a photo sealed for this chat goes out sealed`() = runTest { + storedChat(chat()) + + val result = outgoing().prepare(chatId, listOf(photo), blobSealedFor = chatId).getOrThrow() + + assertEquals(true, result.isSealed) + assertEquals(photo, openAsPeer(result.wire)) + } + + @Test + fun `a plain photo is refused in an encrypted chat`() = runTest { + storedChat(chat()) + + assertEquals(true, outgoing().prepare(chatId, listOf(photo), blobSealedFor = null).isFailure) + } + + @Test + fun `a sealed photo is refused in a chat that does not encrypt, or sealed for another chat`() = runTest { + storedChat(chat(useE2ee = false)) + assertEquals(true, outgoing().prepare(chatId, listOf(photo), blobSealedFor = chatId).isFailure) + + storedChat(chat()) + val other = ChatId(ByteArray(32) { 4 }) + assertEquals(true, outgoing().prepare(chatId, listOf(photo), blobSealedFor = other).isFailure) + } + + @Test + fun `a plain photo goes out plain in a chat that does not encrypt`() = runTest { + storedChat(chat(useE2ee = false)) + + val result = outgoing().prepare(chatId, listOf(photo), blobSealedFor = null).getOrThrow() + + assertEquals(false, result.isSealed) + } + @Test fun `a DM without the flag goes out in plaintext`() = runTest { storedChat(chat(useE2ee = false)) diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlanTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlanTest.kt new file mode 100644 index 0000000000..d5832786c6 --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaSendPlanTest.kt @@ -0,0 +1,24 @@ +package com.flipcash.shared.chat.media + +import kotlin.test.Test +import kotlin.test.assertEquals + +class ChatMediaSendPlanTest { + + @Test + fun `whitespace around the caption is trimmed`() { + assertEquals( + listOf(ChatMediaSendPlan.Message.Media("a", "look", null)), + ChatMediaSendPlan.build(listOf("a"), " look \n", null), + ) + } + + @Test + fun `whitespace alone is no caption and no text message`() { + assertEquals( + listOf(ChatMediaSendPlan.Message.Media("a", null, null)), + ChatMediaSendPlan.build(listOf("a"), " ", null), + ) + assertEquals(emptyList(), ChatMediaSendPlan.build(emptyList(), " ", 5)) + } +} diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaUploadsTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaUploadsTest.kt new file mode 100644 index 0000000000..967a20e33c --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaUploadsTest.kt @@ -0,0 +1,325 @@ +package com.flipcash.shared.chat.media + +import android.net.Uri +import com.flipcash.app.blob.BlobStorageCoordinator +import com.flipcash.app.blob.ChatMediaEncoder +import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.blob.EncryptedConstraints +import com.flipcash.services.models.blob.MimeTypeConstraints +import com.flipcash.services.models.blob.UploadPolicy +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobRejection +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.ModerationResult +import com.flipcash.services.models.chat.RejectionReason +import com.flipcash.shared.chat.internal.BlobSealer +import com.flipcash.shared.chat.internal.OutgoingEncryption +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.io.IOException +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.hours + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +class ChatMediaUploadsTest { + + @get:Rule val folder = TemporaryFolder() + + private val chatId = ChatId(byteArrayOf(1)) + private val blobId = BlobId(byteArrayOf(9, 9)) + private val jpeg = byteArrayOf(1, 2, 3, 4) + private val uri = Uri.parse("content://photo/1") + + private val encoder = mockk { + coEvery { encode(any(), any()) } returns Result.success(ChatMediaEncoder.Encoded(jpeg, 100, 50)) + } + private val blobs = mockk { + every { policy } returns MutableStateFlow(policy(jpeg = true)) + coEvery { awaitChatMediaReady(any()) } returns Result.success(blobId) + } + private var sealer: BlobSealer? = null + private val outgoing = object : OutgoingEncryption by OutgoingEncryption.None { + override suspend fun blobSealer(chatId: ChatId) = Result.success(sealer) + } + + private fun policy(jpeg: Boolean) = UploadPolicy( + version = "v1", + ttl = 1.hours, + mimeTypeConstraints = listOf(MimeTypeConstraints(if (jpeg) "image/jpeg" else "image/png", 1_000_000, null)), + encrypted = EncryptedConstraints(1_000_000, null), + ) + + private fun TestScope.uploads(scope: CoroutineScope = backgroundScope) = ChatMediaUploads( + directory = { File(folder.root, "pending-media") }, + encoder = encoder, + blobs = blobs, + outgoing = outgoing, + sourceSize = { 4000 to 3000 }, + blurhashOf = { "HASH" }, + scope = scope, + newId = { "chip" }, + ) + + private fun stores(block: suspend (onAttempt: (() -> Unit)?, onProgress: ((Long, Long) -> Unit)?) -> Result) { + coEvery { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } coAnswers { + @Suppress("UNCHECKED_CAST") + block(arg(2) as (() -> Unit)?, arg(3) as ((Long, Long) -> Unit)?) + } + } + + @Test + fun `happy path walks preparing, uploading, processing, uploaded`() = runTest { + val seen = ArrayList() + stores { onAttempt, onProgress -> + onAttempt?.invoke() + yield() + onProgress?.invoke(2, 4) + yield() + onProgress?.invoke(4, 4) + yield() + Result.success(blobId) + } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + launchCollect(uploads, id, seen) + runCurrent() + + assertEquals(ChatMediaUploadState.Preparing(4000, 3000), seen.first { it is ChatMediaUploadState.Preparing && it.sourceWidth != null }) + assertTrue(seen.any { it == ChatMediaUploadState.Uploading(0.5f) } || seen.none { it is ChatMediaUploadState.Uploading }) + val done = assertIs(seen.last()) + assertEquals(UploadedPhoto(blobId, 100, 50, null, 4, null), done.photo) + assertEquals(jpeg.toList(), File(folder.root, "pending-media/chip.jpg").readBytes().toList()) + } + + @Test + fun `a shot staged at the shutter is not read until it is written`() = runTest { + stores { _, _ -> Result.success(blobId) } + val ready = CompletableDeferred() + val uploads = uploads() + val id = uploads.stage(chatId, uri, ready) + runCurrent() + + coVerify(exactly = 0) { encoder.encode(any(), any()) } + assertEquals(ChatMediaUploadState.Preparing(null, null), uploads.state(id).first()) + + ready.complete(true) + runCurrent() + + coVerify(exactly = 1) { encoder.encode(any(), any()) } + assertIs(uploads.state(id).first()) + } + + @Test + fun `a shot that failed to write is never read`() = runTest { + val ready = CompletableDeferred() + val uploads = uploads() + val id = uploads.stage(chatId, uri, ready) + ready.complete(false) + runCurrent() + + coVerify(exactly = 0) { encoder.encode(any(), any()) } + assertEquals(ChatMediaUploadState.Preparing(null, null), uploads.state(id).first()) + } + + private fun TestScope.launchCollect(uploads: ChatMediaUploads, id: String, into: MutableList) = + backgroundScope.launch(start = CoroutineStart.UNDISPATCHED) { + uploads.state(id).collect { it?.let(into::add) } + } + + @Test + fun `progress is monotonic within an attempt and ignores unknown length`() = runTest { + val fractions = ArrayList() + stores { onAttempt, onProgress -> + onAttempt?.invoke() + yield() + onProgress?.invoke(1, 4) + yield() + onProgress?.invoke(3, 4) + yield() + onProgress?.invoke(2, 4) // backwards + yield() + onProgress?.invoke(4, 0) // unknown length + yield() + onProgress?.invoke(-1, 4) + yield() + Result.success(blobId) + } + val seen = ArrayList() + val uploads = uploads() + val id = uploads.stage(chatId, uri) + launchCollect(uploads, id, seen) + runCurrent() + fractions += seen.filterIsInstance().map { it.fraction } + assertEquals(listOf(0.25f, 0.75f), fractions) + } + + @Test + fun `bytes reported after the store returned are ignored`() = runTest { + var late: ((Long, Long) -> Unit)? = null + stores { onAttempt, onProgress -> + onAttempt?.invoke() + late = onProgress + Result.success(blobId) + } + coEvery { blobs.awaitChatMediaReady(any()) } coAnswers { + late?.invoke(1, 4) + awaitCancellation() + } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + assertIs(uploads.current(id)) + } + + @Test + fun `a retried store starts over from preparing`() = runTest { + val seen = ArrayList() + stores { onAttempt, onProgress -> + onAttempt?.invoke() + yield() + onProgress?.invoke(3, 4) + yield() + onAttempt?.invoke() + yield() + onProgress?.invoke(1, 4) + yield() + Result.success(blobId) + } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + launchCollect(uploads, id, seen) + runCurrent() + val states = seen.filter { it is ChatMediaUploadState.Preparing || it is ChatMediaUploadState.Uploading } + val resetAt = states.indexOfLast { it is ChatMediaUploadState.Preparing } + assertEquals(ChatMediaUploadState.Uploading(0.75f), states[resetAt - 1]) + assertEquals(ChatMediaUploadState.Uploading(0.25f), states[resetAt + 1]) + } + + @Test + fun `a network failure that outlasts the retries is retryable`() = runTest { + stores { _, _ -> Result.failure(IOException("offline")) } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + val failed = assertIs(uploads.current(id)) + assertTrue(failed.retryable) + assertNull(failed.stored) + } + + @Test + fun `a moderation rejection is terminal`() = runTest { + stores { _, _ -> Result.success(blobId) } + coEvery { blobs.awaitChatMediaReady(any()) } returns Result.failure( + BlobRejectedException(BlobRejection(RejectionReason.MODERATION, ModerationResult.FlaggedCategory.entries.first())), + ) + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + val failed = assertIs(uploads.current(id)) + assertFalse(failed.retryable) + } + + @Test + fun `no jpeg constraint uploads nothing`() = runTest { + every { blobs.policy } returns MutableStateFlow(policy(jpeg = false)) + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + assertFalse(assertIs(uploads.current(id)).retryable) + coVerify(exactly = 0) { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } + } + + @Test + fun `retrying a failed chip resumes the stored blob without uploading again`() = runTest { + stores { _, _ -> Result.success(blobId) } + coEvery { blobs.awaitChatMediaReady(any()) } returns Result.failure(IOException("timed out")) + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + assertEquals(blobId, assertIs(uploads.current(id)).stored?.blobId) + + coEvery { blobs.awaitChatMediaReady(any()) } returns Result.success(blobId) + uploads.retry(id) + runCurrent() + + assertIs(uploads.current(id)) + coVerify(exactly = 1) { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } + coVerify(exactly = 1) { encoder.encode(any(), any()) } + } + + @Test + fun `retrying an unstored chip resends the written bytes without encoding again`() = runTest { + var attempts = 0 + stores { _, _ -> if (attempts++ == 0) Result.failure(IOException("offline")) else Result.success(blobId) } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + uploads.retry(id) + runCurrent() + assertIs(uploads.current(id)) + coVerify(exactly = 1) { encoder.encode(any(), any()) } + } + + @Test + fun `a sealing chat seals and never falls back to plain`() = runTest { + sealer = BlobSealer(chatId) { _, plain -> plain + 0 } + var sealing: com.flipcash.app.blob.ChatMediaSealing? = null + stores { _, _ -> Result.success(blobId) } + coEvery { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } coAnswers { + sealing = secondArg() + Result.success(blobId) + } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + assertEquals(chatId, sealing?.chatId) + val photo = assertIs(uploads.current(id)).photo + assertEquals(chatId, photo.sealedFor) + assertEquals("HASH", photo.blurhash) + } + + @Test + fun `removing a chip cancels the upload and deletes its file`() = runTest { + stores { onAttempt, _ -> + onAttempt?.invoke() + awaitCancellation() + } + val uploads = uploads() + val id = uploads.stage(chatId, uri) + runCurrent() + val file = File(folder.root, "pending-media/chip.jpg") + assertTrue(file.exists()) + + uploads.remove(id) + runCurrent() + + assertNull(uploads.current(id)) + assertFalse(uploads.has(id)) + assertFalse(file.exists()) + } +} diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaVectorTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaVectorTest.kt new file mode 100644 index 0000000000..9cfb1e4dfa --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatMediaVectorTest.kt @@ -0,0 +1,90 @@ +package com.flipcash.shared.chat.media + +import org.json.JSONObject +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * The send-plan and string halves of `chat_media.json` (git blob + * 54f976f2737e37811af0d9bba5a849e7ba3f7ade). The canonical copy is the orchestrator repo's + * `test-vectors/`; this one is synced, so a failure is a regression here or a decision to make in + * the canonical fixture, never a local edit. The blob/encoding half is asserted in `:shared:blob`. + */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE) +class ChatMediaVectorTest { + + private val fixture = JSONObject( + javaClass.classLoader!!.getResourceAsStream("chat_media.json")!!.bufferedReader().use { it.readText() } + ) + + @Test + fun `the attachment ceiling matches the fixture`() { + assertEquals(fixture.getInt("maxAttachments"), ChatMediaSendPlan.MAX_ATTACHMENTS) + } + + @Test + fun `fan-out matches the vectors`() { + val cases = fixture.getJSONArray("fanOut") + assertTrue(cases.length() > 0, "no fanOut vectors loaded") + for (i in 0 until cases.length()) { + val case = cases.getJSONObject(i) + val name = case.getString("name") + val chips = case.getJSONArray("chips").let { a -> (0 until a.length()).map { a.getString(it) } } + val plan = ChatMediaSendPlan.build( + chips = chips, + text = case.getString("text"), + replyTo = case.optNullableString("replyTo")?.id(), + ) + + val expected = case.getJSONArray("messages") + assertEquals(expected.length(), plan.size, "$name: message count") + for (m in 0 until expected.length()) { + val want = expected.getJSONObject(m) + val got = plan[m] + val replyTo = want.optNullableString("replyTo")?.id() + when (want.getString("kind")) { + "text" -> assertEquals( + ChatMediaSendPlan.Message.Text(want.getString("text"), replyTo), got, "$name[$m]", + ) + "media" -> assertEquals( + ChatMediaSendPlan.Message.Media(want.getString("chip"), want.optNullableString("caption"), replyTo), + got, + "$name[$m]", + ) + else -> error("$name[$m]: unknown kind") + } + } + } + } + + @Test + fun `strings match the vectors`() { + val cases = fixture.getJSONArray("strings") + assertTrue(cases.length() > 0, "no strings vectors loaded") + for (i in 0 until cases.length()) { + val case = cases.getJSONObject(i) + val caption = case.optNullableString("caption") + val expected = case.getJSONObject("expected") + assertEquals(expected.getString("snippet"), ChatMediaText.snippet(caption, "Photo"), case.getString("name")) + assertEquals(expected.getString("preview"), ChatMediaText.preview(caption, "Photo"), case.getString("name")) + } + } + + @Test + fun `a push carries the caption or the bare photo`() { + assertEquals("from today", ChatMediaText.pushText("from today")) + assertEquals("📷 Photo", ChatMediaText.pushText(null)) + assertEquals("📷 Photo", ChatMediaText.pushText(" ")) + } + + // The fixture names a cited message "m1"; the plan only needs it to be some id. + private fun String.id(): Long = removePrefix("m").toLong() + + private fun JSONObject.optNullableString(key: String): String? = + if (isNull(key)) null else getString(key) +} diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoaderTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoaderTest.kt new file mode 100644 index 0000000000..b32b58005e --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/ChatPhotoLoaderTest.kt @@ -0,0 +1,112 @@ +package com.flipcash.shared.chat.media + +import com.flipcash.app.persistence.sources.ChatMemberDataSource +import com.flipcash.services.chat.BlobOpenFailure +import com.flipcash.services.chat.ChatContentCrypto +import com.flipcash.services.chat.OpenedBlob +import com.flipcash.services.controllers.BlobStorageController +import com.flipcash.services.models.UserProfile +import com.flipcash.services.models.chat.BlobAccessContext +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobMetadata +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.ChatMember +import com.flipcash.services.models.chat.ImageMetadata +import com.flipcash.services.models.chat.MediaItemRendition +import com.flipcash.services.user.UserManager +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.time.Duration.Companion.hours +import kotlin.time.Instant + +class ChatPhotoLoaderTest { + private val chatId = ChatId(byteArrayOf(1)) + private val blobId = BlobId(byteArrayOf(0x0a, 0x0b)) + private val self = listOf(1) + private val peer = listOf(2) + private val now = Instant.fromEpochSeconds(10_000) + + private val storage = mockk() + private val crypto = mockk() + private val user = mockk { every { accountId } returns self } + private val members = mockk { + coEvery { getMembersForChat(chatId) } returns listOf(self, peer).map { + ChatMember(it, UserProfile(displayName = "n", socialAccounts = emptyList(), phoneNumber = null, email = null), emptyList()) + } + } + private val loader = ChatPhotoLoader(storage, crypto, user, members) { now } + + private fun metadata(url: String, expiresAt: Instant? = null) = BlobMetadata( + mimeType = "image/jpeg", sizeBytes = 3, downloadUrl = url, expiresAtMillis = expiresAt?.toEpochMilliseconds(), + image = ImageMetadata(10, 10, ""), + ) + + private fun photo(url: String?, expiresAt: Instant? = null, sealed: Boolean = false, redacted: Boolean = false) = ChatPhoto( + chatId, MediaItemRendition(MediaItemRendition.Role.ORIGINAL, blobId, url?.let { metadata(it, expiresAt) }), + senderId = peer, sealed = sealed, redacted = redacted, + ) + + @Test + fun `the cache key is the blob id and not the url`() { + assertEquals("chat-media-0a0b", photo("https://a?sig=1").cacheKey) + assertEquals(photo("https://a?sig=1").cacheKey, photo("https://a?sig=2").cacheKey) + } + + @Test + fun `a usable stored url is used without a mint`() = runTest { + assertEquals("https://a", loader.resolveUrl(photo("https://a", expiresAt = now + 1.hours))) + coVerify(exactly = 0) { storage.refreshMetadata(any(), any()) } + } + + @Test + fun `an expired url is minted again for the chat`() = runTest { + coEvery { storage.refreshMetadata(listOf(blobId), BlobAccessContext.Chat(chatId)) } returns + Result.success(mapOf(photo("x").rendition.cacheKey to metadata("https://fresh"))) + + assertEquals("https://fresh", loader.resolveUrl(photo("https://old", expiresAt = now - 1.hours))) + } + + @Test + fun `a missing url is minted and a url that just failed is replaced`() = runTest { + coEvery { storage.refreshMetadata(any(), any()) } returns + Result.success(mapOf(photo("x").rendition.cacheKey to metadata("https://fresh"))) + + assertEquals("https://fresh", loader.resolveUrl(photo(null))) + assertEquals("https://fresh", loader.resolveUrl(photo("https://stale"), failedUrl = "https://stale")) + } + + @Test + fun `a redacted photo never resolves`() = runTest { + assertNull(loader.resolveUrl(photo("https://a", redacted = true))) + coVerify(exactly = 0) { storage.refreshMetadata(any(), any()) } + } + + @Test + fun `a plain photo is passed through`() = runTest { + assertEquals(listOf(1, 2), loader.open(photo("u"), byteArrayOf(1, 2)).toList()) + } + + @Test + fun `a sealed photo is opened for the sender and declared size`() = runTest { + coEvery { crypto.openBlob(chatId, self, peer, peer, blobId.bytes, 3, any()) } returns + OpenedBlob.Plaintext(byteArrayOf(7, 8, 9)) + + assertEquals(listOf(7, 8, 9), loader.open(photo("u", sealed = true), byteArrayOf(1)).toList()) + } + + @Test + fun `a sealed photo that does not open surfaces the reason`() = runTest { + coEvery { crypto.openBlob(any(), any(), any(), any(), any(), any(), any()) } returns + OpenedBlob.Failed(BlobOpenFailure.Authentication) + + val failure = assertFailsWith { loader.open(photo("u", sealed = true), byteArrayOf(1)) } + assertEquals(BlobOpenFailure.Authentication, failure.reason) + } +} diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt new file mode 100644 index 0000000000..5f52266f98 --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt @@ -0,0 +1,364 @@ +package com.flipcash.shared.chat.media + +import android.net.Uri +import com.flipcash.app.blob.BlobStorageCoordinator +import com.flipcash.app.blob.ChatMediaEncoder +import com.flipcash.app.persistence.sources.ChatMessageDataSource +import com.flipcash.app.persistence.sources.ChatMetadataDataSource +import com.flipcash.app.persistence.sources.PendingMediaDataSource +import com.flipcash.app.persistence.sources.PendingMediaRecord +import com.flipcash.services.controllers.ChatMessagingController +import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.ModerationResult +import com.flipcash.services.models.blob.EncryptedConstraints +import com.flipcash.services.models.blob.MimeTypeConstraints +import com.flipcash.services.models.blob.UploadPolicy +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobRejection +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.ChatMessage +import com.flipcash.services.models.chat.ClientMessageId +import com.flipcash.services.models.chat.MessageContent +import com.flipcash.services.models.chat.RejectionReason +import com.flipcash.services.user.UserManager +import com.flipcash.shared.chat.internal.OutgoingEncryption +import com.flipcash.shared.chat.internal.delegates.MediaSendDelegate +import com.flipcash.shared.chat.internal.delegates.MessagingDelegate +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.coVerifyOrder +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.io.IOException +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.hours +import kotlin.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +class MediaSendDelegateTest { + + @get:Rule val folder = TemporaryFolder() + + private val chatId = ChatId(byteArrayOf(1, 1)) + private val self = listOf(7, 7) + + /** `pending_media`, as the data source would hold it. */ + private val records = LinkedHashMap() + private val pendingMedia = mockk().also { m -> + with(m) { stubPending() } + } + + private fun PendingMediaDataSource.stubPending() { + val self = this + coEvery { self.insert(any()) } coAnswers { firstArg>().forEach { records[it.clientIdHex] = it } } + coEvery { self.get(any()) } coAnswers { records[firstArg()] } + coEvery { self.getAll() } coAnswers { records.values.toList() } + coEvery { self.delete(any()) } coAnswers { records.remove(firstArg()); Unit } + coEvery { self.markStored(any(), any(), any(), any(), any()) } coAnswers { + val hex = firstArg() + records[hex] = records.getValue(hex).copy( + storedBlobIdHex = secondArg(), sealedForHex = thirdArg(), + sizeBytes = arg(3), blurhash = arg(4), + ) + } + } + + private class Inserted(val clientMessageId: ClientMessageId, val content: List) + + private val inserted = ArrayList() + private val messageDataSource = mockk(relaxed = true) { + coEvery { insertPending(any(), any(), any(), any(), any()) } coAnswers { + inserted += Inserted(arg(3), secondArg()) + mockk(relaxed = true) + } + coEvery { getRecentSentBy(any(), any(), any()) } returns emptyList() + } + + private val sent = ArrayList>>() + private var sendResult: (ClientMessageId) -> Result = { Result.success(serverMessage(5)) } + private val controller = mockk { + coEvery { sendMessage(any(), any(), any()) } coAnswers { + val id = thirdArg() + sent += id to secondArg() + sendResult(id) + } + } + + private val messaging = mockk(relaxed = true) + private val userManager = mockk { every { accountId } returns self } + + private val encoder = mockk() + private val blobs = mockk() + private val storeBehavior = HashMap Result>() + private var ready: (BlobId) -> Result = { Result.success(it) } + private var next = 0 + + private fun serverMessage(id: Long) = ChatMessage( + messageId = id, senderId = self, content = emptyList(), + timestamp = Instant.fromEpochMilliseconds(1_000), unreadSeq = 0, + ) + + private fun TestScope.delegate(): Pair { + every { blobs.policy } returns MutableStateFlow( + UploadPolicy("v1", 1.hours, listOf(MimeTypeConstraints("image/jpeg", 1_000_000, null)), EncryptedConstraints(1_000_000, null)), + ) + // The photo's bytes carry its name, so a store knows which chip it is for. + coEvery { encoder.encode(any(), any()) } coAnswers { + Result.success(ChatMediaEncoder.Encoded(firstArg().lastPathSegment!!.toByteArray(), 100, 50)) + } + coEvery { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } coAnswers { + val name = String(firstArg()) + storeBehavior[name]?.invoke() ?: Result.success(BlobId(name.toByteArray())) + } + coEvery { blobs.awaitChatMediaReady(any()) } coAnswers { ready(firstArg()) } + val uploads = ChatMediaUploads( + directory = { File(folder.root, "pending-media") }, + encoder = encoder, blobs = blobs, outgoing = OutgoingEncryption.None, + sourceSize = { 4000 to 3000 }, blurhashOf = { null }, + scope = backgroundScope, newId = { "chip${next++}" }, + ) + val delegate = MediaSendDelegate( + messaging = messaging, messagingController = controller, messageDataSource = messageDataSource, + metadataDataSource = mockk(relaxed = true), pendingMedia = pendingMedia, + uploads = uploads, outgoing = OutgoingEncryption.None, userManager = userManager, + scope = backgroundScope, now = { 1_000 }, + ) + return delegate to uploads + } + + private fun stage(uploads: ChatMediaUploads, name: String) = uploads.stage(chatId, Uri.parse("content://photos/$name")) + + private fun hex(id: ClientMessageId) = id.bytes.joinToString("") { "%02x".format(it) } + + @Test + fun `inserts every pending row up front`() = runTest { + val (delegate, uploads) = delegate() + val chips = listOf("a", "b", "c").map { stage(uploads, it) } + runCurrent() + + val ids = delegate.sendMedia(chatId, chips, "hello", 42).getOrThrow() + + assertEquals(3, ids.size) + assertEquals(3, inserted.size) + assertEquals(0, sent.size) + assertEquals(ids.toSet(), records.keys) + val captions = inserted.map { (it.content.single().let { c -> (c as? MessageContent.Reply)?.content?.single() ?: c } as MessageContent.Media).caption } + assertEquals(listOf(null, null, MessageContent.Text("hello")), captions) + assertIs(inserted.first().content.single()) + assertEquals(100, rendition(inserted.first()).blob?.image?.width) + } + + private fun rendition(i: Inserted) = + ((i.content.single().let { (it as? MessageContent.Reply)?.content?.single() ?: it }) as MessageContent.Media) + .items.single().renditions.single() + + @Test + fun `posts strictly in chip order even when a later upload finishes first`() = runTest { + val (delegate, uploads) = delegate() + val gate = kotlinx.coroutines.CompletableDeferred() + storeBehavior["a"] = { gate.await(); Result.success(BlobId("a".toByteArray())) } + val chips = listOf("a", "b", "c").map { stage(uploads, it) } + runCurrent() + val ids = delegate.sendMedia(chatId, chips, "", null).getOrThrow() + runCurrent() + assertEquals(0, sent.size) // b and c are uploaded, but a is first + gate.complete(Unit) + runCurrent() + + assertEquals(ids, sent.map { hex(it.first) }) + assertTrue(records.isEmpty()) + coVerify(exactly = 3) { messageDataSource.confirmPending(any(), any(), any(), true) } + } + + @Test + fun `a failed upload fails only its row`() = runTest { + val (delegate, uploads) = delegate() + storeBehavior["b"] = { Result.failure(IOException("offline")) } + val chips = listOf("a", "b", "c").map { stage(uploads, it) } + val ids = delegate.sendMedia(chatId, chips, "", null).getOrThrow() + runCurrent() + + assertEquals(listOf(ids[0], ids[2]), sent.map { hex(it.first) }) + coVerify(exactly = 1) { messageDataSource.failPending(chatId, match { hex(it) == ids[1] }) } + assertEquals(setOf(ids[1]), records.keys) // kept for retry + } + + @Test + fun `a failed post fails the row and keeps the stored photo`() = runTest { + val (delegate, uploads) = delegate() + sendResult = { Result.failure(IOException("down")) } + val ids = delegate.sendMedia(chatId, listOf(stage(uploads, "a")), "", null).getOrThrow() + runCurrent() + + coVerify { messageDataSource.failPending(chatId, match { hex(it) == ids[0] }) } + assertNotNull(records[ids[0]]?.storedBlobIdHex) + assertEquals(MediaSendProgress.Failed(true), delegate.progressOf(ids[0])) + } + + @Test + fun `the pending row falls back to source pixels before encoding finishes`() = runTest { + val (delegate, uploads) = delegate() + coEvery { encoder.encode(any(), any()) } coAnswers { delay(10_000); Result.failure(IOException()) } + val chip = stage(uploads, "a") + runCurrent() + delegate.sendMedia(chatId, listOf(chip), "", null) + val image = rendition(inserted.single()).blob?.image + assertEquals(4000 to 3000, image?.width to image?.height) + } + + @Test + fun `retry posts a stored photo again under the same client id`() = runTest { + val (delegate, uploads) = delegate() + sendResult = { Result.failure(IOException("down")) } + val ids = delegate.sendMedia(chatId, listOf(stage(uploads, "a")), "", null).getOrThrow() + runCurrent() + sendResult = { Result.success(serverMessage(9)) } + + delegate.retryMedia(chatId, ids[0]).getOrThrow() + runCurrent() + + assertEquals(listOf(ids[0], ids[0]), sent.map { hex(it.first) }) + coVerify(exactly = 1) { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } + assertTrue(records.isEmpty()) + } + + @Test + fun `retry uploads the stored file again when nothing reached storage`() = runTest { + val (delegate, uploads) = delegate() + var attempts = 0 + storeBehavior["a"] = { if (attempts++ == 0) Result.failure(IOException("offline")) else Result.success(BlobId("a".toByteArray())) } + val ids = delegate.sendMedia(chatId, listOf(stage(uploads, "a")), "", null).getOrThrow() + runCurrent() + assertEquals(0, sent.size) + + delegate.retryMedia(chatId, ids[0]).getOrThrow() + runCurrent() + + assertEquals(1, sent.size) + coVerify(exactly = 1) { encoder.encode(any(), any()) } + coVerify(exactly = 2) { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } + } + + @Test + fun `a moderation rejection is not retried and leaves the queue`() = runTest { + val (delegate, uploads) = delegate() + ready = { + Result.failure(BlobRejectedException(BlobRejection(RejectionReason.MODERATION, ModerationResult.FlaggedCategory.entries.first()))) + } + val ids = delegate.sendMedia(chatId, listOf(stage(uploads, "a")), "", null).getOrThrow() + runCurrent() + + assertTrue(records.isEmpty()) + assertFalse(File(folder.root, "pending-media/chip0.jpg").exists()) + assertTrue(delegate.retryMedia(chatId, ids[0]).isFailure) + assertEquals(0, sent.size) + coVerify { messageDataSource.failPending(chatId, match { hex(it) == ids[0] }) } + } + + // region launch reconciliation + + private fun record(hex: String, stored: String?, file: String = "old.jpg") = PendingMediaRecord( + clientIdHex = hex, chatIdHex = "0101", fileName = file, caption = null, replyToMessageId = null, + storedBlobIdHex = stored, sealedForHex = null, width = 100, height = 50, blurhash = null, + sizeBytes = 4, createdAt = 1, + ) + + @Test + fun `an unstored entry is restored failed and waits for a retry`() = runTest { + val (delegate, uploads) = delegate() + File(folder.root, "pending-media").mkdirs() + File(folder.root, "pending-media/old.jpg").writeBytes(byteArrayOf(1)) + records["aa"] = record("aa", stored = null) + + delegate.reconcilePendingMedia() + runCurrent() + + assertEquals(0, sent.size) + assertEquals(MediaSendProgress.Failed(true), delegate.progressOf("aa")) + assertTrue(uploads.has("old")) + assertTrue(records.containsKey("aa")) + } + + @Test + fun `a stored entry resumes and posts without being asked`() = runTest { + val (delegate, _) = delegate() + File(folder.root, "pending-media").mkdirs() + records["aa"] = record("aa", stored = "0909") + + delegate.reconcilePendingMedia() + runCurrent() + + assertEquals(listOf("aa"), sent.map { hex(it.first) }) + coVerify(exactly = 0) { blobs.storeChatMediaUnfinalized(any(), any(), any(), any()) } + coVerify { messageDataSource.retryPending(chatId, "aa") } + assertTrue(records.isEmpty()) + } + + @Test + fun `an entry whose message is already in the chat is dropped`() = runTest { + val (delegate, _) = delegate() + File(folder.root, "pending-media").mkdirs() + File(folder.root, "pending-media/old.jpg").writeBytes(byteArrayOf(1)) + records["aa"] = record("aa", stored = "0909") + val arrived = serverMessage(12).copy( + content = listOf( + MessageContent.Media( + listOf( + com.flipcash.services.models.chat.MediaItem( + listOf( + com.flipcash.services.models.chat.MediaItemRendition( + com.flipcash.services.models.chat.MediaItemRendition.Role.ORIGINAL, + BlobId(byteArrayOf(9, 9)), null, + ), + ), + ), + ), + null, + ), + ), + ) + coEvery { messageDataSource.getRecentSentBy(any(), any(), any()) } returns listOf(arrived) + + delegate.reconcilePendingMedia() + runCurrent() + + assertEquals(0, sent.size) + assertTrue(records.isEmpty()) + assertFalse(File(folder.root, "pending-media/old.jpg").exists()) + coVerify { messageDataSource.confirmPending(chatId, any(), arrived, true) } + } + + @Test + fun `an unstored entry whose file is gone is dropped`() = runTest { + val (delegate, _) = delegate() + records["aa"] = record("aa", stored = null) + delegate.reconcilePendingMedia() + runCurrent() + assertTrue(records.isEmpty()) + } + + // endregion + + private suspend fun MediaSendDelegate.progressOf(hex: String): MediaSendProgress? = + observeMediaSendProgress().first()[hex] +} diff --git a/apps/flipcash/shared/chat/src/test/resources/chat_media.json b/apps/flipcash/shared/chat/src/test/resources/chat_media.json new file mode 100644 index 0000000000..54f976f273 --- /dev/null +++ b/apps/flipcash/shared/chat/src/test/resources/chat_media.json @@ -0,0 +1,639 @@ +{ + "algorithm": "chat-media-photos", + "note": "Behavior fixture for docs/superpowers/specs/2026-09-28-chat-media-photos-design.md. Heights are points or dp as floats; compare with a 0.001 tolerance.", + "maxAttachments": 10, + "minAspect": 0.5, + "maxAspect": 2.0, + "jpegQualityLadder": [ + 0.9, + 0.8, + 0.7, + 0.6 + ], + "uploadMimeType": "image/jpeg", + "fanOut": [ + { + "name": "text-only", + "chips": [], + "text": "hello", + "replyTo": null, + "messages": [ + { + "kind": "text", + "chip": null, + "caption": null, + "text": "hello", + "replyTo": null + } + ], + "note": "No chips: a text message, exactly as today." + }, + { + "name": "text-only-reply", + "chips": [], + "text": "hello", + "replyTo": "m1", + "messages": [ + { + "kind": "text", + "chip": null, + "caption": null, + "text": "hello", + "replyTo": "m1" + } + ], + "note": "No chips, with a reply: a text reply, exactly as today." + }, + { + "name": "empty", + "chips": [], + "text": "", + "replyTo": null, + "messages": [], + "note": "Nothing to send. Send is disabled, so this pins that the builder agrees." + }, + { + "name": "one-photo", + "chips": [ + "a" + ], + "text": "", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "One chip, no text: one media message with no caption." + }, + { + "name": "one-photo-caption", + "chips": [ + "a" + ], + "text": "look", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": "look", + "text": null, + "replyTo": null + } + ], + "note": "One chip with text: the text is the caption." + }, + { + "name": "three-photos-caption", + "chips": [ + "a", + "b", + "c" + ], + "text": "from today", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": "from today", + "text": null, + "replyTo": null + } + ], + "note": "The caption rides on the last message, so it lands under the last image." + }, + { + "name": "three-photos-reply", + "chips": [ + "a", + "b", + "c" + ], + "text": "", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": "m1" + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "The reply attaches to the first message only." + }, + { + "name": "three-photos-caption-reply", + "chips": [ + "a", + "b", + "c" + ], + "text": "these?", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": "m1" + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": "these?", + "text": null, + "replyTo": null + } + ], + "note": "Reply on the first, caption on the last." + }, + { + "name": "one-photo-caption-reply", + "chips": [ + "a" + ], + "text": "this one", + "replyTo": "m1", + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": "this one", + "text": null, + "replyTo": "m1" + } + ], + "note": "With one chip, the same message carries both reply and caption." + }, + { + "name": "ten-photos", + "chips": [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j" + ], + "text": "", + "replyTo": null, + "messages": [ + { + "kind": "media", + "chip": "a", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "b", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "c", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "d", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "e", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "f", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "g", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "h", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "i", + "caption": null, + "text": null, + "replyTo": null + }, + { + "kind": "media", + "chip": "j", + "caption": null, + "text": null, + "replyTo": null + } + ], + "note": "The composer maximum. Order is chip order." + } + ], + "bubble": [ + { + "name": "square", + "imageWidth": 1000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 240.0, + "cropped": false + }, + "note": "1:1 fills the width, same height." + }, + { + "name": "landscape-4-3", + "imageWidth": 4032, + "imageHeight": 3024, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 180.0, + "cropped": false + }, + "note": "Landscape phone photo." + }, + { + "name": "portrait-3-4", + "imageWidth": 3024, + "imageHeight": 4032, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 320.0, + "cropped": false + }, + "note": "Portrait phone photo." + }, + { + "name": "wide-at-limit", + "imageWidth": 2000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 120.0, + "cropped": false + }, + "note": "Exactly 1:2 is not cropped." + }, + { + "name": "panorama", + "imageWidth": 8000, + "imageHeight": 1000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 120.0, + "cropped": true + }, + "note": "Wider than 1:2 clamps to 1:2 and crops." + }, + { + "name": "tall-at-limit", + "imageWidth": 1000, + "imageHeight": 2000, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 480.0, + "cropped": false + }, + "note": "Exactly 2:1 is not cropped." + }, + { + "name": "screenshot", + "imageWidth": 1179, + "imageHeight": 2556, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 480.0, + "cropped": true + }, + "note": "An iPhone screenshot is taller than 2:1 and crops." + }, + { + "name": "tiny-upscaled", + "imageWidth": 40, + "imageHeight": 30, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 180.0, + "cropped": false + }, + "note": "Small images still take the full width. The bubble never shrinks to the image." + }, + { + "name": "missing-metadata", + "imageWidth": 0, + "imageHeight": 0, + "maxWidth": 240, + "expected": { + "width": 240, + "height": 240.0, + "cropped": false + }, + "note": "No dimensions (should not happen after READY): a square placeholder." + }, + { + "name": "android-dp", + "imageWidth": 4032, + "imageHeight": 3024, + "maxWidth": 264.5, + "expected": { + "width": 264.5, + "height": 198.375, + "cropped": false + }, + "note": "Fractional widths are fine; compare with 0.001 tolerance." + } + ], + "downscale": [ + { + "name": "within-bounds", + "sourceWidth": 1200, + "sourceHeight": 900, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 1200, + "height": 900 + }, + "note": "Already fits: unchanged." + }, + { + "name": "edge-bound-landscape", + "sourceWidth": 4032, + "sourceHeight": 3024, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 2048, + "height": 1536 + }, + "note": "Long edge binds." + }, + { + "name": "edge-bound-portrait", + "sourceWidth": 3024, + "sourceHeight": 4032, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 1536, + "height": 2048 + }, + "note": "Long edge binds, portrait." + }, + { + "name": "asymmetric-bounds", + "sourceWidth": 4000, + "sourceHeight": 4000, + "maxWidth": 3000, + "maxHeight": 1000, + "maxPixels": 0, + "expected": { + "width": 1000, + "height": 1000 + }, + "note": "The tighter of width and height binds." + }, + { + "name": "pixels-bind-first", + "sourceWidth": 4032, + "sourceHeight": 3024, + "maxWidth": 4096, + "maxHeight": 4096, + "maxPixels": 4000000, + "expected": { + "width": 2309, + "height": 1732 + }, + "note": "Edges would allow the original; max_pixels binds instead." + }, + { + "name": "pixels-and-edges", + "sourceWidth": 8000, + "sourceHeight": 6000, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 2000000, + "expected": { + "width": 1632, + "height": 1224 + }, + "note": "max_pixels is tighter than the edge limit here." + }, + { + "name": "unbounded", + "sourceWidth": 5000, + "sourceHeight": 3000, + "maxWidth": 0, + "maxHeight": 0, + "maxPixels": 0, + "expected": { + "width": 5000, + "height": 3000 + }, + "note": "No constraints: unchanged." + }, + { + "name": "never-upscale", + "sourceWidth": 300, + "sourceHeight": 200, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 16000000, + "expected": { + "width": 300, + "height": 200 + }, + "note": "Small source: unchanged." + }, + { + "name": "panorama-min-edge", + "sourceWidth": 20000, + "sourceHeight": 10, + "maxWidth": 2048, + "maxHeight": 2048, + "maxPixels": 0, + "expected": { + "width": 2048, + "height": 1 + }, + "note": "An edge never goes below 1." + }, + { + "name": "odd-pixels", + "sourceWidth": 3001, + "sourceHeight": 2999, + "maxWidth": 0, + "maxHeight": 0, + "maxPixels": 1000000, + "expected": { + "width": 1000, + "height": 999 + }, + "note": "Floor rounding still fits max_pixels." + } + ], + "constraintSelection": [ + { + "name": "exact-first", + "patterns": [ + "image/jpeg", + "image/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 0, + "note": "Exact match wins when listed first." + }, + { + "name": "wildcard-image", + "patterns": [ + "image/png", + "image/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 1, + "note": "image/png does not match; image/* does." + }, + { + "name": "catch-all", + "patterns": [ + "video/*", + "*/*" + ], + "mimeType": "image/jpeg", + "expectedIndex": 1, + "note": "Only */* matches." + }, + { + "name": "none", + "patterns": [ + "video/*", + "application/pdf" + ], + "mimeType": "image/jpeg", + "expectedIndex": null, + "note": "No entry: do not upload." + }, + { + "name": "policy-order-wins", + "patterns": [ + "*/*", + "image/jpeg" + ], + "mimeType": "image/jpeg", + "expectedIndex": 0, + "note": "Policy order is authoritative, even if a later entry is more specific." + } + ], + "strings": [ + { + "name": "no-caption", + "caption": null, + "expected": { + "snippet": "Photo", + "preview": "📷 Photo" + }, + "note": "No caption: the word Photo." + }, + { + "name": "caption", + "caption": "from today", + "expected": { + "snippet": "from today", + "preview": "📷 from today" + }, + "note": "The caption replaces the word Photo." + }, + { + "name": "emoji-caption", + "caption": "🎉 done", + "expected": { + "snippet": "🎉 done", + "preview": "📷 🎉 done" + }, + "note": "Captions are used as written." + } + ] +} diff --git a/apps/flipcash/shared/persistence/db/schemas/com.flipcash.app.persistence.FlipcashDatabase/43.json b/apps/flipcash/shared/persistence/db/schemas/com.flipcash.app.persistence.FlipcashDatabase/43.json new file mode 100644 index 0000000000..a2853697ca --- /dev/null +++ b/apps/flipcash/shared/persistence/db/schemas/com.flipcash.app.persistence.FlipcashDatabase/43.json @@ -0,0 +1,1043 @@ +{ + "formatVersion": 1, + "database": { + "version": 43, + "identityHash": "3a2ef31a10e7868c6dce7e3caeb640ec", + "entities": [ + { + "tableName": "messages", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`idBase58` TEXT NOT NULL, `text` TEXT NOT NULL, `amountUsdc` INTEGER, `amountNative` INTEGER, `nativeCurrency` TEXT, `rate` REAL, `state` TEXT NOT NULL, `timestamp` INTEGER NOT NULL, `metadata` TEXT, `mintBase58` TEXT DEFAULT 'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v', `textSubstitutions` TEXT, PRIMARY KEY(`idBase58`))", + "fields": [ + { + "fieldPath": "idBase58", + "columnName": "idBase58", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "amountUsdc", + "columnName": "amountUsdc", + "affinity": "INTEGER" + }, + { + "fieldPath": "amountNative", + "columnName": "amountNative", + "affinity": "INTEGER" + }, + { + "fieldPath": "nativeCurrency", + "columnName": "nativeCurrency", + "affinity": "TEXT" + }, + { + "fieldPath": "rate", + "columnName": "rate", + "affinity": "REAL" + }, + { + "fieldPath": "state", + "columnName": "state", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "metadata", + "columnName": "metadata", + "affinity": "TEXT" + }, + { + "fieldPath": "mintBase58", + "columnName": "mintBase58", + "affinity": "TEXT", + "defaultValue": "'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v'" + }, + { + "fieldPath": "textSubstitutions", + "columnName": "textSubstitutions", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "idBase58" + ] + } + }, + { + "tableName": "tokens", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`address` TEXT NOT NULL, `decimals` INTEGER NOT NULL, `name` TEXT NOT NULL, `symbol` TEXT NOT NULL, `created_at` INTEGER, `description` TEXT NOT NULL, `image_url` TEXT NOT NULL, `social_links` TEXT, `bill_customizations` TEXT, `holder_metrics` TEXT, `market_cap_metrics` TEXT, `vm_vm` TEXT NOT NULL, `vm_authority` TEXT NOT NULL, `vm_lock_duration_days` INTEGER NOT NULL, `lp_currency_config` TEXT, `lp_liquidity_pool` TEXT, `lp_seed` TEXT, `lp_authority` TEXT, `lp_mint_vault` TEXT, `lp_core_mint_vault` TEXT, `lp_circulating_supply_quarks` INTEGER, `lp_sell_fee_bps` INTEGER, `lp_price_amount_usd` REAL, `lp_market_cap_amount_usd` REAL, PRIMARY KEY(`address`))", + "fields": [ + { + "fieldPath": "address", + "columnName": "address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "decimals", + "columnName": "decimals", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "symbol", + "columnName": "symbol", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "created_at", + "affinity": "INTEGER" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "imageUrl", + "columnName": "image_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "socialLinks", + "columnName": "social_links", + "affinity": "TEXT" + }, + { + "fieldPath": "billCustomizationsJson", + "columnName": "bill_customizations", + "affinity": "TEXT" + }, + { + "fieldPath": "holderMetricsJson", + "columnName": "holder_metrics", + "affinity": "TEXT" + }, + { + "fieldPath": "marketCapMetricsJson", + "columnName": "market_cap_metrics", + "affinity": "TEXT" + }, + { + "fieldPath": "vmMetadata.vm", + "columnName": "vm_vm", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "vmMetadata.authority", + "columnName": "vm_authority", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "vmMetadata.lockDurationInDays", + "columnName": "vm_lock_duration_days", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "launchpadMetadata.currencyConfig", + "columnName": "lp_currency_config", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.liquidityPool", + "columnName": "lp_liquidity_pool", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.seed", + "columnName": "lp_seed", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.authority", + "columnName": "lp_authority", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.mintVault", + "columnName": "lp_mint_vault", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.coreMintVault", + "columnName": "lp_core_mint_vault", + "affinity": "TEXT" + }, + { + "fieldPath": "launchpadMetadata.currentCirculatingSupplyQuarks", + "columnName": "lp_circulating_supply_quarks", + "affinity": "INTEGER" + }, + { + "fieldPath": "launchpadMetadata.sellFeeBps", + "columnName": "lp_sell_fee_bps", + "affinity": "INTEGER" + }, + { + "fieldPath": "launchpadMetadata.priceAmount", + "columnName": "lp_price_amount_usd", + "affinity": "REAL" + }, + { + "fieldPath": "launchpadMetadata.marketCapAmount", + "columnName": "lp_market_cap_amount_usd", + "affinity": "REAL" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "address" + ] + } + }, + { + "tableName": "token_social_links", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `token_address` TEXT NOT NULL, `type` TEXT NOT NULL, `value` TEXT NOT NULL, FOREIGN KEY(`token_address`) REFERENCES `tokens`(`address`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tokenAddress", + "columnName": "token_address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_token_social_links_token_address", + "unique": false, + "columnNames": [ + "token_address" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_token_social_links_token_address` ON `${TABLE_NAME}` (`token_address`)" + } + ], + "foreignKeys": [ + { + "table": "tokens", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "token_address" + ], + "referencedColumns": [ + "address" + ] + } + ] + }, + { + "tableName": "token_valuation", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`token_address` TEXT NOT NULL, `balance_quarks` INTEGER NOT NULL, `cost_basis` REAL NOT NULL, PRIMARY KEY(`token_address`), FOREIGN KEY(`token_address`) REFERENCES `tokens`(`address`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "tokenAddress", + "columnName": "token_address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "balanceQuarks", + "columnName": "balance_quarks", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "costBasis", + "columnName": "cost_basis", + "affinity": "REAL", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "token_address" + ] + }, + "indices": [ + { + "name": "index_token_valuation_token_address", + "unique": false, + "columnNames": [ + "token_address" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_token_valuation_token_address` ON `${TABLE_NAME}` (`token_address`)" + } + ], + "foreignKeys": [ + { + "table": "tokens", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "token_address" + ], + "referencedColumns": [ + "address" + ] + } + ] + }, + { + "tableName": "currency_creator_draft", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, `icon_uri` TEXT, `bill_customizations` TEXT, `attestations` TEXT, `current_step` TEXT NOT NULL, `created_mint` TEXT, `saved_at` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "iconUri", + "columnName": "icon_uri", + "affinity": "TEXT" + }, + { + "fieldPath": "billCustomizations", + "columnName": "bill_customizations", + "affinity": "TEXT" + }, + { + "fieldPath": "attestations", + "columnName": "attestations", + "affinity": "TEXT" + }, + { + "fieldPath": "currentStep", + "columnName": "current_step", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdMint", + "columnName": "created_mint", + "affinity": "TEXT" + }, + { + "fieldPath": "savedAt", + "columnName": "saved_at", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "contact_sync_state", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `checksumBytes` BLOB NOT NULL, `lastSyncTimestamp` INTEGER NOT NULL, `needsFullUpload` INTEGER NOT NULL, `hasDiscoveredFlipcashContacts` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "checksumBytes", + "columnName": "checksumBytes", + "affinity": "BLOB", + "notNull": true + }, + { + "fieldPath": "lastSyncTimestamp", + "columnName": "lastSyncTimestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "needsFullUpload", + "columnName": "needsFullUpload", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "hasDiscoveredFlipcashContacts", + "columnName": "hasDiscoveredFlipcashContacts", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "contact_mapping", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`e164` TEXT NOT NULL, `androidContactId` INTEGER NOT NULL, `displayName` TEXT NOT NULL, `photoUri` TEXT, `isOnFlipcash` INTEGER NOT NULL, `displayNumber` TEXT NOT NULL DEFAULT '', `dmChatId` TEXT NOT NULL DEFAULT '', `joinedAtEpochSeconds` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`e164`))", + "fields": [ + { + "fieldPath": "e164", + "columnName": "e164", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "androidContactId", + "columnName": "androidContactId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "displayName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "photoUri", + "columnName": "photoUri", + "affinity": "TEXT" + }, + { + "fieldPath": "isOnFlipcash", + "columnName": "isOnFlipcash", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "displayNumber", + "columnName": "displayNumber", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "dmChatId", + "columnName": "dmChatId", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "joinedAtEpochSeconds", + "columnName": "joinedAtEpochSeconds", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "e164" + ] + } + }, + { + "tableName": "chat_metadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`chat_id_hex` TEXT NOT NULL, `chat_type` TEXT NOT NULL, `last_activity_epoch_ms` INTEGER NOT NULL, `last_message_id` INTEGER, `latest_event_sequence` INTEGER NOT NULL DEFAULT 0, `is_hidden` INTEGER NOT NULL DEFAULT 0, `analytics_counted_through` INTEGER NOT NULL DEFAULT 0, `title` TEXT, `picture_json` TEXT, `member_count` INTEGER NOT NULL DEFAULT 0, `roster_version` INTEGER NOT NULL DEFAULT 0, `rules_json` TEXT, `is_member` INTEGER NOT NULL DEFAULT 1, `mute_until_epoch_ms` INTEGER, `mute_forever` INTEGER NOT NULL DEFAULT 0, `viewer_state_version` INTEGER NOT NULL DEFAULT 0, `can_edit` INTEGER NOT NULL DEFAULT 0, `creator_hex` TEXT, `use_e2ee` INTEGER NOT NULL DEFAULT 0, `description` TEXT, PRIMARY KEY(`chat_id_hex`))", + "fields": [ + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "chatType", + "columnName": "chat_type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastActivityEpochMs", + "columnName": "last_activity_epoch_ms", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastMessageId", + "columnName": "last_message_id", + "affinity": "INTEGER" + }, + { + "fieldPath": "latestEventSequence", + "columnName": "latest_event_sequence", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "isHidden", + "columnName": "is_hidden", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "analyticsCountedThrough", + "columnName": "analytics_counted_through", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "pictureJson", + "columnName": "picture_json", + "affinity": "TEXT" + }, + { + "fieldPath": "memberCount", + "columnName": "member_count", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "rosterVersion", + "columnName": "roster_version", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "rulesJson", + "columnName": "rules_json", + "affinity": "TEXT" + }, + { + "fieldPath": "isMember", + "columnName": "is_member", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "1" + }, + { + "fieldPath": "muteUntilEpochMs", + "columnName": "mute_until_epoch_ms", + "affinity": "INTEGER" + }, + { + "fieldPath": "muteForever", + "columnName": "mute_forever", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "viewerStateVersion", + "columnName": "viewer_state_version", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "canEdit", + "columnName": "can_edit", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "creatorHex", + "columnName": "creator_hex", + "affinity": "TEXT" + }, + { + "fieldPath": "useE2ee", + "columnName": "use_e2ee", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "chat_id_hex" + ] + }, + "indices": [ + { + "name": "index_chat_metadata_last_activity_epoch_ms", + "unique": false, + "columnNames": [ + "last_activity_epoch_ms" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_chat_metadata_last_activity_epoch_ms` ON `${TABLE_NAME}` (`last_activity_epoch_ms`)" + } + ] + }, + { + "tableName": "chat_messages", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`chat_id_hex` TEXT NOT NULL, `message_id` INTEGER NOT NULL, `sender_id_hex` TEXT, `content_json` TEXT, `timestamp_epoch_ms` INTEGER NOT NULL, `unread_seq` INTEGER NOT NULL, `status` TEXT NOT NULL DEFAULT 'SENT', `pending_client_id_hex` TEXT, `event_sequence` INTEGER NOT NULL DEFAULT 0, `last_edited_ts_epoch_ms` INTEGER, `reactions_json` TEXT, `is_deleted` INTEGER NOT NULL DEFAULT 0, `ciphertext_json` TEXT, `encryption_state` TEXT, PRIMARY KEY(`chat_id_hex`, `message_id`))", + "fields": [ + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "messageId", + "columnName": "message_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "senderIdHex", + "columnName": "sender_id_hex", + "affinity": "TEXT" + }, + { + "fieldPath": "contentJson", + "columnName": "content_json", + "affinity": "TEXT" + }, + { + "fieldPath": "timestampEpochMs", + "columnName": "timestamp_epoch_ms", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "unreadSeq", + "columnName": "unread_seq", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'SENT'" + }, + { + "fieldPath": "pendingClientIdHex", + "columnName": "pending_client_id_hex", + "affinity": "TEXT" + }, + { + "fieldPath": "eventSequence", + "columnName": "event_sequence", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "lastEditedTsEpochMs", + "columnName": "last_edited_ts_epoch_ms", + "affinity": "INTEGER" + }, + { + "fieldPath": "reactionsJson", + "columnName": "reactions_json", + "affinity": "TEXT" + }, + { + "fieldPath": "isDeleted", + "columnName": "is_deleted", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "ciphertextJson", + "columnName": "ciphertext_json", + "affinity": "TEXT" + }, + { + "fieldPath": "encryptionState", + "columnName": "encryption_state", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "chat_id_hex", + "message_id" + ] + }, + "indices": [ + { + "name": "index_chat_messages_chat_id_hex_timestamp_epoch_ms", + "unique": false, + "columnNames": [ + "chat_id_hex", + "timestamp_epoch_ms" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_chat_messages_chat_id_hex_timestamp_epoch_ms` ON `${TABLE_NAME}` (`chat_id_hex`, `timestamp_epoch_ms`)" + } + ] + }, + { + "tableName": "chat_members", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`chat_id_hex` TEXT NOT NULL, `user_id_hex` TEXT NOT NULL, `pointers_json` TEXT, PRIMARY KEY(`chat_id_hex`, `user_id_hex`))", + "fields": [ + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userIdHex", + "columnName": "user_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "pointersJson", + "columnName": "pointers_json", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "chat_id_hex", + "user_id_hex" + ] + } + }, + { + "tableName": "chat_draft", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`chat_id_hex` TEXT NOT NULL, `text` TEXT NOT NULL, `reply_target_json` TEXT, `saved_at` INTEGER NOT NULL, PRIMARY KEY(`chat_id_hex`))", + "fields": [ + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "replyTargetJson", + "columnName": "reply_target_json", + "affinity": "TEXT" + }, + { + "fieldPath": "savedAt", + "columnName": "saved_at", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "chat_id_hex" + ] + } + }, + { + "tableName": "chat_archive", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`chat_id_hex` TEXT NOT NULL, `archived_at` INTEGER NOT NULL, PRIMARY KEY(`chat_id_hex`))", + "fields": [ + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "archivedAt", + "columnName": "archived_at", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "chat_id_hex" + ] + } + }, + { + "tableName": "blocked_users", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`user_id_hex` TEXT NOT NULL, `blocked_at_epoch_ms` INTEGER NOT NULL, PRIMARY KEY(`user_id_hex`))", + "fields": [ + { + "fieldPath": "userIdHex", + "columnName": "user_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "blockedAtEpochMs", + "columnName": "blocked_at_epoch_ms", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "user_id_hex" + ] + } + }, + { + "tableName": "user_profiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`user_id_hex` TEXT NOT NULL, `display_name` TEXT NOT NULL, `phone_value` TEXT, `phone_verified` INTEGER, `email_value` TEXT, `email_verified` INTEGER, `social_accounts_json` TEXT, `profile_picture_json` TEXT, `username` TEXT, `pending_migration_json` TEXT, PRIMARY KEY(`user_id_hex`))", + "fields": [ + { + "fieldPath": "userIdHex", + "columnName": "user_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "display_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "phoneValue", + "columnName": "phone_value", + "affinity": "TEXT" + }, + { + "fieldPath": "phoneVerified", + "columnName": "phone_verified", + "affinity": "INTEGER" + }, + { + "fieldPath": "emailValue", + "columnName": "email_value", + "affinity": "TEXT" + }, + { + "fieldPath": "emailVerified", + "columnName": "email_verified", + "affinity": "INTEGER" + }, + { + "fieldPath": "socialAccounts", + "columnName": "social_accounts_json", + "affinity": "TEXT" + }, + { + "fieldPath": "profilePicture", + "columnName": "profile_picture_json", + "affinity": "TEXT" + }, + { + "fieldPath": "username", + "columnName": "username", + "affinity": "TEXT" + }, + { + "fieldPath": "pendingMigrationJson", + "columnName": "pending_migration_json", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "user_id_hex" + ] + } + }, + { + "tableName": "link_previews", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `json` TEXT NOT NULL, `updated_at` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "json", + "columnName": "json", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updated_at", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "key" + ] + } + }, + { + "tableName": "pending_media", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`client_id_hex` TEXT NOT NULL, `chat_id_hex` TEXT NOT NULL, `file_name` TEXT NOT NULL, `caption` TEXT, `reply_to_message_id` INTEGER, `stored_blob_id_hex` TEXT, `sealed_for_hex` TEXT, `width` INTEGER NOT NULL, `height` INTEGER NOT NULL, `blurhash` TEXT, `size_bytes` INTEGER, `created_at` INTEGER NOT NULL, PRIMARY KEY(`client_id_hex`))", + "fields": [ + { + "fieldPath": "clientIdHex", + "columnName": "client_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "chatIdHex", + "columnName": "chat_id_hex", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fileName", + "columnName": "file_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "caption", + "columnName": "caption", + "affinity": "TEXT" + }, + { + "fieldPath": "replyToMessageId", + "columnName": "reply_to_message_id", + "affinity": "INTEGER" + }, + { + "fieldPath": "storedBlobIdHex", + "columnName": "stored_blob_id_hex", + "affinity": "TEXT" + }, + { + "fieldPath": "sealedForHex", + "columnName": "sealed_for_hex", + "affinity": "TEXT" + }, + { + "fieldPath": "width", + "columnName": "width", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "height", + "columnName": "height", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "blurhash", + "columnName": "blurhash", + "affinity": "TEXT" + }, + { + "fieldPath": "sizeBytes", + "columnName": "size_bytes", + "affinity": "INTEGER" + }, + { + "fieldPath": "createdAt", + "columnName": "created_at", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "client_id_hex" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '3a2ef31a10e7868c6dce7e3caeb640ec')" + ] + } +} \ No newline at end of file diff --git a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/FlipcashDatabase.kt b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/FlipcashDatabase.kt index 6b086f2375..e7fd3dcf3c 100644 --- a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/FlipcashDatabase.kt +++ b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/FlipcashDatabase.kt @@ -28,6 +28,7 @@ import com.flipcash.app.persistence.dao.ContactDao import com.flipcash.app.persistence.dao.CurrencyCreatorDraftDao import com.flipcash.app.persistence.dao.LinkPreviewDao import com.flipcash.app.persistence.dao.MessageDao +import com.flipcash.app.persistence.dao.PendingMediaDao import com.flipcash.app.persistence.dao.TokenDao import com.flipcash.app.persistence.dao.UserProfileDao import com.flipcash.app.persistence.entities.BlockedUserEntity @@ -41,6 +42,7 @@ import com.flipcash.app.persistence.entities.ContactSyncStateEntity import com.flipcash.app.persistence.entities.CurrencyCreatorDraftEntity import com.flipcash.app.persistence.entities.LinkPreviewEntity import com.flipcash.app.persistence.entities.MessageEntity +import com.flipcash.app.persistence.entities.PendingMediaEntity import com.flipcash.app.persistence.entities.SocialLinkEntity import com.flipcash.app.persistence.entities.TokenEntity import com.flipcash.app.persistence.entities.TokenValuationEntity @@ -67,6 +69,7 @@ import com.getcode.utils.subByteArray BlockedUserEntity::class, UserProfileEntity::class, LinkPreviewEntity::class, + PendingMediaEntity::class, ], autoMigrations = [ AutoMigration(from = 1, to = 2, spec = FlipcashDatabase.Migration1To2::class), @@ -118,8 +121,11 @@ import com.getcode.utils.subByteArray // below, for the same reason as chat_draft: an archive set cannot be re-fetched. AutoMigration(from = 40, to = 41), // chat_archive table AutoMigration(from = 41, to = 42), // chat_metadata.description (nullable) + // The pending_media table, an AutoMigration for the same reason as chat_draft: an entry + // dropped by the destructive fallback is a photo the viewer queued and never sent. + AutoMigration(from = 42, to = 43), // pending_media table ], - version = 42, + version = 43, ) @TypeConverters(TokenTypeConverters::class, ChatTypeConverters::class) abstract class FlipcashDatabase : RoomDatabase() { @@ -136,6 +142,7 @@ abstract class FlipcashDatabase : RoomDatabase() { abstract fun blockedUserDao(): BlockedUserDao abstract fun userProfileDao(): UserProfileDao abstract fun linkPreviewDao(): LinkPreviewDao + abstract fun pendingMediaDao(): PendingMediaDao class Migration1To2 : Migration(1, 2), AutoMigrationSpec { override fun migrate(db: SupportSQLiteDatabase) { diff --git a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/ChatMessageDao.kt b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/ChatMessageDao.kt index 56c58df1bc..fb204c1d82 100644 --- a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/ChatMessageDao.kt +++ b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/ChatMessageDao.kt @@ -136,6 +136,14 @@ interface ChatMessageDao { @Query("SELECT * FROM chat_messages WHERE chat_id_hex = :chatIdHex AND pending_client_id_hex = :clientIdHex LIMIT 1") suspend fun getByClientId(chatIdHex: String, clientIdHex: String): ChatMessageEntity? + /** The newest [limit] confirmed messages [selfIdHex] sent in [chatIdHex]. */ + @Query(""" + SELECT * FROM chat_messages + WHERE chat_id_hex = :chatIdHex AND sender_id_hex = :selfIdHex AND message_id > 0 + ORDER BY message_id DESC LIMIT :limit + """) + suspend fun getRecentSentBy(chatIdHex: String, selfIdHex: String, limit: Int): List + @Query("SELECT * FROM chat_messages WHERE chat_id_hex = :chatIdHex AND message_id = :messageId LIMIT 1") suspend fun getMessage(chatIdHex: String, messageId: Long): ChatMessageEntity? @@ -271,10 +279,25 @@ interface ChatMessageDao { @Query("DELETE FROM chat_messages WHERE chat_id_hex = :chatIdHex AND pending_client_id_hex = :clientIdHex") suspend fun deletePending(chatIdHex: String, clientIdHex: String) - @Query("SELECT pending_client_id_hex FROM chat_messages WHERE chat_id_hex = :chatIdHex AND status = 'SENDING' AND pending_client_id_hex IS NOT NULL") + @Query("DELETE FROM chat_messages WHERE chat_id_hex = :chatIdHex AND pending_client_id_hex = :clientIdHex AND message_id != :keepMessageId") + suspend fun deletePendingExcept(chatIdHex: String, clientIdHex: String, keepMessageId: Long) + + // A queued photo is excluded from this and from [deleteAllPending]: it can sit in SENDING for as + // long as an upload takes, while a refresh triggered by an earlier photo of the same batch + // arrives. Deleting it would drop a photo whose upload is still running. Its own confirmation + // settles it, see [confirmPendingMessage]. + @Query(""" + SELECT pending_client_id_hex FROM chat_messages + WHERE chat_id_hex = :chatIdHex AND status = 'SENDING' AND pending_client_id_hex IS NOT NULL + AND pending_client_id_hex NOT IN (SELECT client_id_hex FROM pending_media) + """) suspend fun getPendingClientIds(chatIdHex: String): List - @Query("DELETE FROM chat_messages WHERE chat_id_hex = :chatIdHex AND status = 'SENDING'") + @Query(""" + DELETE FROM chat_messages + WHERE chat_id_hex = :chatIdHex AND status = 'SENDING' + AND (pending_client_id_hex IS NULL OR pending_client_id_hex NOT IN (SELECT client_id_hex FROM pending_media)) + """) suspend fun deleteAllPending(chatIdHex: String) /** @@ -293,8 +316,18 @@ interface ChatMessageDao { * the original rather than duplicating it. * * Deliberately not scoped to a chat: the caller runs at login, before it knows which chats exist. + * + * A photo whose `pending_media` entry says its bytes are already stored is left `SENDING`: what + * is left to do is poll and post, which the launch reconciliation does without being asked, and + * flipping the row to failed and back would flash a retry button at the viewer. A photo not yet + * stored is swept like any other, since its retry is the viewer's to ask for. */ - @Query("UPDATE chat_messages SET status = 'FAILED' WHERE status = 'SENDING'") + @Query(""" + UPDATE chat_messages SET status = 'FAILED' + WHERE status = 'SENDING' + AND (pending_client_id_hex IS NULL OR pending_client_id_hex NOT IN ( + SELECT client_id_hex FROM pending_media WHERE stored_blob_id_hex IS NOT NULL)) + """) suspend fun failInterruptedSends() /** @@ -321,8 +354,26 @@ interface ChatMessageDao { newEventSequence: Long, ) + /** + * Settles the pending row of [clientIdHex] as [serverMessage]. + * + * [replaceContent] is for a photo, whose optimistic row carries the local file rather than the + * blob; the confirmed row takes the server's content. When the server's copy of the message + * already arrived over the event stream, which a long upload makes likely, the pending row is + * dropped in its favour instead of being renumbered onto an existing key. + */ @Transaction - suspend fun confirmPendingMessage(chatIdHex: String, clientIdHex: String, serverMessage: ChatMessageEntity) { + suspend fun confirmPendingMessage( + chatIdHex: String, + clientIdHex: String, + serverMessage: ChatMessageEntity, + replaceContent: Boolean = false, + ) { + if (exists(chatIdHex, serverMessage.messageId)) { + // The stream's copy can carry the same client id, so only the pending row goes. + deletePendingExcept(chatIdHex, clientIdHex, serverMessage.messageId) + return + } updatePendingToConfirmed( chatIdHex = chatIdHex, clientIdHex = clientIdHex, @@ -334,10 +385,11 @@ interface ChatMessageDao { // An encrypted send went out as ciphertext; the row keeps its plaintext and gains the // ciphertext beside it. Written as a row rather than in the UPDATE above, since Room would // expand a list parameter into an IN clause. - if (serverMessage.encryptionState != null) { + if (serverMessage.encryptionState != null || replaceContent) { val confirmed = getByClientId(chatIdHex, clientIdHex) ?: return insert( confirmed.copy( + contentJson = if (replaceContent) serverMessage.contentJson else confirmed.contentJson, ciphertextJson = serverMessage.ciphertextJson, encryptionState = serverMessage.encryptionState, ) diff --git a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/PendingMediaDao.kt b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/PendingMediaDao.kt new file mode 100644 index 0000000000..8adbf4f3dc --- /dev/null +++ b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/dao/PendingMediaDao.kt @@ -0,0 +1,39 @@ +package com.flipcash.app.persistence.dao + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import com.flipcash.app.persistence.entities.PendingMediaEntity + +@Dao +interface PendingMediaDao { + + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun upsert(entry: PendingMediaEntity) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun upsert(entries: List) + + @Query("SELECT * FROM pending_media WHERE client_id_hex = :clientIdHex") + suspend fun get(clientIdHex: String): PendingMediaEntity? + + /** Oldest first: entries of one send are written in chip order, and recovery posts in it. */ + @Query("SELECT * FROM pending_media ORDER BY created_at ASC, client_id_hex ASC") + suspend fun getAll(): List + + @Query("UPDATE pending_media SET stored_blob_id_hex = :blobIdHex, sealed_for_hex = :sealedForHex, size_bytes = :sizeBytes, blurhash = :blurhash WHERE client_id_hex = :clientIdHex") + suspend fun markStored( + clientIdHex: String, + blobIdHex: String, + sealedForHex: String?, + sizeBytes: Long, + blurhash: String?, + ) + + @Query("DELETE FROM pending_media WHERE client_id_hex = :clientIdHex") + suspend fun delete(clientIdHex: String) + + @Query("DELETE FROM pending_media") + suspend fun deleteAll() +} diff --git a/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/entities/PendingMediaEntity.kt b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/entities/PendingMediaEntity.kt new file mode 100644 index 0000000000..b0267333f3 --- /dev/null +++ b/apps/flipcash/shared/persistence/db/src/main/kotlin/com/flipcash/app/persistence/entities/PendingMediaEntity.kt @@ -0,0 +1,63 @@ +package com.flipcash.app.persistence.entities + +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.PrimaryKey + +/** + * A photo message the viewer has queued, and what it takes to finish sending it after the process + * dies. The optimistic `chat_messages` row says that a message is on its way; this says how to get + * it there. + * + * The encoded JPEG lives in `filesDir/pending-media/[fileName]`, written once, so a retry resends + * those bytes instead of encoding the photo again. [storedBlobIdHex] is set once the bytes are in + * storage: an entry with one resumes by polling for finalization and posting, an entry without one + * has to upload again. + * + * Keyed by the message's client id, the same one `chat_messages.pending_client_id_hex` carries, so + * a retry posts under the id the server dedupes on. Deleted once the message is confirmed or can + * no longer be sent (the server refused the photo). + * + * An AutoMigration creates it rather than the destructive fallback: an entry that is dropped takes + * a photo the viewer has not sent yet with it. + */ +@Entity(tableName = "pending_media") +data class PendingMediaEntity( + @PrimaryKey + @ColumnInfo(name = "client_id_hex") + val clientIdHex: String, + + @ColumnInfo(name = "chat_id_hex") + val chatIdHex: String, + + @ColumnInfo(name = "file_name") + val fileName: String, + + @ColumnInfo(name = "caption") + val caption: String?, + + @ColumnInfo(name = "reply_to_message_id") + val replyToMessageId: Long?, + + @ColumnInfo(name = "stored_blob_id_hex") + val storedBlobIdHex: String?, + + /** The chat the stored blob was sealed for, or null if it was uploaded plain. */ + @ColumnInfo(name = "sealed_for_hex") + val sealedForHex: String?, + + @ColumnInfo(name = "width") + val width: Int, + + @ColumnInfo(name = "height") + val height: Int, + + @ColumnInfo(name = "blurhash") + val blurhash: String?, + + @ColumnInfo(name = "size_bytes") + val sizeBytes: Long?, + + @ColumnInfo(name = "created_at") + val createdAt: Long, +) diff --git a/apps/flipcash/shared/persistence/db/src/test/kotlin/com/flipcash/app/persistence/dao/PendingMediaDaoTest.kt b/apps/flipcash/shared/persistence/db/src/test/kotlin/com/flipcash/app/persistence/dao/PendingMediaDaoTest.kt new file mode 100644 index 0000000000..9ded5a993c --- /dev/null +++ b/apps/flipcash/shared/persistence/db/src/test/kotlin/com/flipcash/app/persistence/dao/PendingMediaDaoTest.kt @@ -0,0 +1,175 @@ +package com.flipcash.app.persistence.dao + +import android.content.Context +import androidx.room.Room +import androidx.test.core.app.ApplicationProvider +import com.flipcash.app.persistence.FlipcashDatabase +import com.flipcash.app.persistence.converters.MessageContentSerialized +import com.flipcash.app.persistence.entities.ChatMessageEntity +import com.flipcash.app.persistence.entities.MessageStatus +import com.flipcash.app.persistence.entities.PendingMediaEntity +import kotlinx.coroutines.test.runTest +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * A queued photo is the one pending row that outlives a refresh and a restart, so the cases here + * are where the generic pending-row rules (a refresh deletes `SENDING`, the launch sweep fails it) + * have to step aside for an entry in `pending_media`. + */ +@RunWith(RobolectricTestRunner::class) +class PendingMediaDaoTest { + + private lateinit var db: FlipcashDatabase + private lateinit var media: PendingMediaDao + private lateinit var messages: ChatMessageDao + + @Before + fun setUp() { + val context = ApplicationProvider.getApplicationContext() + db = Room.inMemoryDatabaseBuilder(context, FlipcashDatabase::class.java) + .allowMainThreadQueries() + .build() + media = db.pendingMediaDao() + messages = db.chatMessageDao() + } + + @After + fun tearDown() { + db.close() + } + + private fun entry(clientIdHex: String, stored: Boolean = false, createdAt: Long = 1) = PendingMediaEntity( + clientIdHex = clientIdHex, + chatIdHex = CHAT_HEX, + fileName = "$clientIdHex.jpg", + caption = null, + replyToMessageId = null, + storedBlobIdHex = if (stored) "ab".repeat(16) else null, + sealedForHex = null, + width = 800, + height = 600, + blurhash = null, + sizeBytes = null, + createdAt = createdAt, + ) + + private fun pendingRow(clientIdHex: String, messageId: Long) = ChatMessageEntity( + chatIdHex = CHAT_HEX, + messageId = messageId, + senderIdHex = SENDER_HEX, + contentJson = listOf(MessageContentSerialized.Text("photo")), + timestampEpochMs = -messageId, + unreadSeq = 0, + status = MessageStatus.SENDING, + pendingClientIdHex = clientIdHex, + ) + + private fun server(messageId: Long) = ChatMessageEntity( + chatIdHex = CHAT_HEX, + messageId = messageId, + senderIdHex = SENDER_HEX, + contentJson = listOf(MessageContentSerialized.Text("from the server")), + timestampEpochMs = messageId * 1_000, + unreadSeq = messageId, + ) + + @Test + fun `entries come back oldest first`() = runTest { + media.upsert(listOf(entry("b", createdAt = 2), entry("a", createdAt = 1))) + + assertEquals(listOf("a", "b"), media.getAll().map { it.clientIdHex }) + } + + @Test + fun `marking an entry stored keeps the rest of it`() = runTest { + media.upsert(entry("a")) + + media.markStored("a", blobIdHex = "cd".repeat(16), sealedForHex = "ef", sizeBytes = 9, blurhash = "LEHV") + + val stored = media.get("a")!! + assertEquals("cd".repeat(16), stored.storedBlobIdHex) + assertEquals("ef", stored.sealedForHex) + assertEquals(9L, stored.sizeBytes) + assertEquals("a.jpg", stored.fileName) + } + + @Test + fun `a refresh does not delete a queued photo`() = runTest { + messages.upsert(pendingRow("a", -10)) + messages.upsert(pendingRow("text", -20)) + media.upsert(entry("a")) + + messages.upsertAndClearPending(CHAT_HEX, listOf(server(1))) + + assertEquals(MessageStatus.SENDING, messages.getByClientId(CHAT_HEX, "a")?.status) + // The ordinary pending row is still cleared. + assertNull(messages.getByClientId(CHAT_HEX, "text")) + } + + @Test + fun `the sweep spares a stored photo and fails an unstored one`() = runTest { + messages.upsert(pendingRow("stored", -10)) + messages.upsert(pendingRow("unstored", -20)) + messages.upsert(pendingRow("text", -30)) + media.upsert(listOf(entry("stored", stored = true), entry("unstored"))) + + messages.failInterruptedSends() + + assertEquals(MessageStatus.SENDING, messages.getByClientId(CHAT_HEX, "stored")?.status) + assertEquals(MessageStatus.FAILED, messages.getByClientId(CHAT_HEX, "unstored")?.status) + assertEquals(MessageStatus.FAILED, messages.getByClientId(CHAT_HEX, "text")?.status) + } + + @Test + fun `confirming a photo takes the server's content`() = runTest { + messages.upsert(pendingRow("a", -10)) + + messages.confirmPendingMessage(CHAT_HEX, "a", server(7), replaceContent = true) + + val confirmed = messages.getByClientId(CHAT_HEX, "a")!! + assertEquals(7L, confirmed.messageId) + assertEquals(listOf(MessageContentSerialized.Text("from the server")), confirmed.contentJson) + } + + @Test + fun `confirming keeps the local content unless asked`() = runTest { + messages.upsert(pendingRow("a", -10)) + + messages.confirmPendingMessage(CHAT_HEX, "a", server(7)) + + assertEquals(listOf(MessageContentSerialized.Text("photo")), messages.getByClientId(CHAT_HEX, "a")!!.contentJson) + } + + @Test + fun `confirming after the stream delivered the message drops the pending row`() = runTest { + messages.upsert(server(7)) + messages.upsert(pendingRow("a", -10)) + + messages.confirmPendingMessage(CHAT_HEX, "a", server(7), replaceContent = true) + + assertNull(messages.getByClientId(CHAT_HEX, "a")) + assertEquals(7L, messages.getMessage(CHAT_HEX, 7)?.messageId) + } + + @Test + fun `confirming keeps the stream's copy when it carries the same client id`() = runTest { + // A group send whose stream event lands first: the refresh clears the pending row and + // stores the server's copy under the same client id. + messages.upsert(server(7).copy(pendingClientIdHex = "a")) + + messages.confirmPendingMessage(CHAT_HEX, "a", server(7)) + + assertEquals(7L, messages.getMessage(CHAT_HEX, 7)?.messageId) + } + + private companion object { + const val CHAT_HEX = "c0ffee" + const val SENDER_HEX = "5e1f" + } +} diff --git a/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/ChatMessageDataSource.kt b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/ChatMessageDataSource.kt index 3810eec0ac..c93e8f271a 100644 --- a/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/ChatMessageDataSource.kt +++ b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/ChatMessageDataSource.kt @@ -124,6 +124,19 @@ class ChatMessageDataSource @Inject constructor( entities.map { toChatMessage(it) } } ?: emptyFlow() + /** The newest [limit] confirmed messages [selfId] sent in [chatId], newest first. */ + suspend fun getRecentSentBy(chatId: ChatId, selfId: ID, limit: Int): List = + db?.chatMessageDao() + ?.getRecentSentBy(mapper.chatIdHex(chatId), selfId.hexEncodedString(), limit) + .orEmpty() + .map { toChatMessage(it) } + + /** The optimistic row of [clientMessageId], whatever its status; null once it was confirmed or dropped. */ + suspend fun getPending(chatId: ChatId, clientMessageId: ClientMessageId): ChatMessage? = + db?.chatMessageDao() + ?.getByClientId(mapper.chatIdHex(chatId), mapper.clientMessageIdHex(clientMessageId)) + ?.let { toChatMessage(it) } + suspend fun getLatest(chatIdHex: String): ChatMessage? = db?.chatMessageDao()?.getLatest(chatIdHex)?.let { toChatMessage(it) } @@ -341,13 +354,22 @@ class ChatMessageDataSource @Inject constructor( chatId: ChatId, content: List, senderId: ID, + ): PendingMessage = insertPending(chatId, content, senderId, ClientMessageId(RandomId.toByteArray()), ordinal = 0) + + /** [insertPending] under a client id the caller already made, at [ordinal] among rows sent together. */ + suspend fun insertPending( + chatId: ChatId, + content: List, + senderId: ID, + clientMessageId: ClientMessageId, + ordinal: Int, ): PendingMessage { - val clientMessageId = ClientMessageId(RandomId.toByteArray()) val entity = mapper.toPendingEntity( chatIdHex = mapper.chatIdHex(chatId), content = content, senderId = senderId, clientMessageId = clientMessageId, + ordinal = ordinal, ) db?.chatMessageDao()?.upsert(entity) return PendingMessage( @@ -356,12 +378,22 @@ class ChatMessageDataSource @Inject constructor( ) } - suspend fun confirmPending(chatId: ChatId, clientMessageId: ClientMessageId, serverMessage: ChatMessage) { + /** + * [replaceContent] is for a photo, whose optimistic row holds the local file: the confirmed row + * takes [serverMessage]'s content instead of keeping what was written locally. + */ + suspend fun confirmPending( + chatId: ChatId, + clientMessageId: ClientMessageId, + serverMessage: ChatMessage, + replaceContent: Boolean = false, + ) { val hex = mapper.chatIdHex(chatId) db?.chatMessageDao()?.confirmPendingMessage( hex, mapper.clientMessageIdHex(clientMessageId), mapper.toEntity(hex, serverMessage), + replaceContent, ) } diff --git a/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/PendingMediaDataSource.kt b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/PendingMediaDataSource.kt new file mode 100644 index 0000000000..af2181532b --- /dev/null +++ b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/PendingMediaDataSource.kt @@ -0,0 +1,97 @@ +package com.flipcash.app.persistence.sources + +import com.flipcash.app.persistence.FlipcashDatabase +import com.flipcash.app.persistence.entities.PendingMediaEntity +import javax.inject.Inject +import javax.inject.Singleton + +/** + * One queued photo message, as `pending_media` holds it. Ids stay hex strings on this side of the + * boundary: the chat layer that writes them owns what they mean. + */ +data class PendingMediaRecord( + val clientIdHex: String, + val chatIdHex: String, + val fileName: String, + val caption: String?, + val replyToMessageId: Long?, + val storedBlobIdHex: String?, + val sealedForHex: String?, + val width: Int, + val height: Int, + val blurhash: String?, + val sizeBytes: Long?, + val createdAt: Long, +) { + val isStored: Boolean get() = storedBlobIdHex != null +} + +/** + * The `pending_media` table. With no database there is nothing queued, which matches the other data + * sources: the instance is null only before login and after logout. + */ +@Singleton +class PendingMediaDataSource @Inject constructor() { + + private val db: FlipcashDatabase? + get() = FlipcashDatabase.getInstance() + + suspend fun insert(records: List) { + db?.pendingMediaDao()?.upsert(records.map { it.toEntity() }) + } + + suspend fun get(clientIdHex: String): PendingMediaRecord? = + db?.pendingMediaDao()?.get(clientIdHex)?.toRecord() + + suspend fun getAll(): List = + db?.pendingMediaDao()?.getAll().orEmpty().map { it.toRecord() } + + /** Records that the bytes are in storage under [blobIdHex], so a retry polls instead of uploading. */ + suspend fun markStored( + clientIdHex: String, + blobIdHex: String, + sealedForHex: String?, + sizeBytes: Long, + blurhash: String?, + ) { + db?.pendingMediaDao()?.markStored(clientIdHex, blobIdHex, sealedForHex, sizeBytes, blurhash) + } + + suspend fun delete(clientIdHex: String) { + db?.pendingMediaDao()?.delete(clientIdHex) + } + + suspend fun clear() { + db?.pendingMediaDao()?.deleteAll() + } + + private fun PendingMediaRecord.toEntity() = PendingMediaEntity( + clientIdHex = clientIdHex, + chatIdHex = chatIdHex, + fileName = fileName, + caption = caption, + replyToMessageId = replyToMessageId, + storedBlobIdHex = storedBlobIdHex, + sealedForHex = sealedForHex, + width = width, + height = height, + blurhash = blurhash, + sizeBytes = sizeBytes, + createdAt = createdAt, + ) + + private fun PendingMediaEntity.toRecord() = PendingMediaRecord( + clientIdHex = clientIdHex, + chatIdHex = chatIdHex, + fileName = fileName, + caption = caption, + replyToMessageId = replyToMessageId, + storedBlobIdHex = storedBlobIdHex, + sealedForHex = sealedForHex, + width = width, + height = height, + blurhash = blurhash, + sizeBytes = sizeBytes, + createdAt = createdAt, + ) +} diff --git a/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/mapper/chat/ChatEntityMapper.kt b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/mapper/chat/ChatEntityMapper.kt index c0e8377e2a..d426513c74 100644 --- a/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/mapper/chat/ChatEntityMapper.kt +++ b/apps/flipcash/shared/persistence/sources/src/main/kotlin/com/flipcash/app/persistence/sources/mapper/chat/ChatEntityMapper.kt @@ -215,14 +215,17 @@ class ChatEntityMapper @Inject constructor() { content: List, senderId: ID, clientMessageId: ClientMessageId, + ordinal: Int = 0, ): ChatMessageEntity { - val now = Clock.System.now() + // [ordinal] separates rows inserted in the same millisecond, which a batch of photos is: + // the placeholder id is the key, and a collision would replace the earlier row. + val nowMs = Clock.System.now().toEpochMilliseconds() + ordinal return ChatMessageEntity( chatIdHex = chatIdHex, - messageId = -(now.toEpochMilliseconds()), + messageId = -nowMs, senderIdHex = senderId.hexEncodedString(), contentJson = content.map { it.toSerialized() }, - timestampEpochMs = now.toEpochMilliseconds(), + timestampEpochMs = nowMs, unreadSeq = 0, status = MessageStatus.SENDING, pendingClientIdHex = clientMessageId.bytes.toList().hexEncodedString(), diff --git a/services/flipcash/build.gradle.kts b/services/flipcash/build.gradle.kts index 87d42fe9bd..6770471c04 100644 --- a/services/flipcash/build.gradle.kts +++ b/services/flipcash/build.gradle.kts @@ -47,6 +47,7 @@ dependencies { implementation(libs.grpc.protobuf.lite) implementation(libs.protobuf.validate.runtime) implementation(libs.androidx.lifecycle.runtime) + implementation(libs.androidx.lifecycle.process) implementation(libs.androidx.room.runtime) implementation(libs.androidx.room.ktx) implementation(libs.androidx.room.paging) diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/BlobUploader.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/BlobUploader.kt index 5f52538d0c..fd277bbc30 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/BlobUploader.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/BlobUploader.kt @@ -8,5 +8,15 @@ import com.flipcash.services.models.blob.UploadTarget * HTTP PUT (raw body) or POST (multipart/form-data) straight to the storage provider. */ interface BlobUploader { - suspend fun upload(bytes: ByteArray, mimeType: String, target: UploadTarget): Result + /** + * [onProgress] reports `(sentBytes, totalBytes)` over the whole request body — for a multipart + * POST that includes the form fields — as it is written to the socket. It runs on the upload + * thread and may fire many times, so keep it cheap. + */ + suspend fun upload( + bytes: ByteArray, + mimeType: String, + target: UploadTarget, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result } diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/ForegroundGate.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/ForegroundGate.kt new file mode 100644 index 0000000000..c6256c461c --- /dev/null +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/ForegroundGate.kt @@ -0,0 +1,31 @@ +package com.flipcash.services + +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.ProcessLifecycleOwner +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Whether the app is in the foreground. Work whose deadline should count foreground time only + * (chat media finalization) waits on [awaitForeground] before each step, so a backgrounded app + * neither burns its budget nor polls from a process the OS may be about to freeze. + */ +fun interface ForegroundGate { + /** Returns immediately when the app is foregrounded, otherwise suspends until it is. */ + suspend fun awaitForeground() +} + +/** [ForegroundGate] backed by [ProcessLifecycleOwner]: foreground means at least STARTED. */ +@Singleton +internal class ProcessForegroundGate @Inject constructor() : ForegroundGate { + override suspend fun awaitForeground() { + // The process lifecycle registry only accepts observers on the main thread. + withContext(Dispatchers.Main.immediate) { + ProcessLifecycleOwner.get().lifecycle.currentStateFlow + .first { it.isAtLeast(Lifecycle.State.STARTED) } + } + } +} diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/chat/ChatContentCrypto.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/chat/ChatContentCrypto.kt index 3131ebb144..7cf4b6ccb6 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/chat/ChatContentCrypto.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/chat/ChatContentCrypto.kt @@ -34,17 +34,21 @@ class ChatContentCrypto @Inject constructor( private val chatKeys = ConcurrentHashMap() /** - * Encrypts [content] from the viewer to [peerId]. Only Text, and a Reply whose body is Text, - * are sealed; media is not sent encrypted yet. + * Encrypts [content] from the viewer to [peerId]. Text, a photo that meets [SealedMediaContract], + * and a Reply around either are sealed; anything else fails. + * + * A photo's blob is sealed separately, see [sealBlob]; this only carries its id and metadata. */ suspend fun seal(chatId: ChatId, peerId: ID, content: MessageContent): Result { - if (!content.isSealable()) { + // A sealed photo has no download URL, whatever the caller's metadata carried. + val outbound = content.withoutDownloadUrls() + if (!outbound.isSealable()) { return Result.failure(IllegalArgumentException("Cannot encrypt ${content::class.simpleName}")) } val chatKeys = chatKeys(chatId, peerId).getOrElse { return Result.failure(it) } return runCatching { val payload = cipher.encrypt( - content = content.asContent().toByteArray(), + content = outbound.asContent().toByteArray(), chatKey = chatKeys.chatKey, senderPk = chatKeys.ownPk, recipientPk = chatKeys.peerPk, @@ -113,6 +117,64 @@ class ChatContentCrypto @Inject constructor( } } + /** + * Seals [plaintext], the stripped JPEG of the photo [blobId] names, from the viewer to + * [peerId]. Shaped to be the `seal` argument of `BlobStorageController.uploadSealed`. + */ + suspend fun sealBlob(chatId: ChatId, peerId: ID, blobId: ByteArray, plaintext: ByteArray): Result { + val chatKeys = chatKeys(chatId, peerId).getOrElse { return Result.failure(it) } + return runCatching { + cipher.encryptBlob( + image = plaintext, + chatKey = chatKeys.chatKey, + senderPk = chatKeys.ownPk, + recipientPk = chatKeys.peerPk, + chatId = chatId.bytes, + blobId = blobId, + ) + } + } + + /** + * Opens [sealed], the bytes of blob [blobId] that [senderId] uploaded in the DM between + * [selfId] and [peerId], and checks the result is [expectedSize] bytes, the `size_bytes` its + * message declared. + */ + suspend fun openBlob( + chatId: ChatId, + selfId: ID, + peerId: ID, + senderId: ID?, + blobId: ByteArray, + expectedSize: Long, + sealed: ByteArray, + ): OpenedBlob { + val chatKeys = chatKeys(chatId, peerId).getOrElse { cause -> + return OpenedBlob.Failed( + if (cause is ChatCipherException) BlobOpenFailure.Authentication else BlobOpenFailure.KeyPending, + ) + } + val (senderPk, recipientPk) = when (senderId) { + selfId -> chatKeys.ownPk to chatKeys.peerPk + peerId -> chatKeys.peerPk to chatKeys.ownPk + else -> return OpenedBlob.Failed(BlobOpenFailure.Authentication) + } + val plaintext = try { + cipher.decryptBlob( + blob = sealed, + chatKey = chatKeys.chatKey, + senderPk = senderPk, + recipientPk = recipientPk, + chatId = chatId.bytes, + blobId = blobId, + ) + } catch (_: ChatCipherException) { + return OpenedBlob.Failed(BlobOpenFailure.Authentication) + } + if (plaintext.size.toLong() != expectedSize) return OpenedBlob.Failed(BlobOpenFailure.Length) + return OpenedBlob.Plaintext(plaintext) + } + fun clear() { chatKeys.clear() } @@ -139,20 +201,34 @@ class ChatContentCrypto @Inject constructor( } } -private fun MessageContent.isSealable(): Boolean = when (this) { +/** The result of [ChatContentCrypto.openBlob]. */ +sealed interface OpenedBlob { + class Plaintext(val bytes: ByteArray) : OpenedBlob + data class Failed(val reason: BlobOpenFailure) : OpenedBlob +} + +internal fun MessageContent.isSealable(): Boolean = when (this) { is MessageContent.Text -> true - is MessageContent.Reply -> content.isNotEmpty() && content.all { it is MessageContent.Text } + is MessageContent.Media -> SealedMediaContract.isValid(asContent().media) + is MessageContent.Reply -> content.isNotEmpty() && + (content.all { it is MessageContent.Text } || content.singleOrNull()?.let { + it is MessageContent.Media && it.isSealable() + } == true) else -> false } /** - * The spec allows Text, Media, and a Reply of either. Media isn't rendered from an encrypted - * message yet, so it takes the same "update" path as a type this client has never heard of. + * The spec allows Text, Media, and a Reply of either. Media must meet [SealedMediaContract]; + * anything else takes the same "update" path as a type this client has never heard of. */ -private fun MessagingModel.Content.isRenderable(): Boolean = when (typeCase) { +internal fun MessagingModel.Content.isRenderable(): Boolean = when (typeCase) { MessagingModel.Content.TypeCase.TEXT -> true + MessagingModel.Content.TypeCase.MEDIA -> SealedMediaContract.isValid(media) MessagingModel.Content.TypeCase.REPLY -> reply.contentCount > 0 && - reply.contentList.all { it.typeCase == MessagingModel.Content.TypeCase.TEXT } + (reply.contentList.all { it.typeCase == MessagingModel.Content.TypeCase.TEXT } || + (reply.contentCount == 1 && + reply.getContent(0).typeCase == MessagingModel.Content.TypeCase.MEDIA && + reply.getContent(0).isRenderable())) else -> false } diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/chat/SealedMedia.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/chat/SealedMedia.kt new file mode 100644 index 0000000000..7f0eea1712 --- /dev/null +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/chat/SealedMedia.kt @@ -0,0 +1,92 @@ +package com.flipcash.services.chat + +import com.codeinc.flipcash.gen.blob.v1.Model as BlobModel +import com.codeinc.flipcash.gen.messaging.v1.Model as MessagingModel +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.MessageContent + +/** + * What an encrypted message may carry for a photo, and nothing else. A message outside this is + * shown as an update prompt rather than guessed at, which is the contract iOS holds. + */ +internal object SealedMediaContract { + const val BLOB_ID_SIZE = 16 + const val MAX_MIME_LENGTH = 255 + const val MAX_BLURHASH_LENGTH = 64 + const val IMAGE_MIME_PREFIX = "image/" + + fun isValid(media: MessagingModel.MediaContent): Boolean { + if (media.itemsCount != 1) return false + val item = media.getItems(0) + if (item.renditionsCount != 1) return false + val rendition = item.getRenditions(0) + if (rendition.role != BlobModel.Rendition.Role.ORIGINAL) return false + if (!rendition.hasBlobId() || rendition.blobId.value.size() != BLOB_ID_SIZE) return false + if (!rendition.hasBlob()) return false + val blob = rendition.blob + if (blob.mimeType.length !in 1..MAX_MIME_LENGTH) return false + if (!blob.mimeType.startsWith(IMAGE_MIME_PREFIX, ignoreCase = true)) return false + if (blob.sizeBytes < 1) return false + if (blob.kindCase != BlobModel.BlobMetadata.KindCase.IMAGE) return false + val image = blob.image + return image.width >= 1 && image.height >= 1 && image.blurhash.length <= MAX_BLURHASH_LENGTH + } +} + +/** A [ChatContentCrypto.openBlob] failure. Each draws "This photo can't be displayed". */ +sealed interface BlobOpenFailure { + /** The cipher rejected the bytes: wrong key, wrong blob id, wrong sender, or tampering. */ + data object Authentication : BlobOpenFailure + + /** It opened, but to a length other than the `size_bytes` the message declared. */ + data object Length : BlobOpenFailure + + /** It opened to the right length but the image decoder couldn't read it. Raised by the caller that decodes. */ + data object Undecodable : BlobOpenFailure + + /** The chat key isn't available yet. Unlike the others, a later attempt can succeed. */ + data object KeyPending : BlobOpenFailure +} + +/** Why a media send was refused before it reached the wire. */ +class BlobSealingMismatchException(message: String) : IllegalStateException(message) + +/** + * Checks that an uploaded photo's sealing matches the chat it is about to be sent into. + * + * [sealedFor] is the chat the blob was sealed for (the `e2eeChat` it was uploaded with), or null + * for a plain upload. [chatSeals] is whether the send will be encrypted. A plain blob in a + * sealing chat would leak the photo past a promise the chat makes, and a sealed blob outside the + * chat it was sealed for can't be opened by anyone who reads it. + */ +fun checkBlobSealing(sealedFor: ChatId?, sendChatId: ChatId, chatSeals: Boolean): Result = when { + sealedFor == null && chatSeals -> + Result.failure(BlobSealingMismatchException("Plain blob into an encrypted chat")) + sealedFor != null && !chatSeals -> + Result.failure(BlobSealingMismatchException("Sealed blob into a chat that is not encrypted")) + sealedFor != null && sealedFor != sendChatId -> + Result.failure(BlobSealingMismatchException("Blob sealed for a different chat")) + else -> Result.success(Unit) +} + +/** Whether [this] carries a photo, directly or as the body of a reply. */ +fun MessageContent.carriesMedia(): Boolean = when (this) { + is MessageContent.Media -> true + is MessageContent.Reply -> content.any { it is MessageContent.Media } + else -> false +} + +/** A copy with no download URL on any blob, which is what a sealed photo sends. */ +internal fun MessageContent.withoutDownloadUrls(): MessageContent = when (this) { + is MessageContent.Media -> copy( + items = items.map { item -> + item.copy( + renditions = item.renditions.map { r -> + r.copy(blob = r.blob?.copy(downloadUrl = "", expiresAtMillis = null)) + }, + ) + }, + ) + is MessageContent.Reply -> copy(content = content.map { it.withoutDownloadUrls() }) + else -> this +} diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/controllers/BlobStorageController.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/controllers/BlobStorageController.kt index c79da4343f..804a34410b 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/controllers/BlobStorageController.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/controllers/BlobStorageController.kt @@ -1,14 +1,19 @@ package com.flipcash.services.controllers import com.flipcash.services.BlobUploader +import com.flipcash.services.ForegroundGate import com.flipcash.services.internal.extensions.withoutJpegMetadata import com.flipcash.services.models.BlobNotReadyException import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.GetBlobsError +import com.flipcash.services.models.InitiateExternalUploadError import com.flipcash.services.models.blob.UploadPolicy import com.flipcash.services.models.chat.BlobAccessContext import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.chat.BlobMetadata import com.flipcash.services.models.chat.BlobState +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.blob.UploadReservation import com.flipcash.services.repository.BlobStorageRepository import com.flipcash.services.user.UserManager import com.getcode.ed25519.Ed25519 @@ -24,16 +29,30 @@ import kotlin.time.Duration.Companion.seconds * single call — reserve, PUT/POST the bytes, advise completion, and poll until the blob is READY — * returning the durable [BlobId] to hand to e.g. `ProfileController.setProfilePicture`. Callers use * [getUploadPolicy] up front to filter selection and validate size before uploading. + * + * Photos for a chat use [uploadChatMedia] or, when sealed for the chat, [uploadSealed]. */ class BlobStorageController @Inject constructor( private val repository: BlobStorageRepository, private val uploader: BlobUploader, private val userManager: UserManager, + private val foreground: ForegroundGate, ) { - private companion object { - val POLL_INTERVAL = 500.milliseconds - val POLL_TIMEOUT = 30.seconds + companion object { + private val POLL_INTERVAL = 500.milliseconds + private val POLL_TIMEOUT = 30.seconds + + // Chat media finalizes slower than an avatar (moderation, and for sealed bytes nothing to + // transcode), and a user may background the app mid-send. So it polls on a poll budget + // rather than wall time, and only while foregrounded. + internal val CHAT_MEDIA_POLL_INTERVAL = 2.seconds + internal const val CHAT_MEDIA_POLL_BUDGET = 30 + + const val MIME_SEALED = "application/octet-stream" + + /** XChaCha20-Poly1305 framing on a sealed blob: 24-byte nonce plus 16-byte tag. */ + const val SEAL_OVERHEAD = 40 } /** The server's current upload constraints (accepted MIME types + size ceilings). */ @@ -46,8 +65,14 @@ class BlobStorageController @Inject constructor( * Uploads [bytes] to storage and returns the READY [BlobId]. Reserves a presigned target, * PUTs/POSTs the bytes directly to storage, signals completion, and polls until the server * finishes validating/transcoding — so callers never orchestrate the individual steps. + * + * [onProgress] reports `(sentBytes, totalBytes)` of the upload itself. */ - suspend fun upload(bytes: ByteArray, mimeType: String): Result { + suspend fun upload( + bytes: ByteArray, + mimeType: String, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { val owner = owner() ?: return noAccount() // Strip EXIF/GPS/XMP here, at the choke point every upload passes through, rather than @@ -59,13 +84,113 @@ class BlobStorageController @Inject constructor( val reservation = repository.initiateExternalUpload(mimeType, bytes.size.toLong(), owner) .getOrElse { return Result.failure(it) } - uploader.upload(bytes, mimeType, reservation.target) + val stored = store(bytes, mimeType, reservation, owner, onProgress).getOrElse { return Result.failure(it) } + return awaitReady(stored, owner) + } + + /** + * [upload] for a photo going into a chat: same handshake, but finalization polls the way chat + * media needs (see [awaitChatMediaReady]). Plaintext, so the server can moderate it. + */ + suspend fun uploadChatMedia( + bytes: ByteArray, + mimeType: String, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { + val stored = storeChatMedia(bytes, mimeType, onProgress).getOrElse { return Result.failure(it) } + return awaitChatMediaReady(stored) + } + + /** + * The first half of [uploadChatMedia]: reserve, PUT/POST and signal completion, without waiting + * for the server to finalize. Success means the bytes are in storage under the returned id; + * [awaitChatMediaReady] finishes the job. A caller that must tell "never stored" from "stored + * but not ready yet" — to re-upload in one case and only re-poll in the other — uses the pair. + */ + suspend fun storeChatMedia( + bytes: ByteArray, + mimeType: String, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { + val owner = owner() ?: return noAccount() + val bytes = bytes.withoutJpegMetadata() + + val reservation = repository.initiateExternalUpload(mimeType, bytes.size.toLong(), owner) + .getOrElse { return Result.failure(it) } + + return store(bytes, mimeType, reservation, owner, onProgress) + } + + /** Waits for a stored chat photo to finalize. See [awaitReadyChatMedia] for the polling rules. */ + suspend fun awaitChatMediaReady(blobId: BlobId): Result { + val owner = owner() ?: return noAccount() + return awaitReadyChatMedia(blobId, owner) + } + + /** + * Uploads [plaintext] end-to-end encrypted for [e2eeChat]. The bytes are reserved as an opaque + * `application/octet-stream` of `plaintext + SEAL_OVERHEAD`, and [seal] is called with the + * reserved blob id — the cipher binds the ciphertext to it, so it can't run before the + * reservation. The cipher itself stays with the caller. + * + * [seal] must return exactly `plaintext.size + SEAL_OVERHEAD` bytes: the reserved size is + * signed into the upload target, so anything else would be refused by storage after the fact. + * The plaintext is not metadata-stripped here; the encoder already ships clean JPEGs. + */ + suspend fun uploadSealed( + plaintext: ByteArray, + e2eeChat: ChatId, + seal: suspend (blobId: BlobId) -> ByteArray, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { + val stored = storeSealed(plaintext, e2eeChat, seal, onProgress).getOrElse { return Result.failure(it) } + return awaitChatMediaReady(stored) + } + + /** The first half of [uploadSealed]; see [storeChatMedia] for why it is split from finalization. */ + suspend fun storeSealed( + plaintext: ByteArray, + e2eeChat: ChatId, + seal: suspend (blobId: BlobId) -> ByteArray, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null, + ): Result { + val owner = owner() ?: return noAccount() + val reservedSize = plaintext.size + SEAL_OVERHEAD + + val reservation = repository.initiateExternalUpload( + mimeType = MIME_SEALED, + sizeBytes = reservedSize.toLong(), + owner = owner, + e2eeChat = e2eeChat, + ).getOrElse { return Result.failure(it) } + + val sealed = runCatching { seal(reservation.blobId) } + .getOrElse { return Result.failure(InitiateExternalUploadError.Other(it)) } + if (sealed.size != reservedSize) { + return Result.failure( + InitiateExternalUploadError.Other( + IllegalStateException("Sealed ${sealed.size} bytes but reserved $reservedSize") + ) + ) + } + + return store(sealed, MIME_SEALED, reservation, owner, onProgress) + } + + private suspend fun store( + bytes: ByteArray, + mimeType: String, + reservation: UploadReservation, + owner: Ed25519.KeyPair, + onProgress: ((Long, Long) -> Unit)?, + ): Result { + uploader.upload(bytes, mimeType, reservation.target, onProgress) .getOrElse { return Result.failure(it) } // Advisory — the storage-completion event finalizes the blob even if this is skipped/fails. repository.completeExternalUpload(reservation.blobId, owner) - return awaitReady(reservation.blobId, owner) + return Result.success(reservation.blobId) } /** @@ -116,6 +241,32 @@ class BlobStorageController @Inject constructor( return Result.failure(BlobNotReadyException()) } + /** + * Chat-media finalization: [CHAT_MEDIA_POLL_BUDGET] polls [CHAT_MEDIA_POLL_INTERVAL] apart, + * each one waiting for the app to be foregrounded first. The budget counts polls, not time, so + * a backgrounded app doesn't spend it. + * + * A poll that fails in transit is swallowed and still counts — the next one may get through. A + * denial is not transient and ends it. An id the server hasn't finished (or doesn't report a + * status for) is processing. + */ + private suspend fun awaitReadyChatMedia(blobId: BlobId, owner: Ed25519.KeyPair): Result { + repeat(CHAT_MEDIA_POLL_BUDGET) { poll -> + foreground.awaitForeground() + + val polled = repository.getBlobs(listOf(blobId), owner, BlobAccessContext.Owned) + val denied = polled.exceptionOrNull() as? GetBlobsError.Denied + if (denied != null) return Result.failure(denied) + + when (val blob = polled.getOrNull()?.firstOrNull()) { + is BlobState.Ready -> return Result.success(blobId) + is BlobState.Rejected -> return Result.failure(BlobRejectedException(blob.reason)) + null -> if (poll < CHAT_MEDIA_POLL_BUDGET - 1) delay(CHAT_MEDIA_POLL_INTERVAL) + } + } + return Result.failure(BlobNotReadyException()) + } + private fun owner(): Ed25519.KeyPair? = userManager.accountCluster?.authority?.keyPair private fun noAccount(): Result = diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/inject/FlipcashModule.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/inject/FlipcashModule.kt index 69c4014989..ee5fefb67d 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/inject/FlipcashModule.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/inject/FlipcashModule.kt @@ -14,6 +14,8 @@ import com.flipcash.services.internal.domain.TextModerationResponseMapper import com.flipcash.services.internal.domain.UserProfileMapper import com.flipcash.services.internal.domain.ChatMetadataMapper import com.flipcash.services.BlobUploader +import com.flipcash.services.ForegroundGate +import com.flipcash.services.ProcessForegroundGate import com.flipcash.services.internal.network.HttpBlobUploader import com.flipcash.services.internal.network.services.AccountService import com.flipcash.services.internal.network.services.ActivityFeedService @@ -195,6 +197,11 @@ internal object FlipcashModule { uploader: HttpBlobUploader, ): BlobUploader = uploader + @Provides + internal fun providesForegroundGate( + gate: ProcessForegroundGate, + ): ForegroundGate = gate + @Provides internal fun providesAccountRepository( service: AccountService, diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/extensions/JpegMetadata.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/extensions/JpegMetadata.kt index a2e331b4a4..a088c6f3a4 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/extensions/JpegMetadata.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/extensions/JpegMetadata.kt @@ -26,7 +26,7 @@ private const val MARKER_COM = 0xFE * Must agree byte-for-byte with iOS's `JPEGMetadata.stripped` * (`FlipcashCore/Sources/FlipcashCore/Blob/JPEGMetadata.swift`). */ -internal fun ByteArray.withoutJpegMetadata(): ByteArray { +fun ByteArray.withoutJpegMetadata(): ByteArray { val segments = privacySegments() if (segments.isNullOrEmpty()) return this diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/HttpBlobUploader.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/HttpBlobUploader.kt index dd4ade22f2..4375775ece 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/HttpBlobUploader.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/HttpBlobUploader.kt @@ -6,9 +6,14 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.MultipartBody +import okhttp3.MediaType import okhttp3.OkHttpClient import okhttp3.Request +import okhttp3.RequestBody import okhttp3.RequestBody.Companion.toRequestBody +import okio.BufferedSink +import okio.ForwardingSink +import okio.buffer import javax.inject.Inject import javax.inject.Singleton @@ -21,6 +26,7 @@ internal class HttpBlobUploader @Inject constructor() : BlobUploader { bytes: ByteArray, mimeType: String, target: UploadTarget, + onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)?, ): Result = withContext(Dispatchers.IO) { runCatching { val media = mimeType.toMediaTypeOrNull() @@ -29,7 +35,7 @@ internal class HttpBlobUploader @Inject constructor() : BlobUploader { Request.Builder() .url(target.url) .apply { target.headers.forEach { (k, v) -> header(k, v) } } - .put(bytes.toRequestBody(media)) + .put(bytes.toRequestBody(media).counting(onProgress)) .build() } UploadTarget.Method.POST -> { @@ -37,7 +43,7 @@ internal class HttpBlobUploader @Inject constructor() : BlobUploader { val body = MultipartBody.Builder().setType(MultipartBody.FORM).apply { target.formFields.forEach { (k, v) -> addFormDataPart(k, v) } addFormDataPart("file", "upload", bytes.toRequestBody(media)) - }.build() + }.build().counting(onProgress) Request.Builder() .url(target.url) .apply { target.headers.forEach { (k, v) -> header(k, v) } } @@ -55,3 +61,30 @@ internal class HttpBlobUploader @Inject constructor() : BlobUploader { } } } + +/** + * Reports `(written, total)` as [this] is written to the socket. Counts bytes handed to the sink, + * so on a retried or redirected request it can run past [RequestBody.contentLength]; callers clamp. + */ +private fun RequestBody.counting(onProgress: ((Long, Long) -> Unit)?): RequestBody { + if (onProgress == null) return this + val delegate = this + return object : RequestBody() { + override fun contentType(): MediaType? = delegate.contentType() + override fun contentLength(): Long = delegate.contentLength() + + override fun writeTo(sink: BufferedSink) { + val total = contentLength() + var written = 0L + val counting = object : ForwardingSink(sink) { + override fun write(source: okio.Buffer, byteCount: Long) { + super.write(source, byteCount) + written += byteCount + onProgress(written, total) + } + }.buffer() + delegate.writeTo(counting) + counting.flush() + } + } +} diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt index f8740c37e6..4f7374b2c1 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt @@ -11,6 +11,7 @@ import com.flipcash.services.internal.network.extensions.asUserId import com.flipcash.services.internal.network.extensions.authenticate import com.flipcash.services.models.chat.BlobAccessContext import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.ChatId import com.getcode.ed25519.Ed25519 import com.getcode.opencode.internal.network.core.GrpcApi import com.getcode.utils.toByteString @@ -24,7 +25,7 @@ import javax.inject.Singleton /** * Wraps the BlobStorage gRPC service — direct-to-storage uploads. The bytes never travel through * gRPC: [initiateExternalUpload] reserves a [Model.BlobId] and returns a presigned target the client - * uploads to over plain HTTP, [completeExternalUpload] advises the server the upload finished, and + * uploads to over plain HTTP (naming the chat when the bytes are sealed for it), [completeExternalUpload] advises the server the upload finished, and * [getBlobs] resolves ids to their status + a fresh download URL. */ @Singleton @@ -52,10 +53,13 @@ internal class BlobStorageApi @Inject constructor( mimeType: String, sizeBytes: Long, owner: Ed25519.KeyPair, + e2eeChat: ChatId? = null, ): RpcBlobStorageService.InitiateExternalUploadResponse { val request = RpcBlobStorageService.InitiateExternalUploadRequest.newBuilder() .setMimeType(mimeType) .setSizeBytes(sizeBytes) + // Before auth: the signature covers the message as built so far (see getBlobsRequest). + .apply { e2eeChat?.let { setChat(it.asChatId()) } } .apply { setAuth(authenticate(owner)) } .build() diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/LocalToProtobuf.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/LocalToProtobuf.kt index 016f64a26a..03aeefe0b8 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/LocalToProtobuf.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/LocalToProtobuf.kt @@ -157,7 +157,12 @@ internal fun MessageContent.asContent(): MessagingModel.Content { .setMedia( MessagingModel.MediaContent.newBuilder() .addAllItems(items.map { it.asMediaItem() }) - .apply { if (caption != null) setCaption(MessagingModel.TextContent.newBuilder().setText(caption.text)) } + // `this@asContent`: bare `caption` here is the builder's own (always non-null) getter. + .apply { + this@asContent.caption + ?.takeIf { it.text.isNotEmpty() } + ?.let { setCaption(MessagingModel.TextContent.newBuilder().setText(it.text)) } + } ) .build() // Server-authored and receive-only: no client path builds one, so reaching this is a bug. @@ -198,6 +203,45 @@ internal fun com.flipcash.services.models.chat.MediaItemRendition.asRendition(): com.codeinc.flipcash.gen.blob.v1.Model.BlobId.newBuilder() .setValue(blobId.bytes.toByteString()) ) + .apply { this@asRendition.blob?.let { setBlob(it.asBlobMetadata()) } } + .build() +} + +/** + * A sealed photo carries its metadata inline, since the server can't read it, and no download URL: + * there is nothing to download from until the recipient asks `GetBlobs`. A blank [BlobMetadata.downloadUrl] + * is therefore left off rather than written as an empty URL. + */ +internal fun com.flipcash.services.models.chat.BlobMetadata.asBlobMetadata(): com.codeinc.flipcash.gen.blob.v1.Model.BlobMetadata { + val metadata = this + return com.codeinc.flipcash.gen.blob.v1.Model.BlobMetadata.newBuilder() + .setMimeType(mimeType) + .setSizeBytes(sizeBytes) + .apply { + if (metadata.downloadUrl.isNotEmpty()) { + setDownloadUrl( + com.codeinc.flipcash.gen.blob.v1.Model.DownloadUrl.newBuilder() + .setUrl(metadata.downloadUrl) + .apply { + metadata.expiresAtMillis?.let { + setExpiresAt( + com.google.protobuf.Timestamp.newBuilder() + .setSeconds(Math.floorDiv(it, 1_000L)) + .setNanos((Math.floorMod(it, 1_000L) * 1_000_000L).toInt()) + ) + } + } + ) + } + metadata.image?.let { + setImage( + com.codeinc.flipcash.gen.blob.v1.Model.ImageMetadata.newBuilder() + .setWidth(it.width) + .setHeight(it.height) + .setBlurhash(it.blurhash) + ) + } + } .build() } diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/ProtobufToLocal.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/ProtobufToLocal.kt index 2af2f7acf8..0945add42b 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/ProtobufToLocal.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/extensions/ProtobufToLocal.kt @@ -30,6 +30,7 @@ import com.flipcash.services.models.chat.KeyEnvelope import com.flipcash.services.models.chat.LobbyMember import com.flipcash.services.models.chat.LobbyUpdate import com.flipcash.services.models.chat.EmojiReaction +import com.flipcash.services.models.blob.EncryptedConstraints import com.flipcash.services.models.blob.ImageConstraints import com.flipcash.services.models.blob.MimeTypeConstraints import com.flipcash.services.models.blob.UploadPolicy @@ -190,7 +191,8 @@ internal fun MessagingModel.Content.toMessageContent(): MessageContent { ) MessagingModel.Content.TypeCase.MEDIA -> MessageContent.Media( items = media.itemsList.map { it.toMediaItem() }, - caption = if (media.hasCaption()) MessageContent.Text(media.caption.text) else null, + // An empty caption is no caption. + caption = if (media.hasCaption() && media.caption.text.isNotEmpty()) MessageContent.Text(media.caption.text) else null, ) MessagingModel.Content.TypeCase.SYSTEM -> MessageContent.System(system.fallbackText) MessagingModel.Content.TypeCase.DELETED -> MessageContent.Deleted( @@ -661,6 +663,18 @@ internal fun com.codeinc.flipcash.gen.blob.v1.Model.UploadPolicy.toUploadPolicy( } else null, ) }, + encrypted = if (hasEncrypted()) { + EncryptedConstraints( + maxSizeBytes = encrypted.maxSizeBytes, + image = if (encrypted.hasImage()) { + ImageConstraints( + maxWidth = encrypted.image.maxWidth, + maxHeight = encrypted.image.maxHeight, + maxPixels = encrypted.image.maxPixels, + ) + } else null, + ) + } else null, ) } diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/services/BlobStorageService.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/services/BlobStorageService.kt index 3d19b913cc..cdb154b747 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/services/BlobStorageService.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/services/BlobStorageService.kt @@ -16,6 +16,7 @@ import com.flipcash.services.models.chat.BlobAccessContext import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.chat.BlobState import com.flipcash.services.models.chat.BlobStatus +import com.flipcash.services.models.chat.ChatId import com.getcode.ed25519.Ed25519 import com.getcode.opencode.internal.network.extensions.foldWithSuppression import com.getcode.opencode.utils.toValidationOrElse @@ -47,8 +48,9 @@ internal class BlobStorageService @Inject constructor( mimeType: String, sizeBytes: Long, owner: Ed25519.KeyPair, + e2eeChat: ChatId? = null, ): Result { - return runCatching { api.initiateExternalUpload(mimeType, sizeBytes, owner) } + return runCatching { api.initiateExternalUpload(mimeType, sizeBytes, owner, e2eeChat) } .foldWithSuppression( onSuccess = { response -> when (response.result) { diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/repositories/InternalBlobStorageRepository.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/repositories/InternalBlobStorageRepository.kt index 67bf8d0069..12333c5b2c 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/repositories/InternalBlobStorageRepository.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/repositories/InternalBlobStorageRepository.kt @@ -7,6 +7,7 @@ import com.flipcash.services.models.chat.BlobAccessContext import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.chat.BlobState import com.flipcash.services.models.chat.BlobStatus +import com.flipcash.services.models.chat.ChatId import com.flipcash.services.repository.BlobStorageRepository import com.getcode.ed25519.Ed25519 import com.getcode.utils.ErrorUtils @@ -23,7 +24,8 @@ internal class InternalBlobStorageRepository( mimeType: String, sizeBytes: Long, owner: Ed25519.KeyPair, - ): Result = service.initiateExternalUpload(mimeType, sizeBytes, owner) + e2eeChat: ChatId?, + ): Result = service.initiateExternalUpload(mimeType, sizeBytes, owner, e2eeChat) .onFailure { ErrorUtils.handleError(it) } override suspend fun completeExternalUpload( diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/models/blob/UploadPolicy.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/models/blob/UploadPolicy.kt index 6a72398086..0c2b2d898d 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/models/blob/UploadPolicy.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/models/blob/UploadPolicy.kt @@ -14,6 +14,10 @@ data class UploadPolicy( val ttl: Duration, // Ordered most-specific-first: the first entry whose pattern matches wins. val mimeTypeConstraints: List, + // Constraints for end-to-end-encrypted uploads, whose bytes the server can't inspect: one size + // ceiling (on the sealed bytes) and image bounds for the plaintext. Null when the server + // doesn't accept encrypted uploads. + val encrypted: EncryptedConstraints? = null, ) { /** The constraints governing [mimeType], or null if the type is not accepted. */ fun constraintsFor(mimeType: String): MimeTypeConstraints? = @@ -44,6 +48,13 @@ data class MimeTypeConstraints( } } +@Serializable +data class EncryptedConstraints( + val maxSizeBytes: Long, + val image: ImageConstraints?, +) + +// 0 means unbounded. @Serializable data class ImageConstraints( val maxWidth: Int, diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/repository/BlobStorageRepository.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/repository/BlobStorageRepository.kt index 05c6675946..9b743e830d 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/repository/BlobStorageRepository.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/repository/BlobStorageRepository.kt @@ -6,6 +6,7 @@ import com.flipcash.services.models.chat.BlobAccessContext import com.flipcash.services.models.chat.BlobId import com.flipcash.services.models.chat.BlobState import com.flipcash.services.models.chat.BlobStatus +import com.flipcash.services.models.chat.ChatId import com.getcode.ed25519.Ed25519 interface BlobStorageRepository { @@ -15,6 +16,9 @@ interface BlobStorageRepository { mimeType: String, sizeBytes: Long, owner: Ed25519.KeyPair, + // Set when the bytes are sealed for this chat; the server then holds ciphertext it can't + // inspect and applies the policy's encrypted constraints. + e2eeChat: ChatId? = null, ): Result suspend fun completeExternalUpload(blobId: BlobId, owner: Ed25519.KeyPair): Result diff --git a/services/flipcash/src/test/kotlin/com/flipcash/services/chat/ChatContentCryptoTest.kt b/services/flipcash/src/test/kotlin/com/flipcash/services/chat/ChatContentCryptoTest.kt index 967774e60c..85ae66dc44 100644 --- a/services/flipcash/src/test/kotlin/com/flipcash/services/chat/ChatContentCryptoTest.kt +++ b/services/flipcash/src/test/kotlin/com/flipcash/services/chat/ChatContentCryptoTest.kt @@ -1,13 +1,20 @@ package com.flipcash.services.chat import com.codeinc.flipcash.gen.messaging.v1.Model as MessagingModel +import com.flipcash.services.internal.network.extensions.asContent +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobMetadata import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.ImageMetadata +import com.flipcash.services.models.chat.MediaItem +import com.flipcash.services.models.chat.MediaItemRendition import com.flipcash.services.models.chat.MessageContent import com.getcode.ed25519kmp.KeyPair import com.getcode.opencode.model.core.ID import kotlinx.coroutines.test.runTest import org.junit.Test import java.io.IOException +import kotlin.test.assertContentEquals import kotlin.test.assertEquals import kotlin.test.assertIs import kotlin.test.assertTrue @@ -64,12 +71,170 @@ class ChatContentCryptoTest { } @Test - fun `media is not sealed`() = runTest { + fun `media with no items is not sealed`() = runTest { val result = mine.seal(chatId, peer, MessageContent.Media(items = emptyList(), caption = null)) assertTrue(result.isFailure) } + private val blobId = ByteArray(16) { (it + 0xA0).toByte() } + + private fun photo( + role: MediaItemRendition.Role = MediaItemRendition.Role.ORIGINAL, + id: ByteArray? = blobId, + blob: BlobMetadata? = BlobMetadata( + mimeType = "image/jpeg", + sizeBytes = 100, + downloadUrl = "", + image = ImageMetadata(width = 640, height = 480, blurhash = "LEHV6nWB2yk8pyo0adR*.7kCMdnj"), + ), + ) = MessageContent.Media( + items = listOf(MediaItem(listOf(MediaItemRendition(role, BlobId(id ?: ByteArray(0)), blob)))), + caption = MessageContent.Text("look"), + ) + + private fun BlobMetadata.image(i: ImageMetadata?) = copy(image = i) + + @Test + fun `a photo and a reply to a photo are sealed and open for the peer`() = runTest { + val media = photo() + assertEquals(OpenedContent.Plaintext(media), open(sealFromPeer(media))) + + val reply = MessageContent.Reply(repliedMessageId = 9, content = listOf(media)) + assertEquals(OpenedContent.Plaintext(reply), open(sealFromPeer(reply))) + } + + @Test + fun `a download url never goes on the wire for a sealed photo`() = runTest { + val withUrl = photo(blob = photo().items[0].renditions[0].blob!!.copy(downloadUrl = "https://x/y", expiresAtMillis = 5)) + + assertEquals(OpenedContent.Plaintext(photo()), open(sealFromPeer(withUrl))) + } + + @Test + fun `a reply that mixes media with text is not sealed`() = runTest { + val reply = MessageContent.Reply(4, listOf(photo(), MessageContent.Text("x"))) + + assertTrue(mine.seal(chatId, peer, reply).isFailure) + } + + private fun contractCases(): Map { + val ok = photo().items[0].renditions[0].blob!! + val image = ok.image!! + fun withBlob(b: BlobMetadata?) = photo(blob = b) + return mapOf( + "wrong role" to photo(role = MediaItemRendition.Role.DISPLAY), + "short blob id" to photo(id = ByteArray(15)), + "long blob id" to photo(id = ByteArray(17)), + "no blob" to withBlob(null), + "empty mime" to withBlob(ok.copy(mimeType = "")), + "non-image mime" to withBlob(ok.copy(mimeType = "video/mp4")), + "long mime" to withBlob(ok.copy(mimeType = "image/" + "a".repeat(250))), + "zero size" to withBlob(ok.copy(sizeBytes = 0)), + "no image kind" to withBlob(ok.image(null)), + "zero width" to withBlob(ok.image(image.copy(width = 0))), + "zero height" to withBlob(ok.image(image.copy(height = 0))), + "long blurhash" to withBlob(ok.image(image.copy(blurhash = "a".repeat(65)))), + "two items" to photo().let { it.copy(items = it.items + it.items) }, + "two renditions" to photo().let { + it.copy(items = listOf(MediaItem(it.items[0].renditions + it.items[0].renditions))) + }, + "no renditions" to MessageContent.Media(listOf(MediaItem(emptyList())), null), + ) + } + + @Test + fun `a photo outside the contract is neither sealed nor opened`() = runTest { + for ((name, media) in contractCases()) { + assertTrue(mine.seal(chatId, peer, media).isFailure, "seal: $name") + assertTrue(mine.seal(chatId, peer, MessageContent.Reply(1, listOf(media))).isFailure, "seal reply: $name") + // Seal the raw bytes directly, as a peer running other code could. + val raw = media.asContent().toByteArray() + assertEquals( + OpenedContent.Undecryptable(UndecryptableReason.Unsupported), + open(sealRaw(raw)), + "open: $name", + ) + } + } + + @Test + fun `mime prefix is case-insensitive and an empty blurhash and caption are absent`() = runTest { + val ok = photo().items[0].renditions[0].blob!! + val media = photo(blob = ok.copy(mimeType = "IMAGE/PNG").image(ok.image!!.copy(blurhash = ""))) + .copy(caption = null) + + assertEquals(OpenedContent.Plaintext(media), open(sealFromPeer(media))) + + val proto = media.asContent() + val emptyCaption = proto.toBuilder() + .setMedia(proto.media.toBuilder().setCaption(MessagingModel.TextContent.newBuilder().setText(""))) + .build() + assertEquals(OpenedContent.Plaintext(media), open(sealRaw(emptyCaption.toByteArray()))) + } + + @Test + fun `blob sealed by the viewer opens for the peer and for the viewer's other device`() = runTest { + val sealed = mine.sealBlob(chatId, peer, blobId, byteArrayOf(1, 2, 3)).getOrThrow() + + val asPeer = theirs.openBlob(chatId, selfId = peer, peerId = self, senderId = self, blobId = blobId, expectedSize = 3, sealed = sealed) + assertContentEquals(byteArrayOf(1, 2, 3), assertIs(asPeer).bytes) + + val otherDevice = ChatContentCrypto(FakeChatCipher, Keys(selfKeys, peerKeys.publicKey)) + .openBlob(chatId, selfId = self, peerId = peer, senderId = self, blobId = blobId, expectedSize = 3, sealed = sealed) + assertContentEquals(byteArrayOf(1, 2, 3), assertIs(otherDevice).bytes) + } + + @Test + fun `blob key roles follow the sender`() = runTest { + val fromPeer = theirs.sealBlob(chatId, self, blobId, byteArrayOf(9)).getOrThrow() + + // Right: the peer sent it. + assertIs( + mine.openBlob(chatId, self, peer, senderId = peer, blobId = blobId, expectedSize = 1, sealed = fromPeer), + ) + // Wrong: claiming the viewer sent it swaps the roles. + assertEquals( + OpenedBlob.Failed(BlobOpenFailure.Authentication), + mine.openBlob(chatId, self, peer, senderId = self, blobId = blobId, expectedSize = 1, sealed = fromPeer), + ) + // A sender who is neither member can't have sealed it. + assertEquals( + OpenedBlob.Failed(BlobOpenFailure.Authentication), + mine.openBlob(chatId, self, peer, senderId = null, blobId = blobId, expectedSize = 1, sealed = fromPeer), + ) + } + + @Test + fun `blob failures are typed`() = runTest { + val sealed = theirs.sealBlob(chatId, self, blobId, byteArrayOf(1, 2, 3)).getOrThrow() + + assertEquals( + OpenedBlob.Failed(BlobOpenFailure.Length), + mine.openBlob(chatId, self, peer, peer, blobId, expectedSize = 4, sealed = sealed), + ) + assertEquals( + OpenedBlob.Failed(BlobOpenFailure.Authentication), + mine.openBlob(chatId, self, peer, peer, ByteArray(16), expectedSize = 3, sealed = sealed), + ) + val cold = Keys(selfKeys, peerKeys.publicKey).apply { peerFails = IOException("offline") } + assertEquals( + OpenedBlob.Failed(BlobOpenFailure.KeyPending), + ChatContentCrypto(FakeChatCipher, cold).openBlob(chatId, self, peer, peer, blobId, expectedSize = 3, sealed = sealed), + ) + } + + @Test + fun `blob sealing guard matches the chat`() { + val other = ChatId(ByteArray(32) { 8 }) + + assertTrue(checkBlobSealing(sealedFor = chatId, sendChatId = chatId, chatSeals = true).isSuccess) + assertTrue(checkBlobSealing(sealedFor = null, sendChatId = chatId, chatSeals = false).isSuccess) + assertTrue(checkBlobSealing(sealedFor = null, sendChatId = chatId, chatSeals = true).isFailure) + assertTrue(checkBlobSealing(sealedFor = chatId, sendChatId = chatId, chatSeals = false).isFailure) + assertTrue(checkBlobSealing(sealedFor = other, sendChatId = chatId, chatSeals = true).isFailure) + } + @Test fun `an unknown scheme asks for an update without fetching keys`() = runTest { val sealed = sealFromPeer(MessageContent.Text("hi")).copy(scheme = 2) @@ -79,7 +244,7 @@ class ChatContentCryptoTest { } @Test - fun `a plaintext type outside text and reply asks for an update`() = runTest { + fun `a plaintext type outside text, media and reply asks for an update`() = runTest { val media = MessagingModel.Content.newBuilder() .setMedia(MessagingModel.MediaContent.getDefaultInstance()) .build() @@ -89,7 +254,7 @@ class ChatContentCryptoTest { } @Test - fun `a reply to media asks for an update`() = runTest { + fun `a reply to empty media asks for an update`() = runTest { val replyToMedia = MessagingModel.Content.newBuilder() .setReply( MessagingModel.ReplyContent.newBuilder() diff --git a/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerChatMediaTest.kt b/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerChatMediaTest.kt new file mode 100644 index 0000000000..f0c7fac5c0 --- /dev/null +++ b/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerChatMediaTest.kt @@ -0,0 +1,304 @@ +package com.flipcash.services.controllers + +import com.flipcash.services.BlobUploader +import com.flipcash.services.ForegroundGate +import com.flipcash.services.models.BlobNotReadyException +import com.flipcash.services.models.BlobRejectedException +import com.flipcash.services.models.GetBlobsError +import com.flipcash.services.models.InitiateExternalUploadError +import com.flipcash.services.models.ModerationResult +import com.flipcash.services.models.blob.UploadReservation +import com.flipcash.services.models.blob.UploadTarget +import com.flipcash.services.models.chat.BlobAccessContext +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobRejection +import com.flipcash.services.models.chat.BlobState +import com.flipcash.services.models.chat.BlobStatus +import com.flipcash.services.models.chat.ChatId +import com.flipcash.services.models.chat.RejectionReason +import com.flipcash.services.repository.BlobStorageRepository +import com.flipcash.services.user.UserManager +import com.getcode.ed25519.Ed25519 +import com.getcode.opencode.model.accounts.AccountCluster +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertTrue +import kotlin.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class BlobStorageControllerChatMediaTest { + + private val repository = mockk() + private val uploader = mockk() + private val userManager = mockk(relaxed = true) + + private val blobId = BlobId(ByteArray(16) { 7 }) + private val chatId = ChatId(ByteArray(32) { 3 }) + private val target = UploadTarget( + method = UploadTarget.Method.PUT, + url = "https://storage.example/upload", + headers = emptyMap(), + formFields = emptyMap(), + expiresAt = Instant.fromEpochSeconds(10_000), + ) + + private fun controller(foreground: ForegroundGate = ForegroundGate { }) = + BlobStorageController(repository, uploader, userManager, foreground) + + private fun stubHappyUpload() { + val keyPair = mockk(relaxed = true) + val cluster = mockk(relaxed = true) { + every { authority } returns mockk { every { this@mockk.keyPair } returns keyPair } + } + every { userManager.accountCluster } returns cluster + coEvery { repository.initiateExternalUpload(any(), any(), any(), any()) } returns + Result.success(UploadReservation(blobId, target)) + coEvery { uploader.upload(any(), any(), any(), any()) } returns Result.success(Unit) + coEvery { repository.completeExternalUpload(any(), any()) } returns Result.success(BlobStatus.PROCESSING) + } + + private fun ready() = BlobState.Ready(id = blobId, metadata = mockk(relaxed = true)) + private fun rejected(reason: RejectionReason) = BlobState.Rejected( + id = blobId, + reason = BlobRejection(reason, ModerationResult.FlaggedCategory.NONE), + ) + + // MARK: - Sealed upload - + + @Test + fun `sealed upload reserves an opaque blob sized plaintext plus framing for the chat`() = runTest { + stubHappyUpload() + val mime = slot() + val size = slot() + val chat = slot() + coEvery { repository.initiateExternalUpload(capture(mime), capture(size), any(), captureNullable(chat)) } returns + Result.success(UploadReservation(blobId, target)) + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.success(listOf(ready())) + val plaintext = ByteArray(100) { 1 } + val sealed = ByteArray(140) { 9 } + var sealedWith: BlobId? = null + + val result = controller().uploadSealed(plaintext, chatId, seal = { id -> sealedWith = id; sealed }) + + assertEquals(blobId, result.getOrNull()) + assertEquals("application/octet-stream", mime.captured) + assertEquals(140L, size.captured) + assertEquals(chatId, chat.captured) + assertEquals(blobId, sealedWith) + coVerify { uploader.upload(match { it.contentEquals(sealed) }, "application/octet-stream", target, any()) } + } + + @Test + fun `sealed upload fails without uploading when the sealed size differs from the reservation`() = runTest { + stubHappyUpload() + + val result = controller().uploadSealed(ByteArray(100), chatId, seal = { ByteArray(139) }) + + assertIs(result.exceptionOrNull()) + coVerify(exactly = 0) { uploader.upload(any(), any(), any(), any()) } + } + + @Test + fun `sealed upload fails without uploading when sealing throws`() = runTest { + stubHappyUpload() + + val result = controller().uploadSealed(ByteArray(100), chatId, seal = { error("no key") }) + + assertTrue(result.isFailure) + coVerify(exactly = 0) { uploader.upload(any(), any(), any(), any()) } + } + + @Test + fun `plain chat upload sends no chat and declares the given type`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.success(listOf(ready())) + + controller().uploadChatMedia(byteArrayOf(1, 2, 3), "image/jpeg") + + coVerify { repository.initiateExternalUpload("image/jpeg", 3L, any(), null) } + } + + // MARK: - Progress - + + @Test + fun `progress reported by the uploader reaches the caller`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.success(listOf(ready())) + coEvery { uploader.upload(any(), any(), any(), any()) } coAnswers { + arg<((Long, Long) -> Unit)?>(3)?.invoke(50L, 100L) + Result.success(Unit) + } + val seen = mutableListOf>() + + controller().upload(byteArrayOf(1), "image/png") { sent, total -> seen += sent to total } + + assertEquals(listOf(50L to 100L), seen) + } + + // MARK: - Chat media finalization - + + @Test + fun `chat media polls every two seconds until ready, as the owner`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returnsMany listOf( + Result.success(emptyList()), + Result.success(emptyList()), + Result.success(listOf(ready())), + ) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + assertEquals(blobId, result.getOrNull()) + assertEquals(4_000L, testScheduler.currentTime) + coVerify(exactly = 3) { repository.getBlobs(listOf(blobId), any(), BlobAccessContext.Owned) } + } + + @Test + fun `chat media gives up after thirty polls`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.success(emptyList()) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + assertIs(result.exceptionOrNull()) + coVerify(exactly = 30) { repository.getBlobs(any(), any(), any()) } + // Thirty polls, twenty-nine waits: no sleep after the last one. + assertEquals(58_000L, testScheduler.currentTime) + } + + @Test + fun `a failed poll is swallowed and still spends the budget`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returnsMany listOf( + Result.failure(GetBlobsError.Other(RuntimeException("offline"))), + Result.failure(GetBlobsError.Other(RuntimeException("offline"))), + Result.success(listOf(ready())), + ) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + assertEquals(blobId, result.getOrNull()) + coVerify(exactly = 3) { repository.getBlobs(any(), any(), any()) } + } + + @Test + fun `only failed polls exhaust the budget as timed out`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns + Result.failure(GetBlobsError.Other(RuntimeException("offline"))) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + assertIs(result.exceptionOrNull()) + coVerify(exactly = 30) { repository.getBlobs(any(), any(), any()) } + } + + @Test + fun `a rejection ends polling and carries its reason`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns + Result.success(listOf(rejected(RejectionReason.MODERATION))) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + val failure = assertIs(result.exceptionOrNull()) + assertEquals(RejectionReason.MODERATION, failure.rejection.reason) + coVerify(exactly = 1) { repository.getBlobs(any(), any(), any()) } + } + + @Test + fun `a denied poll ends polling`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.failure(GetBlobsError.Denied()) + + val result = controller().uploadChatMedia(byteArrayOf(1), "image/jpeg") + + assertIs(result.exceptionOrNull()) + coVerify(exactly = 1) { repository.getBlobs(any(), any(), any()) } + } + + @Test + fun `polling waits for the foreground and checks it before every poll`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returnsMany listOf( + Result.success(emptyList()), + Result.success(listOf(ready())), + ) + val foregrounded = CompletableDeferred() + var checks = 0 + val gate = ForegroundGate { checks++; foregrounded.await() } + + val upload = async { controller(gate).uploadChatMedia(byteArrayOf(1), "image/jpeg") } + advanceUntilIdle() + + coVerify(exactly = 0) { repository.getBlobs(any(), any(), any()) } + + foregrounded.complete(Unit) + advanceUntilIdle() + + assertEquals(blobId, upload.await().getOrNull()) + assertEquals(2, checks) + } + + @Test + fun `the profile path keeps polling without the foreground gate`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returns Result.success(listOf(ready())) + var checks = 0 + + controller(ForegroundGate { checks++ }).upload(byteArrayOf(1), "image/png") + + assertEquals(0, checks) + } + + // MARK: - Store and finalize, split - + + @Test + fun `storing returns the blob id without polling for finalization`() = runTest { + stubHappyUpload() + + val result = controller().storeChatMedia(byteArrayOf(1), "image/jpeg") + + assertEquals(blobId, result.getOrNull()) + coVerify(exactly = 0) { repository.getBlobs(any(), any(), any()) } + assertEquals(0L, testScheduler.currentTime) + } + + @Test + fun `a store that fails in transit reports the failure and never finalizes`() = runTest { + stubHappyUpload() + coEvery { uploader.upload(any(), any(), any(), any()) } returns Result.failure(RuntimeException("offline")) + + val result = controller().storeSealed(ByteArray(10), chatId, seal = { ByteArray(50) }) + + assertTrue(result.isFailure) + coVerify(exactly = 0) { repository.completeExternalUpload(any(), any()) } + coVerify(exactly = 0) { repository.getBlobs(any(), any(), any()) } + } + + @Test + fun `awaiting a stored blob polls it as the owner until ready`() = runTest { + stubHappyUpload() + coEvery { repository.getBlobs(any(), any(), any()) } returnsMany listOf( + Result.success(emptyList()), + Result.success(listOf(ready())), + ) + + val result = controller().awaitChatMediaReady(blobId) + + assertEquals(blobId, result.getOrNull()) + coVerify(exactly = 0) { repository.initiateExternalUpload(any(), any(), any(), any()) } + coVerify(exactly = 2) { repository.getBlobs(listOf(blobId), any(), BlobAccessContext.Owned) } + } +} diff --git a/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerTest.kt b/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerTest.kt index f8e21309fb..6199bde5b0 100644 --- a/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerTest.kt +++ b/services/flipcash/src/test/kotlin/com/flipcash/services/controllers/BlobStorageControllerTest.kt @@ -1,6 +1,7 @@ package com.flipcash.services.controllers import com.flipcash.services.BlobUploader +import com.flipcash.services.ForegroundGate import com.flipcash.services.internal.extensions.withoutJpegMetadata import com.flipcash.services.models.BlobNotReadyException import com.flipcash.services.models.BlobRejectedException @@ -36,7 +37,8 @@ class BlobStorageControllerTest { private val repository = mockk() private val uploader = mockk() private val userManager = mockk(relaxed = true) - private val controller = BlobStorageController(repository, uploader, userManager) + private val foreground = ForegroundGate { } + private val controller = BlobStorageController(repository, uploader, userManager, foreground) private val blobId = BlobId(ByteArray(16) { 1 }) private val target = UploadTarget( diff --git a/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/extensions/MediaContentMappingTest.kt b/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/extensions/MediaContentMappingTest.kt new file mode 100644 index 0000000000..d34013333d --- /dev/null +++ b/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/extensions/MediaContentMappingTest.kt @@ -0,0 +1,145 @@ +package com.flipcash.services.internal.network.extensions + +import com.codeinc.flipcash.gen.blob.v1.Model as BlobModel +import com.codeinc.flipcash.gen.blob.v1.Model.BlobMetadata.KindCase +import com.codeinc.flipcash.gen.messaging.v1.Model as MessagingModel +import com.flipcash.services.models.chat.BlobId +import com.flipcash.services.models.chat.BlobMetadata +import com.flipcash.services.models.chat.ImageMetadata +import com.flipcash.services.models.chat.MediaItem +import com.flipcash.services.models.chat.MediaItemRendition +import com.flipcash.services.models.chat.MessageContent +import com.google.protobuf.ByteString +import org.junit.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * The photo `Content` plaintexts in `libs/encryption/chat-cipher/.../chat_cipher.json`, shared + * with iOS, must survive proto -> domain -> proto byte for byte, since that is what both clients + * seal and open. + */ +class MediaContentMappingTest { + + private fun rendition(size: Long, width: Int, height: Int, mime: String, blurhash: String) = + BlobModel.Rendition.newBuilder() + .setRole(BlobModel.Rendition.Role.ORIGINAL) + .setBlobId(BlobModel.BlobId.newBuilder().setValue(ByteString.copyFrom(ByteArray(16) { (0xA0 + it).toByte() }))) + .setBlob( + BlobModel.BlobMetadata.newBuilder() + .setMimeType(mime) + .setSizeBytes(size) + .setImage(BlobModel.ImageMetadata.newBuilder().setWidth(width).setHeight(height).setBlurhash(blurhash)) + ) + + private fun media(caption: String?, rendition: BlobModel.Rendition.Builder) = + MessagingModel.MediaContent.newBuilder() + .addItems(BlobModel.Media.newBuilder().addRenditions(rendition)) + .apply { caption?.let { setCaption(MessagingModel.TextContent.newBuilder().setText(it)) } } + + /** + * The values of the `chat_cipher.json` photo plaintexts (reply-to-media, media-with-caption, + * media-no-caption), built with the pinned protos. The fixture's own bytes nest one level + * deeper than `blob.v1.Media` / `messaging.v1.MediaContent` allow, so they cannot be decoded + * into these messages and are not used verbatim. + */ + private val vectors = mapOf( + "media-with-caption" to MessagingModel.Content.newBuilder() + .setMedia(media("a caption", rendition(123456, 1024, 768, "image/jpeg", "LEHV6nWB2yk8pyo0adR*.7kCMdnj"))).build(), + "media-no-caption" to MessagingModel.Content.newBuilder() + .setMedia(media(null, rendition(99, 1, 1, "image/png", ""))).build(), + "reply-to-media" to MessagingModel.Content.newBuilder() + .setReply( + MessagingModel.ReplyContent.newBuilder() + .setRepliedMessageId(MessagingModel.MessageId.newBuilder().setValue(7)) + .addContent( + MessagingModel.Content.newBuilder() + .setMedia(media("look", rendition(128, 640, 480, "image/jpeg", "LEHV6nWB2yk8pyo0adR*.7kCMdnj"))) + ) + ).build(), + ) + + @Test + fun `photo plaintexts round-trip through the domain unchanged`() { + for ((name, content) in vectors) { + val back = content.toMessageContent().asContent().toByteArray() + + assertContentEquals(content.toByteArray(), back, name) + } + } + + @Test + fun `a photo maps to the domain with metadata, caption and no url`() { + val media = vectors.getValue("media-with-caption").toMessageContent() as MessageContent.Media + + val rendition = media.items.single().renditions.single() + assertEquals(MediaItemRendition.Role.ORIGINAL, rendition.role) + assertEquals(16, rendition.blobId.bytes.size) + assertEquals("image/jpeg", rendition.blob?.mimeType) + assertEquals(123456L, rendition.blob?.sizeBytes) + assertEquals("", rendition.blob?.downloadUrl) + assertEquals(MessageContent.Text("a caption"), media.caption) + } + + @Test + fun `a photo with no caption has none, even when the caption is present and empty`() { + val parsed = vectors.getValue("media-no-caption") + val empty = parsed.toBuilder() + .setMedia(parsed.media.toBuilder().setCaption(MessagingModel.TextContent.newBuilder().setText(""))) + .build() + + assertEquals(null, (empty.toMessageContent() as MessageContent.Media).caption) + } + + @Test + fun `a sealed photo emits one original rendition with metadata and no download url`() { + val media = MessageContent.Media( + items = listOf( + MediaItem( + listOf( + MediaItemRendition( + role = MediaItemRendition.Role.ORIGINAL, + blobId = BlobId(ByteArray(16) { it.toByte() }), + blob = BlobMetadata( + mimeType = "image/jpeg", + sizeBytes = 4242, + downloadUrl = "", + image = ImageMetadata(width = 800, height = 600, blurhash = "abc"), + ), + ) + ) + ) + ), + caption = null, + ) + + val proto = media.asContent().media + val rendition = proto.getItems(0).getRenditions(0) + + assertEquals(1, proto.itemsCount) + assertEquals(1, proto.getItems(0).renditionsCount) + assertEquals(com.codeinc.flipcash.gen.blob.v1.Model.Rendition.Role.ORIGINAL, rendition.role) + assertEquals(16, rendition.blobId.value.size()) + assertEquals("image/jpeg", rendition.blob.mimeType) + assertEquals(4242L, rendition.blob.sizeBytes) + assertEquals(KindCase.IMAGE, rendition.blob.kindCase) + assertEquals(800, rendition.blob.image.width) + assertEquals(600, rendition.blob.image.height) + assertEquals("abc", rendition.blob.image.blurhash) + assertFalse(rendition.blob.hasDownloadUrl()) + assertFalse(proto.hasCaption()) + } + + @Test + fun `a rendition with no metadata still emits only its id`() { + val media = MessageContent.Media( + listOf(MediaItem(listOf(MediaItemRendition(MediaItemRendition.Role.ORIGINAL, BlobId(ByteArray(16)), null)))), + null, + ) + + assertFalse(media.asContent().media.getItems(0).getRenditions(0).hasBlob()) + assertTrue(media.asContent().media.getItems(0).getRenditions(0).hasBlobId()) + } +} diff --git a/services/flipcash/src/testFixtures/kotlin/com/flipcash/services/chat/FakeChatCipher.kt b/services/flipcash/src/testFixtures/kotlin/com/flipcash/services/chat/FakeChatCipher.kt index 88f038e2a4..17ef7b1251 100644 --- a/services/flipcash/src/testFixtures/kotlin/com/flipcash/services/chat/FakeChatCipher.kt +++ b/services/flipcash/src/testFixtures/kotlin/com/flipcash/services/chat/FakeChatCipher.kt @@ -43,7 +43,7 @@ object FakeChatCipher : ChatCipher { recipientPk: ByteArray, chatId: ByteArray, blobId: ByteArray, - ): ByteArray = throw UnsupportedOperationException() + ): ByteArray = chatKey + senderPk + recipientPk + blobId + image override fun decryptBlob( blob: ByteArray, @@ -52,5 +52,11 @@ object FakeChatCipher : ChatCipher { recipientPk: ByteArray, chatId: ByteArray, blobId: ByteArray, - ): ByteArray = throw UnsupportedOperationException() + ): ByteArray { + val header = chatKey + senderPk + recipientPk + blobId + if (blob.size < header.size || !blob.copyOfRange(0, header.size).contentEquals(header)) { + throw ChatCipherException("authentication failed") + } + return blob.copyOfRange(header.size, blob.size) + } } From 6b73120a9dce526f4112f370b06437824bab9b6d Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Mon, 5 Oct 2026 21:18:06 -0400 Subject: [PATCH 2/4] fix(chat): keep a sent photo's progress bar up until its row is drawn A photo that finished uploading while it sat in the composer confirms within about 100 ms of the tap. Room hasn't drawn the pending row by then, so the bubble's first frame already reads Sent and the bar never shows. On the Seeker the new bubble landed as Delivered with no bar in any frame. The Sent stage now waits until 800 ms after the row was inserted. The message is still confirmed in the database at once; only the bar's phase waits, and a later post in the same batch isn't held up. --- .../internal/delegates/MediaSendDelegate.kt | 20 ++++++++++++++----- .../chat/media/MediaSendDelegateTest.kt | 16 +++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt index e9431a1c96..fb08add71a 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt @@ -32,6 +32,7 @@ import com.getcode.utils.trace import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.combine @@ -318,7 +319,13 @@ class MediaSendDelegate internal constructor( metadataDataSource.updateLastMessageId(chatId, serverMessage.messageId) metadataDataSource.updateLastActivity(chatId, serverMessage.timestamp.toEpochMilliseconds()) discard(record, keepAs = ChatPhoto.cacheKeyOf(photo.blobId)) - stages.update { it + (hex to Stage.Sent) } + // A photo uploaded while it sat in the composer confirms before Room has drawn its + // row, so the bar would fade out unseen. Hold it until the row has been on screen. + val hold = SENDING_VISIBLE_MILLIS - (now() - record.createdAt) + scope.launch { + delay(hold) + stages.update { it + (hex to Stage.Sent) } + } } .onFailure { cause -> // A chat that stopped taking ciphertext won't take this blob on a retry either. @@ -453,11 +460,14 @@ class MediaSendDelegate internal constructor( // endregion - private companion object { - const val TAG = "MediaSendDelegate" - const val JPEG = "image/jpeg" + internal companion object { + private const val TAG = "MediaSendDelegate" + private const val JPEG = "image/jpeg" /** How far back a launch looks for the message a queued photo may already have become. */ - const val RECENT_WINDOW = 50 + private const val RECENT_WINDOW = 50 + + /** How long a sent photo's progress bar stays up, counted from when its row was inserted. */ + internal const val SENDING_VISIBLE_MILLIS = 800L } } diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt index 5f52266f98..1198e04a47 100644 --- a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt @@ -36,6 +36,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.TestScope import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.advanceTimeBy import kotlinx.coroutines.test.runTest import org.junit.Rule import org.junit.Test @@ -190,6 +191,21 @@ class MediaSendDelegateTest { coVerify(exactly = 3) { messageDataSource.confirmPending(any(), any(), any(), true) } } + @Test + fun `a send that confirms at once holds its progress until the row has been seen`() = runTest { + val (delegate, uploads) = delegate() + val chip = stage(uploads, "a") + runCurrent() // uploaded before the send, so the post confirms straight away + val ids = delegate.sendMedia(chatId, listOf(chip), "", null).getOrThrow() + runCurrent() + + assertEquals(1, sent.size) + assertEquals(MediaSendProgress.Sending, delegate.progressOf(ids[0])) + advanceTimeBy(MediaSendDelegate.SENDING_VISIBLE_MILLIS) + runCurrent() + assertEquals(MediaSendProgress.Sent, delegate.progressOf(ids[0])) + } + @Test fun `a failed upload fails only its row`() = runTest { val (delegate, uploads) = delegate() From 6e14bb5756cf2d541fdaf6b1530f7c2e224ce0b7 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Tue, 6 Oct 2026 10:32:41 -0400 Subject: [PATCH 3/4] fix(services): give blob storage calls a 15s deadline BlobStorageApi queued every call with wait-for-ready and no deadline, so a photo sent while the phone was offline never left "Sending": the reserve call waited for a channel that could not resolve its host, and the upload's retries and the bubble's "Not sent" never ran. Unary blob calls now fail after 15s, the same deadline iOS puts on them (CallOptions.unaryDefault). --- .../internal/network/api/BlobStorageApi.kt | 21 ++++- .../network/api/BlobStorageApiDeadlineTest.kt | 77 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/api/BlobStorageApiDeadlineTest.kt diff --git a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt index 4f7374b2c1..2f4a7f4654 100644 --- a/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt +++ b/services/flipcash/src/main/kotlin/com/flipcash/services/internal/network/api/BlobStorageApi.kt @@ -19,8 +19,11 @@ import dev.bmcreations.protovalidate.orThrow import io.grpc.ManagedChannel import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit import javax.inject.Inject import javax.inject.Singleton +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds /** * Wraps the BlobStorage gRPC service — direct-to-storage uploads. The bytes never travel through @@ -29,14 +32,22 @@ import javax.inject.Singleton * [getBlobs] resolves ids to their status + a fresh download URL. */ @Singleton -internal class BlobStorageApi @Inject constructor( - @FlipcashManagedChannel +internal class BlobStorageApi( managedChannel: ManagedChannel, + private val unaryDeadline: Duration, ) : GrpcApi(managedChannel) { - private val api = BlobStorageGrpcKt.BlobStorageCoroutineStub(managedChannel) + @Inject + constructor(@FlipcashManagedChannel managedChannel: ManagedChannel) : this(managedChannel, UNARY_DEADLINE) + + private val stub = BlobStorageGrpcKt.BlobStorageCoroutineStub(managedChannel) .withWaitForReady() + // Wait-for-ready queues a call until the channel connects, which offline is never: a photo + // send would sit at "Sending" with nothing to retry. The deadline, as on iOS, turns that into + // a failure the upload's own retries and the bubble's "Not sent" can act on. + private val api get() = stub.withDeadlineAfter(unaryDeadline.inWholeMilliseconds, TimeUnit.MILLISECONDS) + suspend fun getUploadPolicy(owner: Ed25519.KeyPair): RpcBlobStorageService.GetUploadPolicyResponse { val request = RpcBlobStorageService.GetUploadPolicyRequest.newBuilder() .apply { setAuth(authenticate(owner)) } @@ -99,6 +110,10 @@ internal class BlobStorageApi @Inject constructor( api.getBlobs(request) } } + + private companion object { + val UNARY_DEADLINE = 15.seconds + } } /** diff --git a/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/api/BlobStorageApiDeadlineTest.kt b/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/api/BlobStorageApiDeadlineTest.kt new file mode 100644 index 0000000000..34861e284a --- /dev/null +++ b/services/flipcash/src/test/kotlin/com/flipcash/services/internal/network/api/BlobStorageApiDeadlineTest.kt @@ -0,0 +1,77 @@ +package com.flipcash.services.internal.network.api + +import androidx.lifecycle.ProcessLifecycleOwner +import com.codeinc.flipcash.gen.common.v1.Common +import com.getcode.ed25519.Ed25519 +import io.grpc.ManagedChannel +import io.grpc.Status +import io.grpc.StatusException +import io.grpc.okhttp.OkHttpChannelBuilder +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.unmockkStatic +import io.mockk.unmockkObject +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import com.flipcash.services.internal.network.extensions.authenticate +import com.getcode.utils.toByteString +import org.junit.After +import org.junit.Before +import org.junit.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class BlobStorageApiDeadlineTest { + + // Nothing listens on port 1, so the channel never becomes ready. With wait-for-ready and no + // deadline, a call there waits for as long as the device stays offline. + private val channel: ManagedChannel = OkHttpChannelBuilder.forAddress("127.0.0.1", 1) + .usePlaintext() + .build() + + private val owner = mockk(relaxed = true) + + @Before + fun setUp() { + // GrpcApi registers with the process lifecycle on construction, which needs a main looper. + mockkObject(ProcessLifecycleOwner.Companion) + every { ProcessLifecycleOwner.get() } returns mockk(relaxed = true) + // Signing needs the native Ed25519 library; the request only has to get as far as the wire. + mockkStatic(AUTHENTICATE) + every { any>().authenticate(any()) } returns signedAuth + } + + @After + fun tearDown() { + channel.shutdownNow() + unmockkObject(ProcessLifecycleOwner.Companion) + unmockkStatic(AUTHENTICATE) + } + + @Test + fun `a call that cannot reach the server fails at the deadline`() = runBlocking { + val api = BlobStorageApi(channel, unaryDeadline = 200.milliseconds) + + val error = assertFailsWith { + withTimeout(5.seconds) { api.getUploadPolicy(owner) } + } + + assertEquals(Status.Code.DEADLINE_EXCEEDED, error.status.code) + } + + private companion object { + const val AUTHENTICATE = "com.flipcash.services.internal.network.extensions.AuthenticateMessageKt" + + val signedAuth: Common.Auth = Common.Auth.newBuilder() + .setKeyPair( + Common.Auth.KeyPair.newBuilder() + .setPubKey(Common.PublicKey.newBuilder().setValue(ByteArray(32) { 1 }.toByteString())) + .setSignature(Common.Signature.newBuilder().setValue(ByteArray(64) { 2 }.toByteString())), + ) + .build() + } +} From 014c6a85d951eb245cabc193676c9cc1a4d689b4 Mon Sep 17 00:00:00 2001 From: Brandon McAnsh Date: Tue, 6 Oct 2026 10:50:37 -0400 Subject: [PATCH 4/4] fix(chat): upload a failed photo again when it is sent A photo whose upload failed before the send (offline, say) went straight to a failed row, because the send only waited on the existing upload. The send now restarts a retryable failure, so the row shows Sending until the upload settles. --- .../chat/internal/delegates/MediaSendDelegate.kt | 5 +++++ .../shared/chat/media/MediaSendDelegateTest.kt | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt index fb08add71a..95b2d3f9b1 100644 --- a/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt +++ b/apps/flipcash/shared/chat/src/main/kotlin/com/flipcash/shared/chat/internal/delegates/MediaSendDelegate.kt @@ -161,6 +161,11 @@ class MediaSendDelegate internal constructor( ) } chips.update { it + queued.associate { q -> hex(q.clientMessageId) to q.message.chip } } + // A chip that failed before the send (offline, say) goes up again; its bubble waits on it. + for (q in queued) { + val state = uploads.current(q.message.chip) + if (state is ChatMediaUploadState.Failed && state.retryable) uploads.retry(q.message.chip) + } val hexes = queued.map { hex(it.clientMessageId) } synchronized(inFlight) { inFlight.addAll(hexes) } diff --git a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt index 1198e04a47..dcf4951584 100644 --- a/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt +++ b/apps/flipcash/shared/chat/src/test/kotlin/com/flipcash/shared/chat/media/MediaSendDelegateTest.kt @@ -258,6 +258,22 @@ class MediaSendDelegateTest { assertTrue(records.isEmpty()) } + @Test + fun `sending a photo whose upload already failed uploads it again`() = runTest { + val (delegate, uploads) = delegate() + var attempts = 0 + storeBehavior["a"] = { if (attempts++ == 0) Result.failure(IOException("offline")) else Result.success(BlobId("a".toByteArray())) } + val chip = stage(uploads, "a") + runCurrent() + assertTrue(uploads.current(chip) is ChatMediaUploadState.Failed) + + delegate.sendMedia(chatId, listOf(chip), "", null).getOrThrow() + runCurrent() + + assertEquals(1, sent.size) + coVerify(exactly = 0) { messageDataSource.failPending(any(), any()) } + } + @Test fun `retry uploads the stored file again when nothing reached storage`() = runTest { val (delegate, uploads) = delegate()