diff --git a/.agent-artifacts/ui-evidence/sub-ship-now/001-desktop-macos-saved-waiting-light.png b/.agent-artifacts/ui-evidence/sub-ship-now/001-desktop-macos-saved-waiting-light.png new file mode 100644 index 00000000000..6debb813223 Binary files /dev/null and b/.agent-artifacts/ui-evidence/sub-ship-now/001-desktop-macos-saved-waiting-light.png differ diff --git a/.agent-artifacts/ui-evidence/sub-ship-now/002-desktop-macos-saved-waiting-dark.png b/.agent-artifacts/ui-evidence/sub-ship-now/002-desktop-macos-saved-waiting-dark.png new file mode 100644 index 00000000000..f6933bb92fc Binary files /dev/null and b/.agent-artifacts/ui-evidence/sub-ship-now/002-desktop-macos-saved-waiting-dark.png differ diff --git a/.agent-artifacts/ui-evidence/sub-ship-now/003-mobile-ios-local-speech-permission.png b/.agent-artifacts/ui-evidence/sub-ship-now/003-mobile-ios-local-speech-permission.png new file mode 100644 index 00000000000..5a6a4fa200e Binary files /dev/null and b/.agent-artifacts/ui-evidence/sub-ship-now/003-mobile-ios-local-speech-permission.png differ diff --git a/.agent-artifacts/ui-evidence/sub-ship-now/004-mobile-ios-local-speech-unavailable.png b/.agent-artifacts/ui-evidence/sub-ship-now/004-mobile-ios-local-speech-unavailable.png new file mode 100644 index 00000000000..31ee7e8fbc1 Binary files /dev/null and b/.agent-artifacts/ui-evidence/sub-ship-now/004-mobile-ios-local-speech-unavailable.png differ diff --git a/.agent-artifacts/ui-evidence/sub-ship-now/evidence.json b/.agent-artifacts/ui-evidence/sub-ship-now/evidence.json new file mode 100644 index 00000000000..4a8297b5ebb --- /dev/null +++ b/.agent-artifacts/ui-evidence/sub-ship-now/evidence.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "images": [ + { + "file": "001-desktop-macos-saved-waiting-light.png", + "platform": "desktop-macos", + "description": "Actual ConversationStatusBadge(.locked) rendered in light mode with SwiftUI ImageRenderer; isolated synthetic fixture, no account data", + "captured_by": "agent", + "source": "visual-audit" + }, + { + "file": "002-desktop-macos-saved-waiting-dark.png", + "platform": "desktop-macos", + "description": "Actual ConversationStatusBadge(.locked) rendered in dark mode with SwiftUI ImageRenderer; isolated synthetic fixture, no account data", + "captured_by": "agent", + "source": "visual-audit" + }, + { + "file": "003-mobile-ios-local-speech-permission.png", + "platform": "mobile-ios", + "description": "Actual local speech recovery notice with SF Pro, iOS platform and existing localized capture/custody/actions; synthetic UI fixture", + "captured_by": "agent", + "source": "visual-audit" + }, + { + "file": "004-mobile-ios-local-speech-unavailable.png", + "platform": "mobile-ios", + "description": "Actual local speech recovery notice with SF Pro, iOS platform and existing localized capture/custody/actions; synthetic UI fixture", + "captured_by": "agent", + "source": "visual-audit" + } + ] +} diff --git a/.github/agent-docs/on-device-speech.md b/.github/agent-docs/on-device-speech.md index cfb54e9e99e..55d33608d29 100644 --- a/.github/agent-docs/on-device-speech.md +++ b/.github/agent-docs/on-device-speech.md @@ -3,3 +3,11 @@ - `app/ios/Runner/SpeechDeadline.swift` owns deadlines for SpeechAnalyzer availability and recognition. A timeout may leave a shared model download running, but must finish recognition cleanup before replying to Dart. Native speech results and timeout callbacks use the main queue for one-shot completion. - `PurePollingSocket` keeps one provider operation in flight. Do not wrap it in `Future.timeout`: that cannot cancel native/HTTP work and permits duplicate transcription. Providers own timeouts and throw failures so retained audio can be retried; an empty successful result means no speech. - Native deadline regression: `ruby app/ios/test/speech_deadline_test.rb` (local/CI manifest). Dart regressions (from `app/`): `flutter test test/unit/pure_polling_test.dart test/unit/on_device_transcription_upgrade_test.dart`. + +## Mobile freemium engine rollback + +The existing iOS speech channel selects SpeechAnalyzer/SpeechTranscriber on supported iOS 26 hardware and locales, reserving and installing its OS-managed asset through AssetInventory. Unsupported hardware/locales, model-install or recognition failures fall back to the existing on-device SFSpeech path. Existing fallback UX/status strings are retained; there is no Parakeet setup in this path. + +`mobile-speech-analyzer-disabled` is an inverted PostHog operational kill switch, registered in `config/feature-flags.yaml`. Missing/false/error/offline keeps the improved engine selected; true forces SFSpeech for both availability probing and transcription. It is read per operation through the existing identity/consent-fenced analytics adapter. Users with analytics disabled retain the shipped new-engine default. The flag does not affect cloud STT, overflow, summaries, schedules or plans. + +Android's existing Whisper provider splits canonical PCM WAV input into at most 30-second independent plugin calls. The pinned plugin creates/frees a whisper_context for each call; failure propagates so polling retains the complete input for retry. diff --git a/.github/failure-classes/FC-held-content-tap-bypasses-access-guard.json b/.github/failure-classes/FC-held-content-tap-bypasses-access-guard.json new file mode 100644 index 00000000000..2cabb78068c --- /dev/null +++ b/.github/failure-classes/FC-held-content-tap-bypasses-access-guard.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "id": "FC-held-content-tap-bypasses-access-guard", + "violated_contract": "Held content with subscription UI enabled must not open a full-content destination through any row or preview tap. Obscuring the child and guarding only the action button does not guard an enclosing row recognizer.", + "canonical_prevention": "Route the preview action and enclosing row through the same access guard, give the preview a full-area guarded tap recognizer, and test off-button taps on every preview surface with subscription UI enabled and disabled. Preserve interactive row menus and the App Review read-only memory exception.", + "canonical_prevention_artifact": [ + "app/test/widgets/waiting_memory_preview_test.dart", + "app/test/widgets/conversation_locked_preview_test.dart" + ], + "evidence_prs": [], + "scope_hints": [ + "app/lib/ui/components/omi_locked_preview.dart", + "app/lib/pages/memories/widgets/memory_item.dart", + "app/lib/pages/conversations/widgets/conversation_list_item.dart" + ], + "status": "open" +} diff --git a/app/android/build.gradle b/app/android/build.gradle index ee9190244bd..90e98e7e8c0 100644 --- a/app/android/build.gradle +++ b/app/android/build.gradle @@ -54,3 +54,16 @@ gradle.projectsEvaluated { tasks.register("clean", Delete) { delete rootProject.buildDir } + +// The pinned FFI plugin must free model/WAV resources on every error path. +gradle.projectsEvaluated { + def sub = rootProject.subprojects.find { it.name == 'whisper_flutter_new' } + if (sub != null) { + def source = new File(sub.projectDir, '../src/main.cpp') + def patch = new File(rootProject.projectDir, '../scripts/patch_whisper_cleanup.py') + def proc = ['python3', patch.absolutePath, source.canonicalPath].execute(null, rootProject.projectDir) + def out = new StringBuffer(); def err = new StringBuffer() + proc.consumeProcessOutput(out, err); proc.waitFor() + if (proc.exitValue() != 0) throw new GradleException("Whisper cleanup patch failed: ${err} ${out}") + } +} diff --git a/app/changelog/unreleased/20261014-freemium-speech-engines.json b/app/changelog/unreleased/20261014-freemium-speech-engines.json new file mode 100644 index 00000000000..d4c796a15cd --- /dev/null +++ b/app/changelog/unreleased/20261014-freemium-speech-engines.json @@ -0,0 +1,3 @@ +{ + "change": "On-device fallback uses Apple’s newer speech engine on supported iPhones and decodes Android recordings in bounded chunks for better transcription." +} diff --git a/app/changelog/unreleased/20261014-subscription-waiting-state.json b/app/changelog/unreleased/20261014-subscription-waiting-state.json new file mode 100644 index 00000000000..967d120ea58 --- /dev/null +++ b/app/changelog/unreleased/20261014-subscription-waiting-state.json @@ -0,0 +1 @@ +{"change": "Saved conversations and memories waiting on your allowance link to usage without an upgrade prompt"} diff --git a/app/integration_test/visual_audit/scenarios/capture.dart b/app/integration_test/visual_audit/scenarios/capture.dart index 425cd289d0c..98622cfe938 100644 --- a/app/integration_test/visual_audit/scenarios/capture.dart +++ b/app/integration_test/visual_audit/scenarios/capture.dart @@ -19,6 +19,7 @@ import 'package:omi/pages/home/widgets/home_daily_recaps.dart'; import 'package:omi/pages/home/widgets/home_tab_switcher.dart'; import 'package:omi/backend/schema/daily_summary.dart'; import 'package:omi/providers/capture_provider.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'package:omi/providers/conversation_provider.dart'; import 'package:omi/backend/schema/structured.dart'; import 'package:omi/providers/device_provider.dart'; @@ -38,6 +39,8 @@ enum AuditLive { idle, pendant, pendantPaused, phone, phonePaused, phoneAfterPen class AuditCaptureProvider extends ChangeNotifier implements CaptureProvider { AuditCaptureProvider(this.live); final AuditLive live; + @override + LocalSttFailure? get localSttFailure => null; @override bool get pendantCaptureVerified => true; diff --git a/app/integration_test/visual_audit/scenarios/conversations.dart b/app/integration_test/visual_audit/scenarios/conversations.dart index 80725cba5a4..a34769ac34a 100644 --- a/app/integration_test/visual_audit/scenarios/conversations.dart +++ b/app/integration_test/visual_audit/scenarios/conversations.dart @@ -94,10 +94,10 @@ const _twoSourceGroup = CaptureGroup( final conversationsScenarios = [ AuditScenario( id: 'conversations-locked-preview', - title: 'Locked conversations with an upgrade action', + title: 'Saved conversations waiting on their allowance', page: _page, state: 'An unlocked conversation followed by three consecutive locked previews, which share one ' - 'frosted card and one upgrade action; synthetic titles only', + 'frosted card, a waiting status and one usage action; synthetic titles only', run: (a) async { final items = [ auditConversation('unlocked', title: 'Design catch-up with Alex'), @@ -114,8 +114,11 @@ final conversationsScenarios = [ await a.pump(const ConversationsPage(requestInitialLoad: false), providers: _listProviders(items)); expect(find.byType(ConversationListItem), findsNWidgets(4)); expect(find.byType(LockedConversationRun), findsOneWidget); - expect(find.text('Upgrade to Unlimited'), findsOneWidget); - await a.shot('Conversation list with frosted locked previews'); + expect(find.text('Pending'), findsOneWidget); + expect(find.text('View Usage'), findsOneWidget); + expect(find.byIcon(Icons.lock_outline), findsNothing); + expect(find.byIcon(Icons.hourglass_empty), findsOneWidget); + await a.shot('Conversation list with saved-content waiting previews'); }, ), AuditScenario( diff --git a/app/ios/Podfile b/app/ios/Podfile index 387e583feca..933586f6708 100644 --- a/app/ios/Podfile +++ b/app/ios/Podfile @@ -61,6 +61,12 @@ else end post_install do |installer| + # Fail closed on native dependency drift; patch both copies compiled by the pod. + plugin = File.realpath(File.join(__dir__, '.symlinks/plugins/whisper_flutter_new')) + patch = File.expand_path('../scripts/patch_whisper_cleanup.py', __dir__) + sources = [File.join(plugin, 'src/main.cpp'), File.join(plugin, 'ios/Classes/whisper_flutter_new.cpp')] + raise 'Whisper cleanup patch failed' unless system('python3', patch, *sources) + installer.pods_project.targets.each do |target| flutter_additional_ios_build_settings(target) target.build_configurations.each do |config| diff --git a/app/ios/Runner/AppDelegate.swift b/app/ios/Runner/AppDelegate.swift index 58a3b9c58a6..7051b3007a9 100644 --- a/app/ios/Runner/AppDelegate.swift +++ b/app/ios/Runner/AppDelegate.swift @@ -1030,6 +1030,13 @@ enum SpeechAnalyzerTranscription { /// Installs the model for `locale` if it is not already on the device. /// Concurrent callers wait on the same download. static func ensureModel(for locale: Locale) async throws { + // This app keeps one locale reservation; OS-installed assets remain OS-owned. + for previous in await AssetInventory.reservedLocales where previous.identifier(.bcp47) != locale.identifier(.bcp47) { + installTasks[previous.identifier(.bcp47)]?.cancel() + installTasks[previous.identifier(.bcp47)] = nil + _ = await AssetInventory.release(reservedLocale: previous) + } + _ = try await AssetInventory.reserve(locale: locale) if await isInstalled(locale) { return } let key = locale.identifier(.bcp47) let task: Task @@ -1054,11 +1061,10 @@ enum SpeechAnalyzerTranscription { /// Whether transcription can run for `language`: the locale is supported /// and its model is installed, or finishes installing within - /// `installWait`. A download still running after that counts as available - /// too; it continues in the background and transcribe() waits for it. - /// Only a failed download (no network, unsupported locale) reports false. + /// `installWait`. A pending install continues, but reports not ready. + /// The caller can offer setup/retry while captured audio stays retained. static func isAvailable(language: String, installWait: Double = 8) async -> Bool { - guard let locale = await locale(for: language) else { + guard SpeechTranscriber.isAvailable, let locale = await locale(for: language) else { NSLog("[SpeechAnalyzer] no supported locale for language %@", language) return false } @@ -1066,22 +1072,21 @@ enum SpeechAnalyzerTranscription { return await SpeechDeadline.run(seconds: installWait, operation: { do { try await ensureModel(for: locale) - return true + return await isInstalled(locale) } catch { NSLog("[SpeechAnalyzer] model install failed for %@: %@", locale.identifier(.bcp47), error.localizedDescription) return false } }, onTimeout: { - // The shared download keeps running; only this availability waiter - // has a deadline. A later transcription shares the same download. - true + // The shared install continues, but readiness must not claim success. + false }) } /// Transcribes a whole audio file (the Dart side writes 16 kHz mono WAV /// clips) and returns the text, empty when no speech was recognized. static func transcribe(fileURL: URL, language: String) async throws -> String { - guard let locale = await locale(for: language) else { + guard SpeechTranscriber.isAvailable, let locale = await locale(for: language) else { throw TranscriptionError.unsupportedLanguage(language) } let installed: Result = await SpeechDeadline.run(seconds: 20, operation: { @@ -1135,53 +1140,72 @@ enum SpeechAnalyzerTranscription { class SpeechRecognitionHandler: NSObject { func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { - if call.method == "transcribe" { - guard let args = call.arguments as? [String: Any], - let path = args["filePath"] as? String else { - result(FlutterError(code: "INVALID_ARGS", message: "Missing arguments", details: nil)) - return - } - - let language = args["language"] as? String ?? "en-US" - if #available(iOS 26, *) { - // SpeechAnalyzer needs no Dictation setting; fall back to - // SFSpeechRecognizer only if it cannot handle this clip. - Task { - do { - let text = try await SpeechAnalyzerTranscription.transcribe( - fileURL: URL(fileURLWithPath: path), language: language) - DispatchQueue.main.async { result(text) } - } catch { - NSLog("[SpeechAnalyzer] transcribe failed, falling back to SFSpeechRecognizer: %@", error.localizedDescription) - DispatchQueue.main.async { - self.transcribe(filePath: path, language: language, result: result) - } + let args = call.arguments as? [String: Any] ?? [:] + let language = args["language"] as? String ?? "en-US" + let preferAnalyzer = args["useSpeechAnalyzer"] as? Bool != false + Task { @MainActor in + let coordinator = coordinator(language: language, path: args["filePath"] as? String) + switch call.method { + case "onDeviceAvailable", "readiness", "prepareSpeech": + var readiness = await coordinator.readiness(preferAnalyzer: preferAnalyzer) + if call.method == "prepareSpeech", readiness.status == "permission_required" { + _ = await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0) } } + readiness = await coordinator.readiness(preferAnalyzer: preferAnalyzer) } - return - } - transcribe(filePath: path, language: language, result: result) - } else if call.method == "onDeviceAvailable" { - let args = call.arguments as? [String: Any] - let language = args?["language"] as? String ?? "en-US" - if #available(iOS 26, *) { - Task { - if await SpeechAnalyzerTranscription.isAvailable(language: language) { - DispatchQueue.main.async { result(true) } - } else { - DispatchQueue.main.async { - self.probeOnDeviceRecognition(language: language, result: result) - } - } + result(call.method == "onDeviceAvailable" ? readiness.ready : readiness.wire) + case "transcribe": + guard args["filePath"] is String else { + result(FlutterError(code: "INVALID_ARGS", message: "Missing audio file", details: nil)); return } - return + do { result(try await coordinator.transcribe(preferAnalyzer: preferAnalyzer)) } + catch let error as LocalSpeechUnavailable { + let status = error.readiness.status + result(FlutterError(code: status == "permission_required" || status == "permission_denied" ? "UNAUTHORIZED" : "UNAVAILABLE", + message: "Selected local speech engine needs setup", details: error.readiness.wire)) + } catch { result(FlutterError(code: "RECOGNITION_ERROR", message: error.localizedDescription, details: nil)) } + default: result(FlutterMethodNotImplemented) } - probeOnDeviceRecognition(language: language, result: result) - } else { - result(FlutterMethodNotImplemented) } } + @MainActor + private func coordinator(language: String, path: String?) -> SpeechFallbackCoordinator { + SpeechFallbackCoordinator(analyzerReady: { + if #available(iOS 26, *) { return await SpeechAnalyzerTranscription.isAvailable(language: language) } + return false + }, legacyReady: { + let permission = SFSpeechRecognizer.authorizationStatus() + if permission != .authorized { + return LocalSpeechReadiness(engine: "sfspeech", status: permission == .notDetermined ? "permission_required" : "permission_denied") + } + let available: Bool = await withCheckedContinuation { continuation in + self.probeOnDeviceRecognition(language: language) { value in + continuation.resume(returning: value as? Bool ?? false) + } + } + return LocalSpeechReadiness(engine: "sfspeech", status: available ? "ready" : "unavailable") + }, analyze: { + if #available(iOS 26, *), let path { + return try await SpeechAnalyzerTranscription.transcribe(fileURL: URL(fileURLWithPath: path), language: language) + } + throw LocalSpeechUnavailable(readiness: LocalSpeechReadiness(engine: "speech_analyzer", status: "unavailable")) + }, legacy: { + guard let path else { throw LocalSpeechUnavailable(readiness: LocalSpeechReadiness(engine: "sfspeech", status: "unavailable")) } + return try await withCheckedThrowingContinuation { continuation in + self.transcribe(filePath: path, language: language) { value in + if let error = value as? FlutterError { + if error.code == "UNAUTHORIZED" || error.code == "UNAVAILABLE" { + continuation.resume(throwing: LocalSpeechUnavailable(readiness: + LocalSpeechReadiness(engine: "sfspeech", status: error.code == "UNAUTHORIZED" ? "permission_denied" : "unavailable"))) + } else { continuation.resume(throwing: NSError(domain: error.code, code: 1)) } + } else { continuation.resume(returning: value as? String ?? "") } + } + } + }) + } + /// Resolve the app's language setting to a locale whose recognizer can run /// on-device. The app passes bare language codes ("en"); on-device assets /// are installed per full locale ("en-US"), and a recognizer built from a @@ -1230,7 +1254,7 @@ class SpeechRecognitionHandler: NSObject { return } guard let silence = SpeechRecognitionHandler.writeSilentWav(seconds: 0.6) else { - result(true) // Could not build a probe clip; do not block the fallback on that. + result(false) // Unknown readiness is not a working recognizer. return } @@ -1265,7 +1289,7 @@ class SpeechRecognitionHandler: NSObject { } DispatchQueue.main.asyncAfter(deadline: .now() + 5) { guard !finished else { return } - finish(true) // Slow but not failing; let the real request decide. + finish(false) // A timed-out probe cannot attest readiness. } } diff --git a/app/ios/Runner/SpeechDeadline.swift b/app/ios/Runner/SpeechDeadline.swift index dc98841bb69..c2b198d43da 100644 --- a/app/ios/Runner/SpeechDeadline.swift +++ b/app/ios/Runner/SpeechDeadline.swift @@ -58,3 +58,42 @@ final class SpeechDeadline { complete(await onTimeout()) } } + +/// Shared by the real channel and hermetic native tests. A fallback is only +/// selected after its own permission/availability check, never assumed ready. +@MainActor +struct LocalSpeechReadiness { + let engine: String + let status: String + var ready: Bool { status == "ready" } + var wire: [String: String] { ["engine": engine, "status": status] } +} + +@MainActor +struct LocalSpeechUnavailable: Error { + let readiness: LocalSpeechReadiness +} + +@MainActor +struct SpeechFallbackCoordinator { + let analyzerReady: () async -> Bool + let legacyReady: () async -> LocalSpeechReadiness + let analyze: () async throws -> String + let legacy: () async throws -> String + + func readiness(preferAnalyzer: Bool) async -> LocalSpeechReadiness { + if preferAnalyzer, await analyzerReady() { + return LocalSpeechReadiness(engine: "speech_analyzer", status: "ready") + } + return await legacyReady() + } + + func transcribe(preferAnalyzer: Bool) async throws -> String { + if preferAnalyzer, await analyzerReady() { + do { return try await analyze() } catch { /* Check the fallback before using it. */ } + } + let readiness = await legacyReady() + guard readiness.ready else { throw LocalSpeechUnavailable(readiness: readiness) } + return try await legacy() + } +} diff --git a/app/ios/test/speech_deadline_test.rb b/app/ios/test/speech_deadline_test.rb index c707bc58fc3..3334b8964ac 100644 --- a/app/ios/test/speech_deadline_test.rb +++ b/app/ios/test/speech_deadline_test.rb @@ -60,3 +60,5 @@ def test_deadline_returns_without_waiting_for_shared_work_and_owns_cleanup end end end + +require_relative "speech_fallback_coordinator_test" diff --git a/app/ios/test/speech_fallback_coordinator_test.rb b/app/ios/test/speech_fallback_coordinator_test.rb new file mode 100644 index 00000000000..63bf8eefb04 --- /dev/null +++ b/app/ios/test/speech_fallback_coordinator_test.rb @@ -0,0 +1,57 @@ +require 'minitest/autorun' +require 'open3' +require 'tmpdir' + +class SpeechFallbackCoordinatorTest < Minitest::Test + def test_actual_coordinator_install_recognition_permission_and_rollback_paths + Dir.mktmpdir('speech-coordinator') do |dir| + source = File.expand_path('../Runner/SpeechDeadline.swift', __dir__) + harness = File.join(dir, 'Harness.swift') + File.write(harness, <<~SWIFT) + import Foundation + @main struct Harness { + @MainActor static func main() async throws { + var analyzerAvailable = true + var legacyStatus = "permission_denied" + var failRecognition = false + var analyzerCalls = 0 + var legacyCalls = 0 + enum Fault: Error { case recognition } + let coordinator = SpeechFallbackCoordinator(analyzerReady: { analyzerAvailable }, + legacyReady: { LocalSpeechReadiness(engine: "sfspeech", status: legacyStatus) }, + analyze: { analyzerCalls += 1; if failRecognition { throw Fault.recognition }; return "new engine" }, + legacy: { legacyCalls += 1; return "legacy engine" }) + let ready = await coordinator.readiness(preferAnalyzer: true) + precondition(ready.ready && ready.engine == "speech_analyzer") + let newText = try await coordinator.transcribe(preferAnalyzer: true) + precondition(newText == "new engine" && legacyCalls == 0) + // Actual rollback: denied permission must not execute legacy or claim readiness. + let denied = await coordinator.readiness(preferAnalyzer: false) + precondition(!denied.ready && denied.status == "permission_denied") + do { _ = try await coordinator.transcribe(preferAnalyzer: false); preconditionFailure() } + catch let error as LocalSpeechUnavailable { precondition(error.readiness.status == "permission_denied") } + precondition(legacyCalls == 0 && analyzerCalls == 1) + // Install/hardware failure checks the usable fallback and executes it. + analyzerAvailable = false; legacyStatus = "ready" + let oldText = try await coordinator.transcribe(preferAnalyzer: true) + precondition(oldText == "legacy engine" && legacyCalls == 1) + // A native recognition error takes the same verified fallback. + analyzerAvailable = true; failRecognition = true + let recovered = try await coordinator.transcribe(preferAnalyzer: true) + precondition(recovered == "legacy engine" && legacyCalls == 2) + // Neither engine works: actionable unavailability, no false empty success. + legacyStatus = "unavailable" + do { _ = try await coordinator.transcribe(preferAnalyzer: true); preconditionFailure() } + catch let error as LocalSpeechUnavailable { precondition(error.readiness.status == "unavailable") } + precondition(legacyCalls == 2) + print("fallback behavioral paths passed") + } + } + SWIFT + output, status = Open3.capture2e('swiftc', '-parse-as-library', source, harness, '-o', File.join(dir, 'test')) + assert status.success?, output + output, status = Open3.capture2e(File.join(dir, 'test')) + assert status.success?, output + end + end +end diff --git a/app/lib/models/stt_result.dart b/app/lib/models/stt_result.dart index 93bcae0fa5e..aaed4710a24 100644 --- a/app/lib/models/stt_result.dart +++ b/app/lib/models/stt_result.dart @@ -99,10 +99,12 @@ class SttSegment { } class SttTranscriptionResult { + /// End of successfully consumed audio, including silence. + final double? audioEndSeconds; final List segments; final String? rawText; - SttTranscriptionResult({this.segments = const [], this.rawText}); + SttTranscriptionResult({this.segments = const [], this.rawText, this.audioEndSeconds}); bool get isEmpty => segments.isEmpty && (rawText == null || rawText!.trim().isEmpty); bool get isNotEmpty => !isEmpty; diff --git a/app/lib/pages/conversation_capturing/page.dart b/app/lib/pages/conversation_capturing/page.dart index 2770a36cffc..4bc3e3fa08c 100644 --- a/app/lib/pages/conversation_capturing/page.dart +++ b/app/lib/pages/conversation_capturing/page.dart @@ -1,3 +1,6 @@ +import 'package:omi/utils/other/temp.dart'; +import 'package:omi/ui/components/local_stt_recovery_notice.dart'; +import 'package:omi/pages/settings/transcription_settings_page.dart'; import 'dart:async'; import 'package:omi/utils/platform/platform_manager.dart'; @@ -134,6 +137,7 @@ class _ConversationCapturingPageState extends State { /// is Paused, capture recovering on its own (a dropped socket, a mic stall) is Reconnecting, a /// terminal transcription failure or offline buffering is named as such, otherwise Listening. CaptureDisplayState _displayState(CaptureProvider provider, {required bool capturingPhotos}) { + if (provider.localSttFailure != null) return CaptureDisplayState.recording; final interruption = captureInterruption( interrupted: provider.recordingState == RecordingState.interrupted, readerPaused: provider.isPaused, @@ -143,7 +147,7 @@ class _ConversationCapturingPageState extends State { audioInterrupted: interruption == CaptureInterruption.micTaken, reconnecting: interruption == CaptureInterruption.recovering, paused: provider.isPaused || provider.isCallActive, - transcriptionUnavailable: provider.terminalTranscriptionFailure != null, + transcriptionUnavailable: provider.terminalTranscriptionFailure != null || provider.localSttFailure != null, bufferingFor: provider.customSttBufferingDuration, capturingPhotos: capturingPhotos, ); @@ -177,6 +181,11 @@ class _ConversationCapturingPageState extends State { body: Column( children: [ const CaptureRecoveryBanner(), + if (provider.localSttFailure case final failure?) + LocalSttRecoveryNotice( + failure: failure, + onRetry: provider.retryLocalTranscription, + onSetup: () => routeToPage(context, const TranscriptionSettingsPage())), _buildUnsyncedWalIndicator(provider), Expanded( child: provider.segments.isEmpty && provider.photos.isEmpty @@ -661,6 +670,7 @@ class _ConversationCapturingPageState extends State { required bool transcriptionInterrupted, }) { if (!provider.pendantCaptureVerified) return ''; + if (provider.localSttFailure != null) return context.l10n.transcriptionUnavailableRecordingSaved; if (usage.isOutOfCredits) return context.l10n.transcriptionUnavailableRecordingSaved; if (provider.terminalTranscriptionFailure != null) { return context.l10n.transcriptionUnavailableRecordingContinues; diff --git a/app/lib/pages/conversations/widgets/conversation_list_item.dart b/app/lib/pages/conversations/widgets/conversation_list_item.dart index 96213ff55ee..755b8ce3568 100644 --- a/app/lib/pages/conversations/widgets/conversation_list_item.dart +++ b/app/lib/pages/conversations/widgets/conversation_list_item.dart @@ -71,7 +71,7 @@ class ConversationListItem extends StatefulWidget { /// (the Home preview), so selection mode can never start without a way to act on it or leave. final bool allowSelection; - /// Drawn inside a [LockedConversationRun]: the run blurs the card under its one upgrade action, + /// Drawn inside a [LockedConversationRun]: the run blurs the card under its one usage action, /// so the row draws no padding or lock of its own but keeps its gestures (open, long-press menu, /// swipe-to-delete, selection). final bool inLockedRun; @@ -176,11 +176,7 @@ class _ConversationListItemState extends State { Future _open(BuildContext context, ConversationProvider provider) async { if (widget.conversation.isLocked) { if (!context.read().showSubscriptionUI) return; - PlatformManager.instance.analytics.paywallOpened( - 'locked_conversation', - currentPlan: context.read().subscription?.subscription.plan.wireName ?? 'unknown', - ); - routeToPage(context, const UsagePage(showUpgradeDialog: true, paywallSource: 'locked_conversation')); + routeToPage(context, const UsagePage()); return; } HapticFeedback.selectionClick(); @@ -429,7 +425,12 @@ class _ConversationListItemState extends State { ); // A run frosts its rows together under one action. if (!widget.conversation.isLocked || widget.inLockedRun) return content; - return OmiLockedPreview(label: context.l10n.upgradeToUnlimited, onPressed: onTap, child: content); + return OmiLockedPreview( + status: context.l10n.statusPending, + label: context.l10n.viewUsage, + onPressed: onTap, + child: content, + ); } /// Time and length, with the New badge beside them (hub audit #16) and the star. @@ -838,7 +839,7 @@ class _MergingIndicatorState extends State with SingleTickerPr } /// Two or more locked rows in a row on the free plan: one frosted card holding all of them, with a -/// single "Upgrade to Unlimited" instead of one per row. A lone locked row stays a +/// single usage action instead of one per row. A lone locked row stays a /// [ConversationListItem] with its own [OmiLockedPreview]. class LockedConversationRun extends StatelessWidget { const LockedConversationRun({super.key, required this.conversations, required this.date}); @@ -846,7 +847,7 @@ class LockedConversationRun extends StatelessWidget { final List conversations; final DateTime date; - Future _upgrade(BuildContext context) async { + Future _openUsage(BuildContext context) async { // Same as a lone locked row's action: while merging, a locked row can't be picked. if (context.read().isSelectionModeActive) { HapticFeedback.lightImpact(); @@ -854,11 +855,7 @@ class LockedConversationRun extends StatelessWidget { return; } if (!context.read().showSubscriptionUI) return; - PlatformManager.instance.analytics.paywallOpened( - 'locked_conversation', - currentPlan: context.read().subscription?.subscription.plan.wireName ?? 'unknown', - ); - routeToPage(context, const UsagePage(showUpgradeDialog: true, paywallSource: 'locked_conversation')); + routeToPage(context, const UsagePage()); } @override @@ -869,8 +866,9 @@ class LockedConversationRun extends StatelessWidget { key: const Key('locked_conversation_run'), // Each row keeps its own long-press menu, swipe-to-delete and selection handling. interactive: true, - label: context.l10n.upgradeToUnlimited, - onPressed: () => _upgrade(context), + status: context.l10n.statusPending, + label: context.l10n.viewUsage, + onPressed: () => _openUsage(context), child: Column( mainAxisSize: MainAxisSize.min, children: [ diff --git a/app/lib/pages/conversations/widgets/processing_capture.dart b/app/lib/pages/conversations/widgets/processing_capture.dart index d4b65fd1aeb..b886cdf7ca6 100644 --- a/app/lib/pages/conversations/widgets/processing_capture.dart +++ b/app/lib/pages/conversations/widgets/processing_capture.dart @@ -1,3 +1,5 @@ +import 'package:omi/ui/components/local_stt_recovery_notice.dart'; +import 'package:omi/pages/settings/transcription_settings_page.dart'; import 'dart:async'; import 'package:omi/utils/platform/platform_manager.dart'; @@ -138,39 +140,46 @@ class _ConversationCaptureWidgetState extends State { return const SizedBox.shrink(); } - return GestureDetector( - onTap: () async { - // Offline/batch mode has no live transcript — the card is informational, - // so swallow taps instead of opening the (empty) capturing page. Covers both - // device batch and the phone-mic Transcribe Later session. - if (provider.isPhoneMicBatchRecording || - (SharedPreferencesUtil().batchModeEnabled && provider.havingRecordingDevice)) { - return; - } - final isCaptureActive = provider.recordingState == RecordingState.record || - provider.recordingState == RecordingState.systemAudioRecord || - provider.recordingState == RecordingState.deviceRecord || - provider.recordingState == RecordingState.initialising || - provider.recordingState == RecordingState.interrupted || - provider.recordingState == RecordingState.pause || - provider.havingRecordingDevice || - provider.isPaused; - if (!isCaptureActive && provider.segments.isEmpty && provider.photos.isEmpty) return; - PlatformManager.instance.analytics.liveTranscriptCardClicked( - hasSegments: provider.segments.isNotEmpty, - hasPhotos: provider.photos.isNotEmpty, - segmentCount: provider.segments.length, - photoCount: provider.photos.length, - ); - routeToPage(context, ConversationCapturingPage(topConversationId: provider.topConversationId)); - }, - child: Semantics( - button: !batch, - hint: batch ? null : context.l10n.liveTranscript, - child: _cardShell(_buildUnifiedRecordingUI(provider), - padding: batch ? _liveCardPadding : EdgeInsets.zero, decorated: batch), + return Column(children: [ + if (provider.localSttFailure case final failure?) + LocalSttRecoveryNotice( + failure: failure, + onRetry: provider.retryLocalTranscription, + onSetup: () => routeToPage(context, const TranscriptionSettingsPage())), + GestureDetector( + onTap: () async { + // Offline/batch mode has no live transcript — the card is informational, + // so swallow taps instead of opening the (empty) capturing page. Covers both + // device batch and the phone-mic Transcribe Later session. + if (provider.isPhoneMicBatchRecording || + (SharedPreferencesUtil().batchModeEnabled && provider.havingRecordingDevice)) { + return; + } + final isCaptureActive = provider.recordingState == RecordingState.record || + provider.recordingState == RecordingState.systemAudioRecord || + provider.recordingState == RecordingState.deviceRecord || + provider.recordingState == RecordingState.initialising || + provider.recordingState == RecordingState.interrupted || + provider.recordingState == RecordingState.pause || + provider.havingRecordingDevice || + provider.isPaused; + if (!isCaptureActive && provider.segments.isEmpty && provider.photos.isEmpty) return; + PlatformManager.instance.analytics.liveTranscriptCardClicked( + hasSegments: provider.segments.isNotEmpty, + hasPhotos: provider.photos.isNotEmpty, + segmentCount: provider.segments.length, + photoCount: provider.photos.length, + ); + routeToPage(context, ConversationCapturingPage(topConversationId: provider.topConversationId)); + }, + child: Semantics( + button: !batch, + hint: batch ? null : context.l10n.liveTranscript, + child: _cardShell(_buildUnifiedRecordingUI(provider), + padding: batch ? _liveCardPadding : EdgeInsets.zero, decorated: batch), + ), ), - ); + ]); }, ); } @@ -281,7 +290,8 @@ class _ConversationCaptureWidgetState extends State { osHoldsMic: provider.isCallActive, ); final micTaken = interruption == CaptureInterruption.micTaken; - final hasTerminalTranscriptionFailure = provider.terminalTranscriptionFailure != null; + final hasTerminalTranscriptionFailure = + provider.terminalTranscriptionFailure != null || provider.localSttFailure != null; final bufferingFor = provider.customSttBufferingDuration; // Determine if this is an OmiGlass-type device (captures photos) @@ -300,12 +310,15 @@ class _ConversationCaptureWidgetState extends State { final starting = isPhoneRecording && provider.recordingState == RecordingState.initialising && displayState == CaptureDisplayState.listening; - final copy = unverified - ? CaptureCardCopy(context.l10n.captureSourcePendant) - : starting - ? CaptureCardCopy(context.l10n.captureStarting) - : captureCardCopy(context.l10n, displayState, - micTaken: micTaken, socketDown: !provider.transcriptServiceReady); + final copy = provider.localSttFailure != null + ? CaptureCardCopy(context.l10n.transcriptionUnavailable, + detail: context.l10n.setupOnDevice, explanation: context.l10n.transcriptionUnavailableRecordingSaved) + : unverified + ? CaptureCardCopy(context.l10n.captureSourcePendant) + : starting + ? CaptureCardCopy(context.l10n.captureStarting) + : captureCardCopy(context.l10n, displayState, + micTaken: micTaken, socketDown: !provider.transcriptServiceReady); // When recording is active: the one capture status and control surface. if (isDeviceRecording || isPhoneRecording) { diff --git a/app/lib/pages/memories/widgets/memory_item.dart b/app/lib/pages/memories/widgets/memory_item.dart index 89650734198..324541f20ac 100644 --- a/app/lib/pages/memories/widgets/memory_item.dart +++ b/app/lib/pages/memories/widgets/memory_item.dart @@ -1,6 +1,3 @@ -import 'dart:ui'; - -import 'package:omi/utils/platform/platform_manager.dart'; import 'package:flutter/material.dart'; import 'package:font_awesome_flutter/font_awesome_flutter.dart'; @@ -53,10 +50,129 @@ class MemoryItem extends StatelessWidget { final provenanceLabel = _resolveProvenanceLabel(context, provenanceType); final temporalLabel = _temporalLabel(context, memory); final editable = memoryIsEditable(memory); + void openMemory() { + if (memory.isLocked) { + if (!context.read().showSubscriptionUI) { + showMemoryQuickEditSheet(context, memory, provider, readOnly: true); + return; + } + routeToPage(context, const UsagePage()); + return; + } + if (editable) { + onTap(context, memory, provider); + } else { + showMemoryQuickEditSheet(context, memory, provider); + } + } + + final rowContent = Row( + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + if (memory.isKnowledgeLedger) ...[ + Padding( + padding: const EdgeInsets.only(top: 2, right: 8), + child: Icon( + _ledgerIcon(memory), + size: 15, + color: memory.isHistoricalKnowledgeLedgerRow + ? OmiColors.textPrimary.withValues(alpha: 0.6) + : OmiColors.textPrimary, + ), + ), + ], + Expanded( + child: Text(memory.content.decodeString, style: OmiType.subhead.copyWith(height: 1.4)), + ), + if (editable) + Padding( + padding: const EdgeInsetsDirectional.only(start: 12), + child: Icon( + Icons.edit_outlined, + size: 16, + color: OmiColors.textTertiary, + semanticLabel: context.l10n.editMemoryTitle, + ), + ), + ], + ), + if (memory.ledgerSlot != null && memory.ledgerSlot!.trim().isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 4), + child: Text( + memory.ledgerSlot!, + style: OmiType.caption.copyWith(color: OmiColors.textTertiary), + ), + ), + if (memory.isLedgerPlaybook && (memory.ledgerBody ?? '').trim().isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 6), + child: Text( + memory.ledgerBody!.trim(), + maxLines: 3, + overflow: TextOverflow.ellipsis, + style: OmiType.footnote.copyWith(color: OmiColors.textSecondary), + ), + ), + if (provenanceLabel != null) + Padding( + padding: const EdgeInsets.only(top: 4), + child: Text(provenanceLabel, style: OmiType.caption.copyWith(color: OmiColors.textTertiary)), + ), + if (temporalLabel != null) + Padding( + padding: const EdgeInsets.only(top: 4), + child: Text(temporalLabel, style: OmiType.caption.copyWith(color: OmiColors.textTertiary)), + ), + ], + ), + ), + const SizedBox(width: AppStyles.spacingM), + Row( + mainAxisSize: MainAxisSize.min, + children: [ + if (memory.isBaseline) ...[ + Icon( + Icons.flag, + color: OmiColors.textSecondary, + size: 20, + semanticLabel: context.l10n.baselineMemory, + ), + const SizedBox(width: AppStyles.spacingS), + ], + if (memory.conversationId != null) ...[ + _buildConversationLinkButton(context), + const SizedBox(width: AppStyles.spacingS), + ], + if (_canReviewLedgerRow(memory)) ...[ + _buildReviewButton(context, accepted: true), + const SizedBox(width: AppStyles.spacingS), + _buildReviewButton(context, accepted: false), + const SizedBox(width: AppStyles.spacingS), + ], + if (provider.memoryBeliefEnabled && _canSetMemoryUse(memory)) ...[ + _buildMemoryUseButton(context), + const SizedBox(width: AppStyles.spacingS), + ], + if (provider.canRevertSupersededFact(memory)) ...[ + _buildRevertButton(context), + const SizedBox(width: AppStyles.spacingS), + ], + // _buildVisibilityButton(context), + ], + ), + ], + ); final Widget memoryWidget = GestureDetector( - // Every memory opens: editable ones into the edit sheet, the rest read-only. - onTap: - editable ? () => onTap(context, memory, provider) : () => showMemoryQuickEditSheet(context, memory, provider), + // Padding and frosted-content taps obey the same access guard as the action. + onTap: openMemory, // A locked row's content is behind the paywall overlay; the row menu would show it in full. onLongPress: memory.isLocked ? null : () => _showRowMenu(context, editable), child: AnimatedContainer( @@ -68,148 +184,14 @@ class MemoryItem extends StatelessWidget { borderRadius: OmiRadius.xlAll, boxShadow: [BoxShadow(color: Colors.black.withValues(alpha: 0.1), blurRadius: 4, offset: const Offset(0, 2))], ), - child: Stack( - children: [ - Row( - crossAxisAlignment: CrossAxisAlignment.center, - children: [ - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Row( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - if (memory.isKnowledgeLedger) ...[ - Padding( - padding: const EdgeInsets.only(top: 2, right: 8), - child: Icon( - _ledgerIcon(memory), - size: 15, - color: memory.isHistoricalKnowledgeLedgerRow - ? OmiColors.textPrimary.withValues(alpha: 0.6) - : OmiColors.textPrimary, - ), - ), - ], - Expanded( - child: Text(memory.content.decodeString, style: OmiType.subhead.copyWith(height: 1.4)), - ), - if (editable) - Padding( - padding: const EdgeInsetsDirectional.only(start: 12), - child: Icon( - Icons.edit_outlined, - size: 16, - color: OmiColors.textTertiary, - semanticLabel: context.l10n.editMemoryTitle, - ), - ), - ], - ), - if (memory.ledgerSlot != null && memory.ledgerSlot!.trim().isNotEmpty) - Padding( - padding: const EdgeInsets.only(top: 4), - child: Text( - memory.ledgerSlot!, - style: OmiType.caption.copyWith(color: OmiColors.textTertiary), - ), - ), - if (memory.isLedgerPlaybook && (memory.ledgerBody ?? '').trim().isNotEmpty) - Padding( - padding: const EdgeInsets.only(top: 6), - child: Text( - memory.ledgerBody!.trim(), - maxLines: 3, - overflow: TextOverflow.ellipsis, - style: OmiType.footnote.copyWith(color: OmiColors.textSecondary), - ), - ), - if (provenanceLabel != null) - Padding( - padding: const EdgeInsets.only(top: 4), - child: Text(provenanceLabel, style: OmiType.caption.copyWith(color: OmiColors.textTertiary)), - ), - if (temporalLabel != null) - Padding( - padding: const EdgeInsets.only(top: 4), - child: Text(temporalLabel, style: OmiType.caption.copyWith(color: OmiColors.textTertiary)), - ), - ], - ), - ), - const SizedBox(width: AppStyles.spacingM), - Row( - mainAxisSize: MainAxisSize.min, - children: [ - if (memory.isBaseline) ...[ - Icon( - Icons.flag, - color: OmiColors.textSecondary, - size: 20, - semanticLabel: context.l10n.baselineMemory, - ), - const SizedBox(width: AppStyles.spacingS), - ], - if (memory.conversationId != null) ...[ - _buildConversationLinkButton(context), - const SizedBox(width: AppStyles.spacingS), - ], - if (_canReviewLedgerRow(memory)) ...[ - _buildReviewButton(context, accepted: true), - const SizedBox(width: AppStyles.spacingS), - _buildReviewButton(context, accepted: false), - const SizedBox(width: AppStyles.spacingS), - ], - if (provider.memoryBeliefEnabled && _canSetMemoryUse(memory)) ...[ - _buildMemoryUseButton(context), - const SizedBox(width: AppStyles.spacingS), - ], - if (provider.canRevertSupersededFact(memory)) ...[ - _buildRevertButton(context), - const SizedBox(width: AppStyles.spacingS), - ], - // _buildVisibilityButton(context), - ], - ), - ], - ), - if (memory.isLocked) - Positioned.fill( - child: ClipRRect( - child: BackdropFilter( - filter: ImageFilter.blur(sigmaX: 2.0, sigmaY: 2.0), - child: GestureDetector( - onTap: () { - if (!context.read().showSubscriptionUI) { - // No upgrade path to offer: still let the reader open what is visible. - showMemoryQuickEditSheet(context, memory, provider, readOnly: true); - return; - } - PlatformManager.instance.analytics.paywallOpened( - 'locked_memory', - currentPlan: - context.read().subscription?.subscription.plan.wireName ?? 'unknown', - ); - routeToPage(context, const UsagePage(showUpgradeDialog: true, paywallSource: 'locked_memory')); - return; - }, - child: Container( - alignment: Alignment.center, - decoration: BoxDecoration( - color: Colors.black.withValues(alpha: 0.01), - borderRadius: OmiRadius.smAll, - ), - child: context.watch().showSubscriptionUI - ? Text(context.l10n.upgradeToUnlimited, style: OmiType.headline) - : const SizedBox.shrink(), - ), - ), - ), - ), - ), - ], - ), + child: memory.isLocked + ? OmiLockedPreview( + status: context.l10n.statusPending, + label: context.watch().showSubscriptionUI ? context.l10n.viewUsage : context.l10n.saved, + onPressed: openMemory, + child: rowContent, + ) + : rowContent, ), ); diff --git a/app/lib/pages/onboarding/guided_voice_io.dart b/app/lib/pages/onboarding/guided_voice_io.dart index 2f5bf03a2c7..015d938adfd 100644 --- a/app/lib/pages/onboarding/guided_voice_io.dart +++ b/app/lib/pages/onboarding/guided_voice_io.dart @@ -12,6 +12,7 @@ import 'package:omi/backend/preferences.dart'; import 'package:omi/services/services.dart'; import 'package:omi/services/siri_integration.dart'; import 'package:omi/services/account_cutover/account_cutover_runtime.dart'; +import 'package:omi/services/experiments/apple_speech_analyzer_gate.dart'; import 'package:omi/services/sockets/on_device_apple_provider.dart'; import 'package:omi/utils/audio/wav_bytes.dart'; import 'package:omi/utils/hard_secret_detector.dart'; @@ -33,6 +34,10 @@ class DeviceGuidedVoiceIO implements GuidedVoiceIO, GuidedVoiceExperienceIO { bool _selectionLocked = false; bool _recording = false; bool _closed = false; + // Engine choice resolved once by preflight and injected into the recognizer + // created by prepare(), so a flag change mid-session cannot make the provider + // transcribe with a different engine than availability validated. + bool? _useSpeechAnalyzer; // Non-null while a start() call is awaiting the recorder's own start-up. A stop() // that lands in this window can't stop a recorder that isn't running yet, so it // defers to start(): once the recorder comes up, start() sees _recording is false @@ -52,10 +57,17 @@ class DeviceGuidedVoiceIO implements GuidedVoiceIO, GuidedVoiceExperienceIO { final prefs = SharedPreferencesUtil(); _language = prefs.hasSetPrimaryLanguage ? prefs.userPrimaryLanguage : 'en'; if (_language == 'multi') _language = 'en'; - final local = Platform.isIOS && await OnDeviceAppleProvider.isOnDeviceAvailable(_language); + // Resolve the engine gate once here; the provider created in prepare() + // reuses this exact choice instead of re-reading the flag per operation. + final analyzer = Platform.isIOS ? await AppleSpeechAnalyzerGate.useNewEngine() : true; + final local = Platform.isIOS && + await OnDeviceAppleProvider.isOnDeviceAvailable(_language, useSpeechAnalyzer: () async => analyzer); // A slow preflight may finish after recording has been allowed. Never // switch its engine after prepare selected the recognizer/cloud path. - if (!_selectionLocked && !_closed) _local = local; + if (!_selectionLocked && !_closed) { + _local = local; + _useSpeechAnalyzer = analyzer; + } return _local || await guidedVoicePreflight(); } @@ -81,8 +93,15 @@ class DeviceGuidedVoiceIO implements GuidedVoiceIO, GuidedVoiceExperienceIO { throw GuidedVoiceFailure(permission.isPermanentlyDenied ? 'permission_permanently_denied' : 'permission_denied'); } // Keep the language/engine selected by preflight: changing to cloud here - // would start a recording without checking that transcription path. - if (_local) _recognizer = OnDeviceAppleProvider(language: _language); + // would start a recording without checking that transcription path. The + // pinned engine choice is injected so transcribe() never re-reads the + // flag and diverges from what availability validated. + if (_local) { + _recognizer = OnDeviceAppleProvider( + language: _language, + useSpeechAnalyzer: () async => _useSpeechAnalyzer ?? true, + ); + } _directory = await (await getTemporaryDirectory()).createTemp('omi-introduction-'); if (_closed) await close(); } diff --git a/app/lib/pages/settings/transcription_settings_page.dart b/app/lib/pages/settings/transcription_settings_page.dart index 639701e7ef0..37be3b6fdff 100644 --- a/app/lib/pages/settings/transcription_settings_page.dart +++ b/app/lib/pages/settings/transcription_settings_page.dart @@ -1,3 +1,7 @@ +import 'package:permission_handler/permission_handler.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/ui/components/local_stt_recovery_notice.dart'; import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -1208,6 +1212,20 @@ class _TranscriptionSettingsPageState extends State { ), const SizedBox(height: OmiSpacing.sm), TranscriptionHelpText(context.l10n.nativeEngineNoDownload), + FutureBuilder( + future: OnDeviceAppleProvider.readiness(_currentLanguage), + builder: (context, snapshot) { + final readiness = snapshot.data; + if (readiness == null || readiness.ready) return const SizedBox.shrink(); + return LocalSttRecoveryNotice( + language: _currentLanguage, + failure: LocalSttFailure(readiness.status, canProbeRecovery: true), + onRetry: () async { + setState(() {}); + }, + onSetup: openAppSettings); + }, + ), const SizedBox(height: OmiSpacing.lg), _buildLanguageSelector(), ], diff --git a/app/lib/services/capture/capture_controller.dart b/app/lib/services/capture/capture_controller.dart index 48f06e200d8..3253547ce69 100644 --- a/app/lib/services/capture/capture_controller.dart +++ b/app/lib/services/capture/capture_controller.dart @@ -1,3 +1,7 @@ +import 'package:omi/models/stt_provider.dart'; +import 'package:omi/services/capture/local_stt_custody.dart'; +import 'package:omi/services/freemium_transcription_service.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'dart:async'; import 'package:omi/services/capture/uplink_silence_timer.dart'; @@ -247,6 +251,18 @@ class CaptureController extends ChangeNotifier /// How long the custom STT endpoint has been unreachable, or null if it is /// not in use or is currently healthy. + LocalSttFailure? _localSetupFailure; + LocalSttFailure? get localSttFailure => customSttPollingSocket(_socket?.socket)?.localFailure ?? _localSetupFailure; + Future retryLocalTranscription() async { + final polling = customSttPollingSocket(_socket?.socket); + if (polling != null) { + await polling.retryLocalTranscription(); + } else { + await onRecordProfileSettingChanged(); + } + notifyListeners(); + } + Duration? get customSttBufferingDuration => customSttBufferingFor(_socket?.socket); // Phone mic WAL: buffer for splitting variable-sized PCM chunks into fixed-size frames @@ -1563,6 +1579,12 @@ class CaptureController extends ChangeNotifier 'custom=${customSttConfig.isEnabled} provider=${customSttConfig.provider}', ); + LocalSttCustody.selectedLocalRoute = + decision.allowanceOnDevice || decision.customSttConfig?.provider == SttProvider.onDeviceWhisper; + _localSetupFailure = decision.blockSocket && decision.allowanceOnDevice + ? LocalSttFailure(FreemiumTranscriptionService().appleReadiness?.status ?? 'unavailable', + canProbeRecovery: true) + : null; if (decision.blockSocket) { Logger.warning('[SttMode] Blocking transcription socket (${decision.reason})'); await _abandonTranscriptionSocket(reason: 'stt mode blocked: ${decision.reason}'); @@ -2217,6 +2239,12 @@ class CaptureController extends ChangeNotifier final customSttConfig = _preferences.customSttConfig; final decision = await SttModeResolver.instance.decide(persistedCustomStt: customSttConfig, codec: codec); if (_deviceIdentityStale(deviceRevision)) return; + LocalSttCustody.selectedLocalRoute = + decision.allowanceOnDevice || decision.customSttConfig?.provider == SttProvider.onDeviceWhisper; + _localSetupFailure = decision.blockSocket && decision.allowanceOnDevice + ? LocalSttFailure(FreemiumTranscriptionService().appleReadiness?.status ?? 'unavailable', + canProbeRecovery: true) + : null; if (decision.blockSocket) { await _abandonTranscriptionSocket(reason: 'stt mode blocked: ${decision.reason}'); return; @@ -3292,6 +3320,11 @@ class CaptureController extends ChangeNotifier } Future _socketErrorBody(Object err) async { + if (err is LocalSttFailure) { + FreemiumTranscriptionService().reportLocalFailure(err.code); + notifyListeners(); // WAL/microphone and the healthy transcript-ingest socket continue. + return; + } _recordingTelemetry.observeSocketError(); _transcriptionServiceStatuses = []; _transcriptServiceReady = false; diff --git a/app/lib/services/capture/local_stt_custody.dart b/app/lib/services/capture/local_stt_custody.dart new file mode 100644 index 00000000000..07c87d6f17c --- /dev/null +++ b/app/lib/services/capture/local_stt_custody.dart @@ -0,0 +1,7 @@ +/// Selected capture route, not an entitlement decision. Local speech is an +/// asynchronous consumer: enqueueing to it is never an audio-delivery receipt. +/// CaptureController supplies its resolved route; no preference/service lookup. +abstract final class LocalSttCustody { + static bool selectedLocalRoute = false; + static bool get keepsPhoneAudio => selectedLocalRoute; +} diff --git a/app/lib/services/capture/stt_mode_resolver.dart b/app/lib/services/capture/stt_mode_resolver.dart index f36999fa480..d1f7a753949 100644 --- a/app/lib/services/capture/stt_mode_resolver.dart +++ b/app/lib/services/capture/stt_mode_resolver.dart @@ -1,3 +1,4 @@ +import 'package:omi/models/stt_provider.dart'; import 'package:flutter/foundation.dart'; import 'package:omi/models/custom_stt_config.dart'; @@ -57,13 +58,13 @@ class SttModeResolver { final bool Function() flagReader; final TranscriptionAllowanceSnapshot? Function() allowanceReader; - final Future Function() readinessReader; + final Future Function({String? language}) readinessReader; final CustomSttConfig? Function() onDeviceConfigBuilder; SttModeResolver({ bool Function()? flagReader, TranscriptionAllowanceSnapshot? Function()? allowanceReader, - Future Function()? readinessReader, + Future Function({String? language})? readinessReader, CustomSttConfig? Function()? onDeviceConfigBuilder, }) : flagReader = flagReader ?? _defaultFlag, allowanceReader = allowanceReader ?? _defaultAllowance, @@ -79,7 +80,8 @@ class SttModeResolver { static TranscriptionAllowanceSnapshot? _defaultAllowance() => TranscriptionAllowanceCache.current; - static Future _defaultReadiness() => FreemiumTranscriptionService().checkReadiness(); + static Future _defaultReadiness({String? language}) => + FreemiumTranscriptionService().checkReadiness(language: language); static CustomSttConfig? _defaultOnDeviceConfig() { final config = FreemiumTranscriptionService().getFreemiumConfig(); @@ -91,8 +93,14 @@ class SttModeResolver { FreemiumReadiness readiness = FreemiumReadiness.ready; final flag = flagReader(); final allowance = allowanceReader(); - if (flag && !persistedCustomStt.isEnabled && _needsOnDeviceReadiness(allowance)) { - readiness = await readinessReader(); + if ((persistedCustomStt.provider == SttProvider.onDeviceWhisper && + !kIsWeb && + defaultTargetPlatform == TargetPlatform.iOS) || + (flag && !persistedCustomStt.isEnabled && _needsOnDeviceReadiness(allowance))) { + readiness = await readinessReader( + language: persistedCustomStt.isEnabled && persistedCustomStt.provider == SttProvider.onDeviceWhisper + ? persistedCustomStt.language + : null); } return resolve( flagEnabled: flag, @@ -116,6 +124,10 @@ class SttModeResolver { required BleAudioCodec codec, }) { if (persistedCustomStt.isEnabled) { + if (persistedCustomStt.provider == SttProvider.onDeviceWhisper && readiness != FreemiumReadiness.ready) { + return const SttModeDecision( + path: SttResolvedPath.blocked, reason: 'on_device_not_ready', allowanceOnDevice: true); + } return SttModeDecision( path: SttResolvedPath.honorCustom, customSttConfig: persistedCustomStt, diff --git a/app/lib/services/experiments/apple_speech_analyzer_gate.dart b/app/lib/services/experiments/apple_speech_analyzer_gate.dart new file mode 100644 index 00000000000..0c58caf945e --- /dev/null +++ b/app/lib/services/experiments/apple_speech_analyzer_gate.dart @@ -0,0 +1,17 @@ +import 'package:omi/utils/analytics/analytics_manager.dart'; + +/// Operational rollback: false/missing/offline keeps the improved engine on. +/// Lifecycle: ops_kill; owner: dazheng. No overflow or plan dependency. +abstract final class AppleSpeechAnalyzerGate { + static const enabledFlag = 'mobile-speech-analyzer-disabled'; + + static Future useNewEngine({Future Function(String)? readFlag}) async { + try { + final killed = + await (readFlag ?? AnalyticsManager().isFeatureEnabled)(enabledFlag).timeout(const Duration(seconds: 2)); + return !killed; + } catch (_) { + return true; + } + } +} diff --git a/app/lib/services/freemium_transcription_service.dart b/app/lib/services/freemium_transcription_service.dart index 49fa2142aa5..4f6c8e4f92d 100644 --- a/app/lib/services/freemium_transcription_service.dart +++ b/app/lib/services/freemium_transcription_service.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:flutter/material.dart'; +import 'package:flutter/foundation.dart'; import 'package:path_provider/path_provider.dart'; @@ -8,6 +9,8 @@ import 'package:omi/backend/preferences.dart'; import 'package:omi/models/custom_stt_config.dart'; import 'package:omi/models/stt_provider.dart'; import 'package:omi/utils/logger.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/services/sockets/apple_speech_readiness.dart'; /// Enum representing the current freemium transcription state enum FreemiumTranscriptionState { @@ -36,6 +39,11 @@ class FreemiumTranscriptionService extends ChangeNotifier { static final FreemiumTranscriptionService _instance = FreemiumTranscriptionService._internal(); factory FreemiumTranscriptionService() => _instance; FreemiumTranscriptionService._internal(); + @visibleForTesting + FreemiumTranscriptionService.testing({this.iosOverride, this.appleReadinessReader}); + bool? iosOverride; + Future Function(String)? appleReadinessReader; + AppleSpeechReadiness? appleReadiness; FreemiumTranscriptionState _state = FreemiumTranscriptionState.premium; FreemiumReadiness _readiness = FreemiumReadiness.requiresSetup; @@ -60,10 +68,21 @@ class FreemiumTranscriptionService extends ChangeNotifier { } /// Check if freemium (on-device) STT is ready to use - Future checkReadiness() async { - // iOS: Native Apple Speech Recognition is always available - if (Platform.isIOS) { - _readiness = FreemiumReadiness.ready; + static String effectiveLanguage({String? language}) { + final preferences = SharedPreferencesUtil(); + final config = preferences.customSttConfig; + final selected = language ?? + (config.isEnabled && config.provider == SttProvider.onDeviceWhisper + ? (config.language ?? 'en') + : preferences.userPrimaryLanguage); + return selected.isEmpty ? 'multi' : selected; + } + + Future checkReadiness({String? language}) async { + if (iosOverride ?? Platform.isIOS) { + appleReadiness = + await (appleReadinessReader ?? OnDeviceAppleProvider.readiness)(effectiveLanguage(language: language)); + _readiness = appleReadiness!.ready ? FreemiumReadiness.ready : FreemiumReadiness.requiresSetup; notifyListeners(); return _readiness; } @@ -86,6 +105,12 @@ class FreemiumTranscriptionService extends ChangeNotifier { return _readiness; } + void reportLocalFailure(String status) { + _readiness = FreemiumReadiness.requiresSetup; + if (Platform.isIOS) appleReadiness = AppleSpeechReadiness('sfspeech', status); + notifyListeners(); + } + /// Find any downloaded Whisper model in the models directory /// Returns the path to the first found model, preferring smaller models Future _findDownloadedModelPath() async { diff --git a/app/lib/services/sockets/apple_speech_readiness.dart b/app/lib/services/sockets/apple_speech_readiness.dart new file mode 100644 index 00000000000..effe4a3d50d --- /dev/null +++ b/app/lib/services/sockets/apple_speech_readiness.dart @@ -0,0 +1,8 @@ +class AppleSpeechReadiness { + final String engine; + final String status; + const AppleSpeechReadiness(this.engine, this.status); + bool get ready => status == 'ready'; + factory AppleSpeechReadiness.fromMap(Map? data) => + AppleSpeechReadiness(data?['engine'] as String? ?? 'unknown', data?['status'] as String? ?? 'unavailable'); +} diff --git a/app/lib/services/sockets/composite_transcription_socket.dart b/app/lib/services/sockets/composite_transcription_socket.dart index f5f80f200b0..2ce410b7b9d 100644 --- a/app/lib/services/sockets/composite_transcription_socket.dart +++ b/app/lib/services/sockets/composite_transcription_socket.dart @@ -1,3 +1,4 @@ +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'dart:async'; import 'dart:convert'; @@ -135,6 +136,10 @@ class CompositeTranscriptionSocket implements IPureSocket { return; } + if (err is LocalSttFailure) { + onError(err, trace); // Keep both sockets and capture custody alive for setup recovery. + return; + } CustomSttLogService.instance.error('Composite', '$name socket error: $err'); DebugLogManager.logError(err, trace, 'composite_socket_child_error', {'child_socket': name}); diff --git a/app/lib/services/sockets/local_stt_failure.dart b/app/lib/services/sockets/local_stt_failure.dart new file mode 100644 index 00000000000..ad3d94a6093 --- /dev/null +++ b/app/lib/services/sockets/local_stt_failure.dart @@ -0,0 +1,13 @@ +/// A local transcription failure requiring explicit setup or input repair. +/// Audio stays in the polling buffer and the existing WAL; no billed fallback. +class LocalSttFailure implements Exception { + final String code; + final bool canProbeRecovery; + const LocalSttFailure(this.code, {this.canProbeRecovery = false}); + @override + String toString() => 'Local transcription needs attention ($code)'; +} + +abstract interface class RecoverableLocalSttProvider { + Future isReadyToRetry(); +} diff --git a/app/lib/services/sockets/on_device_apple_provider.dart b/app/lib/services/sockets/on_device_apple_provider.dart index c0a5a9fc0c3..ee3c4e9028b 100644 --- a/app/lib/services/sockets/on_device_apple_provider.dart +++ b/app/lib/services/sockets/on_device_apple_provider.dart @@ -1,30 +1,39 @@ import 'dart:io'; import 'dart:typed_data'; -import 'package:flutter/services.dart' show MethodChannel; +import 'package:flutter/services.dart' show MethodChannel, PlatformException; import 'package:path_provider/path_provider.dart'; import 'package:omi/models/stt_result.dart'; +import 'package:omi/utils/audio/whisper_pcm.dart'; +import 'package:omi/services/sockets/apple_speech_readiness.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/services/experiments/apple_speech_analyzer_gate.dart'; import 'package:omi/services/custom_stt_log_service.dart'; import 'package:omi/services/sockets/on_device_transcript_quality_gate.dart'; import 'package:omi/services/sockets/pure_polling.dart'; -class OnDeviceAppleProvider implements ISttProvider { +class OnDeviceAppleProvider implements ISttProvider, RecoverableLocalSttProvider { final String language; + final Future Function()? useSpeechAnalyzer; static const MethodChannel _channel = MethodChannel('com.omi.ios/speech'); final OnDeviceTranscriptQualityGate _qualityGate = OnDeviceTranscriptQualityGate(); - OnDeviceAppleProvider({this.language = 'en'}); + OnDeviceAppleProvider({this.language = 'en', this.useSpeechAnalyzer}); /// Whether iOS has an on-device recognizer (language model installed, not /// network-backed) for [language]. Bare codes like "en" are resolved /// natively to an installed full locale such as en-US. Returns false on any /// channel error so callers treat "unknown" as "unavailable". - static Future isOnDeviceAvailable(String language) async { - if (!Platform.isIOS) return false; + static Future isOnDeviceAvailable(String language, + {Future Function()? useSpeechAnalyzer, bool? isIOS}) async { + if (!(isIOS ?? Platform.isIOS)) return false; try { - final available = await _channel.invokeMethod('onDeviceAvailable', {'language': language}); + final available = await _channel.invokeMethod('onDeviceAvailable', { + 'language': language, + 'useSpeechAnalyzer': await (useSpeechAnalyzer ?? AppleSpeechAnalyzerGate.useNewEngine)() + }); return available ?? false; } catch (e) { CustomSttLogService.instance.error('OnDeviceApple', 'Availability check failed: $e'); @@ -32,6 +41,25 @@ class OnDeviceAppleProvider implements ISttProvider { } } + static Future readiness( + String language, { + Future Function()? useSpeechAnalyzer, + bool prepare = false, + }) async { + try { + final data = await _channel.invokeMapMethod(prepare ? 'prepareSpeech' : 'readiness', { + 'language': language, + 'useSpeechAnalyzer': await (useSpeechAnalyzer ?? AppleSpeechAnalyzerGate.useNewEngine)(), + }); + return AppleSpeechReadiness.fromMap(data); + } catch (_) { + return const AppleSpeechReadiness('unknown', 'unavailable'); + } + } + + @override + Future isReadyToRetry() async => (await readiness(language, useSpeechAnalyzer: useSpeechAnalyzer)).ready; + @override Future transcribe( Uint8List audioData, { @@ -40,10 +68,11 @@ class OnDeviceAppleProvider implements ISttProvider { }) async { try { final sw = Stopwatch()..start(); + final normalized = WhisperPcm.normalize(audioData); final tempDir = await getTemporaryDirectory(); final tempFile = File('${tempDir.path}/temp_apple_audio_${DateTime.now().millisecondsSinceEpoch}.wav'); - await tempFile.writeAsBytes(audioData); + await tempFile.writeAsBytes(normalized.wav); String effectiveLanguage = language ?? this.language; if (effectiveLanguage == 'multi') { @@ -54,20 +83,21 @@ class OnDeviceAppleProvider implements ISttProvider { final String? result = await _channel.invokeMethod('transcribe', { 'filePath': tempFile.path, 'language': effectiveLanguage, + 'useSpeechAnalyzer': await (useSpeechAnalyzer ?? AppleSpeechAnalyzerGate.useNewEngine)(), }); if (result == null || result.isEmpty) { - return null; + return SttTranscriptionResult(audioEndSeconds: audioOffsetSeconds + normalized.duration); } - final duration = audioData.lengthInBytes / 32000.0; + final duration = normalized.duration; final filteredText = _qualityGate.filter( result, audioData: audioData, ); if (filteredText == null) { CustomSttLogService.instance.warning('OnDeviceApple', 'Dropped low-quality local transcript: $result'); - return null; + return SttTranscriptionResult(audioEndSeconds: audioOffsetSeconds + duration); } CustomSttLogService.instance.info( @@ -76,6 +106,7 @@ class OnDeviceAppleProvider implements ISttProvider { ); return SttTranscriptionResult( + audioEndSeconds: audioOffsetSeconds + duration, segments: [ SttSegment(text: filteredText, start: audioOffsetSeconds, end: audioOffsetSeconds + duration, speakerId: 0), ], @@ -87,6 +118,11 @@ class OnDeviceAppleProvider implements ISttProvider { } } } catch (e) { + if (e is PlatformException && ['UNAUTHORIZED', 'UNAVAILABLE', 'INVALID_ARGS'].contains(e.code)) { + final details = e.details; + final status = details is Map ? details['status'] as String? : null; + throw LocalSttFailure(status ?? e.code.toLowerCase(), canProbeRecovery: e.code != 'INVALID_ARGS'); + } CustomSttLogService.instance.error('OnDeviceApple', 'Transcription error: $e'); rethrow; // Keep failed/timed-out audio in the polling buffer for retry. } diff --git a/app/lib/services/sockets/on_device_whisper_provider.dart b/app/lib/services/sockets/on_device_whisper_provider.dart index 04b3b12b5cb..62e3ac839df 100644 --- a/app/lib/services/sockets/on_device_whisper_provider.dart +++ b/app/lib/services/sockets/on_device_whisper_provider.dart @@ -7,6 +7,9 @@ import 'package:path_provider/path_provider.dart'; import 'package:whisper_flutter_new/whisper_flutter_new.dart'; import 'package:omi/models/stt_result.dart'; +import 'package:omi/utils/audio/wav_bytes.dart'; +import 'package:omi/utils/audio/whisper_pcm.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'package:omi/services/custom_stt_log_service.dart'; import 'package:omi/services/sockets/on_device_transcript_quality_gate.dart'; import 'package:omi/services/sockets/pure_polling.dart'; @@ -24,7 +27,7 @@ class OnDeviceWhisperProvider implements ISttProvider { if (_isInitialized) return; try { if (!await File(modelPath).exists()) { - throw Exception('Model file not found at $modelPath'); + throw const LocalSttFailure('model_missing'); } final dir = p.dirname(modelPath); @@ -101,11 +104,59 @@ class OnDeviceWhisperProvider implements ISttProvider { return pcmBytes / (sampleRate * 2); } + /// Each plugin call allocates and frees a fresh whisper_context. Never pass + /// more than 30 seconds, including when retained polling audio grows on retry. + @visibleForTesting + static Iterable boundedAudioChunks(Uint8List bytes) sync* { + if (audioDurationSeconds(bytes) <= 30) { + yield bytes; + return; + } + final header = ByteData.sublistView(bytes); + final rate = header.getUint32(24, Endian.little); + final pcmSize = header.getUint32(40, Endian.little); + if (rate == 0 || + header.getUint16(22, Endian.little) != 1 || + header.getUint16(34, Endian.little) != 16 || + pcmSize > bytes.length - 44 || + pcmSize.isOdd) { + throw const FormatException('Expected canonical mono 16-bit PCM WAV'); + } + final limit = 44 + pcmSize; + final chunkBytes = rate * 2 * 30; + for (var start = 44; start < limit; start += chunkBytes) { + final end = (start + chunkBytes).clamp(0, limit); + yield WavBytes.fromPcm(Uint8List.sublistView(bytes, start, end), sampleRate: rate, numChannels: 1).asBytes(); + } + } + @override Future transcribe( Uint8List audioData, { double audioOffsetSeconds = 0, String? language, + }) async { + final normalized = WhisperPcm.normalize(audioData); + final segments = []; + final text = []; + var offset = audioOffsetSeconds; + for (final chunk in boundedAudioChunks(normalized.wav)) { + final result = await transcribeChunk(chunk, audioOffsetSeconds: offset, language: language); + if (result != null) { + segments.addAll(result.segments); + if (result.rawText?.isNotEmpty == true) text.add(result.rawText!); + } + offset += audioDurationSeconds(chunk); + } + return SttTranscriptionResult( + segments: segments, rawText: text.join(' '), audioEndSeconds: audioOffsetSeconds + normalized.duration); + } + + @visibleForTesting + Future transcribeChunk( + Uint8List audioData, { + double audioOffsetSeconds = 0, + String? language, }) async { try { final sw = Stopwatch()..start(); @@ -159,7 +210,10 @@ class OnDeviceWhisperProvider implements ISttProvider { } } catch (e) { CustomSttLogService.instance.error('OnDeviceWhisper', 'Transcription error: $e'); - return null; + final message = e.toString(); + if (message.contains('OMI_STT_PERMANENT_INPUT:')) throw const LocalSttFailure('invalid_audio'); + if (message.contains('OMI_STT_MODEL_UNAVAILABLE:')) throw const LocalSttFailure('model_invalid'); + rethrow; // Transient decoder failures retain the complete polling buffer. } } diff --git a/app/lib/services/sockets/pure_polling.dart b/app/lib/services/sockets/pure_polling.dart index e4d87c806aa..50ed05d850e 100644 --- a/app/lib/services/sockets/pure_polling.dart +++ b/app/lib/services/sockets/pure_polling.dart @@ -3,6 +3,7 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:omi/models/stt_result.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'package:omi/services/custom_stt_log_service.dart'; import 'package:omi/services/sockets/pure_socket.dart'; import 'package:omi/utils/audio/audio_transcoder.dart'; @@ -69,6 +70,8 @@ class PurePollingSocket implements IPureSocket { final List _audioFrames = []; bool _isProcessing = false; + LocalSttFailure? localFailure; + DateTime? _nextRecoveryProbe; int _connectionGeneration = 0; double _audioOffsetSeconds = 0; @@ -140,6 +143,23 @@ class PurePollingSocket implements IPureSocket { return; } + if (localFailure != null) { + if (!localFailure!.canProbeRecovery || sttProvider is! RecoverableLocalSttProvider) return; + if (_nextRecoveryProbe != null && DateTime.now().isBefore(_nextRecoveryProbe!)) return; + _isProcessing = true; + final recoveryGeneration = _connectionGeneration; + try { + _nextRecoveryProbe = DateTime.now().add(const Duration(seconds: 30)); + if (!await (sttProvider as RecoverableLocalSttProvider).isReadyToRetry()) return; + if (recoveryGeneration != _connectionGeneration || _status != PurePollingStatus.connected) return; + localFailure = null; + onConnected(); + } catch (_) { + return; + } finally { + _isProcessing = false; + } + } _isProcessing = true; final generation = _connectionGeneration; @@ -176,9 +196,10 @@ class PurePollingSocket implements IPureSocket { if (generation != _connectionGeneration) return; _bufferingSince = null; _consecutiveFailures = 0; + if (result?.audioEndSeconds != null) _audioOffsetSeconds = result!.audioEndSeconds!; if (result != null && result.isNotEmpty) { if (result.segments.isNotEmpty) { - _audioOffsetSeconds = result.segments.last.end; + _audioOffsetSeconds = result.audioEndSeconds ?? result.segments.last.end; } final segmentsJson = result.segments.where((s) => s.text.trim().isNotEmpty).map((s) => s.toTranscriptSegmentJson()).toList(); @@ -193,6 +214,11 @@ class PurePollingSocket implements IPureSocket { _consecutiveFailures++; _bufferingSince ??= DateTime.now(); _requeueFrames(frames); + if (e is LocalSttFailure) { + localFailure = e; + _nextRecoveryProbe = DateTime.now().add(const Duration(seconds: 30)); + onError(e, trace); // Local setup is actionable, not a disconnect. + } // Do NOT call onError()/propagate this as a fatal // socket error here. sttProvider.transcribe() already retries // transient failures internally; a failure this far up means the STT @@ -217,7 +243,9 @@ class PurePollingSocket implements IPureSocket { var droppedBytes = 0; while (_totalBufferBytes > config.maxBufferBytes && _audioFrames.isNotEmpty) { - droppedBytes += _audioFrames.removeAt(0).length; + final dropped = _audioFrames.removeAt(0); + droppedBytes += dropped.length; + _audioOffsetSeconds += (config.transcoder ?? PassThroughTranscoder()).sourceSeconds(dropped); } if (droppedBytes > 0) { final serviceId = config.serviceId ?? 'Polling'; @@ -288,13 +316,21 @@ class PurePollingSocket implements IPureSocket { void send(dynamic message) { if (message is Uint8List) { _audioFrames.add(message); + if (localFailure != null) _requeueFrames([]); } else if (message is List) { _audioFrames.add(Uint8List.fromList(message)); + if (localFailure != null) _requeueFrames([]); } else { Logger.debug("[Polling] Unsupported message type: ${message.runtimeType}"); } } + Future retryLocalTranscription() async { + _nextRecoveryProbe = null; + if (localFailure?.canProbeRecovery == false) localFailure = null; + await _flushBuffer(); + } + Future flushNow() async { await _flushBuffer(); } diff --git a/app/lib/services/wals/local_wal_sync.dart b/app/lib/services/wals/local_wal_sync.dart index 2b64f371c9d..851efe7e00b 100644 --- a/app/lib/services/wals/local_wal_sync.dart +++ b/app/lib/services/wals/local_wal_sync.dart @@ -1,3 +1,4 @@ +import 'package:omi/services/capture/local_stt_custody.dart'; import 'dart:async'; import 'dart:io'; import 'dart:math'; @@ -86,7 +87,10 @@ bool canClaimLiveCapture(List batch, List pendingForConversation, int /// ([LocalWalSyncImpl.syncWal]) deliberately ignores this budget. @visibleForTesting bool isAutoUploadEligible(Wal wal) => - wal.status == WalStatus.miss && wal.storage == WalStorage.disk && wal.retryCount < walMaxAutoRetries; + !wal.onDeviceOnly && + wal.status == WalStatus.miss && + wal.storage == WalStorage.disk && + wal.retryCount < walMaxAutoRetries; /// Pause tolerance for saved utterance gaps and WAL edges; coverage only suppresses automatic /// repair, never deletes audio, so short gaps are treated as pauses while the copy stays recoverable. @@ -143,6 +147,7 @@ typedef WalCoverageTelemetryEmitter = void Function(Map fields) class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { List _wals = []; + bool _pendingPhoneOnly = false; List _frames = []; String? _captureEvidenceRoot; int _captureEvidenceGeneration = 0; @@ -274,6 +279,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { _retiredWals.addAll(_wals); _wals = []; _frames = []; + _pendingPhoneOnly = false; _frameSynced = []; _captureEvidenceRoot = null; _captureEvidenceGeneration++; @@ -386,6 +392,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { // is owned by the signed-in account. Stamp only when unstamped so a // record surfaced by native recovery keeps its original owner. wal.ownerUid ??= _currentWalOwnerUid(); + wal.repairLegacyImportedCustody(); // Native-storage recovery can surface old WALs while a new recording is // active. Only inherit the current session's location for WALs that began // at (or after) this session, never for historical recordings. @@ -585,6 +592,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { await _enqueueBuffer(() => _flush(generation)); _frames = []; + _pendingPhoneOnly = false; _frameSynced = []; } @@ -598,6 +606,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { await _enqueueBuffer(() => _flush(generation)); if (!_isCurrent(generation)) return; _frames = []; + _pendingPhoneOnly = false; _frameSynced = []; _framesPerSecond = codec.getFramesPerSecond(); @@ -683,6 +692,8 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { timerStart: timerStart, data: chunk, storage: WalStorage.mem, + onDeviceOnly: _pendingPhoneOnly, + capturedOnDeviceOnly: _pendingPhoneOnly, status: syncedOffset == chunkFrameCount ? WalStatus.synced : WalStatus.miss, device: _deviceId ?? "omi", deviceModel: _deviceModel ?? "Omi", @@ -796,6 +807,8 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { timerStart: timerStart, data: runFrames.map((f) => f.payload).toList(), storage: WalStorage.mem, + onDeviceOnly: _pendingPhoneOnly, + capturedOnDeviceOnly: _pendingPhoneOnly, status: syncedOffset == frameCount ? WalStatus.synced : WalStatus.miss, device: _deviceId ?? "omi", deviceModel: _deviceModel ?? "Omi", @@ -901,6 +914,8 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { timerStart: timerStart, data: frames.map((f) => f.payload).toList(), storage: WalStorage.mem, + onDeviceOnly: _pendingPhoneOnly, + capturedOnDeviceOnly: _pendingPhoneOnly, status: syncedOffset == frameCount ? WalStatus.synced : WalStatus.miss, device: _deviceId ?? "omi", deviceModel: _deviceModel ?? "Omi", @@ -1552,6 +1567,8 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { timerStart: timerStart, data: chunk, storage: WalStorage.mem, + onDeviceOnly: _pendingPhoneOnly, + capturedOnDeviceOnly: _pendingPhoneOnly, status: syncedOffset == chunkFrameCount ? WalStatus.synced : WalStatus.miss, device: _deviceId ?? "omi", deviceModel: _deviceModel ?? "Omi", @@ -1573,6 +1590,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { } _frames = []; + _pendingPhoneOnly = false; _frameSynced = []; // Flush all in-memory WALs to disk immediately @@ -1785,6 +1803,13 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { @override WalFrame onFrameCaptured(WalFrame frame, {String? captureRoot}) { + if (LocalSttCustody.keepsPhoneAudio) { + _pendingPhoneOnly = true; + for (final wal in _wals.where((w) => w.storage == WalStorage.mem)) { + wal.onDeviceOnly = true; + wal.capturedOnDeviceOnly = true; + } + } if (captureRoot != _captureEvidenceRoot) { _captureEvidenceRoot = captureRoot; // A restored WAL for the same root is the durable high-water mark. @@ -1882,6 +1907,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { @override void markFrameSynced(FrameSyncKey key) { + if (_pendingPhoneOnly || LocalSttCustody.keepsPhoneAudio) return; for (int i = _frames.length - 1; i >= 0; i--) { if (_frames[i].syncKey == key) { _frameSynced[i] = true; @@ -2234,6 +2260,7 @@ class LocalWalSyncImpl with WidgetsBindingObserver implements LocalWalSync { @override Future syncWal({required Wal wal, IWalSyncProgressListener? progress}) async { + if (wal.onDeviceOnly) return null; final generation = _sessionGeneration; await _enqueueBuffer(() => _flush(generation)); if (!_isCurrent(generation)) return null; diff --git a/app/lib/services/wals/wal.dart b/app/lib/services/wals/wal.dart index c8ea90d5296..ea69553d230 100644 --- a/app/lib/services/wals/wal.dart +++ b/app/lib/services/wals/wal.dart @@ -174,6 +174,21 @@ class Wal { WalStatus status; WalStorage storage; + /// Explicit phone-only custody. Never admitted to a cloud audio drain. + bool onDeviceOnly; + + /// Recording-time custody evidence. Null identifies pre-fix dev records. + bool? capturedOnDeviceOnly; + + /// Imported device storage was never part of the phone's live STT route. + /// Repair only the erroneous pre-fix stamp, preserving explicit capture consent. + void repairLegacyImportedCustody() { + if (capturedOnDeviceOnly == null && + (originalStorage == WalStorage.sdcard || originalStorage == WalStorage.flashPage)) { + onDeviceOnly = false; + } + } + String? filePath; List> data; int storageOffset = 0; @@ -338,6 +353,8 @@ class Wal { this.channel = 1, this.status = WalStatus.inProgress, this.storage = WalStorage.mem, + this.onDeviceOnly = false, + this.capturedOnDeviceOnly, this.filePath, this.device = "phone", this.deviceModel, @@ -376,6 +393,8 @@ class Wal { sampleRate: json['sample_rate'] ?? 16000, status: WalStatus.values.asNameMap()[json['status']] ?? WalStatus.inProgress, storage: WalStorage.values.asNameMap()[json['storage']] ?? WalStorage.mem, + onDeviceOnly: json['on_device_only'] == true, + capturedOnDeviceOnly: json['captured_on_device_only'] as bool?, filePath: json['file_path'], seconds: json['seconds'] ?? chunkSizeInSeconds, device: json['device'] ?? "phone", @@ -405,7 +424,7 @@ class Wal { uploadedAt: json['uploaded_at'] ?? 0, syncedAt: json['synced_at'] ?? 0, keptForTranscriptRecovery: json['kept_for_transcript_recovery'] == true, - ); + )..repairLegacyImportedCustody(); } Map toJson() { @@ -416,6 +435,8 @@ class Wal { 'sample_rate': sampleRate, 'status': status.name, 'storage': storage.name, + 'on_device_only': onDeviceOnly, + if (capturedOnDeviceOnly != null) 'captured_on_device_only': capturedOnDeviceOnly, 'file_path': filePath, 'seconds': seconds, 'device': device, diff --git a/app/lib/ui/components/local_stt_recovery_notice.dart b/app/lib/ui/components/local_stt_recovery_notice.dart new file mode 100644 index 00000000000..247f2ea0729 --- /dev/null +++ b/app/lib/ui/components/local_stt_recovery_notice.dart @@ -0,0 +1,83 @@ +import 'package:flutter/material.dart'; +import 'package:permission_handler/permission_handler.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/services/freemium_transcription_service.dart'; +import 'package:omi/ui/ui.dart'; +import 'package:omi/utils/l10n_extensions.dart'; + +/// A local setup problem leaves capture and its WAL active. Reuses existing +/// fallback copy and offers permission/settings plus an explicit retained retry. +class LocalSttRecoveryNotice extends StatefulWidget { + const LocalSttRecoveryNotice( + {super.key, + required this.failure, + required this.onRetry, + required this.onSetup, + this.language, + this.preparePermission, + this.openSettings}); + final LocalSttFailure failure; + final String? language; + final Future Function() onRetry; + final VoidCallback onSetup; + final Future Function()? preparePermission; + final Future Function()? openSettings; + @override + State createState() => _LocalSttRecoveryNoticeState(); +} + +class _LocalSttRecoveryNoticeState extends State { + bool _busy = false; + Future _retry() async { + setState(() => _busy = true); + try { + await widget.onRetry(); + } finally { + if (mounted) setState(() => _busy = false); + } + } + + Future _setup() async { + if (widget.failure.code == 'permission_required') { + final ready = await (widget.preparePermission ?? + () async => (await OnDeviceAppleProvider.readiness( + FreemiumTranscriptionService.effectiveLanguage(language: widget.language), + prepare: true)) + .ready)(); + if (ready) { + await _retry(); + return; + } + await (widget.openSettings ?? openAppSettings)(); + } else if (widget.failure.code == 'permission_denied' || widget.failure.code == 'unauthorized') { + await (widget.openSettings ?? openAppSettings)(); + } else { + widget.onSetup(); + } + } + + @override + Widget build(BuildContext context) { + final permission = widget.failure.code.startsWith('permission') || widget.failure.code == 'unauthorized'; + return Semantics( + liveRegion: true, + child: Padding( + padding: const EdgeInsets.all(OmiSpacing.md), + child: Column(crossAxisAlignment: CrossAxisAlignment.start, children: [ + Text(context.l10n.transcriptionUnavailableRecordingSaved, style: OmiType.footnote), + const SizedBox(height: OmiSpacing.sm), + Wrap(spacing: OmiSpacing.sm, runSpacing: OmiSpacing.sm, children: [ + OmiButton.secondary( + key: const ValueKey('local_stt_setup'), + label: permission ? context.l10n.openSettings : context.l10n.setupOnDevice, + onPressed: _busy ? null : _setup), + OmiButton.secondary( + key: const ValueKey('local_stt_retry'), + label: context.l10n.tryAgain, + onPressed: _busy ? null : _retry), + ]), + ]), + )); + } +} diff --git a/app/lib/ui/components/omi_locked_preview.dart b/app/lib/ui/components/omi_locked_preview.dart index ed6d05584ef..608582922d4 100644 --- a/app/lib/ui/components/omi_locked_preview.dart +++ b/app/lib/ui/components/omi_locked_preview.dart @@ -13,11 +13,13 @@ class OmiLockedPreview extends StatelessWidget { required this.child, required this.label, required this.onPressed, + this.status, this.interactive = false, }); final Widget child; final String label; + final String? status; final FutureOr Function() onPressed; /// Lets gestures reach the obscured [child] (for example a row's long-press menu or swipe). The @@ -26,44 +28,55 @@ class OmiLockedPreview extends StatelessWidget { @override Widget build(BuildContext context) { - return ClipRRect( - borderRadius: OmiRadius.xlAll, - child: Stack( - alignment: Alignment.center, - children: [ - ExcludeSemantics( - child: IgnorePointer( - ignoring: !interactive, - // Filter only this card's content, not the scrolling backdrop. - child: ImageFiltered( - imageFilter: ImageFilter.blur(sigmaX: 6, sigmaY: 6), - // This obscured content is decorative. The readable action below - // keeps the user's text scale and determines its own height. - child: MediaQuery.withNoTextScaling(child: child), + return GestureDetector( + behavior: HitTestBehavior.opaque, + // Child recognizers retain row long-press/swipe; gaps use the guarded action. + onTap: () => onPressed(), + child: ClipRRect( + borderRadius: OmiRadius.xlAll, + child: Stack( + alignment: Alignment.center, + children: [ + ExcludeSemantics( + child: IgnorePointer( + ignoring: !interactive, + // Filter only this card's content, not the scrolling backdrop. + child: ImageFiltered( + imageFilter: ImageFilter.blur(sigmaX: 6, sigmaY: 6), + // This obscured content is decorative. The readable action below + // keeps the user's text scale and determines its own height. + child: MediaQuery.withNoTextScaling(child: child), + ), ), ), - ), - Positioned.fill( - child: IgnorePointer( - ignoring: interactive, - child: ColoredBox( - key: const Key('locked_preview_tint'), - color: OmiColors.surface1.withValues(alpha: 0.65), + Positioned.fill( + child: IgnorePointer( + ignoring: interactive, + child: ColoredBox( + key: const Key('locked_preview_tint'), + color: OmiColors.surface1.withValues(alpha: 0.65), + ), ), ), - ), - // Non-positioned so the action can also determine the card's height. - Padding( - padding: const EdgeInsets.symmetric(horizontal: OmiSpacing.md, vertical: OmiSpacing.sm), - child: OmiButton.tertiary( - key: const Key('locked_preview_action'), - label: label, - icon: Icons.lock_outline, - size: OmiButtonSize.compact, - onPressed: onPressed, + // Non-positioned so the action can also determine the card's height. + Padding( + padding: const EdgeInsets.symmetric(horizontal: OmiSpacing.md, vertical: OmiSpacing.sm), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + if (status != null) Text(status!, style: OmiType.subhead), + OmiButton.tertiary( + key: const Key('locked_preview_action'), + label: label, + icon: Icons.hourglass_empty, + size: OmiButtonSize.compact, + onPressed: onPressed, + ), + ], + ), ), - ), - ], + ], + ), ), ); } diff --git a/app/lib/utils/audio/audio_transcoder.dart b/app/lib/utils/audio/audio_transcoder.dart index 9e137261b2e..0078453048b 100644 --- a/app/lib/utils/audio/audio_transcoder.dart +++ b/app/lib/utils/audio/audio_transcoder.dart @@ -297,3 +297,45 @@ class AudioTranscoderFactory { } } } + +/// Source time, without decoding (which would mutate the Opus decoder). +/// Opus TOC duration follows RFC 6716 section 3.1; padding is not audio time. +extension AudioSourceClock on IAudioTranscoder { + double sourceSeconds(Uint8List frame) { + final transcoder = this; + if (transcoder is PcmToWavTranscoder) { + return frame.length / (transcoder.sampleRate * transcoder.channels * transcoder.bitsPerSample / 8); + } + if (transcoder is RawPcmTranscoder) return frame.length / (transcoder.sampleRate * transcoder.channels * 2); + if (transcoder is OpusToWavTranscoder || + transcoder is OpusToRawPcmTranscoder || + transcoder is OpusFramesToWavTranscoder) { + return opusPacketSeconds(frame); + } + // Pass-through WAV inputs include their own source clock. + if (frame.length >= 44 && frame[0] == 82 && frame[8] == 87) { + final bytes = ByteData.sublistView(frame); + final bytesPerSecond = bytes.getUint32(28, Endian.little); + if (bytesPerSecond > 0) return bytes.getUint32(40, Endian.little) / bytesPerSecond; + } + return 0; // Opaque formats have no duration contract. + } +} + +double opusPacketSeconds(Uint8List packet) { + if (packet.isEmpty) return 0; + final toc = packet[0]; + final frames = switch (toc & 3) { + 0 => 1, + 1 || 2 => 2, + _ => packet.length > 1 ? packet[1] & 63 : 0, + }; + final size = (toc >> 3) & 3; + final milliseconds = toc & 128 != 0 + ? 2.5 * (1 << size) + : toc & 96 == 96 + ? (toc & 8 != 0 ? 20.0 : 10.0) + : (size == 3 ? 60.0 : 10.0 * (1 << size)); + final duration = frames * milliseconds / 1000; + return duration <= 0.12 ? duration : 0; +} diff --git a/app/lib/utils/audio/whisper_pcm.dart b/app/lib/utils/audio/whisper_pcm.dart new file mode 100644 index 00000000000..a43feebf6aa --- /dev/null +++ b/app/lib/utils/audio/whisper_pcm.dart @@ -0,0 +1,58 @@ +import 'dart:typed_data'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/utils/audio/wav_bytes.dart'; + +/// Canonical PCM input from app transcoders: validates before entering native +/// code, downmixes and resamples to Whisper's mandatory 16 kHz / 16-bit mono. +class WhisperPcm { + final Uint8List wav; + final double duration; + WhisperPcm(this.wav, this.duration); + + static WhisperPcm normalize(Uint8List bytes) { + if (bytes.length < 44) throw const LocalSttFailure('invalid_audio'); + final data = ByteData.sublistView(bytes); + bool tag(int offset, String value) => String.fromCharCodes(bytes.sublist(offset, offset + 4)) == value; + final channels = data.getUint16(22, Endian.little); + final rate = data.getUint32(24, Endian.little); + final size = data.getUint32(40, Endian.little); + if (!tag(0, 'RIFF') || + !tag(8, 'WAVE') || + !tag(12, 'fmt ') || + !tag(36, 'data') || + data.getUint32(16, Endian.little) != 16 || + data.getUint16(20, Endian.little) != 1 || + channels < 1 || + channels > 2 || + rate < 8000 || + rate > 48000 || + data.getUint16(34, Endian.little) != 16 || + size > bytes.length - 44 || + size % (channels * 2) != 0) { + throw const LocalSttFailure('invalid_audio'); + } + final frames = size ~/ (channels * 2); + final duration = frames / rate; + final count = (frames * 16000 / rate).round(); + final pcm = ByteData(count * 2); + double sample(int frame) { + if (frames == 0) return 0; + frame = frame.clamp(0, frames - 1); + var sum = 0; + for (var ch = 0; ch < channels; ch++) { + sum += data.getInt16(44 + (frame * channels + ch) * 2, Endian.little); + } + return sum / channels; + } + + for (var i = 0; i < count; i++) { + final position = i * rate / 16000; + final base = position.floor(); + final fraction = position - base; + final value = sample(base) * (1 - fraction) + sample(base + 1) * fraction; + pcm.setInt16(i * 2, value.round().clamp(-32768, 32767), Endian.little); + } + return WhisperPcm( + WavBytes.fromPcm(pcm.buffer.asUint8List(), sampleRate: 16000, numChannels: 1).asBytes(), duration); + } +} diff --git a/app/scripts/patch_whisper_cleanup.py b/app/scripts/patch_whisper_cleanup.py new file mode 100644 index 00000000000..b76137c56cb --- /dev/null +++ b/app/scripts/patch_whisper_cleanup.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Fail-closed, idempotent native cleanup patch for pinned whisper_flutter_new. + +Like the foreground-task patch, invoked by native build setup. No dependency +revision change: RAII owns both resources across errors and C++ exceptions. +""" +from pathlib import Path +import sys +import os +import tempfile + +MARKER = 'OMI_WHISPER_RESOURCE_GUARDS_V1' + +def patch(source: str) -> str: + if MARKER in source: + return source + replacements = { + 'json transcribe(json jsonBody) noexcept': 'json transcribe(json jsonBody)', + '#include ': '#include \n#include \n// ' + MARKER, + ' struct whisper_context *ctx = whisper_init_from_file(params.model.c_str());': ''' std::unique_ptr context( + whisper_init_from_file(params.model.c_str()), &whisper_free); + auto *ctx = context.get(); + if (!ctx) { + jsonResult["@type"] = "error"; + jsonResult["message"] = "OMI_STT_MODEL_UNAVAILABLE: model could not be loaded"; + return jsonResult; + }''', + ' if (wav.channels != 1 && wav.channels != 2)': ''' auto wav_cleanup = [&wav](drwav *) { drwav_uninit(&wav); }; + std::unique_ptr wav_guard(&wav, wav_cleanup); + if (wav.channels != 1 && wav.channels != 2)''', + ' drwav_uninit(&wav);': '', + ' whisper_free(ctx);': '', + ' failed to open WAV file ': 'OMI_STT_PERMANENT_INPUT: failed to open WAV file', + 'must be mono or stereo': 'OMI_STT_PERMANENT_INPUT: must be mono or stereo', + 'WAV file must be 16 kHz': 'OMI_STT_PERMANENT_INPUT: WAV file must be 16 kHz', + 'WAV file must be 16 bit': 'OMI_STT_PERMANENT_INPUT: WAV file must be 16 bit', + 'error: unknown language = ': 'OMI_STT_PERMANENT_INPUT: unknown language = ', + 'failed to process audio': 'OMI_STT_RETRYABLE: failed to process audio', + } + for old, new in replacements.items(): + if source.count(old) != 1: + raise ValueError(f'Pinned Whisper source drift: expected exactly one {old!r}') + source = source.replace(old, new) + return source + +if __name__ == '__main__': + for arg in sys.argv[1:]: + path = Path(arg) + old = path.read_text() + new = patch(old) + if new != old: + with tempfile.NamedTemporaryFile(mode='w', dir=path.parent, delete=False) as output: + output.write(new) + temporary = output.name + os.chmod(temporary, path.stat().st_mode) + os.replace(temporary, path) + print(f'Whisper cleanup verified: {path.name}') diff --git a/app/scripts/test_whisper_cleanup.py b/app/scripts/test_whisper_cleanup.py new file mode 100644 index 00000000000..62d9c6eeee5 --- /dev/null +++ b/app/scripts/test_whisper_cleanup.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Exercise patched pinned native getTextFromWavFile, real WAV parsing and RAII. +Only the heavyweight whisper kernel is fault-injected; no Dart method override. +""" +import json +import subprocess +import tempfile +from pathlib import Path +from urllib.parse import urlparse, unquote +from patch_whisper_cleanup import patch + +app = Path(__file__).resolve().parents[1] +config = app / '.dart_tool/package_config.json' +pkg = next(p for p in json.loads(config.read_text())['packages'] if p['name'] == 'whisper_flutter_new') +uri = urlparse(pkg['rootUri']) +root = Path(unquote(uri.path)) if uri.scheme == 'file' else (config.parent / pkg['rootUri']).resolve() +with tempfile.TemporaryDirectory(prefix='whisper-cleanup-') as directory: + temp = Path(directory) + (temp / 'main.cpp').write_text(patch((root / 'src/main.cpp').read_text())) + harness = r''' +#include "whisper.cpp/whisper.h" +#include +#include +#include +#include +#include "main.cpp" +static int live = 0; +static int fault = 0; +extern "C" { +struct whisper_context *whisper_init_from_file(const char *path) { + if (std::string(path) == "bad-model") return nullptr; + ++live; return reinterpret_cast(new char[1]); +} +void whisper_free(struct whisper_context *ctx) { --live; delete[] reinterpret_cast(ctx); } +int whisper_lang_id(const char *) { return 0; } +struct whisper_full_params whisper_full_default_params(enum whisper_sampling_strategy) { return {}; } +int whisper_full(struct whisper_context *, struct whisper_full_params, const float *, int) { + if (fault == 2) throw std::runtime_error("injected inference exception"); + return fault; +} +int whisper_full_n_segments(struct whisper_context *) { return 0; } +const char *whisper_full_get_segment_text(struct whisper_context *, int) { return ""; } +int64_t whisper_full_get_segment_t0(struct whisper_context *, int) { return 0; } +int64_t whisper_full_get_segment_t1(struct whisper_context *, int) { return 0; } +} +int descriptors() { + DIR *d = opendir("/dev/fd"); int n = 0; + while (readdir(d)) ++n; + closedir(d); return n; +} +void wav(const char *path, int rate, int channels = 1, int bits = 16) { + std::ofstream f(path, std::ios::binary); + auto u16 = [&](uint16_t x) { f.write(reinterpret_cast(&x), 2); }; + auto u32 = [&](uint32_t x) { f.write(reinterpret_cast(&x), 4); }; + f.write("RIFF",4); u32(36+320); f.write("WAVEfmt ",8); u32(16); u16(1); + u16(channels); u32(rate); u32(rate*channels*bits/8); u16(channels*bits/8); u16(bits); + f.write("data",4); u32(320); char pcm[320] = {}; f.write(pcm,320); +} +int main() { + wav("pcm8.wav",8000); wav("channels.wav",16000,3); wav("bits.wav",16000,1,8); wav("valid.wav",16000); + auto request = [](const char *file, const char *model = "model") { + whisper_params p; p.audio = file; p.model = model; p.no_timestamps = true; + return transcribe(json{{"model",model},{"audio",file},{"language","en"},{"threads",1},{"is_verbose",false},{"is_translate",false},{"is_special_tokens",false},{"is_no_timestamps",true},{"split_on_word",false}}); + }; + const int before = descriptors(); + for (int i=0; i<100; ++i) { + for (const char *file : {"pcm8.wav","channels.wav","bits.wav","missing.wav"}) { + auto result = request(file); + assert(result["@type"] == "error"); + assert(result["message"].get().find("OMI_STT_PERMANENT_INPUT:") == 0); + assert(live == 0 && descriptors() == before); + } + fault = 1; assert(request("valid.wav")["@type"] == "error"); assert(live == 0); + fault = 2; + try { request("valid.wav"); assert(false); } catch (const std::runtime_error &) {} + assert(live == 0 && descriptors() == before); + } + fault = 0; assert(request("valid.wav")["text"] == ""); assert(live == 0); + assert(request("valid.wav","bad-model")["@type"] == "error"); assert(live == 0); + assert(descriptors() == before); +} +''' + (temp / 'test.cpp').write_text(harness) + subprocess.run(['clang++', '-std=c++20', '-fsanitize=address,undefined', '-I'+str(root/'src'), '-I'+str(temp), + str(temp/'test.cpp'), '-o', str(temp/'test')], check=True) + subprocess.run([str(temp/'test')], cwd=temp, check=True) +print('Pinned native error cleanup passed: 600 error/exception paths, no live contexts or leaked WAV handles') diff --git a/app/test.sh b/app/test.sh index 7f1e6335276..ff78db9e61a 100755 --- a/app/test.sh +++ b/app/test.sh @@ -64,6 +64,9 @@ fi bash "$ROOT_DIR/scripts/check_hermetic_test_env.sh" --app-dir "$ROOT_DIR" +# Exercise the pinned native resource-error paths before Dart mocks. +python3 scripts/test_whisper_cleanup.py + # Execute registration with the real compile-time opt-in in a separate isolate. # Keep the ordinary suite (including the pinned fail-closed guard) unopted-in. flutter test --dart-define=OMI_DEV_CONTROLS=1 test/spine diff --git a/app/test/models/subscription_plan_type_test.dart b/app/test/models/subscription_plan_type_test.dart index 35195341b5e..74aa4fa4470 100644 --- a/app/test/models/subscription_plan_type_test.dart +++ b/app/test/models/subscription_plan_type_test.dart @@ -9,7 +9,8 @@ Subscription _subFromWirePlan(String plan) { 'features': [], 'cancel_at_period_end': false, 'deprecated': false, - 'limits': {}, + 'future_subscription_field': {'enabled': true}, + 'limits': {'future_allowance': 42}, }); } diff --git a/app/test/services/capture/stt_mode_capture_entry_test.dart b/app/test/services/capture/stt_mode_capture_entry_test.dart index 8b23411bf3e..64f62c5de8a 100644 --- a/app/test/services/capture/stt_mode_capture_entry_test.dart +++ b/app/test/services/capture/stt_mode_capture_entry_test.dart @@ -56,7 +56,7 @@ void main() { flagReader: () => true, allowanceReader: () => const TranscriptionAllowanceSnapshot(mode: 'on_device', reason: 'plan_allowance_exhausted'), - readinessReader: () async => FreemiumReadiness.ready, + readinessReader: ({String? language}) async => FreemiumReadiness.ready, onDeviceConfigBuilder: () => const CustomSttConfig(provider: SttProvider.onDeviceWhisper, identity: SttModeResolver.freemiumOnDeviceId), ); @@ -73,7 +73,7 @@ void main() { SttModeResolver.instance = SttModeResolver( flagReader: () => true, allowanceReader: () => const TranscriptionAllowanceSnapshot(mode: 'on_device', reason: 'subscription_inactive'), - readinessReader: () async => FreemiumReadiness.requiresSetup, + readinessReader: ({String? language}) async => FreemiumReadiness.requiresSetup, onDeviceConfigBuilder: () => const CustomSttConfig(provider: SttProvider.onDeviceWhisper), ); final capture = _RecordingSocketCapture(); @@ -87,7 +87,7 @@ void main() { flagReader: () => true, allowanceReader: () => const TranscriptionAllowanceSnapshot(mode: 'on_device', reason: 'plan_allowance_exhausted'), - readinessReader: () async => FreemiumReadiness.ready, + readinessReader: ({String? language}) async => FreemiumReadiness.ready, onDeviceConfigBuilder: () => const CustomSttConfig(provider: SttProvider.onDeviceWhisper, identity: SttModeResolver.freemiumOnDeviceId), ); @@ -102,7 +102,7 @@ void main() { flagReader: () => false, allowanceReader: () => const TranscriptionAllowanceSnapshot(mode: 'on_device', reason: 'plan_allowance_exhausted'), - readinessReader: () async => FreemiumReadiness.ready, + readinessReader: ({String? language}) async => FreemiumReadiness.ready, onDeviceConfigBuilder: () => const CustomSttConfig(provider: SttProvider.onDeviceWhisper), ); final capture = _RecordingSocketCapture(); diff --git a/app/test/services/sockets/on_device_whisper_chunking_test.dart b/app/test/services/sockets/on_device_whisper_chunking_test.dart new file mode 100644 index 00000000000..f973a60e7d4 --- /dev/null +++ b/app/test/services/sockets/on_device_whisper_chunking_test.dart @@ -0,0 +1,73 @@ +import 'dart:typed_data'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:omi/models/stt_result.dart'; +import 'package:omi/services/sockets/on_device_whisper_provider.dart'; +import 'package:omi/utils/audio/wav_bytes.dart'; + +class RecordingWhisper extends OnDeviceWhisperProvider { + RecordingWhisper({this.failAt, this.silentTailChunks = const {}}) : super(modelPath: '/unused'); + final int? failAt; + + /// Chunks at these indices return null (silence / quality-gate rejection). + final Set silentTailChunks; + final calls = []; + @override + Future transcribeChunk( + Uint8List data, { + double audioOffsetSeconds = 0, + String? language, + }) async { + if (calls.length == failAt) throw StateError('decoder failed'); + final duration = OnDeviceWhisperProvider.audioDurationSeconds(data); + final index = calls.length; + calls.add(duration); + expect(language, 'de'); + if (silentTailChunks.contains(index)) return null; + return SttTranscriptionResult(rawText: 'chunk', segments: [ + SttSegment(text: 'chunk', start: audioOffsetSeconds, end: audioOffsetSeconds + duration), + ]); + } +} + +void main() { + for (final rate in [8000, 16000]) { + test('65 seconds at $rate Hz decode as 30/30/5 with exact offsets and custody', () async { + final pcm = Uint8List(rate * 2 * 65); + for (var i = 0; i < pcm.length; i++) { + pcm[i] = i % 251; + } + final wav = WavBytes.fromPcm(pcm, sampleRate: rate, numChannels: 1).asBytes(); + final parts = OnDeviceWhisperProvider.boundedAudioChunks(wav).toList(); + final restored = BytesBuilder(); + for (final part in parts) { + restored.add(part.sublist(44)); + } + expect(restored.toBytes(), pcm); + final provider = RecordingWhisper(); + final result = await provider.transcribe(wav, language: 'de', audioOffsetSeconds: 10); + expect(provider.calls, [30, 30, 5]); + expect(result!.segments.map((s) => s.start), [10, 40, 70]); + expect(result.segments.last.end, 75); + }); + } + test('failed middle chunk propagates for whole-buffer retry', () async { + final wav = WavBytes.fromPcm(Uint8List(16000 * 2 * 65), sampleRate: 16000, numChannels: 1).asBytes(); + await expectLater(RecordingWhisper(failAt: 1).transcribe(wav, language: 'de'), throwsStateError); + }); + test('short recordings stay one call', () async { + final provider = RecordingWhisper(); + final wav = WavBytes.fromPcm(Uint8List(32000 * 5), sampleRate: 16000, numChannels: 1).asBytes(); + await provider.transcribe(wav, language: 'de'); + expect(provider.calls, [5]); + }); + test('a silent trailing chunk still advances the consumed end', () async { + // 65s → chunks 30/30/5; the final 5s chunk returns null (trailing silence). + final provider = RecordingWhisper(silentTailChunks: {2}); + final wav = WavBytes.fromPcm(Uint8List(16000 * 2 * 65), sampleRate: 16000, numChannels: 1).asBytes(); + final result = await provider.transcribe(wav, language: 'de', audioOffsetSeconds: 10); + expect(provider.calls, [30, 30, 5]); + // Consumed silence advances the polling clock without stretching speech. + expect(result!.audioEndSeconds, 75); + expect(result.segments.last.end, 70); + }); +} diff --git a/app/test/unit/apple_speech_analyzer_gate_test.dart b/app/test/unit/apple_speech_analyzer_gate_test.dart new file mode 100644 index 00000000000..b7bb48374e5 --- /dev/null +++ b/app/test/unit/apple_speech_analyzer_gate_test.dart @@ -0,0 +1,79 @@ +import 'dart:io'; +import 'dart:typed_data'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:omi/services/experiments/apple_speech_analyzer_gate.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/utils/audio/wav_bytes.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + const channel = MethodChannel('com.omi.ios/speech'); + final messenger = TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + tearDown(() => messenger.setMockMethodCallHandler(channel, null)); + + test('missing/false kill switch keeps improved engine on', () async { + expect( + await AppleSpeechAnalyzerGate.useNewEngine(readFlag: (key) async { + expect(key, 'mobile-speech-analyzer-disabled'); + return false; + }), + isTrue); + }); + test('true kill switch rolls back to SFSpeech', () async { + expect(await AppleSpeechAnalyzerGate.useNewEngine(readFlag: (_) async => true), isFalse); + }); + test('unavailable flag read retains new-engine default', () async { + expect(await AppleSpeechAnalyzerGate.useNewEngine(readFlag: (_) async => throw Exception('offline')), isTrue); + }); + for (final enabled in [true, false]) { + test('availability passes engine choice $enabled to native', () async { + messenger.setMockMethodCallHandler(channel, (call) async { + expect(call.method, 'onDeviceAvailable'); + expect(call.arguments['useSpeechAnalyzer'], enabled); + expect(call.arguments['language'], 'en'); + return true; // Either analyzer or legacy availability can satisfy this. + }); + expect(await OnDeviceAppleProvider.isOnDeviceAvailable('en', isIOS: true, useSpeechAnalyzer: () async => enabled), + isTrue); + }); + } + for (final available in [false, null]) { + test('native unavailable or unsupported reports $available safely', () async { + messenger.setMockMethodCallHandler(channel, (_) async => available); + expect(await OnDeviceAppleProvider.isOnDeviceAvailable('zz', isIOS: true, useSpeechAnalyzer: () async => true), + isFalse); + }); + } + test('availability channel failure and non-iOS are unavailable', () async { + messenger.setMockMethodCallHandler(channel, (_) async => throw PlatformException(code: 'unavailable')); + expect(await OnDeviceAppleProvider.isOnDeviceAvailable('en', isIOS: true), isFalse); + expect(await OnDeviceAppleProvider.isOnDeviceAvailable('en', isIOS: false), isFalse); + }); + for (final enabled in [true, false]) { + test('transcription passes engine choice $enabled and cleans up audio', () async { + final dir = await Directory.systemTemp.createTemp('speech-engine-choice-'); + const pathChannel = MethodChannel('plugins.flutter.io/path_provider'); + messenger.setMockMethodCallHandler(pathChannel, (_) async => dir.path); + messenger.setMockMethodCallHandler(channel, (call) async { + expect(call.arguments['useSpeechAnalyzer'], enabled); + expect(File(call.arguments['filePath'] as String).existsSync(), isTrue); + return 'We are testing speech recognition today.'; + }); + try { + final pcm = Uint8List(32000); + final samples = ByteData.sublistView(pcm); + for (var i = 0; i < pcm.length; i += 2) { + samples.setInt16(i, i % 4 == 0 ? 9000 : -9000, Endian.little); + } + final wav = WavBytes.fromPcm(pcm, sampleRate: 16000, numChannels: 1).asBytes(); + final result = await OnDeviceAppleProvider(useSpeechAnalyzer: () async => enabled).transcribe(wav); + expect(result!.rawText, contains('testing speech')); + expect(await dir.list().toList(), isEmpty); + } finally { + messenger.setMockMethodCallHandler(pathChannel, null); + await dir.delete(recursive: true); + } + }); + } +} diff --git a/app/test/unit/composite_transcription_socket_test.dart b/app/test/unit/composite_transcription_socket_test.dart index 647d57b150f..f2cb25ff1dd 100644 --- a/app/test/unit/composite_transcription_socket_test.dart +++ b/app/test/unit/composite_transcription_socket_test.dart @@ -1,3 +1,4 @@ +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'dart:convert'; import 'dart:typed_data'; @@ -22,6 +23,22 @@ void main() { await SharedPreferencesUtil.init(); }); + test('permanent local failure keeps capture forwarding and secondary socket alive', () async { + final primary = _FakeSocket(); + final secondary = _FakeSocket(); + final socket = CompositeTranscriptionSocket( + primarySocket: primary, secondarySocket: secondary, forwardRawAudioToSecondary: false); + await socket.connect(); + primary.onError(const LocalSttFailure('permission_denied', canProbeRecovery: true), StackTrace.current); + expect(socket.status, PureSocketStatus.connected); + expect(secondary.status, PureSocketStatus.connected); + final audio = Uint8List.fromList([1, 2, 3]); + socket.send(audio); + expect(primary.sent, [same(audio)]); + expect(secondary.sent, isEmpty); + await socket.stop(); + }); + test('does not send input frames to the Omi socket when disabled', () async { final primary = _FakeSocket(); final secondary = _FakeSocket(); diff --git a/app/test/unit/local_stt_recovery_test.dart b/app/test/unit/local_stt_recovery_test.dart new file mode 100644 index 00000000000..aa0f0a39fe7 --- /dev/null +++ b/app/test/unit/local_stt_recovery_test.dart @@ -0,0 +1,254 @@ +import 'package:flutter/foundation.dart'; +import 'package:omi/models/custom_stt_config.dart'; +import 'package:omi/models/stt_provider.dart'; +import 'package:omi/services/capture/stt_mode_resolver.dart'; +import 'package:omi/backend/schema/bt_device/bt_device.dart'; +import 'dart:io'; +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:omi/backend/preferences.dart'; +import 'package:omi/models/stt_result.dart'; +import 'package:omi/services/freemium_transcription_service.dart'; +import 'package:omi/services/sockets/on_device_apple_provider.dart'; +import 'package:omi/services/sockets/apple_speech_readiness.dart'; +import 'package:omi/services/sockets/on_device_whisper_provider.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/services/sockets/pure_polling.dart'; +import 'package:omi/services/sockets/pure_socket.dart'; +import 'package:omi/utils/audio/audio_transcoder.dart'; +import 'package:omi/utils/audio/wav_bytes.dart'; +import 'package:omi/utils/audio/whisper_pcm.dart'; + +class Listener implements IPureSocketListener { + final errors = []; + final messages = []; + @override + void onConnected() {} + @override + void onClosed([int? code]) {} + @override + void onError(Object err, StackTrace trace) { + errors.add(err); + } + + @override + void onMessage(dynamic message) { + messages.add(message); + } +} + +class SpeechThenSilence extends OnDeviceWhisperProvider { + SpeechThenSilence() : super(modelPath: '/unused'); + final offsets = []; + @override + Future transcribeChunk(Uint8List bytes, + {double audioOffsetSeconds = 0, String? language}) async { + offsets.add(audioOffsetSeconds); + return offsets.length == 1 + ? SttTranscriptionResult( + segments: [SttSegment(text: 'speech', start: audioOffsetSeconds, end: audioOffsetSeconds + 30)]) + : null; + } +} + +class PausedSpeech implements ISttProvider, RecoverableLocalSttProvider { + bool ready = false; + final offsets = []; + @override + Future isReadyToRetry() async => ready; + @override + Future transcribe(Uint8List bytes, {double audioOffsetSeconds = 0}) async { + if (!ready) throw const LocalSttFailure('permission_denied', canProbeRecovery: true); + offsets.add(audioOffsetSeconds); + final seconds = PassThroughTranscoder().sourceSeconds(bytes); + return SttTranscriptionResult( + audioEndSeconds: audioOffsetSeconds + seconds, + segments: [SttSegment(text: 'recovered', start: audioOffsetSeconds, end: audioOffsetSeconds + seconds)]); + } + + @override + void dispose() {} +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + setUp(() async { + SharedPreferences.setMockInitialValues({}); + await SharedPreferencesUtil.init(); + }); + test('real polling consumes speech then 35 seconds silence, then more silence', () async { + final provider = SpeechThenSilence(); + final socket = PurePollingSocket(config: const AudioPollingConfig(minBufferSizeBytes: 1), sttProvider: provider); + await socket.connect(); + socket.send(WavBytes.fromPcm(Uint8List(32000 * 65), sampleRate: 16000, numChannels: 1).asBytes()); + await socket.flushNow(); + expect(socket.audioOffset, 65); + socket.send(WavBytes.fromPcm(Uint8List(32000 * 5), sampleRate: 16000, numChannels: 1).asBytes()); + await socket.flushNow(); + expect(provider.offsets, [0, 30, 60, 65]); + expect(socket.audioOffset, 70); + socket.stop(); + }); + for (final rate in [8000, 16000]) { + test('real polling overflow advances source clock before recovery at $rate Hz', () async { + final provider = PausedSpeech(); + final listener = Listener(); + final socket = PurePollingSocket( + config: AudioPollingConfig( + minBufferSizeBytes: 1, maxBufferBytes: rate * 2 * 2, transcoder: PcmToWavTranscoder(sampleRate: rate)), + sttProvider: provider); + socket.setListener(listener); + await socket.connect(); + socket.setAudioOffset(10); + socket.send(Uint8List(rate * 2)); + await socket.flushNow(); // Actual failure retains the first second. + socket.send(Uint8List(rate * 2)); + socket.send(Uint8List(rate * 2)); // Drop exactly the oldest source second. + expect(socket.audioOffset, 11); + expect(socket.bufferedBytes, rate * 4); + provider.ready = true; + await socket.retryLocalTranscription(); + expect(provider.offsets, [11]); + expect(listener.messages.single, contains('"start":11')); + expect(socket.audioOffset, 13); + await socket.stop(); + }); + } + test('Opus source clock counts TOC duration, not compressed byte size', () { + expect(opusPacketSeconds(Uint8List.fromList([0x98, 0])), 0.02); + expect(opusPacketSeconds(Uint8List.fromList([0x99, 0, 0, 0])), 0.04); + expect(opusPacketSeconds(Uint8List.fromList([0x9b, 3, 0])), 0.06); + }); + for (final configuredSupported in [true, false]) { + test('explicit language readiness overrides primary in both directions: $configuredSupported', () async { + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + addTearDown(() => debugDefaultTargetPlatformOverride = null); + final configured = configuredSupported ? 'en' : 'zz'; + SharedPreferencesUtil().userPrimaryLanguage = configuredSupported ? 'zz' : 'en'; + final config = CustomSttConfig(provider: SttProvider.onDeviceWhisper, language: configured); + await SharedPreferencesUtil().saveCustomSttConfig(config); + final probed = []; + final service = FreemiumTranscriptionService.testing( + iosOverride: true, + appleReadinessReader: (language) async { + probed.add(language); + return AppleSpeechReadiness('speech_analyzer', language == 'en' ? 'ready' : 'unavailable'); + }); + final resolver = SttModeResolver(flagReader: () => false, readinessReader: service.checkReadiness); + final decision = await resolver.decide(persistedCustomStt: config, codec: BleAudioCodec.pcm16); + expect(probed, [configured]); + expect(decision.blockSocket, !configuredSupported); + // Default readiness/preparation callers use the same configured language. + await service.checkReadiness(); + expect(probed.last, configured); + expect(FreemiumTranscriptionService.effectiveLanguage(), configured); + }); + } + test('pcm8 and stereo normalize before native entry without changing duration', () { + for (final rate in [8000, 16000, 44100, 48000]) { + for (final channels in [1, 2]) { + final pcm = Uint8List(rate * channels * 2); + for (var i = 0; i < pcm.length; i += 2) { + ByteData.sublistView(pcm).setInt16(i, 12000, Endian.little); + } + final result = WhisperPcm.normalize(WavBytes.fromPcm(pcm, sampleRate: rate, numChannels: channels).asBytes()); + final header = ByteData.sublistView(result.wav); + expect(header.getUint32(24, Endian.little), 16000); + expect(header.getUint16(22, Endian.little), 1); + expect(result.duration, 1); + expect(result.wav.length, 32044); + expect(header.getInt16(44, Endian.little), 12000); + } + } + }); + test('actual malformed-input provider path pauses polling and retains frames', () async { + final socket = PurePollingSocket( + config: const AudioPollingConfig(minBufferSizeBytes: 1), + sttProvider: OnDeviceWhisperProvider(modelPath: '/unused')); + final listener = Listener(); + socket.setListener(listener); + await socket.connect(); + socket.send(Uint8List.fromList([1, 2, 3])); + await socket.flushNow(); + expect(socket.localFailure!.code, 'invalid_audio'); + expect(socket.bufferedBytes, 3); + expect(socket.status, PureSocketStatus.connected); + await socket.flushNow(); + await socket.flushNow(); + expect(listener.errors.length, 1, reason: 'permanent bad input never loops through native allocation'); + socket.stop(); + }); + for (final denied in ['permission_denied', 'unavailable']) { + test('actual Apple provider rollback $denied preserves audio and recovers without reconnect', () async { + final temp = await Directory.systemTemp.createTemp('speech-recovery'); + const pathChannel = MethodChannel('plugins.flutter.io/path_provider'); + const speechChannel = MethodChannel('com.omi.ios/speech'); + final messenger = TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + var rollback = false; + var calls = 0; + messenger.setMockMethodCallHandler(pathChannel, (_) async => temp.path); + messenger.setMockMethodCallHandler(speechChannel, (call) async { + if (call.method == 'readiness') { + return {'engine': rollback ? 'sfspeech' : 'speech_analyzer', 'status': rollback ? denied : 'ready'}; + } + calls++; + if (rollback) { + throw PlatformException( + code: denied == 'permission_denied' ? 'UNAUTHORIZED' : 'UNAVAILABLE', details: {'status': denied}); + } + return 'We are testing local speech today.'; + }); + final provider = OnDeviceAppleProvider(useSpeechAnalyzer: () async => !rollback); + final service = FreemiumTranscriptionService.testing( + iosOverride: true, + appleReadinessReader: (lang) => + OnDeviceAppleProvider.readiness(lang, useSpeechAnalyzer: () async => !rollback)); + final socket = PurePollingSocket( + config: AudioPollingConfig(minBufferSizeBytes: 1, transcoder: PcmToWavTranscoder()), sttProvider: provider); + final listener = Listener(); + socket.setListener(listener); + try { + expect(await service.checkReadiness(), FreemiumReadiness.ready); + await socket.connect(); + rollback = true; + expect(await service.checkReadiness(), FreemiumReadiness.requiresSetup); + expect(service.getFreemiumConfig(), isNull); + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + final resolver = SttModeResolver(flagReader: () => false, readinessReader: service.checkReadiness); + final decision = await resolver.decide( + persistedCustomStt: const CustomSttConfig(provider: SttProvider.onDeviceWhisper), + codec: BleAudioCodec.pcm16); + expect(decision.blockSocket, isTrue, + reason: 'today’s persisted free fallback checks readiness even with plan flag off'); + expect(decision.allowanceOnDevice, isTrue); + debugDefaultTargetPlatformOverride = null; + final pcm = Uint8List(32000); + for (var i = 0; i < pcm.length; i += 2) { + ByteData.sublistView(pcm).setInt16(i, i % 4 == 0 ? 9000 : -9000, Endian.little); + } + socket.send(pcm); + await socket.flushNow(); + expect(socket.localFailure!.code, denied); + expect(socket.bufferedBytes, pcm.length); + expect(listener.errors.single, isA()); + await socket.flushNow(); + expect(calls, 1); + socket.send(pcm); + expect(socket.bufferedBytes, pcm.length * 2); + rollback = false; + await socket.retryLocalTranscription(); + expect(socket.localFailure, isNull); + expect(socket.bufferedBytes, 0); + expect(listener.messages, hasLength(1)); + expect(calls, 2); + } finally { + debugDefaultTargetPlatformOverride = null; + socket.stop(); + messenger.setMockMethodCallHandler(pathChannel, null); + messenger.setMockMethodCallHandler(speechChannel, null); + await temp.delete(recursive: true); + } + }); + } +} diff --git a/app/test/unit/local_wal_sync_test.dart b/app/test/unit/local_wal_sync_test.dart index 46f100389e1..cd6b48d6321 100644 --- a/app/test/unit/local_wal_sync_test.dart +++ b/app/test/unit/local_wal_sync_test.dart @@ -1,3 +1,6 @@ +import 'package:omi/models/custom_stt_config.dart'; +import 'package:omi/models/stt_provider.dart'; +import 'package:omi/services/capture/local_stt_custody.dart'; import 'dart:io'; import 'dart:convert'; import 'dart:typed_data'; @@ -57,6 +60,7 @@ void main() { }, ); + await WalFileManager.init(); listener = _MockListener(); sync = LocalWalSyncImpl(listener); }); @@ -67,6 +71,83 @@ void main() { SyncJobPoller.instance.resetForTesting(); }); + for (final origin in [WalStorage.sdcard, WalStorage.flashPage]) { + for (final legacyStamp in [false, true]) { + test( + 'historical $origin (legacy=$legacyStamp) import while local remains uploadable after route change and explicit retry', + () async { + final directory = await Directory.systemTemp.createTemp('historical-import'); + addTearDown(() => directory.delete(recursive: true)); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + const MethodChannel('plugins.flutter.io/path_provider'), (_) async => directory.path); + var uploads = 0; + var accept = false; + final persisted = >[]; + final local = LocalWalSyncImpl(_MockListener(), + loadWals: () async => [], + persistWals: (wals) async { + persisted.clear(); + persisted.addAll(wals.map((w) => w.toJson())); + }, + uploadGate: SyncUploadGate( + limiter: SyncRateLimiter.instance, + fairUseStatusLoader: () async => null, + uploader: (files, + {onUploadProgress, + conversationId, + captureEvidence, + recordingSessionId, + audioStartSeconds, + audioEndSeconds, + claimLiveCapture = false, + geolocation}) async { + uploads++; + if (!accept) throw StateError('synthetic refused upload'); + return UploadFilesResult.queued('historical-job'); + })); + addTearDown(local.stop); + LocalSttCustody.selectedLocalRoute = true; + addTearDown(() => LocalSttCustody.selectedLocalRoute = false); + local.start(); + await local.walReady; + const name = 'historical_audio.bin'; + await File('${directory.path}/$name').writeAsBytes([1, 2, 3]); + final imported = Wal( + timerStart: 100, + codec: BleAudioCodec.pcm16, + seconds: 1, + storage: WalStorage.disk, + originalStorage: origin, + onDeviceOnly: legacyStamp, + filePath: name, + status: WalStatus.miss); + final historical = legacyStamp ? Wal.fromJson(imported.toJson()) : imported; + await local.addExternalWal(historical, admittedGeneration: local.sessionGeneration); + expect(historical.onDeviceOnly, isFalse); + LocalSttCustody.selectedLocalRoute = false; + expect(isAutoUploadEligible(historical), isTrue); + await local.syncAll(); + expect(uploads, 1); + accept = true; + await local.syncWal(wal: historical); + expect(uploads, 2, reason: 'explicit retry reaches the real uploader after a refused upload'); + expect(historical.status, WalStatus.uploaded); + expect(persisted.single['on_device_only'], isFalse); + // Repair a pre-fix persisted stamp without weakening recording-time local consent. + final old = historical.toJson()..['on_device_only'] = true; + old.remove('captured_on_device_only'); + final repaired = Wal.fromJson(old); + expect(repaired.onDeviceOnly, isFalse); + expect(repaired.toJson()['on_device_only'], isFalse); + old['captured_on_device_only'] = true; + expect(Wal.fromJson(old).onDeviceOnly, isTrue); + old.remove('original_storage'); + old.remove('captured_on_device_only'); + expect(Wal.fromJson(old).onDeviceOnly, isTrue, reason: 'legacy phone-local custody is preserved'); + }); + } + } + group('onFrameCaptured', () { test('adds frame with synced=false', () { final frame = WalFrame(payload: [0xAA, 0xBB], syncKey: FrameSyncKey([1])); @@ -175,6 +256,36 @@ void main() { }); group('markFrameSynced', () { + test('local enqueue is not a receipt: audio persists phone-only and cannot cloud-upload', () async { + LocalSttCustody.selectedLocalRoute = true; + await SharedPreferencesUtil() + .saveCustomSttConfig(const CustomSttConfig(provider: SttProvider.onDeviceWhisper, sendRawAudioToOmi: false)); + addTearDown(() async { + LocalSttCustody.selectedLocalRoute = false; + await SharedPreferencesUtil().saveCustomSttConfig(CustomSttConfig.defaultConfig); + }); + await sync.onAudioCodecChanged(BleAudioCodec.pcm16); + sync.setDeviceInfo('phone', 'Phone'); + for (var i = 0; i < 100; i++) { + final key = FrameSyncKey([i]); + sync.onFrameCaptured(WalFrame(payload: List.filled(320, i), syncKey: key)); + sync.markFrameSynced(key); + } + expect(sync.testFrameSynced, everyElement(false)); + await sync.finalizeCurrentSession(); + LocalSttCustody.selectedLocalRoute = false; + final wals = await sync.getAllWals(); + expect(wals, isNotEmpty); + for (final wal in wals) { + expect(wal.onDeviceOnly, isTrue); + expect(wal.capturedOnDeviceOnly, isTrue); + expect(isAutoUploadEligible(wal), isFalse); + expect(await sync.syncWal(wal: wal), isNull); + expect(Wal.fromJson(wal.toJson()).onDeviceOnly, isTrue); + expect(await sync.hasDurableWal(wal, admittedGeneration: sync.sessionGeneration), isTrue); + } + }); + test('marks matching frame as synced', () { final key = FrameSyncKey([0x10, 0x20, 0x30]); sync.onFrameCaptured(WalFrame(payload: [1], syncKey: key)); diff --git a/app/test/unit/on_device_transcription_upgrade_test.dart b/app/test/unit/on_device_transcription_upgrade_test.dart index 380d523ea50..86ad71499e4 100644 --- a/app/test/unit/on_device_transcription_upgrade_test.dart +++ b/app/test/unit/on_device_transcription_upgrade_test.dart @@ -92,7 +92,8 @@ void main() { final result = await provider.transcribe(_silentPcmWav()); - expect(result, isNull); + expect(result!.isEmpty, isTrue); + expect(result.audioEndSeconds, closeTo(0.1, 0.001)); } finally { TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( pathProviderChannel, diff --git a/app/test/widgets/capture_home_ui_test.dart b/app/test/widgets/capture_home_ui_test.dart index b04a71a2fde..c43e81f9c59 100644 --- a/app/test/widgets/capture_home_ui_test.dart +++ b/app/test/widgets/capture_home_ui_test.dart @@ -18,6 +18,7 @@ import 'package:omi/pages/conversations/widgets/processing_capture.dart'; import 'package:omi/pages/conversation_capturing/page.dart'; import 'package:omi/pages/home/widgets/battery_info_widget.dart'; import 'package:omi/providers/capture_provider.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; import 'package:omi/providers/connectivity_provider.dart'; import 'package:omi/providers/device_provider.dart'; import 'package:omi/providers/phone_call_provider.dart'; @@ -33,6 +34,11 @@ class _Capture extends ChangeNotifier implements CaptureProvider { this.interrupted = false, this.callActive = false, this.readerPaused = false}); + @override + LocalSttFailure? localSttFailure; + int localRetries = 0; + @override + Future retryLocalTranscription() async => localRetries++; bool verified = true; int? offlineElapsedOverride; @override @@ -212,6 +218,21 @@ void main() { await tester.pump(); } + testWidgets('permanent local failure keeps phone capture visible with saved audio and a working retry', + (tester) async { + final capture = _Capture(_Live.phone) + ..localSttFailure = const LocalSttFailure('permission_denied', canProbeRecovery: true); + await pump(tester, const ConversationCaptureWidget(showsCall: true), capture: capture); + expect(find.text(en.transcriptionUnavailableRecordingSaved), findsWidgets); + expect(find.text(en.openSettings), findsOneWidget); + await tester.tap(find.byKey(const ValueKey('local_stt_retry'))); + await tester.pumpAndSettle(); + expect(capture.localRetries, 1); + expect(capture.recordingState, RecordingState.record); + expect(capture.liveCaptureSource, 'phone'); + expect(tester.takeException(), isNull); + }); + group('live card', () { testWidgets('names the source as a glyph, the state and the latest line, with Pause', (tester) async { final capture = _Capture(_Live.pendant); diff --git a/app/test/widgets/conversation_locked_preview_test.dart b/app/test/widgets/conversation_locked_preview_test.dart index c1d1a462abb..bcbf2cc34da 100644 --- a/app/test/widgets/conversation_locked_preview_test.dart +++ b/app/test/widgets/conversation_locked_preview_test.dart @@ -14,7 +14,8 @@ import 'package:omi/providers/usage_provider.dart'; import 'package:omi/ui/ui.dart'; const _title = 'Planning the next team meeting'; -const _upgrade = 'Upgrade to Unlimited'; +// The locked preview routes to the usage page, so its action says so. +const _upgrade = 'View Usage'; class _Routes extends NavigatorObserver { final pushed = >[]; @@ -23,6 +24,13 @@ class _Routes extends NavigatorObserver { void didPush(Route route, Route? previousRoute) => pushed.add(route); } +class _Usage extends UsageProvider { + _Usage(this.visible); + final bool visible; + @override + bool get showSubscriptionUI => visible; +} + void main() { late ConversationProvider conversations; late UsageProvider usage; @@ -103,6 +111,9 @@ void main() { expect(find.descendant(of: find.byType(ImageFiltered), matching: find.text(_title)), findsOneWidget); expect(find.descendant(of: find.byType(ImageFiltered), matching: find.text(_upgrade)), findsNothing); expect(find.byType(BackdropFilter), findsNothing); + expect(find.text('Pending'), findsOneWidget); + expect(find.byIcon(Icons.lock_outline), findsNothing); + expect(find.byIcon(Icons.hourglass_empty), findsOneWidget); expect(find.bySemanticsLabel(RegExp(_title)), findsNothing); expect(find.bySemanticsLabel(_upgrade), findsOneWidget); @@ -139,7 +150,7 @@ void main() { expect(find.text(_upgrade), findsNothing); }); - testWidgets('the upgrade action keeps the existing paywall destination', (tester) async { + testWidgets('the waiting preview opens usage without forcing a paywall', (tester) async { final routes = _Routes(); await pumpRow(tester, observer: routes); await tester.tap(find.byKey(const Key('locked_preview_action'))); @@ -147,7 +158,7 @@ void main() { final route = routes.pushed.last as MaterialPageRoute; final destination = route.builder(tester.element(find.byType(ConversationListItem))); expect(destination, isA()); - expect((destination as UsagePage).showUpgradeDialog, isTrue); + expect((destination as UsagePage).showUpgradeDialog, isFalse); // Check navigation without building the network-backed usage screen. await tester.pumpWidget(const SizedBox.shrink()); }); @@ -214,7 +225,9 @@ void main() { tester, body: LockedConversationRun(conversations: run, date: run.first.createdAt), ); - await tester.longPress(find.text('$_title 1')); + final row = tester.getRect(find.byKey(const ValueKey('locked_locked-1'))); + // The central usage action is interactive; the leading edge still opens this row's menu. + await tester.longPressAt(Offset(row.left + 16, row.center.dy)); await tester.pumpAndSettle(); expect(find.byKey(const Key('conversation_action_delete')), findsOneWidget); }); @@ -235,4 +248,55 @@ void main() { expect(conversations.selectedConversationIds, isEmpty); expect(tester.takeException(), isNull); }); + for (final visible in [true, false]) { + for (final grouped in [true, false]) { + testWidgets('frost tap on ${grouped ? 'grouped' : 'single'} held conversation visibility $visible', + (tester) async { + usage.dispose(); + usage = _Usage(visible); + final routes = _Routes(); + final run = lockedRun(); + await pumpRow(tester, + observer: routes, + body: grouped ? LockedConversationRun(conversations: run, date: run.first.createdAt) : null); + final rect = + tester.getRect(grouped ? find.byKey(const ValueKey('locked_locked-1')) : find.byType(OmiLockedPreview)); + await tester.tapAt(Offset(rect.left + 16, rect.center.dy)); + expect(routes.pushed, hasLength(visible ? 2 : 1)); + if (visible) { + expect((routes.pushed.last as MaterialPageRoute).builder(tester.element(find.byType(OmiLockedPreview))), + isA()); + await tester.pumpWidget(const SizedBox.shrink()); + } else { + await tester.pumpAndSettle(); + expect(find.byType(UsagePage), findsNothing); + expect(find.bySemanticsLabel(RegExp(_title)), findsNothing); + expect(tester.takeException(), isNull); + } + }); + } + } + for (final visible in [true, false]) { + testWidgets('gap in held conversation run uses guarded action visibility $visible', (tester) async { + usage.dispose(); + usage = _Usage(visible); + final routes = _Routes(); + final run = lockedRun(); + await pumpRow(tester, + observer: routes, body: LockedConversationRun(conversations: run, date: run.first.createdAt)); + final first = tester.getRect(find.byKey(const ValueKey('locked_locked-0'))); + final second = tester.getRect(find.byKey(const ValueKey('locked_locked-1'))); + await tester.tapAt(Offset(first.left + 16, (first.bottom + second.top) / 2)); + expect(routes.pushed, hasLength(visible ? 2 : 1)); + if (visible) { + expect((routes.pushed.last as MaterialPageRoute).builder(tester.element(find.byType(LockedConversationRun))), + isA()); + await tester.pumpWidget(const SizedBox.shrink()); + } else { + await tester.pumpAndSettle(); + expect(find.bySemanticsLabel(RegExp(_title)), findsNothing); + expect(tester.takeException(), isNull); + } + }); + } } diff --git a/app/test/widgets/local_stt_recovery_notice_test.dart b/app/test/widgets/local_stt_recovery_notice_test.dart new file mode 100644 index 00000000000..1c6cfc7c4c6 --- /dev/null +++ b/app/test/widgets/local_stt_recovery_notice_test.dart @@ -0,0 +1,90 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:omi/backend/preferences.dart'; +import 'package:omi/models/custom_stt_config.dart'; +import 'package:omi/models/stt_provider.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:omi/l10n/app_localizations.dart'; +import 'package:omi/services/sockets/local_stt_failure.dart'; +import 'package:omi/ui/components/local_stt_recovery_notice.dart'; + +void main() { + testWidgets('permission action probes explicit configured language through actual channel', (tester) async { + SharedPreferences.setMockInitialValues({}); + await SharedPreferencesUtil.init(); + SharedPreferencesUtil().userPrimaryLanguage = 'zz'; + await SharedPreferencesUtil() + .saveCustomSttConfig(const CustomSttConfig(provider: SttProvider.onDeviceWhisper, language: 'en')); + final languages = []; + const channel = MethodChannel('com.omi.ios/speech'); + final messenger = TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + messenger.setMockMethodCallHandler(channel, (call) async { + expect(call.method, 'prepareSpeech'); + languages.add(call.arguments['language'] as String); + return {'engine': 'sfspeech', 'status': 'ready'}; + }); + addTearDown(() => messenger.setMockMethodCallHandler(channel, null)); + var retries = 0; + await tester.pumpWidget(MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: Scaffold( + body: LocalSttRecoveryNotice( + failure: const LocalSttFailure('permission_required'), + onRetry: () async { + retries++; + }, + onSetup: () {})))); + await tester.tap(find.byKey(const ValueKey('local_stt_setup'))); + await tester.pumpAndSettle(); + expect(languages, ['en']); + expect(retries, 1); + }); + for (final status in ['permission_denied', 'permission_required', 'unavailable']) { + testWidgets('$status explains custody and exposes working recovery actions at 200% text', (tester) async { + var settings = 0; + var setup = 0; + var retries = 0; + var requests = 0; + tester.view.physicalSize = const Size(640, 1000); + tester.view.devicePixelRatio = 2; + addTearDown(tester.view.resetPhysicalSize); + addTearDown(tester.view.resetDevicePixelRatio); + await tester.pumpWidget(MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + builder: (context, child) => + MediaQuery(data: MediaQuery.of(context).copyWith(textScaler: const TextScaler.linear(2)), child: child!), + home: Scaffold( + body: SingleChildScrollView( + child: LocalSttRecoveryNotice( + failure: LocalSttFailure(status, canProbeRecovery: true), + onRetry: () async { + retries++; + }, + onSetup: () { + setup++; + }, + openSettings: () async { + settings++; + return true; + }, + preparePermission: () async { + requests++; + return true; + }, + ))))); + expect(find.text('Transcription is unavailable — recording continues and your audio is saved.'), findsOneWidget); + await tester.tap(find.byKey(const ValueKey('local_stt_setup'))); + await tester.pumpAndSettle(); + expect(settings, status == 'permission_denied' ? 1 : 0); + expect(requests, status == 'permission_required' ? 1 : 0); + expect(setup, status == 'unavailable' ? 1 : 0); + await tester.tap(find.byKey(const ValueKey('local_stt_retry'))); + await tester.pumpAndSettle(); + expect(retries, status == 'permission_required' ? 2 : 1); + expect(tester.takeException(), isNull); + }); + } +} diff --git a/app/test/widgets/waiting_memory_preview_test.dart b/app/test/widgets/waiting_memory_preview_test.dart new file mode 100644 index 00000000000..691b4245e4b --- /dev/null +++ b/app/test/widgets/waiting_memory_preview_test.dart @@ -0,0 +1,133 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:provider/provider.dart'; +import 'package:omi/backend/schema/memory.dart'; +import 'package:omi/l10n/app_localizations.dart'; +import 'package:omi/pages/memories/widgets/memory_item.dart'; +import 'package:omi/pages/memories/widgets/memory_edit_sheet.dart'; +import 'package:omi/pages/settings/usage_page.dart'; +import 'package:omi/providers/memories_provider.dart'; +import 'package:omi/providers/usage_provider.dart'; +import 'package:omi/ui/ui.dart'; + +class _Usage extends UsageProvider { + _Usage(this.visible); + final bool visible; + @override + bool get showSubscriptionUI => visible; +} + +class _Routes extends NavigatorObserver { + final pushed = >[]; + @override + void didPush(Route route, Route? previousRoute) => pushed.add(route); +} + +void main() { + testWidgets('short waiting memory fits its status and action at 200% text', (tester) async { + final provider = MemoriesProvider(); + final usage = UsageProvider(); + addTearDown(provider.dispose); + addTearDown(usage.dispose); + final memory = Memory( + id: 'held', + uid: 'fixture', + content: 'OK', + category: MemoryCategory.system, + createdAt: DateTime.utc(2026), + updatedAt: DateTime.utc(2026), + visibility: MemoryVisibility.private, + isLocked: true); + await tester.pumpWidget(ChangeNotifierProvider.value( + value: usage, + child: MaterialApp( + theme: buildOmiTheme(), + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: const [Locale('en')], + builder: (context, child) => + MediaQuery(data: MediaQuery.of(context).copyWith(textScaler: const TextScaler.linear(2)), child: child!), + home: Scaffold( + body: SizedBox( + width: 340, + child: MemoryItem(memory: memory, provider: provider, showDismissible: false, onTap: (_, __, ___) {}))), + ), + )); + expect(find.text('Pending'), findsOneWidget); + expect(find.text('View Usage'), findsOneWidget); + expect(find.byIcon(Icons.lock_outline), findsNothing); + expect(find.bySemanticsLabel('OK'), findsNothing); + final preview = tester.getRect(find.byType(OmiLockedPreview)); + final action = tester.getRect(find.byKey(const Key('locked_preview_action'))); + expect(preview.contains(action.topLeft), isTrue); + expect(preview.contains(action.bottomRight), isTrue); + expect(tester.takeException(), isNull); + }); + for (final visible in [true, false]) { + for (final target in ['frost', 'padding', 'button']) { + testWidgets('held memory $target tap obeys subscription visibility $visible', (tester) async { + final provider = MemoriesProvider(); + final usage = _Usage(visible); + final routes = _Routes(); + addTearDown(provider.dispose); + addTearDown(usage.dispose); + var editCalls = 0; + final memory = Memory( + id: 'held', + uid: 'fixture', + content: 'Private held memory', + category: MemoryCategory.system, + createdAt: DateTime.utc(2026), + updatedAt: DateTime.utc(2026), + visibility: MemoryVisibility.private, + isLocked: true, + ); + await tester.pumpWidget(ChangeNotifierProvider.value( + value: usage, + child: MaterialApp( + theme: buildOmiTheme(), + navigatorObservers: [routes], + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: const [Locale('en')], + home: Scaffold( + body: Align( + alignment: Alignment.topLeft, + child: SizedBox( + width: 340, + child: MemoryItem( + memory: memory, + provider: provider, + showDismissible: false, + onTap: (_, __, ___) => editCalls++)))), + ), + )); + final preview = tester.getRect(find.byType(OmiLockedPreview)); + if (target == 'button') { + await tester.tap(find.byKey(const Key('locked_preview_action'))); + } else { + // Left edge is outside See plans; padding exercises the enclosing row handler. + await tester.tapAt(Offset(preview.left + (target == 'padding' ? -10 : 2), preview.center.dy)); + } + expect(editCalls, 0); + expect(routes.pushed, hasLength(2)); + if (visible) { + final destination = + (routes.pushed.last as MaterialPageRoute).builder(tester.element(find.byType(MemoryItem))); + expect(destination, isA()); + expect((destination as UsagePage).showUpgradeDialog, isFalse); + expect(find.byType(MemoryEditSheet), findsNothing); + expect(find.byType(SelectableText), findsNothing); + expect(find.bySemanticsLabel('Private held memory'), findsNothing); + // Do not build the network-backed usage page. + await tester.pumpWidget(const SizedBox.shrink()); + } else { + await tester.pumpAndSettle(); + expect(tester.widget(find.byType(MemoryEditSheet)).readOnly, isTrue); + expect(find.widgetWithText(SelectableText, 'Private held memory'), findsOneWidget); + expect(find.byKey(const ValueKey('memory_edit_field')), findsNothing); + expect(find.byType(UsagePage), findsNothing); + expect(tester.takeException(), isNull); + } + }); + } + } +} diff --git a/backend/docs/feature-flag-registry.md b/backend/docs/feature-flag-registry.md index ab5ac5eaf7d..464daaaa829 100644 --- a/backend/docs/feature-flag-registry.md +++ b/backend/docs/feature-flag-registry.md @@ -1,4 +1,4 @@ - + # Feature-flag authority registry @@ -113,7 +113,7 @@ entries are exempt: they are queued for removal, not running. ## Overdue for a decision -None as of 2026-10-13. +None as of 2026-10-14. ## Flags @@ -335,6 +335,7 @@ and an explicit empty literal renders as `''`. | `exp-002-desktop-identity-kill-v1` | EXP-002 desktop identity arm emergency stop | backend, macos | posthog | inverted | — | — | — | expected (kill) | keep | — | unowned | | `free-tier-kill-switch-v1` | Remote free-tier stop | backend | posthog | inverted | — | — | — | expected (kill) | keep | — | unowned | | `jit-processing-kill-switch-v1` | JIT remote kill for allowlisted accounts | backend | posthog | inverted | — | — | — | expected (kill) | keep | — | unowned | +| `mobile-speech-analyzer-disabled` | Force existing iOS freemium fallback to SFSpeech; false or absent selects SpeechAnalyzer on supported iOS 26 devices/locales | mobile | posthog | inverted | — | — | — | expected (kill) | keep | — | dazheng | ### config_switch diff --git a/backend/tests/unit/test_subscription_ingestion_characterization.py b/backend/tests/unit/test_subscription_ingestion_characterization.py index 2df85765a86..0c6f7374da1 100644 --- a/backend/tests/unit/test_subscription_ingestion_characterization.py +++ b/backend/tests/unit/test_subscription_ingestion_characterization.py @@ -672,3 +672,26 @@ def test_lifecycle_observation_precedes_entitlement_effects(lane, monkeypatch): monkeypatch.setattr(payment, 'record_subscription_event', lambda **kw: lane.calls.append(('observation', ()))) run(lane) assert [name for name, _ in lane.calls][:2] == ['observation', 'customer'] + + +@pytest.mark.parametrize('ledger_enabled', [False, True]) +@pytest.mark.parametrize('stored_plan', [PlanType.basic, PlanType.plus]) +def test_unresolved_checkout_preserves_cache_and_valid_stored_plan_effects( + lane, monkeypatch, ledger_enabled, stored_plan +): + """#21372: retry admission must follow cache invalidation and valid-paid unlocks.""" + lane.current = Subscription(plan=stored_plan, current_period_end=NOW + 100) + lane.stripe_obj = event(EVENTS[1], 'active')['data']['object'] + lane.event = _checkout_event() + monkeypatch.setattr(payment, '_update_subscription_from_session', lambda uid, session: False) + monkeypatch.setattr(payment, 'webhook_ledger_enabled', lambda: ledger_enabled) + result = asyncio.run(payment._handle_stripe_event(lane.event)) + expected = ['credits', 'trial_cache'] + if stored_plan == PlanType.plus: + expected += ['conversations', 'memories', 'actions', 'fair_use'] + assert _call_names(lane)[: len(expected)] == expected + assert 'write' not in _call_names(lane) + assert lane.current.plan == stored_plan + assert isinstance(result, payment.RetryableBillingSkip) == ledger_enabled + if ledger_enabled: + assert _call_names(lane) == expected diff --git a/config/feature-flags.yaml b/config/feature-flags.yaml index ffbd9030d1d..b3c003e84fb 100644 --- a/config/feature-flags.yaml +++ b/config/feature-flags.yaml @@ -1,4 +1,5 @@ flags: + - {"key":"mobile-speech-analyzer-disabled","aliases":[],"kind":"posthog","lifecycle":"ops_kill","surfaces":["mobile"],"summary":"Force existing iOS freemium fallback to SFSpeech; false or absent selects SpeechAnalyzer on supported iOS 26 devices/locales","fail":"inverted","owner":"dazheng","created":"2026-10-14","decision":"keep","posthog":{"row":"expected","role":"kill"},"notes":"Default false: improved engine ships on. True rolls back both availability probing and transcription without a release. Registering as a kill role lets `scripts/posthog_flag_sync.py --apply-missing-kills` pre-arm the row at 0% so the emergency rollback is available before an incident. Read per operation through the existing identity/consent-fenced analytics adapter; unavailable/offline/tracking-disabled flag reads retain the new-engine default. No remote row is created or changed by this PR."} - {"key":"STT_LIVE_DEMAND_ENABLED","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Default off; qualified English hidden capture uses window STT; visible or active consumers require vendors","fail":"closed","owner":"dazheng","created":"2026-10-14","review_by":"2026-11-13","decision":"pending"} - {"key":"STT_LIVE_DEMAND_PERCENT","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Stable UID live demand allocation, default zero; lead caps first ramp at five percent with a twelve-hour bake","fail":"closed","owner":"dazheng","created":"2026-10-14","review_by":"2026-11-13","decision":"pending"} - {"key":"STT_LIVE_DEMAND_UID_ALLOWLIST","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Explicit live demand canary UIDs, comma separated; empty by default; requires global enable","fail":"closed","owner":"dazheng","created":"2026-10-14","review_by":"2026-11-13","decision":"pending"} diff --git a/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationRowView.swift b/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationRowView.swift index 0ea0bbce235..044a001bb0b 100644 --- a/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationRowView.swift +++ b/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationRowView.swift @@ -387,11 +387,15 @@ struct ConversationRowView: View { // Title + metadata below VStack(alignment: .leading, spacing: OmiSpacing.hairline) { HStack(spacing: OmiSpacing.sm) { - Text(conversation.displayTitle) - .scaledFont(size: OmiType.body, weight: .medium) - .foregroundColor(titleColor) - .lineLimit(1) - .help(conversation.displayTitle) + // The locked badge already says "Saved and waiting"; the title slot + // stays quiet so the pill isn't duplicated beside itself. + if conversation.displayState != .locked { + Text(conversation.displayTitle) + .scaledFont(size: OmiType.body, weight: .medium) + .foregroundColor(titleColor) + .lineLimit(1) + .help(conversation.displayTitle) + } statusCluster(phase: phase) @@ -433,11 +437,14 @@ struct ConversationRowView: View { // Title + time/duration below VStack(alignment: .leading, spacing: OmiSpacing.hairline) { HStack(spacing: OmiSpacing.sm) { - Text(conversation.displayTitle) - .scaledFont(size: OmiType.subheading, weight: .medium) - .foregroundColor(titleColor) - .lineLimit(1) - .help(conversation.displayTitle) + // The locked badge carries the waiting copy; no duplicate title. + if conversation.displayState != .locked { + Text(conversation.displayTitle) + .scaledFont(size: OmiType.subheading, weight: .medium) + .foregroundColor(titleColor) + .lineLimit(1) + .help(conversation.displayTitle) + } statusCluster(phase: phase) diff --git a/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationStatusBadge.swift b/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationStatusBadge.swift index f6744d87e97..f829c381a8b 100644 --- a/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationStatusBadge.swift +++ b/desktop/macos/Desktop/Sources/MainWindow/Components/ConversationStatusBadge.swift @@ -23,10 +23,10 @@ struct ConversationStatusBadge: View { ) case .locked: Pill( - icon: "lock.fill", - text: "Locked", + icon: "hourglass", + text: "Saved and waiting", color: PageGlass.warning, - help: "This conversation is locked until your subscription is active." + help: "This conversation is saved. Check your plan and usage to see when processing is available." ) case .failed: Pill( diff --git a/desktop/macos/Desktop/Sources/MainWindow/Pages/ConversationDetailHeader.swift b/desktop/macos/Desktop/Sources/MainWindow/Pages/ConversationDetailHeader.swift index 0c43ca90f74..5134ee297be 100644 --- a/desktop/macos/Desktop/Sources/MainWindow/Pages/ConversationDetailHeader.swift +++ b/desktop/macos/Desktop/Sources/MainWindow/Pages/ConversationDetailHeader.swift @@ -123,13 +123,16 @@ struct ConversationDetailHeader {isUnknownPlan ? 'Plan unavailable' - : isUnlimited && !isCanceling_ - ? 'Manage Your Plan' + : isPaid && !isCanceling_ + ? currentPlanName : 'Choose Your Plan'} - {isUnlimited && subscription?.current_period_end && ( + {isPaid && subscription?.current_period_end && (

{isCanceling_ ? `Cancels on ${formatDate( @@ -389,6 +393,8 @@ export function PlansSheet({

)} + + {/* Features List */}

@@ -420,7 +426,7 @@ export function PlansSheet({ disabled={ isLoading || !selectedPriceId || - (isUnlimited && !isCanceling_ && selectedOption?.is_active) + (isPaid && !isCanceling_ && selectedOption?.is_active) } className={cn( 'w-full rounded-xl py-3 font-medium transition-colors', @@ -436,7 +442,7 @@ export function PlansSheet({ ) : isCanceling_ ? ( 'Reactivate Subscription' - ) : isUnlimited ? ( + ) : isPaid ? ( selectedOption?.is_active ? ( 'Current Plan' ) : ( @@ -447,7 +453,7 @@ export function PlansSheet({ )} - {!isUnlimited && subscription?.stripe_subscription_id && ( + {!isPaid && subscription?.stripe_subscription_id && ( {subscription?.stripe_subscription_id && (

- {/* Monthly Listening Usage */} -
-
- - - Monthly Listening Limit - -
-
- - {monthlyUsage - ? Math.round(monthlyUsage.transcription_seconds / 60) - : 0} - - / 1,200 min - - - - {monthlyUsage - ? 1200 - Math.round(monthlyUsage.transcription_seconds / 60) - : 1200}{' '} - min left - -
-
-
-
-
+ {/* What's Included - Checklist */}

What's included

-
-
-
- -
- - - 1,200 minutes - {' '} - of listening per month - (limited) - -
-
-
- -
- - Unlimited{' '} - words transcribed - -
-
-
- -
- - Unlimited{' '} - insights - -
-
-
- -
- - Unlimited{' '} - memories - -
-
+
    + {DEFAULT_PLAN_FEATURES.map((feature) => ( +
  • + + {feature} +
    • + ))} +
@@ -1192,9 +1133,7 @@ function UsageSectionContent({

Choose a Plan

-

- Unlock unlimited listening time -

+

Choose your plan