diff --git a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json new file mode 100644 index 00000000000..5bec7819a79 --- /dev/null +++ b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "id": "FC-stt-allowance-not-enforced-at-dispatch", + "violated_contract": "The transcription cap is computed but not enforced where Omi-funded provider audio is dispatched; replay, fallback and delayed execution can bypass request-time allowance checks.", + "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it. Fence provider-bound recovery receipts before side effects and atomically prove ingestion, clear known non-calls, and reconcile ambiguous attempts with one bounded re-dispatch while retaining terminal failures and finalizing siblings. Exercise both flag states at provider and queue boundaries, including oversized empty retries without spending failure attempts when held.", + "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_batch_pipeline_worker.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_sync_cap_recovery_checkpoint.py", "backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py", "backend/tests/unit/test_sync_cap_recovery_plan.py", "backend/tests/unit/test_sync_cap_reconciliation_metrics.py", "backend/tests/unit/test_prerecorded_dispatch_selection.py", "backend/tests/unit/test_sync_cap_content.py"], + "evidence_prs": [], + "scope_hints": ["backend/routers/listen/", "backend/utils/stt/", "backend/utils/sync/", "backend/utils/batch_pipeline/"], + "status": "open" +} diff --git a/app/changelog/unreleased/20261016-stt-cap-response-dto.json b/app/changelog/unreleased/20261016-stt-cap-response-dto.json new file mode 100644 index 00000000000..bb56a275667 --- /dev/null +++ b/app/changelog/unreleased/20261016-stt-cap-response-dto.json @@ -0,0 +1,3 @@ +{ + "kind": "none" +} diff --git a/app/lib/backend/schema/gen/conversation_wire.g.dart b/app/lib/backend/schema/gen/conversation_wire.g.dart index 05dac78c0ad..2e2046fa8f7 100644 --- a/app/lib/backend/schema/gen/conversation_wire.g.dart +++ b/app/lib/backend/schema/gen/conversation_wire.g.dart @@ -1673,6 +1673,7 @@ class GeneratedSearchConversationsResponse { class GeneratedSyncLocalFilesResultResponse { final List? errors; final int failedSegments; + final bool? heldAudio; final List? newMemories; final int totalSegments; final List? updatedMemories; @@ -1680,6 +1681,7 @@ class GeneratedSyncLocalFilesResultResponse { const GeneratedSyncLocalFilesResultResponse({ this.errors, this.failedSegments = 0, + this.heldAudio, this.newMemories, this.totalSegments = 0, this.updatedMemories, @@ -1689,6 +1691,7 @@ class GeneratedSyncLocalFilesResultResponse { return GeneratedSyncLocalFilesResultResponse( errors: _readFieldValue>(_readField(json, const ["errors"]), "errors", _readStringList, requiredField: false, nullable: true), failedSegments: _required(_readFieldValue(_readField(json, const ["failed_segments"]), "failed_segments", _readInt, requiredField: false, nullable: false, defaultValue: 0), "failed_segments"), + heldAudio: _readFieldValue(_readField(json, const ["held_audio"]), "held_audio", _readBool, requiredField: false, nullable: true), newMemories: _readFieldValue>(_readField(json, const ["new_memories"]), "new_memories", _readStringList, requiredField: false, nullable: true), totalSegments: _required(_readFieldValue(_readField(json, const ["total_segments"]), "total_segments", _readInt, requiredField: false, nullable: false, defaultValue: 0), "total_segments"), updatedMemories: _readFieldValue>(_readField(json, const ["updated_memories"]), "updated_memories", _readStringList, requiredField: false, nullable: true), @@ -1699,6 +1702,7 @@ class GeneratedSyncLocalFilesResultResponse { return { 'errors': errors, 'failed_segments': failedSegments, + 'held_audio': heldAudio, 'new_memories': newMemories, 'total_segments': totalSegments, 'updated_memories': updatedMemories, diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 91a59ada47b..ef520fb69ef 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -148,7 +148,7 @@ STT policy: `config/stt_provider_policy.py`. Windowed live rollout: `docs/operat - **monitoring** (`backend/charts/monitoring/`) — Prometheus/Grafana, Loki/Alloy, alerts and HPA adapters. Sync export flags, sizing and CPU audit: [runbook](docs/runbooks/sync-cloudrun-rightsizing.md). - **backend-secrets** (`backend/charts/backend-secrets/`) — ExternalSecret and SecretStore resources that sync backend runtime secrets into GKE namespaces. -[Reset recovery](docs/locked-content-reset-recovery.md). +[Reset recovery](docs/locked-content-reset-recovery.md). Dark funded-STT admission and held-audio recovery: [dispatch contract](docs/stt-cap-dispatch.md). Runtime env contract: keep `backend/deploy/runtime_env.yaml` aligned with GKE Helm values and Cloud Run runtime env; run `backend/scripts/pre-deploy-check.sh` after backend runtime env or deploy workflow changes. The backend deploy must validate the exact JSON state emitted by `render_backend_runtime_env.py --state-output`, and any Cloud Run export/replace editor must preserve those literal values before apply. The `llm_gateway` manifest section owns the release, ingress, and static-address identity; a reserved address alone is never an endpoint contract. Gateway-mode promotion requires the control-plane gate plus `probe-llm-gateway-from-cloud-run.sh` before Cloud Run revisions are created. diff --git a/backend/config/stt_cap_dispatch.py b/backend/config/stt_cap_dispatch.py new file mode 100644 index 00000000000..59f05f96f18 --- /dev/null +++ b/backend/config/stt_cap_dispatch.py @@ -0,0 +1,7 @@ +"""Dark dispatch policy. No environment/deployment changes enable this implicitly.""" + +import os + + +def enabled() -> bool: + return os.getenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false').lower() == 'true' diff --git a/backend/database/sync_cap_holds.py b/backend/database/sync_cap_holds.py new file mode 100644 index 00000000000..011cd82ad15 --- /dev/null +++ b/backend/database/sync_cap_holds.py @@ -0,0 +1,106 @@ +"""Transfer fenced sync-ledger ownership to durable, recoverable audio holds. + +A held upload is acknowledged, but is not proof of completed transcription. +Only settlement of every referenced hold can publish that proof. +""" + +from datetime import datetime, timedelta, timezone +from typing import Any + +from google.cloud import firestore + +from database._client import get_firestore_client +from database.sync_ledger import ( + sync_content_reference, + sync_content_owner_matches, + is_valid_completed_sync_content_result, + is_valid_held_sync_content_result, + LEDGER_RETENTION_DAYS, +) + +PROTOCOL = 'stt_cap_hold_v1' + + +def mark_content_held( + uid: str, + content_id: str, + job_id: str, + result: dict, + hold_ids: list[str], + *, + run_token: str | None = None, + run_epoch: int | None = None, +) -> bool: + if not hold_ids or not is_valid_held_sync_content_result(result): + raise ValueError('durable held audio receipts required') + client = get_firestore_client() + ref = sync_content_reference(client, uid, content_id) + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if not sync_content_owner_matches(row, job_id, run_token, run_epoch): + return False + tx.update( + ref, + { + 'status': 'held', + 'dispatch_protocol': PROTOCOL, + 'result': result, + 'cap_hold_ids': sorted(set(hold_ids)), + 'pending_cap_hold_ids': sorted(set(hold_ids)), + 'updated_at': datetime.now(timezone.utc), + # An unprocessed capture must survive inactive accounts. + 'expires_at': firestore.DELETE_FIELD, + }, + ) + return True + + return commit(client.transaction()) + + +def recovery_status(uid: str, content_id: str, hold_id: str) -> str: + row = sync_content_reference(get_firestore_client(), uid, content_id).get().to_dict() or {} + if row.get('status') == 'completed' and is_valid_completed_sync_content_result(row.get('result')): + return 'completed' + if row.get('dispatch_protocol') == PROTOCOL and row.get('status') == 'held': + if hold_id not in row.get('cap_hold_ids', []): + return 'waiting' + return 'ready' if hold_id in row.get('pending_cap_hold_ids', []) else 'settled' + return 'waiting' # The original worker still owns preparation/retry. + + +def settle_hold(uid: str, content_id: str, hold_id: str, response: dict, pieces: int) -> bool: + """Idempotently join one recovered hold; never overwrite another owner.""" + client = get_firestore_client() + ref = sync_content_reference(client, uid, content_id) + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if row.get('status') == 'completed': + return is_valid_completed_sync_content_result(row.get('result')) + if row.get('dispatch_protocol') != PROTOCOL or row.get('status') != 'held': + return False + if hold_id not in row.get('cap_hold_ids', []): + return False + pending = set(row.get('pending_cap_hold_ids') or []) + if hold_id not in pending: + return True + pending.remove(hold_id) + result = dict(row['result']) + for key in ('new_memories', 'updated_memories'): + result[key] = sorted(set(result.get(key) or []) | set(response.get(key) or [])) + result['recovered_segments'] = int(result.get('recovered_segments', 0)) + pieces + result['held_audio'] = bool(pending) + now = datetime.now(timezone.utc) + updates: dict[str, Any] = {'result': result, 'pending_cap_hold_ids': sorted(pending), 'updated_at': now} + if not pending: + # All source audio has reached content/usage settlement; only now + # can ordinary duplicate admission converge a completed result. + result.update(failed_segments=0, errors=[], outcome='success') + updates.update(status='completed', expires_at=now + timedelta(days=LEDGER_RETENTION_DAYS)) + tx.update(ref, updates) + return True + + return commit(client.transaction()) diff --git a/backend/database/sync_cap_recovery.py b/backend/database/sync_cap_recovery.py new file mode 100644 index 00000000000..cc8d65c42bb --- /dev/null +++ b/backend/database/sync_cap_recovery.py @@ -0,0 +1,73 @@ +"""Piece-lineage proof committed in the exact conversation-ingest transaction.""" + +import time +from typing import Any + +from database._client import get_firestore_client + + +class RecoveryIngestFenced(RuntimeError): + """An expired/replaced recovery attempt cannot append conversation content.""" + + +def ingest_hooks(uid: str, context: dict, incoming: dict): + ref = ( + get_firestore_client() + .collection('users') + .document(uid) + .collection('stt_cap_holds') + .document(context['hold_id']) + ) + current = {} + piece = context['piece'] + + def before(transaction): + row = ref.get(transaction=transaction).to_dict() or {} + receipt = (row.get('piece_receipts') or {}).get(piece) or {} + if ( + row.get('lease_token') != context['lease_token'] + or row.get('lease_until', 0) <= time.time() + or receipt.get('state') != 'dispatched' + or receipt.get('attempt', 1) != context['attempt'] + ): + raise RecoveryIngestFenced('held piece ingest ownership changed') + current.clear() + current.update(row.get('piece_ingests') or {}) + + def after(transaction, assigned): + conversation, created, _ = assigned + cid = conversation['id'] + response: dict[str, Any] = {'new_memories': [], 'updated_memories': []} + repeat = conversation.get('_sync_lineage_repeat_only') + if not repeat: + response['new_memories' if created else 'updated_memories'] = [cid] + if conversation.get('sync_relevance') == 'keep' and ( + not repeat or conversation.get('_sync_lineage_completion_pending') + ): + response['_merged'] = {cid: incoming.get('language')} + current[piece] = { + **context['receipt'], + 'state': 'transcribed', + 'has_content': True, + 'response': response, + 'lineage_anchor': incoming['id'], + 'conversation_id': cid, + } + # No reads here: assignment has already staged its conversation writes. + transaction.update(ref, {'piece_ingests': dict(current)}) + + return before, after + + +def ingested_piece(uid: str, hold_id: str, piece: str) -> dict | None: + row = ( + get_firestore_client() + .collection('users') + .document(uid) + .collection('stt_cap_holds') + .document(hold_id) + .get() + .to_dict() + or {} + ) + return (row.get('piece_ingests') or {}).get(piece) diff --git a/backend/database/sync_ledger.py b/backend/database/sync_ledger.py index a54e57422f3..d22ad59d37e 100644 --- a/backend/database/sync_ledger.py +++ b/backend/database/sync_ledger.py @@ -73,6 +73,7 @@ class SyncContentRunBindingOutcome(str, Enum): BOUND = 'bound' COMPLETED = 'completed' + HELD = 'held' LOST = 'lost' @@ -91,6 +92,10 @@ def bound(self) -> bool: def completed(self) -> bool: return self.outcome is SyncContentRunBindingOutcome.COMPLETED + @property + def held(self) -> bool: + return self.outcome is SyncContentRunBindingOutcome.HELD + def _ledger_ref(client: Any, uid: str, content_id: str) -> Any: collection = sync_stage.collection_name('sync_content_ledger') @@ -122,10 +127,24 @@ def _ledger_owner_matches( return bound_token == run_token and bound_epoch == run_epoch +def sync_content_reference(client: Any, uid: str, content_id: str) -> Any: + """Share the canonical owner-scoped reference with a custody transfer.""" + return _ledger_ref(client, uid, content_id) + + +def sync_content_owner_matches( + existing: Dict[str, Any], job_id: str, run_token: str | None, run_epoch: int | None = None +) -> bool: + """Custody transfer uses the same token/epoch fence as normal completion.""" + return _ledger_owner_matches(existing, job_id, run_token, run_epoch) + + def is_valid_completed_sync_content_result(result: Any) -> bool: """A ledger completion may converge Redis only for an all-success result.""" if not isinstance(result, dict): return False + if result.get('held_audio') is True: + return False failed_segments = result.get('failed_segments') total_segments = result.get('total_segments') errors = result.get('errors') @@ -152,6 +171,12 @@ def is_valid_completed_sync_content_result(result: Any) -> bool: return outcome in {'success', 'expected_silence'} +def is_valid_held_sync_content_result(result: Any) -> bool: + if not isinstance(result, dict) or result.get('held_audio') is not True: + return False + return is_valid_completed_sync_content_result({**result, 'held_audio': False}) + + def _processing_claim_updates( job_id: str, lane: str, @@ -193,6 +218,10 @@ def _claim_transaction( ) -> Dict[str, Any]: snapshot = ref.get(transaction=transaction) existing = cast(Dict[str, Any], snapshot.to_dict() or {}) if getattr(snapshot, 'exists', False) else {} + if existing.get('dispatch_protocol') == 'stt_cap_hold_v1' and existing.get('status') == 'held': + if is_valid_held_sync_content_result(existing.get('result')): + return {'outcome': 'held', 'result': existing['result'], 'job_id': existing.get('job_id')} + return {'outcome': 'busy', 'job_id': existing.get('job_id')} if existing.get('dispatch_protocol') == 'batch_v1': result = existing.get('result') if existing.get('status') == 'completed' and is_valid_completed_sync_content_result(result): @@ -293,6 +322,10 @@ def _bind_run_token_transaction( """ snapshot = ref.get(transaction=transaction) existing = cast(Dict[str, Any], snapshot.to_dict() or {}) if getattr(snapshot, 'exists', False) else {} + if existing.get('dispatch_protocol') == 'stt_cap_hold_v1' and existing.get('status') == 'held': + if is_valid_held_sync_content_result(existing.get('result')): + return SyncContentRunBinding(SyncContentRunBindingOutcome.HELD, existing['result']) + return SyncContentRunBinding(SyncContentRunBindingOutcome.LOST) if existing.get('dispatch_protocol') == 'batch_v1': result = existing.get('result') if existing.get('status') == 'completed' and is_valid_completed_sync_content_result(result): diff --git a/backend/diarizer/batch_pipeline.py b/backend/diarizer/batch_pipeline.py index f92f89baa39..5c4fa7ddce8 100644 --- a/backend/diarizer/batch_pipeline.py +++ b/backend/diarizer/batch_pipeline.py @@ -12,9 +12,11 @@ import threading from typing import Any, Callable, Mapping, Sequence +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import record_audio from config.batch_pipeline_providers import ProviderConfig from config.sync_live_dedupe import sync_live_dedupe_active_for -from utils.batch_pipeline.contracts import WorkItem, WorkResult +from utils.batch_pipeline.contracts import WorkItem, WorkResult, HeldWork from utils.batch_pipeline.errors import FatalGpuError from utils.observability.fallback import record_fallback @@ -234,7 +236,7 @@ def __init__( self._unavailable = False self.counts = {'items': 0, 'reused_live': 0, 'silence': 0, 'failed': 0} - def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: + def process_batch(self, items: Sequence[WorkItem], *, authorize_dispatch=None) -> Sequence[WorkResult | HeldWork]: if len(items) > 32 or len({i.segment_id for i in items}) != len(items): raise ValueError('pull must contain at most 32 distinct conversations') if any(item.duration_seconds > self.config.max_unit_seconds for item in items): @@ -250,19 +252,31 @@ def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: # Bound decoded host memory to two audio-hours (~460 MB float32). for item in items: if group and seconds + item.duration_seconds > 7200: - results.extend(self._process(group)) + results.extend(self._process(group, authorize_dispatch=authorize_dispatch)) group, seconds = [], 0 group.append(item) seconds += item.duration_seconds - results.extend(self._process(group)) + results.extend(self._process(group, authorize_dispatch=authorize_dispatch)) return results - def _process(self, items): + def _process(self, items, *, authorize_dispatch=None): states, windows, failed, results = {}, [], set(), [] failure_records = [] + held = set() recovery_records = [] shapes = {item.segment_id: {'samples': -1, 'vad_spans': -1, 'windows': -1, 'asr_words': 0} for item in items} + def allowed(required): + try: + return set(authorize_dispatch(required)) if authorize_dispatch is not None else set() + except Exception: + return set() + + def accepted(segment_id, seconds): + record_audio('sync', 'accepted', self.config.words, seconds) + state = states[segment_id] + state['cap_provider_audio_seconds'] = state.get('cap_provider_audio_seconds', 0) + seconds + def fail(item, exc, stage): # Quarantine poisoned CUDA state without importing a second GPU owner. if is_fatal_cuda_error(exc): @@ -344,11 +358,21 @@ def fail(item, exc, stage): fail(item, exc, stage) for batch in pack_windows(windows, self.config): - batch = [w for w in batch if w.segment_id not in failed] + batch = [w for w in batch if w.segment_id not in failed and w.segment_id not in held] + if cap_dispatch_enabled(): + # Authorization belongs immediately before ASR, after potentially + # slow decode/VAD and between successive microbatches. Missing + # policy authority is a hold, never provider permission. + required = {w.segment_id for w in batch} + held.update(required - allowed(required)) + batch = [w for w in batch if w.segment_id not in held] if not batch: continue try: hypotheses = self.words.transcribe_batch([w.samples for w in batch]) + if cap_dispatch_enabled(): + for window in batch: + accepted(window.segment_id, window.duration) if len(hypotheses) != len(batch): raise ValueError('provider result cardinality mismatch') except Exception as exc: @@ -369,6 +393,9 @@ def fail(item, exc, stage): for state in states.values(): item = state['item'] + if item.segment_id in held: + results.append(HeldWork(item.segment_id)) + continue if item.segment_id in failed: continue stage = 'assemble_words' @@ -381,9 +408,22 @@ def fail(item, exc, stage): # with a second hypothesis or publish a fabricated silence. reason = 'no_owned_words' if shapes[item.segment_id]['asr_words'] else 'empty_asr' stage = 'asr_recovery' - words = validate_words( - self.recover_words(state['samples'], self.config.max_padded_seconds), duration - ) + if cap_dispatch_enabled(): + recovered = self.recover_words( + state['samples'], + self.config.max_padded_seconds, + before_dispatch=lambda: item.segment_id in allowed({item.segment_id}), + after_dispatch=lambda seconds: accepted(item.segment_id, seconds), + ) + if recovered is None: + held.add(item.segment_id) + results.append(HeldWork(item.segment_id)) + continue + words = validate_words(recovered, duration) + else: + words = validate_words( + self.recover_words(state['samples'], self.config.max_padded_seconds), duration + ) recovery_records.append( ( item.segment_id, @@ -432,6 +472,11 @@ def fail(item, exc, stage): 'speech_seconds': state['speech_seconds'], 'speech_spans': state['speech_spans'], } + if cap_dispatch_enabled(): + payload['cap_dispatch'] = { + 'omi_funded_accepted_provider_audio_seconds': state.get('cap_provider_audio_seconds', 0), + 'admission': 'authorized_compute', + } stage = 'persist' uri = self.persist(item, payload) results.append(WorkResult(item.segment_id, uri, 'silence' if state['silence'] else 'success')) diff --git a/backend/diarizer/batch_pipeline_models.py b/backend/diarizer/batch_pipeline_models.py index 4c8d9f7dbe7..4aeee9244f7 100644 --- a/backend/diarizer/batch_pipeline_models.py +++ b/backend/diarizer/batch_pipeline_models.py @@ -83,7 +83,7 @@ def transcribe_batch(self, samples): result.append(words) return result - def recover_empty(self, samples, max_padded_seconds): + def recover_empty(self, samples, max_padded_seconds, *, before_dispatch=None, after_dispatch=None): """One full-capture, single-item retry with synthetic silence at both edges. TDT can emit all blanks for a fragment starting mid-utterance. Retain @@ -97,9 +97,16 @@ def recover_empty(self, samples, max_padded_seconds): right = max(2 * SAMPLE_RATE, 8 * SAMPLE_RATE - len(samples) - left) padded_seconds = (len(samples) + left + right) / SAMPLE_RATE if padded_seconds > min(120, max_padded_seconds): - return [] + # The callback identifies enforced dispatch. Main's unflagged + # adapter still returns an empty hypothesis; policy holds must not + # spend a provider-failure attempt for audio never submitted. + return None if before_dispatch is not None else [] padded = np.pad(samples, (left, right)) + if before_dispatch is not None and not before_dispatch(): + return None # Retain raw work without spending a provider-failure attempt. hypotheses = self.transcribe_batch([padded]) + if after_dispatch is not None: + after_dispatch(padded_seconds) if len(hypotheses) != 1: raise ValueError('provider recovery result cardinality mismatch') words = validate_words(hypotheses[0], padded_seconds) diff --git a/backend/diarizer/batch_pipeline_runtime.py b/backend/diarizer/batch_pipeline_runtime.py index ed1c5bee3b8..b63a033703d 100644 --- a/backend/diarizer/batch_pipeline_runtime.py +++ b/backend/diarizer/batch_pipeline_runtime.py @@ -15,6 +15,8 @@ from prometheus_client import CollectorRegistry, CONTENT_TYPE_LATEST, generate_latest +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import AUDIO_SECONDS from config.batch_pipeline_providers import provider_config from database._client import get_data_plane_firestore_client from database.batch_pipeline_store import FirestoreStore @@ -57,13 +59,13 @@ def __init__( self.memory_probe = memory_probe self._unavailable = False - def process_batch(self, items): + def process_batch(self, items, **kwargs): if self._unavailable: raise FatalGpuError('GPU chain unavailable; restart required') try: baseline = self.memory_probe.begin() if self.memory_probe else None try: - return self.worker.process_batch(items) + return self.worker.process_batch(items, **kwargs) finally: # Python completion is not CUDA completion. Never publish before # quiescence, even for a dedicated GPU with no live reservation. @@ -86,6 +88,9 @@ def process_batch(self, items): def health_server(state: RuntimeState, registry: CollectorRegistry, port: int) -> ThreadingHTTPServer: + if cap_dispatch_enabled(): + registry.register(AUDIO_SECONDS) + class Handler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/metrics': diff --git a/backend/docs/feature-flag-registry.md b/backend/docs/feature-flag-registry.md index e424b446202..89baa11b6f7 100644 --- a/backend/docs/feature-flag-registry.md +++ b/backend/docs/feature-flag-registry.md @@ -262,6 +262,7 @@ and an explicit empty literal renders as `''`. | `SPEAKER_GROUPING_MODE` | incumbent\|provider_strict\|provider_cannot_link; owner_link is shadow-only and rejected; default incumbent everywhere; David gates promotion | backend | env | open | — | — | incumbent (backend-listen (chart), cloud_run/backend, cloud_run/backend-integration, cloud_run/backend-sync, cloud_run/backend-sync-backfill, gke/backend-listen, gke/pusher, pusher (chart)) | — | pending | 2026-11-08 | dazheng | | `SPEAKER_MATCH_SCORES_ENABLED` | Persist bounded internal voice-match evidence on existing conversation writes; dev/local/offline default on; production hosts explicitly on | backend | env | closed | — | — | true (backend-listen (chart), cloud_run/backend, cloud_run/backend-integration, cloud_run/backend-sync, cloud_run/backend-sync-backfill, gke/backend-listen, gke/pusher, pusher (chart)) | — | pending | 2026-11-04 | dazheng | | `STOREFRONT_LADDER_V1_ENABLED` | Default-off versioned unified plans presentation; exact true admits v2 only; installed-client endpoints unchanged | backend | env | closed | — | — | — | — | pending | 2026-11-13 | dazheng | +| `STT_CAP_DISPATCH_ENFORCEMENT` | Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation | backend | env | closed | — | — | — | — | pending | 2026-11-15 | dazheng | | `STT_FAILOVER_RECOVERY_ENABLED` | Enable live-STT failover recovery on prod listen | backend | env | closed | declared | false (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-11-03 | backend | | `STT_LEARNED_LANGUAGE_PROFILE` | Use a bounded per-user spoken-language history for live STT routing and Soniox hints | backend | env | closed | true | true (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-10-28 | backend | | `STT_LIVE_DEMAND_ENABLED` | Default off; qualified English hidden capture uses window STT; visible or active consumers require vendors | backend | env | closed | declared | false (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-11-13 | dazheng | @@ -545,6 +546,7 @@ their code default (`fail` tells you which way a missing value resolves). - `SONIOX_ELAPSED_AXIS` — Measure Soniox elapsed timestamps before enabling speaker windows (fail: closed) - `SPEAKER_GROUPING_CROSS_SCOPE_THRESHOLD` — G2 cross-scope cosine-distance threshold; default 0.60; invalid configuration uses 0.60 (fail: closed) - `STOREFRONT_LADDER_V1_ENABLED` — Default-off versioned unified plans presentation; exact true admits v2 only; installed-client endpoints unchanged (fail: closed) +- `STT_CAP_DISPATCH_ENFORCEMENT` — Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation (fail: closed) - `STT_NO_TEXT_RESCUE_SECONDS` — Maximum paid wall and admitted audio seconds per no-text rescue; default 60 bounded 5-120 (fail: closed) - `STT_PAID_SPILLOVER_BUDGET_ENABLED` — Default-off fleet budget for paid router promotions after Parakeet capacity refusal; denial restores static order (fail: closed) - `STT_PAID_SPILLOVER_DEEPGRAM_PER_MINUTE` — Deepgram fleet paid router promotion cap per UTC minute; default 30; zero refuses promotions (fail: closed) diff --git a/backend/docs/listen_pusher_pipeline.mdx b/backend/docs/listen_pusher_pipeline.mdx index 7fac7b14833..2f3920e33ed 100644 --- a/backend/docs/listen_pusher_pipeline.mdx +++ b/backend/docs/listen_pusher_pipeline.mdx @@ -6,6 +6,22 @@ description: "Sequence diagrams and finalization policy for the /v4/listen WebSo # Listen + Pusher Pipeline — Sequence Diagrams +## Default-off funded transcription cap + +`STT_CAP_DISPATCH_ENFORCEMENT` gates provider creation and audio sends through a +shared live admission state, including multichannel, replay and replacement legs. +Denial preserves receiving and existing audio storage paths; accepted provider +work drains. Custom/on-device STT and flag-off metering retain their behavior. +The [dispatch contract](stt-cap-dispatch.md) also documents sync holds, recovery, +BYOK funding, additive metrics and the per-client activation decision. +Sync recovery fences a provider-bound `dispatched` receipt before paid STT or +ingestion, and atomically checkpoints ingest proof alongside the conversation. +Known non-calls clear the intent. After 30 minutes, ambiguous work reconciles +from that proof or gets one second dispatch; a second ambiguity retains the audio +as failed reconciliation. Successful siblings finalize independently. Flagged +oversized GPU empty-result retries become held work without a failure attempt; +the unflagged retry follows main's behavior. + ## Windowed live transcription deadlines Parakeet packing is independently gated by `PARAKEET_PACKING_ENABLED=false` diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md new file mode 100644 index 00000000000..72617fb3506 --- /dev/null +++ b/backend/docs/stt-cap-dispatch.md @@ -0,0 +1,185 @@ +# Transcription cap at provider dispatch + +`STT_CAP_DISPATCH_ENFORCEMENT` is a default-off backend rollout switch. This PR +adds no deployment binding and does not enable it. Enabling changes the policy +for existing capped paid plans: exhausted sync uploads wait as audio rather than +being transcribed into locked conversations. Product approval is required before +activation. No ladder-v1 code or prices change. + +## Boundaries + +Live receiver creation installs one funded admission state for the session. +Actual Deepgram, Soniox, Modulate, Parakeet and window-provider constructors use +that state, including direct registry endpoints and fallback/recovery legs. +The state debits accepted PCM seconds across channels, replay and successors. +The existing 60-second usage flush refreshes it from ordinary funded allowance; +root counters retain their received-audio semantics. Timestamp adapters suppress +before recording accepted-provider spans. Denial is a policy pause, not provider +death: receiving, pusher audio and existing private-cloud/raw archive paths +continue. Accepted provider queues drain. Nothing enables cloud audio retention +for live users who did not already opt into it. Custom/on-device STT bypasses the +new state. The existing Plus threshold event still uses its existing wire shape. + +Deepgram cloud with a request key is customer funded. Every other provider, +including self-hosted Deepgram and an Omi-funded fallback from BYOK, uses the +ordinary allowance with both BYOK quota and desktop-paywall exemptions disabled. +No key or identity is a metric label. + +Sync v1 intake and v2 execution hold decoded WAVs before VAD when denied. V2 +applies ordinary source-frame and safety-WAL coverage first, so already committed +audio is not enrolled as new recoverable work. Source maps, recording identity +and geolocation travel with the encrypted hold. +The shared segment processor rechecks immediately before each STT invocation, +including its empty-result retry. Late exhaustion holds the remaining segment. +No denied segment reaches cloud STT. Initial holds coordinate database and storage +leaves asynchronously; GCS work does not occupy the sync pipeline pool. +Transient admission/configuration holds do not claim credits are exhausted. A successfully persisted hold is acknowledged +without creating a locked transcript. Hold failures fail the upload/job; callers +must retain/retry their source audio. V1 adds `held_audio` and `held_files` to the +exhausted acknowledgment; v2 adds `held_audio` to its durable result. + +Batch compute authorization checks current funded allowance. The worker calls +that authority again after decode/VAD, before each ASR microbatch and immediately +before the full-capture empty-ASR recovery dispatch. With enforcement on, +oversized recovery returns held work without spending a provider-failure attempt; +the unflagged path retains main's empty-result/failure behavior. Oversized recovery +does not reauthorize or record audio it never submits; accepted recovery counts actual +synthetic padding. Missing +worker policy authority holds work. Held work uses the scheduler's existing +`defer`, preserving raw artifacts and refunding the provider-failure attempt. +Quarantine recovery rechecks both before fetching audio and before claiming its +sole provider attempt; a denied prepared intent remains recoverable. + +## Durable sync recovery + +Encrypted source audio lives under the existing private-cloud bucket's +`chunks/{uid}/cap-held/` owner prefix; encrypted metadata and progress live under +`users/{uid}/stt_cap_holds`. The checkpoint has no application TTL. Existing +account deletion recursively removes the manifest and purges this chunk prefix. +A failed object upload cannot publish a successful manifest or acknowledgment. +Concurrent identical uploads cannot overwrite a recovery checkpoint. + +WAV hashing, encryption, upload, download and decryption use bounded chunks; +large decoded recordings are never materialized as whole-file byte buffers. +Large source-map metadata uses an encrypted sidecar under the same owner prefix. +Late per-segment denials are aggregated into the v1/v2 held result and excluded +from post-STT subscription/DG quantities until recovered. Root meter writers and +all flag-off call semantics remain unchanged. + +V2 transfers the fenced content ledger into `held`, without a TTL or a completed +transcription proof. Duplicate intake can acknowledge custody without dispatching +another provider. A preparing/failed source job still owns its retry; its holds +cannot race it into STT. Only settlement of every referenced hold completes the +content ledger. Holds that were never acknowledged await the source job's retry. + +The existing `locked_content_recovery` authenticated-request path kicks an async +recovery pass independently of its once-per-month unlock receipt and legacy reset +flag. The flagged API sync router registers the recovery callback; auth-only and +pusher images do not load the sync pipeline. A distributed 60-second cooldown +bounds requests; token-fenced renewable leases prevent concurrent processing of +a hold. At reset or upgrade, current +funded allowance and the existing hard abuse restrictions must pass again before +every new recovered provider request. Already persisted receipts can settle at +the cap without another provider call. Recovery +replays WAL coverage and lineage planning, loads speaker embeddings, and reuses +VAD, segment intake, enrichment and optional playback finalization. Token-fenced +piece receipts persist a `dispatched` intent with funding, provider, model, +`dispatched_at` and `attempt=1` **before** STT or ingestion, under the lease fence. +Provider selection is pinned. Before dispatch, recovery freezes encrypted VAD +piece bytes and their lineage context under the existing owner prefix. A later +coverage/VAD decision cannot redefine a piece or suppress a sibling checkpoint. +The original full audio remains retained throughout. + +`process_segment` explicitly reports `provider_called`, set immediately before +`prerecorded`, never inferred from an error class. A known non-call atomically +removes its intent (or reverts an unspent second attempt). The next claim retries +normally. If a call may have happened, the piece waits 30 minutes. Then: + +1. A proof written in the **same transaction as conversation ingestion** identifies + the hold, piece and deterministic conversation lineage. Recovery marks that + piece transcribed, settles its existing idempotent usage key and finalizes it + without another provider call. +2. Without a proof, current allowance/abuse admission permits **one** additional + application dispatch, persisted as `attempt=2`. Recovery owns retries; the + ordinary segment's local empty-result retry is disabled for these checkpointed + recovery attempts. Provider-internal SDK retries remain outside this ledger. +3. A second ambiguous attempt gets the same cooldown/proof check, then becomes + `failed_reconciliation`. Its audio and manifest remain retained. Successful + siblings still settle usage and finalize independently; their paths checkpoint + finalization progress. A failed piece never publishes an all-success content + ledger result. + +A reconciled ingest also retains raw audio, since playback storage may not have +finished before the interrupted processor returned. Terminal retained manifests +sort after actionable work and cannot dispatch. Successful ledger settlement is +retryable before that terminal transition. Neither retained form has an +application TTL or is silently erased. Ordinary fully successful recovery deletes +source and frozen pieces only after content/usage settlement. + +`omi_stt_cap_recovery_reconciliation_total{outcome}` counts `cleared_noncall`, +`reconciled_ingested`, `redispatched`, and `failed`. Structured +`event=stt_cap_recovery_reconciliation` logs include hold id and piece name, never +content. Metric labels contain no uid, hold id or piece name. New recovery and +its signals are inert when the dispatch flag is off. WS-only and inactive +accounts wait for their next authenticated HTTP request; there is no new cron. +Rolling back the dispatch flag pauses this new recovery, preserving its audio. + +## Launch configuration and measurement + +With enforcement off, launch-date handling is unchanged. With it on, missing or +invalid `SUBSCRIPTION_LAUNCH_DATE` logs `event=stt_cap_launch_config` and increments +`omi_stt_cap_launch_config_errors_total{reason="missing|invalid"}`. Finite-plan +allowance fails conservatively instead of inventing zero usage. A valid explicit +future date continues to mean intentionally unlaunched. + +`omi_stt_cap_provider_audio_seconds_total` is additive provenance: + +- `path`: `live` or `sync`; +- `outcome`: `accepted`, `suppressed` (live), or `held` (sync); +- `provider`: a bounded serving family or `unknown`; +- `allowance`: `exhausted` or `other`, at the admission/acceptance observation; `other` + includes available allowance and untrusted lookups, without claiming availability. + +Accepted live quantities count PCM accepted by a provider adapter, including +replay; failed sends do not count. Suppressed quantities count input PCM, before +VAD. Sync successful provider calls count WAV duration, including retries; +quarantine recovery counts its accepted request duration. The GPU worker counts +actual padded microbatch audio and persists additive `cap_dispatch` provenance +with `omi_funded_accepted_provider_audio_seconds` and its compute admission origin. +Held sync quantities count durably retained audio; queue denials count attempted +held quantities and can repeat across policy retries. They are not a distinct +captured-content counter. Provider-internal retries/uncertain failures are not an +exact vendor invoice ledger. Existing root counters and unit projections do not +change, and no new provenance is emitted with the flag off. + +Admission checks are not distributed quota reservations. Concurrent already +admitted work can drain across the cap; live refresh is bounded by its existing +flush cadence. Do not interpret a zero violation metric as proof against every +cross-process admission race. + +## What users see when the flag is on + +- **Mobile, on-device overflow flag off:** capture continues as main permits, but + exhausted managed sessions stop producing new cloud transcripts. Existing Plus + threshold handling remains. Uploaded offline audio waits durably; this PR adds + no held-content card or automatic on-device processing of that backlog. +- **Mobile, on-device overflow flag on:** eligible clients keep the existing + on-device offer/mode switch. Custom/on-device sessions continue normally. + Already-held server uploads still wait for reset/upgrade recovery. +- **macOS:** the existing threshold handler can stop capture, as it does today. + The backend also refuses new funded sends if the client continues; accepted work + drains. There is no new macOS on-device overflow flow in this PR. +- **Web:** no new on-device offer is added. Exhausted funded live transcription + pauses; supported sync uploads wait for recovery. Existing content stays as it + was, and current clients do not gain a new waiting-audio view from these additive + backend fields. + +These visible gaps and the change from locked transcripts to held audio are the +activation decision for David; default-off deployment preserves today's policy. + +## Follow-ups in #21488 + +Findings #6 and #7 remain outside this PR: budget-aware live refresh when ladder +allowance returns `remaining_seconds=None`, and conservative admission when usage +unit projection has coverage gaps. Carry this funded-dispatch/held-audio contract +into that ladder cutover without changing the legacy meter here. diff --git a/backend/models/sync_job_status.py b/backend/models/sync_job_status.py index ee792da689b..14c78350723 100644 --- a/backend/models/sync_job_status.py +++ b/backend/models/sync_job_status.py @@ -12,6 +12,7 @@ class SyncLocalFilesResultResponse(BaseModel): failed_segments: int = 0 total_segments: int = 0 errors: list[str] = Field(default_factory=list) + held_audio: bool | None = None class SyncJobStartResponse(BaseModel): diff --git a/backend/routers/listen/contracts.py b/backend/routers/listen/contracts.py index 988c4227876..feae93cabba 100644 --- a/backend/routers/listen/contracts.py +++ b/backend/routers/listen/contracts.py @@ -101,6 +101,7 @@ class ListenSessionState: active: bool = True close_code: int = 1001 stt_terminal_failure: bool = False + stt_cap_admission: Any = None shutdown_event: asyncio.Event = field(default_factory=asyncio.Event) audio_ring_buffer: Any = None speaker_id_enabled: bool = False diff --git a/backend/routers/listen/legacy_recovery.py b/backend/routers/listen/legacy_recovery.py index 0fbcf4c28ec..d9c93a54b92 100644 --- a/backend/routers/listen/legacy_recovery.py +++ b/backend/routers/listen/legacy_recovery.py @@ -156,6 +156,11 @@ async def monitor_stt_death(self: Any) -> None: (``stt_failed`` + WebSocket 1011) as soon as the death latch flips (#10028). """ while self.host.state.active and not self.host.state.stt_terminal_failure: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + if await self.host.wait(owner.STT_DEATH_POLL_INTERVAL_SECONDS): + return + continue socket = self.stt_socket outcome = getattr(socket, 'leg_outcome', None) if outcome is not None and outcome.owner_closing: @@ -163,6 +168,8 @@ async def monitor_stt_death(self: Any) -> None: if socket is not None and owner.live_stt_socket_is_dead(socket): if await self._failover_stt_socket(): continue + if admission is not None and admission.funded_paused is True: + continue if outcome is not None and outcome.owner_closing: return if self.host.state.active and not self.host.state.stt_terminal_failure: diff --git a/backend/routers/listen/receiver.py b/backend/routers/listen/receiver.py index 11ec2cf925c..03537e61875 100644 --- a/backend/routers/listen/receiver.py +++ b/backend/routers/listen/receiver.py @@ -9,6 +9,9 @@ import time import uuid +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.stt_cap_dispatch import LiveAdmission, current_live_admission, suppress_socket_audio +from utils.observability.stt_cap_dispatch import record_audio as record_cap_audio from utils.manual_speaker_assignments import acknowledged_teaching from utils.other.live_audio_archive import LiveAudioArchive from collections import OrderedDict, deque @@ -202,6 +205,8 @@ def __getattr__(self, name: str) -> Any: return getattr(self._conn, name) def send(self, data: bytes, start_sample: Optional[int] = None) -> bool: + if suppress_socket_audio(self._conn, data): + return True try: accepted = ( self._conn.send(data, start_sample=start_sample) if start_sample is not None else self._conn.send(data) @@ -250,6 +255,14 @@ class ListenReceiver(ReplayFilterMixin): def __init__(self, host: Any, channel_configs: List[ChannelConfig], channel_id_to_index: Dict[int, int]): self.soniox_idle_budget = SonioxIdleBudget() self.host = host + self.cap_admission = ( + LiveAdmission(host.request.uid, host.request.source) + if stt_cap_dispatch_enabled() and not host.use_custom_stt + else None + ) + self.cap_paused = False + if self.cap_admission is not None: + host.state.stt_cap_admission = self.cap_admission self.raw_archive = LiveAudioArchive(host) try: self.lane_meter = LiveLaneMeter() @@ -908,6 +921,10 @@ async def _create_stt_socket( ) -> Any: if not isinstance(getattr(self, 'recovery_enabled', None), bool): self.recovery_enabled = session_recovery_enabled(self) + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.reason == 'allowance_unavailable': + await admission.refresh() + cap_token = current_live_admission.set(admission) enabled_token = current_recovery_enabled.set(self.recovery_enabled) token = current_recovery.set(getattr(self, 'recovery', None) if self.recovery_enabled else None) try: @@ -923,6 +940,7 @@ async def _create_stt_socket( finally: current_recovery.reset(token) current_recovery_enabled.reset(enabled_token) + current_live_admission.reset(cap_token) async def _connect_stt_socket( self, @@ -1226,6 +1244,14 @@ def _record_selected_epoch(self, epoch: Optional[ProviderEpochTranslator], socke ).inc() async def initialize_stt(self) -> bool: + admission = getattr(self, 'cap_admission', None) + if admission is not None: + await admission.refresh() + if not admission.allowed(self.host.stt_service): + admission.omi_selected = True + self.cap_paused = True + return True # Receive/persist without consulting provider routing. + self.cap_paused = False self.live_demand = attach_live_demand(self) if self.live_demand is not None: await self.live_demand.publish() @@ -1309,6 +1335,9 @@ def callback(segments: List[Dict[str, Any]], channel: ChannelConfig = config) -> return True except ProviderChainUnavailable as error: + if admission is not None and admission.funded_paused is True: + self.cap_paused = True + return True await self._drain_stt_sockets() await terminate_live_stt_backoff( request.websocket, @@ -1318,6 +1347,9 @@ def callback(segments: List[Dict[str, Any]], channel: ChannelConfig = config) -> ) return False except Exception as error: + if admission is not None and admission.funded_paused is True: + self.cap_paused = True + return True await self._drain_stt_sockets() await terminate_live_stt_session( request.websocket, @@ -1744,6 +1776,11 @@ async def _monitor_stt_death(self) -> None: if getattr(self, 'recovery', None) is None: return await legacy_recovery.monitor_stt_death(self) while self.host.state.active and not self.host.state.stt_terminal_failure: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + if await self.host.wait(STT_DEATH_POLL_INTERVAL_SECONDS): + return + continue socket = self.stt_socket outcome = getattr(socket, 'leg_outcome', None) if outcome is not None and outcome.owner_closing: @@ -1751,6 +1788,8 @@ async def _monitor_stt_death(self) -> None: if socket is not None and live_stt_socket_is_dead(socket): if await self._failover_stt_socket(): continue + if admission is not None and admission.funded_paused is True: + continue recovery = getattr(self, 'recovery', None) if recovery is not None: if recovery.client_has_left() or (outcome is not None and outcome.owner_closing): @@ -1820,6 +1859,18 @@ async def _handle_image_chunk(self, payload: Dict[str, Any]) -> None: self.host.spawn(self._process_photo(image, chunk.id), name='photo_process') async def _flush_stt_buffer(self, buffer: bytearray, *, force: bool = False) -> None: + admission = getattr(self, 'cap_admission', None) + if admission is not None and not admission.allowed(self.host.stt_service): + record_cap_audio( + 'live', + 'suppressed', + self.host.stt_service, + len(buffer) / (2 * self.host.request.sample_rate), + admission.reason == 'plan_allowance_exhausted', + ) + buffer.clear() + self._stt_buffer_start_sample = None + return if getattr(self, 'recovery', None) is None: return await legacy_recovery.flush_stt_buffer(self, buffer, force=force) request = self.host.request @@ -1957,6 +2008,9 @@ async def _flush_stt_buffer(self, buffer: bytearray, *, force: bool = False) -> return async def _recover_ingested_audio(self) -> None: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + return if ( not await self._failover_stt_socket() and self.host.state.active @@ -2034,7 +2088,7 @@ async def _handle_multi_channel_audio(self, data: bytes, now: float | None = Non pcm_len=len(pcm), fair_use_track_dg_usage=self.host.state.fair_use_track_dg_usage, ) - if should_send: + if should_send and not suppress_socket_audio(self.stt_sockets_multi[channel_index], pcm): sent = await send_live_stt_audio( request.websocket, self.host.state, diff --git a/backend/routers/listen/runtime.py b/backend/routers/listen/runtime.py index 69ecff109ba..2f37b6994ee 100644 --- a/backend/routers/listen/runtime.py +++ b/backend/routers/listen/runtime.py @@ -108,6 +108,7 @@ from .conversations import LiveConversationController from .persistence import ListenPersistence from .parity_capture import ListenParityCapture +from utils.stt_cap_dispatch import customer_funded from . import receiver as receiver_module from .receiver import ListenReceiver from .registry import register as register_listen_session @@ -799,6 +800,11 @@ async def _record_usage_periodically(self) -> None: await self._refresh_credits(transcription_seconds=transcription_seconds) async def _refresh_credits(self, *, transcription_seconds: int = 0) -> None: + admission = getattr(getattr(self, 'receiver', None), 'cap_admission', None) + if admission is not None: + await admission.refresh() + if getattr(self.receiver, 'cap_paused', False) and admission.managed: + await self.receiver.initialize_stt() now = time.time() invalidated = await self.persistence.call(check_credits_invalidation, self.request.uid) needs_refresh = ( @@ -820,6 +826,8 @@ async def _refresh_credits(self, *, transcription_seconds: int = 0) -> None: elif self.state.remaining_seconds_cache is not None and transcription_seconds > 0: self.state.remaining_seconds_cache = max(0, self.state.remaining_seconds_cache - transcription_seconds) remaining = self.state.remaining_seconds_cache + if admission is not None and (admission.omi_selected or not customer_funded(self.stt_service)): + remaining = admission.remaining subscription = await self.persistence.call(user_db.get_user_valid_subscription, self.request.uid) if remaining is not None and remaining <= FREEMIUM_THRESHOLD_SECONDS and not self.state.freemium_threshold_sent: if should_emit_plus_meter_warning(subscription): diff --git a/backend/routers/sync.py b/backend/routers/sync.py index e492e314936..41efd7fe505 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -60,6 +60,8 @@ from models.geolocation import geolocation_from_private_header from models.sync_audio import AudioPrecacheResponse, AudioUrlsResponse from routers.listen.contracts import persisted_started_seconds +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.sync.cap_hold import hold_capture_if_exhausted from utils.sync.admission_responses import sync_content_in_progress_response from utils.analytics import record_sync_v1_usage as record_usage from utils.other import endpoints as auth @@ -187,6 +189,18 @@ from utils.sync import backfill_cutover, legacy_fresh, stage as sync_stage, uid_sequencer from utils.sync.provenance import capture_matches_server_conversation as _capture_matches_server_conversation +# Only the API process importing this router owns held-audio STT recovery. +# Pusher/auth-only images retain the lightweight locked-content recovery path. +if cap_dispatch_enabled(): + from utils.locked_content_recovery import register_cap_audio_recovery + + def _kick_cap_hold_recovery(uid: str) -> None: + from utils.sync.cap_hold_recovery import kick + + kick(uid) + + register_cap_audio_recovery(_kick_cap_hold_recovery) + logger = logging.getLogger(__name__) # Audio constants @@ -644,6 +658,30 @@ async def sync_local_files( except HTTPException as e: raise HTTPException(status_code=e.status_code, detail=e.detail, headers=_V1_DEPRECATION_HEADERS) + cap_held = False + cap_admission: dict = {} + if cap_dispatch_enabled(): + cap_held = await hold_capture_if_exhausted( + uid, + wav_paths, + source, + admission=cap_admission, + target_conversation_id=conversation_id, + client_device_id=client_device_context.client_device_id, + client_platform=client_device_context.platform, + ) + if cap_held: + return JSONResponse( + status_code=200, + headers=_V1_DEPRECATION_HEADERS, + content={ + 'new_memories': [], + 'updated_memories': [], + 'credits_exhausted': cap_admission.get('reason') == 'plan_allowance_exhausted', + 'held_audio': True, + 'held_files': len(wav_paths), + }, + ) vad_errors = [] vad_segment_lock = threading.Lock() @@ -798,6 +836,9 @@ def _run_vad(path): ] ) + cap_holds = response.get('_cap_holds', {}) if cap_dispatch_enabled() else {} + cap_held_seconds = sum(item['seconds'] for item in cap_holds.values()) + await run_blocking(sync_executor, _reprocess_merged_conversations, uid, response) if private_cloud_sync_enabled: await run_blocking(sync_executor, _finalize_sync_audio_files, uid, response) @@ -807,6 +848,8 @@ def _run_vad(path): if fair_use_restrict_dg: try: dg_ms = int(total_speech_seconds * 1000) + if cap_holds: + dg_ms = int(max(0, total_speech_seconds - cap_held_seconds) * 1000) if dg_ms > 0: await run_blocking(db_executor, record_dg_usage_ms, uid, dg_ms) except Exception as e: @@ -826,9 +869,17 @@ def _run_vad(path): 'new_memories': sorted(response['new_memories']), 'updated_memories': sorted(response['updated_memories']), } + if cap_holds: + result.update( + held_audio=True, + credits_exhausted=any(item.get('reason') == 'plan_allowance_exhausted' for item in cap_holds.values()), + held_segments=len(cap_holds), + ) failed_segments = len(segment_errors) successful_segments = total_segments - failed_segments + if cap_holds: + successful_segments -= len(cap_holds) if failed_segments > 0: result['failed_segments'] = failed_segments @@ -841,15 +892,18 @@ def _run_vad(path): if total_segments > 0 and successful_segments == 0: # All segments failed — return 500 (consistent with VAD error behavior) - raise HTTPException( - status_code=500, - detail=f"All {total_segments} segment(s) failed processing: {'; '.join(segment_errors[:3])}", - headers=_V1_DEPRECATION_HEADERS, - ) + if not cap_holds: + raise HTTPException( + status_code=500, + detail=f"All {total_segments} segment(s) failed processing: {'; '.join(segment_errors[:3])}", + headers=_V1_DEPRECATION_HEADERS, + ) # Record subscription usage only when at least one segment succeeded try: usage_seconds = int(total_speech_seconds) + if cap_holds: + usage_seconds = int(max(0, total_speech_seconds - cap_held_seconds)) if successful_segments > 0 else 0 if usage_seconds > 0: await run_blocking( db_executor, record_usage, uid, transcription_seconds=usage_seconds, speech_seconds=usage_seconds @@ -1176,7 +1230,7 @@ async def sync_local_files_v2( job_id, lane_decision.lane.value, ) - if claim.get('outcome') == 'completed': + if claim.get('outcome') in {'completed', 'held'}: cached_result = claim.get('result') or {} await run_blocking(db_executor, mark_job_completed, job_id, cached_result) if backfill_slot_acquired: diff --git a/backend/tests/support/firestore_query_driver_registry.py b/backend/tests/support/firestore_query_driver_registry.py index e31bb6f81e7..73aad2c04ed 100644 --- a/backend/tests/support/firestore_query_driver_registry.py +++ b/backend/tests/support/firestore_query_driver_registry.py @@ -2927,6 +2927,13 @@ def _messaging_retention_candidate(client, combo, trial): ) +_add( + DriverEntry( + 'utils.sync.cap_hold.pending_holds', + base={'uid': UID}, + neutrals={'limit': (10, 'bounded recovery page; does not affect filters')}, + ) +) _add(DriverEntry('database.locked_content.has_recoverable_locked_content', base={'uid': UID})) _add( CoveredByEntry( diff --git a/backend/tests/unit/test_batch_pipeline_admission.py b/backend/tests/unit/test_batch_pipeline_admission.py index fb3e76e1a32..8bd5f25919d 100644 --- a/backend/tests/unit/test_batch_pipeline_admission.py +++ b/backend/tests/unit/test_batch_pipeline_admission.py @@ -96,3 +96,39 @@ def test_policy_backend_failure_and_missing_duration_fail_closed(monkeypatch): reserve.reset_mock() assert admission.authorize(store, tokens(lease), now=1)['allowed'] == [] reserve.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_rechecked_after_queueing_before_gpu_authorization(monkeypatch, enforce): + from utils import stt_cap_dispatch as cap + + store, queue, lease, reserve = setup(monkeypatch, fresh=True) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + receipt = admission.authorize(store, tokens(lease), now=1) + if enforce: + assert receipt == {'allowed': [], 'retry_after': {'segment': 60}} + assert queue.defer('segment', lease.token, 1, 60) + assert store.rows['jobs', 'segment']['attempt'] == 0 + assert 'compute_admission_token' not in store.rows['jobs', 'segment'] + reserve.assert_not_called() + else: + assert receipt['allowed'] == ['segment'] + cap.funded_allowance.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('duration', [None, 'invalid']) +def test_flag_off_never_parses_new_dispatch_duration(monkeypatch, enforce, duration): + store, _, lease, _ = setup(monkeypatch, fresh=True) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + store.rows['jobs', 'segment']['duration_seconds'] = duration + gate = Mock(side_effect=AssertionError('unparseable duration must not reach dispatch policy')) + monkeypatch.setattr(admission, 'sync_allowed', gate) + receipt = admission.authorize(store, tokens(lease), now=1) + assert receipt == ( + {'allowed': [], 'retry_after': {'segment': 30}} if enforce else {'allowed': ['segment'], 'retry_after': {}} + ) + gate.assert_not_called() diff --git a/backend/tests/unit/test_batch_pipeline_pump.py b/backend/tests/unit/test_batch_pipeline_pump.py index 3bd8bb26042..fbff6f64eee 100644 --- a/backend/tests/unit/test_batch_pipeline_pump.py +++ b/backend/tests/unit/test_batch_pipeline_pump.py @@ -234,3 +234,27 @@ def process_batch(self, items): assert store.rows['jobs', 'first']['state'] == 'committed' assert store.rows['jobs', 'second']['state'] == 'committed' assert store.rows['results', 'second']['previous_segment_id'] == 'first' + + +@pytest.mark.asyncio +async def test_worker_dispatch_denial_defers_without_quarantining_or_losing_audio(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + calls = [] + + def authorize(leases): + calls.append([v.item.segment_id for v in leases]) + return {'allowed': calls[-1] if len(calls) == 1 else []} + + class RecheckingWorker: + def process_batch(self, items, *, authorize_dispatch): + assert not authorize_dispatch({v.segment_id for v in items}) + return [HeldWork(v.segment_id) for v in items] + + store, _, pump = setup(RecheckingWorker(), authorize_compute=authorize) + assert await pump.run_once() == 1 + job = store.rows['jobs', 'first'] + assert job['state'] == 'retry' and job['attempt'] == 0 + assert ('results', 'first') not in store.rows + assert calls == [['first'], ['first']] diff --git a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py index a6b9cee920d..8a8450c2e5b 100644 --- a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py +++ b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py @@ -420,3 +420,49 @@ def test_recovery_cas_lost_token_and_artifact_registration_are_fenced(setup): with pytest.raises(BatchPrivacyBlocked): register_batch_artifact(store, binding.owner_id, 'bad', 'gs://synthetic/' + 'b' * 32 + '.enc', 'results', 0) stt.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_exhausted_recovery_never_starts_a_provider_call(setup, monkeypatch, enforce): + from types import SimpleNamespace + from utils import stt_cap_dispatch as cap + + _, _, _, artifacts, stt, recovery = setup + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + result = recovery.recover('bad', 'synthetic-owner') + if enforce: + assert result == 'held' + stt.assert_not_called() + artifacts.fetch_audio.assert_not_called() + else: + stt.assert_called_once() + cap.funded_allowance.assert_not_called() + + +def test_recovery_rechecks_after_audio_download(setup, monkeypatch): + from utils.batch_pipeline import quarantine_recovery as module + + _, _, _, artifacts, stt, recovery = setup + monkeypatch.setattr(module, 'sync_allowed', Mock(side_effect=[True, False])) + assert recovery.recover('bad', 'synthetic-owner') == 'held' + artifacts.fetch_audio.assert_called_once() + stt.assert_not_called() + + +def test_unavailable_completion_lookup_is_not_reported_as_genuine_exhaustion(setup, monkeypatch): + from types import SimpleNamespace + from utils.batch_pipeline import quarantine_recovery as module + + _, _, _, _, _, recovery = setup + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(module, 'sync_allowed', Mock(return_value=True)) + monkeypatch.setattr( + module, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='allowance_unavailable')) + ) + metric = Mock() + monkeypatch.setattr(module, 'record_audio', metric) + assert recovery.recover('bad', 'synthetic-owner') == 'recovered' + metric.assert_called_once_with('sync', 'accepted', 'unknown', 5, False) diff --git a/backend/tests/unit/test_batch_pipeline_runtime.py b/backend/tests/unit/test_batch_pipeline_runtime.py index ed67bf217ae..dbd8e797c90 100644 --- a/backend/tests/unit/test_batch_pipeline_runtime.py +++ b/backend/tests/unit/test_batch_pipeline_runtime.py @@ -289,3 +289,21 @@ def test_memory_probe_separates_host_and_device_peaks(): assert result['gpu_allocated_peak_bytes'] == 30 assert result['gpu_reserved_peak_bytes'] == 40 assert result['host_process_peak_bytes'] > 0 + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_audio_counter_is_exported_by_the_worker_metrics_registry(monkeypatch, enforce): + from utils.observability.stt_cap_dispatch import record_audio + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + state = runtime.RuntimeState() + server = runtime.health_server(state, CollectorRegistry(), 0) + try: + record_audio('sync', 'accepted', 'parakeet', 5) + body = urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}/metrics').read().decode() + assert ('omi_stt_cap_provider_audio_seconds_total' in body) is enforce + if enforce: + assert 'path="sync"' in body and 'provider="parakeet"' in body + finally: + server.shutdown() + server.server_close() diff --git a/backend/tests/unit/test_batch_pipeline_worker.py b/backend/tests/unit/test_batch_pipeline_worker.py index f264be2cae4..0064e307d55 100644 --- a/backend/tests/unit/test_batch_pipeline_worker.py +++ b/backend/tests/unit/test_batch_pipeline_worker.py @@ -5,6 +5,7 @@ import sys import wave from types import SimpleNamespace +from unittest.mock import Mock import numpy as np import pytest @@ -715,3 +716,127 @@ def diarize(samples): assert part_payload['words'] == [{'word': ['first', 'second'][index], 'timestamp': [0.25, 0.75]}] assert part_payload['diarization']['turns'] == [{'start': 0.0, 'end': 2.0, 'speaker': ['left', 'right'][index]}] assert part_payload['speech_seconds'] == 2.0 + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_provenance_is_additive_and_uses_actual_padded_provider_audio(monkeypatch, enforce): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + worker, words, saved, _, _, _ = setup_worker() + worker.process_batch([item('a', 2)], authorize_dispatch=lambda ids: ids) + if enforce: + assert saved['a']['cap_dispatch'] == { + 'omi_funded_accepted_provider_audio_seconds': 2, + 'admission': 'authorized_compute', + } + else: + assert 'cap_dispatch' not in saved['a'] + assert saved['a']['speech_seconds'] == 2 + assert words.calls == [[2]] + + +def test_worker_rechecks_immediately_before_asr_and_holds_without_failure(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + worker, words, saved, _, diarized, failures = setup_worker() + checks = [] + + def authorize(ids): + checks.append(ids) + return set() + + result = worker.process_batch([item('a', 2)], authorize_dispatch=authorize) + assert result == [HeldWork('a')] + assert checks == [{'a'}] + assert not words.calls and not saved and not diarized and not failures + + +def test_worker_missing_authority_does_not_spend_at_cap(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + worker, words, saved, _, _, _ = setup_worker() + assert worker.process_batch([item('a', 2)]) == [HeldWork('a')] + assert not words.calls and not saved + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('recovery_allowed', [False, True]) +def test_full_capture_retry_rechecks_cap_and_counts_actual_padding(monkeypatch, enforce, recovery_allowed): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + calls, checks = [], [] + + class Model: + def transcribe(self, arrays, **kwargs): + calls.append([len(samples) / 16000 for samples in arrays]) + entries = [] if len(calls) == 1 else [{'word': 'recovered', 'start': 2.25, 'end': 2.75}] + return [SimpleNamespace(text='recovered' if entries else '', timestamp={'word': entries})] + + provider = ParakeetWords(Model()) + worker, _, saved, _, diarized, failures = setup_worker( + decode=lambda work: np.ones(int(work.duration_seconds * 16000), dtype=np.float32), + words_providers={'parakeet': provider}, + recover_words=provider.recover_empty, + ) + + def authorize(ids): + checks.append(ids) + return ids if len(checks) == 1 or recovery_allowed else set() + + result = worker.process_batch([item('a', 2)], authorize_dispatch=authorize) + if enforce and not recovery_allowed: + assert result == [HeldWork('a')] + assert calls == [[2]] + assert not saved and not diarized and not failures + else: + assert result[0].outcome == 'success' + assert calls == [[2], [8]] + assert saved['a']['words'] == [{'word': 'recovered', 'timestamp': [0.25, 0.75]}] + assert not failures + if enforce: + assert saved['a']['cap_dispatch']['omi_funded_accepted_provider_audio_seconds'] == 10 + else: + assert 'cap_dispatch' not in saved['a'] + assert checks == ([{'a'}, {'a'}] if enforce else []) + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_oversized_recovery_neither_rechecks_nor_counts_unsubmitted_audio(enforce): + provider = ParakeetWords(SimpleNamespace(transcribe=lambda *a, **kw: pytest.fail('oversized provider request'))) + before, after = Mock(), Mock() + result = provider.recover_empty( + np.ones(117 * 16000, dtype=np.float32), + 272, + **({'before_dispatch': before, 'after_dispatch': after} if enforce else {}), + ) + assert result is None if enforce else result == [] + before.assert_not_called() + after.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_oversized_empty_retry_holds_only_when_enforced(monkeypatch, enforce): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + model = Mock() + model.transcribe.side_effect = lambda arrays, **_: [ + SimpleNamespace(text='', timestamp={'word': []}) for _ in arrays + ] + provider = ParakeetWords(model) + worker, _, saved, _, diarized, failures = setup_worker( + decode=lambda work: np.ones(int(work.duration_seconds * 16000), dtype=np.float32), + words_providers={'parakeet': provider}, + recover_words=provider.recover_empty, + ) + result = worker.process_batch([item('a', 117)], authorize_dispatch=lambda ids: ids) + assert model.transcribe.call_count == 1 # Initial ASR only; 121s padded retry cannot fit. + assert not saved and not diarized + if enforce: + assert result == [HeldWork('a')] + assert not failures + else: + # Exact main behavior: an empty retry enters ordinary provider failure. + assert not result and failures == [('a', 'BatchValidationError')] diff --git a/backend/tests/unit/test_listen_parity_runtime.py b/backend/tests/unit/test_listen_parity_runtime.py index cee5aae05d1..e65b4e16ff2 100644 --- a/backend/tests/unit/test_listen_parity_runtime.py +++ b/backend/tests/unit/test_listen_parity_runtime.py @@ -16,6 +16,7 @@ from utils.executors import storage_executor from utils.listen_session_bootstrap import ListenConnectBase from utils.stt.streaming import STTService +from utils.stt import language_policy _PRINCIPAL = 'allowed-runtime-principal' _TRANSCRIPT = 'synthetic runtime transcript' @@ -172,6 +173,9 @@ async def run_blocking_fake(executor, function, *args, **kwargs): monkeypatch.setattr(runtime_module, 'PUSHER_ENABLED', False) monkeypatch.setattr(runtime_module, 'should_load_speech_profile', lambda **_kwargs: False) monkeypatch.setattr(runtime_module, 'should_enable_speaker_identification', lambda **_kwargs: False) + # This seam verifies capture/persistence, with synthetic STT. Language + # inference has its own suite; do not cold-load its models in a worker. + monkeypatch.setattr(language_policy, 'classify_output', lambda *_args: ('in_profile', 'en')) monkeypatch.setattr(receiver_module, 'should_initialize_vad_gate', lambda **_kwargs: False) monkeypatch.setattr(ListenReceiver, '_create_stt_socket', create_stt_socket) monkeypatch.setattr(runtime, '_build_components', build_components) diff --git a/backend/tests/unit/test_listen_stt_death_monitor.py b/backend/tests/unit/test_listen_stt_death_monitor.py index af8e059df09..b08157645d6 100644 --- a/backend/tests/unit/test_listen_stt_death_monitor.py +++ b/backend/tests/unit/test_listen_stt_death_monitor.py @@ -97,3 +97,22 @@ async def test_monitor_exits_without_terminating_once_already_terminal(): with patch.object(receiver_mod, 'terminate_live_stt_session', new=AsyncMock()) as terminate: await ListenReceiver._monitor_stt_death(monitor_self) terminate.assert_not_awaited() + + +@pytest.mark.parametrize('legacy', [False, True]) +@pytest.mark.anyio +async def test_cap_pause_keeps_receiving_when_provider_dies(legacy): + from utils.stt_cap_dispatch import LiveAdmission + + receiver = _monitor_self(dead=True) + receiver.cap_admission = LiveAdmission('synthetic') + receiver.cap_admission.omi_selected = True + receiver.host.state.stt_cap_admission = receiver.cap_admission + if legacy: + receiver.recovery = None + receiver._failover_stt_socket = AsyncMock(side_effect=AssertionError('cap pause must not dial a replacement')) + with patch.object(receiver_mod, 'terminate_live_stt_session', new=AsyncMock()) as terminate: + await ListenReceiver._monitor_stt_death(receiver) + terminate.assert_not_awaited() + receiver._failover_stt_socket.assert_not_awaited() + assert receiver.host.state.active diff --git a/backend/tests/unit/test_locked_content_reset_recovery.py b/backend/tests/unit/test_locked_content_reset_recovery.py index 2794d42ee63..649e9fd07e6 100644 --- a/backend/tests/unit/test_locked_content_reset_recovery.py +++ b/backend/tests/unit/test_locked_content_reset_recovery.py @@ -295,3 +295,17 @@ def test_initial_enablement_drains_backlog_midmonth(env): env.module.recover_locked_content('test-user', now=midmonth) assert not any(env.locks.values()) assert env.state_data['completed_period'] == '2026-11' + + +def test_cap_audio_recovery_is_kicked_even_with_legacy_reset_flag_off(env, monkeypatch): + from unittest.mock import Mock + + fake = ModuleType('utils.sync.cap_hold_recovery') + fake.kick = Mock() + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setenv('LOCKED_CONTENT_RESET_RECOVERY_ENABLED', 'false') + env.module.register_cap_audio_recovery(fake.kick) + env.module.recover_locked_content('test-user', now=NOW) + env.module.recover_locked_content('test-user', now=NOW) + assert fake.kick.call_count == 2 + env.state.get.assert_not_called() diff --git a/backend/tests/unit/test_parakeet_prerecorded.py b/backend/tests/unit/test_parakeet_prerecorded.py index 2dd0e218484..d0825d35cf6 100644 --- a/backend/tests/unit/test_parakeet_prerecorded.py +++ b/backend/tests/unit/test_parakeet_prerecorded.py @@ -18,6 +18,7 @@ os.environ.setdefault('HOSTED_PARAKEET_API_URL', 'http://fake-parakeet:8080') import utils.stt.pre_recorded as pr # noqa: E402 +from utils.stt.streaming import STTService, get_stt_service_for_language # noqa: E402 def _cdist(a, b, metric='cosine'): @@ -522,8 +523,6 @@ def make_stream_resp(): class TestStreamingFactoryRouting: def test_parakeet_in_stt_service_models(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet']), patch.dict( os.environ, {'HOSTED_PARAKEET_API_URL': 'http://fake-parakeet:8080'} ): @@ -532,8 +531,6 @@ def test_parakeet_in_stt_service_models(self): assert model == 'parakeet' def test_parakeet_streaming_fallback_for_cjk_uses_deepgram(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet', 'dg-nova-3']), patch.dict( os.environ, {'HOSTED_PARAKEET_API_URL': 'http://fake-parakeet:8080'} ): @@ -542,8 +539,6 @@ def test_parakeet_streaming_fallback_for_cjk_uses_deepgram(self): assert model == 'nova-3' def test_parakeet_fallback_without_url_uses_policy_default(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet']), patch.dict(os.environ, {}, clear=False): env_backup = os.environ.pop('HOSTED_PARAKEET_API_URL', None) try: diff --git a/backend/tests/unit/test_prerecorded_dispatch_selection.py b/backend/tests/unit/test_prerecorded_dispatch_selection.py new file mode 100644 index 00000000000..f9fac9b507e --- /dev/null +++ b/backend/tests/unit/test_prerecorded_dispatch_selection.py @@ -0,0 +1,34 @@ +"""A held piece must use the provider named by its durable dispatch intent.""" + +from unittest.mock import Mock + +import pytest + +from utils.stt import pre_recorded as stt + + +@pytest.mark.parametrize('pinned', [False, True]) +def test_url_dispatch_preserves_default_selection_or_uses_checkpointed_provider(monkeypatch, pinned): + monkeypatch.delenv('OMI_STT_STUB', raising=False) + selection = Mock(return_value=('modulate', 'en', 'velma-2')) + monkeypatch.setattr(stt, 'get_prerecorded_service', selection) + modulate, parakeet = Mock(), Mock() + monkeypatch.setattr(stt, 'ModulatePrerecordedProvider', lambda: modulate) + monkeypatch.setattr(stt, 'ParakeetPrerecordedProvider', lambda: parakeet) + options = {'selected_service': ('parakeet', 'multi', 'parakeet')} if pinned else {} + selected, unused = (parakeet, modulate) if pinned else (modulate, parakeet) + assert stt.prerecorded('synthetic-url', language='multi', **options) is selected.transcribe_url.return_value + selected.transcribe_url.assert_called_once_with( + 'synthetic-url', + speakers_count=None, + attempts=0, + return_language=False, + diarize=True, + language='multi', + keywords=None, + ) + unused.transcribe_url.assert_not_called() + if pinned: + selection.assert_not_called() + else: + selection.assert_called_once_with('multi') diff --git a/backend/tests/unit/test_stt_cap_dispatch.py b/backend/tests/unit/test_stt_cap_dispatch.py new file mode 100644 index 00000000000..60d2b31bbfa --- /dev/null +++ b/backend/tests/unit/test_stt_cap_dispatch.py @@ -0,0 +1,226 @@ +"""Dispatch-level regression matrix: no billed provider work after denial.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from config import stt_cap_dispatch as flag +from utils import stt_cap_dispatch as cap +from routers.listen.receiver import _RecordingSTTSocket +from utils.stt.live_session import LiveLegSocket +from utils.stt.vad_gate import GatedSTTSocket + + +@pytest.fixture(autouse=True) +def isolation(monkeypatch): + monkeypatch.delenv('STT_CAP_DISPATCH_ENFORCEMENT', raising=False) + monkeypatch.setattr(cap, 'get_byok_key', lambda _: None) + monkeypatch.setattr(cap, 'record_audio', Mock()) + + +@pytest.mark.parametrize('value', [None, '', 'false', '1', 'yes']) +def test_dark_default(value, monkeypatch): + if value is not None: + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', value) + resolver = Mock(side_effect=AssertionError('flag off performed admission IO')) + monkeypatch.setattr(cap, 'funded_allowance', resolver) + assert not flag.enabled() + assert cap.sync_allowed('synthetic') + resolver.assert_not_called() + + +@pytest.mark.parametrize('provider', ['deepgram', 'soniox', 'modulate', 'parakeet']) +@pytest.mark.parametrize('enforce', [False, True]) +async def test_provider_creation_and_sends_flag_matrix(provider, enforce, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + admission = cap.LiveAdmission('synthetic') + admission.reason = 'plan_allowance_exhausted' + raw = Mock(send=Mock(return_value=True), is_connection_dead=False) + factory = AsyncMock(return_value=raw) + connect = cap.live_provider(provider, 0)(factory) + token = cap.current_live_admission.set(admission) + try: + socket = await connect(16000) + finally: + cap.current_live_admission.reset(token) + socket.send(b'\0' * 32000) + assert factory.await_count == (0 if enforce else 1) + assert raw.send.call_count == (0 if enforce else 1) + assert not socket.is_connection_dead + if enforce: + cap.record_audio.assert_called_with('live', 'suppressed', provider, 1, True) + + +async def test_one_balance_covers_multichannel_replay_and_replacement(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = cap.LiveAdmission('synthetic') + state.managed, state.remaining = True, 2 + state.reason = 'plan_within_allowance' + providers = [Mock(send=Mock(return_value=True)) for _ in range(4)] + sockets = [cap.CapSocket(state, 'soniox', 16000, AsyncMock(), raw) for raw in providers] + # Two channel sends consume the shared balance. Replay and successor sends + # cannot restart it. Policy refusal never publishes transport death. + for socket in sockets: + assert socket.send(b'\0' * 32000) + assert [raw.send.call_count for raw in providers] == [1, 1, 0, 0] + assert state.remaining == 0 + assert state.reason == 'plan_allowance_exhausted' + + +async def test_byok_exempts_only_customer_deepgram_then_reauthorizes_fallback(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(cap, 'get_byok_key', lambda _: 'synthetic-key') + state = cap.LiveAdmission('synthetic') + dg, fallback = Mock(), Mock() + state.send(dg, b'00', 'deepgram', 16000) + state.send(fallback, b'00', 'modulate', 16000) + dg.send.assert_called_once() + fallback.send.assert_not_called() + monkeypatch.setenv('DEEPGRAM_SELF_HOSTED_ENABLED', 'true') + state.send(dg, b'00', 'deepgram', 16000) + assert dg.send.call_count == 1 + + +async def test_held_constructor_resumes_after_reset_without_reopening_capture(monkeypatch): + state = cap.LiveAdmission('synthetic') + raw = Mock() + factory = AsyncMock(return_value=raw) + socket = cap.CapSocket(state, 'modulate', 16000, factory) + state.send(raw, b'00', 'modulate', 16000) + factory.assert_not_called() + state.managed, state.remaining = True, 10 + await socket.resume() + assert socket.raw is raw + assert socket.send(b'00') is raw.send.return_value + socket.finish() + raw.finish.assert_called_once() + + +async def test_resumed_registry_leg_unwraps_nested_provider_and_debits_once(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = cap.LiveAdmission('synthetic') + raw = Mock(send=Mock(return_value=True)) + inner = cap.live_provider('modulate', 0)(AsyncMock(return_value=raw)) + socket = cap.CapSocket(state, 'modulate', 16000, lambda: inner(16000)) + state.managed, state.remaining = True, 2 + token = cap.current_live_admission.set(state) + try: + await socket.resume() + assert socket.send(b'\0' * 32000) + finally: + cap.current_live_admission.reset(token) + assert socket.raw is raw + raw.send.assert_called_once() + assert state.remaining == 1 + assert cap.record_audio.call_count == 1 + + +@pytest.mark.parametrize('managed', [False, True]) +def test_queued_sync_always_resolves_omi_funding(managed, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + resolver = Mock(return_value=SimpleNamespace(managed=managed, reason='plan_allowance_exhausted')) + monkeypatch.setattr('utils.subscription.resolve_transcription_allowance', resolver) + assert cap.sync_allowed('synthetic', 60, 'parakeet') is managed + resolver.assert_called_once_with('synthetic', source=None, byok_active=False, omi_funded=True) + + +@pytest.mark.parametrize('kind', ['recording', 'gated', 'managed', 'replay', 'idle-replay']) +def test_suppression_never_fabricates_provider_timeline_spans(kind): + state = cap.LiveAdmission('synthetic') + state.reason = 'plan_allowance_exhausted' + raw = Mock() + guarded = cap.CapSocket(state, 'parakeet', 16000, AsyncMock(), raw) + tracker = Mock() + if kind == 'recording': + socket = _RecordingSTTSocket(guarded, tracker) + assert socket.send(b'00', start_sample=900) + elif kind == 'gated': + socket = object.__new__(GatedSTTSocket) + socket._conn, socket._send_tracker = guarded, tracker + assert socket.send(b'00', start_sample=900) + else: + socket = object.__new__(LiveLegSocket) + socket.raw, socket._send_tracker = guarded, tracker + socket.session = SimpleNamespace(receiver=SimpleNamespace(_window_ring=lambda: None)) + if kind == 'replay': + assert socket.replay_send(b'00', 900) + elif kind == 'idle-replay': + assert socket.send_admitted_audio(b'00', ((900, 1),)) + else: + assert socket.send(b'00', start_sample=900) + raw.send.assert_not_called() + tracker.note_accepted.assert_not_called() + tracker.note_accepted_spans.assert_not_called() + + +@pytest.mark.parametrize('provider', ['deepgram', 'soniox', 'modulate', 'parakeet']) +async def test_real_provider_constructor_wiring_refuses_before_sdk_or_wire(provider, monkeypatch): + from utils.stt import streaming + import websockets + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + sdk = Mock(side_effect=AssertionError('denied constructor reached SDK')) + wire = AsyncMock(side_effect=AssertionError('denied constructor reached network')) + monkeypatch.setattr(streaming, 'DeepgramClient', sdk) + monkeypatch.setattr(websockets, 'connect', wire) + state = cap.LiveAdmission('synthetic') + token = cap.current_live_admission.set(state) + try: + if provider == 'deepgram': + socket = await streaming.process_audio_dg(Mock(), 'en', 16000, 1) + elif provider == 'soniox': + socket = await streaming.process_audio_soniox(Mock(), 16000, 'en') + elif provider == 'modulate': + socket = await streaming.process_audio_modulate(Mock(), 16000, 'en') + else: + socket = await streaming.process_audio_parakeet(Mock(), 'en', 16000, 1) + finally: + cap.current_live_admission.reset(token) + assert isinstance(socket, cap.CapSocket) + assert socket.raw is None + sdk.assert_not_called() + wire.assert_not_called() + + +async def test_initial_exhaustion_receives_without_entering_provider_router(monkeypatch): + from routers.listen.receiver import ListenReceiver + + state = cap.LiveAdmission('synthetic') + state.refresh = AsyncMock() + receiver = object.__new__(ListenReceiver) + receiver.cap_admission = state + receiver.host = SimpleNamespace(stt_service='modulate') + assert await receiver.initialize_stt() + assert receiver.cap_paused and state.funded_paused + assert not hasattr(receiver, 'live_demand') # No selection, model or provider initialization. + + +async def test_terminal_provider_error_cannot_close_capture_during_cap_pause(): + from utils.stt.live_failure import terminate_live_stt_session, terminate_live_stt_backoff + + state = cap.LiveAdmission('synthetic') + state.omi_selected = True + session = SimpleNamespace(stt_cap_admission=state, stt_terminal_failure=False, active=True) + websocket = SimpleNamespace(send_json=AsyncMock(), close=AsyncMock()) + assert not await terminate_live_stt_session( + websocket, session, failure=Mock(), reason='connection_lost', platform='ios' + ) + assert not await terminate_live_stt_backoff(websocket, session, reason='provider_unavailable', retry_after=30) + assert session.active and not session.stt_terminal_failure + websocket.send_json.assert_not_called() + websocket.close.assert_not_called() + + +async def test_paused_soniox_binds_wire_ledger_before_resumed_audio(): + state = cap.LiveAdmission('synthetic') + raw = Mock() + socket = cap.CapSocket(state, 'soniox', 16000, AsyncMock(return_value=raw)) + ledger = object() + socket.set_wire_ledger(ledger) + socket.set_capture_axis_ledger(ledger) + raw.set_wire_ledger.assert_not_called() + state.managed, state.remaining = True, 10 + await socket.resume() + raw.set_wire_ledger.assert_called_once_with(ledger) + raw.set_capture_axis_ledger.assert_called_once_with(ledger) diff --git a/backend/tests/unit/test_stt_cap_dispatch_http.py b/backend/tests/unit/test_stt_cap_dispatch_http.py new file mode 100644 index 00000000000..25ff5e2f20b --- /dev/null +++ b/backend/tests/unit/test_stt_cap_dispatch_http.py @@ -0,0 +1,152 @@ +"""Actual sync HTTP routing with local fake persistence/provider boundaries.""" + +from types import SimpleNamespace +from unittest.mock import Mock, AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +import pytest + +from routers import sync as router +from utils.sync.lanes import SyncLane + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.setenv('OMI_ENV_STAGE', 'dev') + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(router, 'get_sync_ledger_fence_mode', lambda: router.SyncLedgerFenceMode.ACTIVE) + monkeypatch.setattr( + router, + 'classify_sync_lane', + lambda *a, **kw: SimpleNamespace( + lane=SyncLane.FRESH, + trust=SimpleNamespace(value='untrusted'), + maximum_age_seconds=0, + reason='synthetic', + automatic_recovery_allowed=True, + ), + ) + monkeypatch.setattr(router, 'get_hard_restriction_status', lambda _: (False, None)) + monkeypatch.setattr(router, 'is_daily_audio_ceiling_exceeded', lambda _: False) + monkeypatch.setattr(router, 'has_transcription_credits', lambda _: False) + monkeypatch.setattr( + router, 'resolve_transcription_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted') + ) + monkeypatch.setattr(router, 'retrieve_file_paths', lambda *_: ['synthetic.bin']) + monkeypatch.setattr(router, 'archive_mobile_batch_raw_files', lambda *_: []) + monkeypatch.setattr(router, 'decode_files_to_wav', lambda _: ['1792152000.wav']) + monkeypatch.setattr(router, '_cleanup_files', Mock()) + + async def held(*args, **kwargs): + kwargs['admission']['reason'] = router.resolve_transcription_allowance('synthetic').reason + return True + + monkeypatch.setattr(router, 'hold_capture_if_exhausted', AsyncMock(side_effect=held)) + monkeypatch.setattr(router, 'retrieve_vad_segments', Mock(side_effect=AssertionError('denied audio reached VAD'))) + app = FastAPI() + app.include_router(router.router) + app.dependency_overrides[router.auth.get_current_user_uid] = lambda: 'synthetic' + with TestClient(app, raise_server_exceptions=False) as test_client: + yield test_client + + +def test_exhausted_upload_acknowledges_only_after_durable_hold(client): + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json() == { + 'new_memories': [], + 'updated_memories': [], + 'credits_exhausted': True, + 'held_audio': True, + 'held_files': 1, + } + router.hold_capture_if_exhausted.assert_called_once() + router.retrieve_vad_segments.assert_not_called() + + +def test_failed_hold_is_not_reported_as_an_accepted_upload(client): + router.hold_capture_if_exhausted.side_effect = RuntimeError('synthetic storage unavailable') + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 500 + router.retrieve_vad_segments.assert_not_called() + + +def test_v2_typed_result_preserves_flag_off_wire_and_exposes_only_present_hold(): + from models.sync_job_status import SyncJobStatusResponse + + body = {'job_id': 'job', 'status': 'completed', 'result': {'new_memories': [], 'updated_memories': []}} + result = SyncJobStatusResponse(**body).model_dump(exclude_none=True)['result'] + assert result == { + 'new_memories': [], + 'updated_memories': [], + 'failed_segments': 0, + 'total_segments': 0, + 'errors': [], + } + body['result']['held_audio'] = True + assert SyncJobStatusResponse(**body).model_dump(exclude_none=True)['result']['held_audio'] is True + + +def test_flag_off_http_result_retains_main_shape_without_hold_dispatch(client, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false') + monkeypatch.setattr(router, 'FAIR_USE_ENABLED', False) + monkeypatch.setattr(router, 'retrieve_vad_segments', Mock()) + monkeypatch.setattr(router.users_db, 'get_user_transcription_preferences', Mock(return_value={})) + monkeypatch.setattr(router.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(router, 'build_person_embeddings_cache', Mock(return_value={})) + monkeypatch.setattr(router, '_reprocess_merged_conversations', Mock()) + monkeypatch.setattr(router, 'schedule_person_voice_learning_retries', Mock()) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.content == b'{"new_memories":[],"updated_memories":[]}' + router.hold_capture_if_exhausted.assert_not_called() + + +@pytest.mark.parametrize('accepted', [False, True]) +def test_late_v1_hold_is_reported_and_only_processed_speech_is_metered(client, monkeypatch, accepted): + monkeypatch.setattr(router, 'FAIR_USE_ENABLED', False) + router.hold_capture_if_exhausted.side_effect = None + router.hold_capture_if_exhausted.return_value = False + paths = ['1792152000.wav', '1792152020.wav'] if accepted else ['1792152020.wav'] + monkeypatch.setattr(router, 'retrieve_vad_segments', lambda path, segments, errors, **kw: segments.update(paths)) + monkeypatch.setattr(router, 'get_wav_duration', lambda path: 20) + monkeypatch.setattr(router.users_db, 'get_user_transcription_preferences', Mock(return_value={})) + monkeypatch.setattr(router.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(router, 'build_person_embeddings_cache', Mock(return_value={})) + monkeypatch.setattr(router, '_reprocess_merged_conversations', Mock()) + monkeypatch.setattr(router, 'schedule_person_voice_learning_retries', Mock()) + usage = Mock() + monkeypatch.setattr(router, 'record_usage', usage) + + def process(path, uid, response, *args, **kwargs): + if path == '1792152000.wav': + response['new_memories'].add('accepted') + return True + response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 20, 'reason': 'plan_allowance_exhausted'}} + return False + + monkeypatch.setattr(router, 'process_segment', process) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json() == { + 'new_memories': ['accepted'] if accepted else [], + 'updated_memories': [], + 'held_audio': True, + 'credits_exhausted': True, + 'held_segments': 1, + } + if accepted: + assert usage.call_args.kwargs == {'transcription_seconds': 20, 'speech_seconds': 20} + else: + usage.assert_not_called() + + +@pytest.mark.parametrize('reason', ['allowance_unavailable', 'launch_config_missing', 'launch_config_invalid']) +def test_transient_policy_hold_does_not_claim_credits_exhausted(client, monkeypatch, reason): + monkeypatch.setattr(router, 'resolve_transcription_allowance', lambda _: SimpleNamespace(reason=reason)) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json()['held_audio'] is True + assert response.json()['credits_exhausted'] is False + router.retrieve_vad_segments.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_content.py b/backend/tests/unit/test_sync_cap_content.py new file mode 100644 index 00000000000..859533efdea --- /dev/null +++ b/backend/tests/unit/test_sync_cap_content.py @@ -0,0 +1,99 @@ +"""A held source is durable custody, never a completed-transcription proof.""" + +from copy import deepcopy + +import pytest + +from database import sync_cap_holds as holds, sync_ledger as ledger +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore + + +@pytest.fixture +def state(monkeypatch): + monkeypatch.setenv('OMI_ENV_STAGE', 'prod') + client = StrictFirestore() + ref = ledger.sync_content_reference(client, 'synthetic', 'content') + row = { + 'status': 'processing', + 'job_id': 'original', + 'ledger_run_token': 'owner-token', + 'ledger_run_epoch': 1, + 'expires_at': 'old-ttl', + } + client.rows[ref.path] = row + monkeypatch.setattr(holds, 'get_firestore_client', lambda: client) + return client, ref + + +def _hold(): + return holds.mark_content_held( + 'synthetic', + 'content', + 'original', + { + 'held_audio': True, + 'new_memories': [], + 'updated_memories': [], + 'failed_segments': 0, + 'total_segments': 0, + 'errors': [], + 'outcome': 'success', + }, + ['a', 'b'], + run_token='owner-token', + run_epoch=1, + ) + + +def test_hold_is_fenced_and_never_manufactures_completion(state): + client, ref = state + before = deepcopy(client.rows[ref.path]) + assert not holds.mark_content_held( + 'synthetic', + 'content', + 'stale', + {'held_audio': True, 'failed_segments': 0, 'total_segments': 0, 'errors': [], 'outcome': 'success'}, + ['a'], + run_token='stale', + run_epoch=0, + ) + assert client.rows[ref.path] == before + assert _hold() + row = client.rows[ref.path] + assert row['status'] == 'held' + # The strict transaction fixture records transforms rather than applying + # DELETE_FIELD; assert the removal request, not a simulated TTL deletion. + assert str(row['expires_at']) == str(holds.firestore.DELETE_FIELD) + assert not ledger.is_valid_completed_sync_content_result(row['result']) + claim = ledger.claim_sync_content('synthetic', 'content', 'retry', 'fresh', firestore_client=client) + assert claim['outcome'] == 'held' and claim['result']['held_audio'] + assert holds.recovery_status('synthetic', 'content', 'a') == 'ready' + + +def test_only_all_recovered_holds_can_complete_and_settlement_is_idempotent(state): + client, ref = state + assert _hold() + assert holds.settle_hold('synthetic', 'content', 'a', {'new_memories': ['one']}, 2) + first = deepcopy(client.rows[ref.path]) + assert first['status'] == 'held' + assert first['result']['held_audio'] + assert holds.settle_hold('synthetic', 'content', 'a', {'new_memories': ['wrong']}, 99) + assert client.rows[ref.path] == first + assert holds.settle_hold('synthetic', 'content', 'b', {'updated_memories': ['two']}, 1) + row = client.rows[ref.path] + assert row['status'] == 'completed' + assert row['result']['new_memories'] == ['one'] + assert row['result']['updated_memories'] == ['two'] + assert row['result']['recovered_segments'] == 3 + assert ledger.is_valid_completed_sync_content_result(row['result']) + assert 'expires_at' in row + + +def test_recovery_does_not_overwrite_a_different_content_owner(state): + client, ref = state + assert _hold() + client.rows[ref.path].update(status='processing', dispatch_protocol='replacement', job_id='replacement') + before = deepcopy(client.rows[ref.path]) + assert holds.recovery_status('synthetic', 'content', 'a') == 'waiting' + assert not holds.settle_hold('synthetic', 'content', 'a', {}, 1) + assert client.rows[ref.path] == before diff --git a/backend/tests/unit/test_sync_cap_coordinator.py b/backend/tests/unit/test_sync_cap_coordinator.py new file mode 100644 index 00000000000..c6be217c8a6 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_coordinator.py @@ -0,0 +1,116 @@ +"""Exercise real v2 coordination: coverage precedes custody and late holds do not bill.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +from database import sync_cap_holds +from utils.sync import cap_hold +from tests.unit.test_wal_audio_coverage_pipeline import ( + coordinator, + _wire, + _coverage_env, + _committed_envelope, + _claim, + _read_payload, + _write_labeled_wav, + _frame_bytes, + FRAME_SAMPLES, + BIN_NAME, + ORIGIN, + generation, +) + + +async def _run(module, tmp_path): + await module._run_full_pipeline_background_async( + 'job-coverage', + 'uid', + [f'/tmp/{BIN_NAME}'], + 'omi', + False, + str(tmp_path / 'job'), + client_device_id='pendant', + recording_session_id=ORIGIN, + task_mode=True, + content_id='content', + capture_evidence_claims={BIN_NAME: _claim()}, + ) + + +def _ledger(pipeline, monkeypatch): + pipeline.get_sync_content_partial_result = Mock(return_value={}) + pipeline.get_processed_sync_segment_ids = Mock(return_value=set()) + pipeline.compute_sync_segment_id = lambda uid, path: Path(path).name + pipeline.checkpoint_sync_content_partial_result = Mock(return_value=True) + pipeline.mark_sync_content_completed = Mock(return_value=True) + writer = Mock(return_value=True) + monkeypatch.setattr(sync_cap_holds, 'mark_content_held', writer) + return writer + + +async def test_full_hold_contains_only_wal_audio_not_already_committed(coordinator, monkeypatch, tmp_path): + module, stubs = coordinator + pipeline = stubs['pipeline'] + _coverage_env(monkeypatch) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = _wire( + pipeline, + monkeypatch, + tmp_path, + lineage_rows=[generation(1, capture_evidence=_committed_envelope(range(7)))], + ) + writer = _ledger(pipeline, monkeypatch) + monkeypatch.setattr( + cap_hold, 'funded_allowance', lambda uid: SimpleNamespace(managed=False, reason='plan_allowance_exhausted') + ) + retained = [] + + async def retain(uid, path, metadata, reason, receipt): + retained.append((_read_payload(path), metadata)) + receipt.update(id='held-source', seconds=1.5) + + monkeypatch.setattr(cap_hold, '_hold_capture', retain) + await _run(module, tmp_path) + assert len(retained) == 1 + assert retained[0][0] == b''.join(_frame_bytes(v, FRAME_SAMPLES) for v in (7, 8, 9)) + assert retained[0][1]['source_frame_map']['claim']['source_frame_start'] == 7 + assert retained[0][1]['recording_session_id'] == ORIGIN + assert state.vad_seen == state.processed == [] + pipeline.record_usage.assert_not_called() + pipeline.mark_sync_content_completed.assert_not_called() + assert writer.call_args.args[3]['held_audio'] is True + assert writer.call_args.kwargs['hold_ids'] == ['held-source'] + + +async def test_late_hold_reports_custody_and_charges_only_dispatched_speech(coordinator, monkeypatch, tmp_path): + module, stubs = coordinator + pipeline = stubs['pipeline'] + _coverage_env(monkeypatch) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + _wire(pipeline, monkeypatch, tmp_path, lineage_rows=[]) + writer = _ledger(pipeline, monkeypatch) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda uid: SimpleNamespace(managed=True, reason='available')) + paths = [tmp_path / '1760000000.wav', tmp_path / '1760000002.5.wav'] + for path in paths: + _write_labeled_wav(path, range(5)) + pipeline.retrieve_vad_segments = lambda path, segments, errors, **kw: segments.update(map(str, paths)) + + def process(path, uid, response, lock, errors, *args, **kwargs): + if path == str(paths[0]): + response['updated_memories'].add('accepted') + return True + response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 2.5}} + kwargs['deferred_outcome']['cap_held'] = True + return False + + pipeline.process_segment = process + await _run(module, tmp_path) + pipeline.mark_sync_content_completed.assert_not_called() + result = writer.call_args.args[3] + assert result['held_audio'] is True + assert result['failed_segments'] == 0 + assert result['updated_memories'] == ['accepted'] + assert writer.call_args.kwargs['hold_ids'] == ['late'] + assert pipeline.record_usage.call_args.kwargs['transcription_seconds'] == 2 + assert pipeline.record_usage.call_args.kwargs['speech_seconds'] == 2 diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py new file mode 100644 index 00000000000..fcd8af910ac --- /dev/null +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -0,0 +1,274 @@ +"""Both sync versions execute process_segment; prove the last pre-STT gate.""" + +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock +import wave + +import pytest + +from utils.sync import cap_hold, pipeline +from utils import stt_cap_dispatch as cap + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('queued', [False, True], ids=['sync-v1', 'sync-v2-worker']) +def test_exhaustion_holds_before_url_or_provider_dispatch(tmp_path, monkeypatch, enforce, queued): + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\0' * 32000) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + hold = Mock(side_effect=lambda *a, **kw: kw['receipt'].update(id='held', seconds=1)) + url, stt = Mock(return_value='synthetic-url'), Mock(return_value=([], 'en')) + monkeypatch.setattr(cap_hold, 'hold_segment', hold) + monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', url) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr(pipeline, 'keyterms', AsyncMock(return_value=[])) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', lambda _: ('modulate', None, 'velma-2')) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + monkeypatch.setattr(pipeline, 'sync_safety_wal_enabled', lambda: False) + errors = [] + assert ( + pipeline.process_segment( + str(path), + 'synthetic', + {'new_memories': set(), 'updated_memories': set()}, + threading.Lock(), + errors, + is_locked=True, + job_id='queued-job' if queued else None, + ) + is False + ) + assert not errors + if enforce: + hold.assert_called_once() + assert hold.call_args.args[2]['timestamp'] == 1792152000 + url.assert_not_called() + stt.assert_not_called() + else: + hold.assert_not_called() + assert stt.call_count == 2 # Main's bounded empty-result retry remains unchanged. + cap.funded_allowance.assert_not_called() + + +def test_hold_failure_cannot_acknowledge_audio_or_dispatch_provider(tmp_path, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr(pipeline, 'sync_allowed', lambda *_: False) + monkeypatch.setattr(cap_hold, 'hold_segment', Mock(side_effect=RuntimeError('synthetic upload failed'))) + stt = Mock() + monkeypatch.setattr(pipeline, 'prerecorded', stt) + errors = [] + pipeline.process_segment(str(tmp_path / '1792152000.wav'), 'synthetic', {}, threading.Lock(), errors) + assert errors + stt.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_recovery_pins_checkpointed_provider_only_with_enforcement(tmp_path, monkeypatch, enforce): + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\0' * 32000) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr(pipeline, 'sync_allowed', lambda *_: True) + monkeypatch.setattr(pipeline, 'funded_allowance', lambda *_: SimpleNamespace(reason='available')) + monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', Mock(return_value='synthetic-url')) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr(pipeline, 'keyterms', AsyncMock(return_value=[])) + selection = Mock(return_value=('modulate', None, 'velma-2')) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', selection) + stt = Mock(return_value=([], 'en')) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + errors = [] + service = ('parakeet', 'multi', 'parakeet') + assert not pipeline.process_segment( + str(path), 'synthetic', {}, threading.Lock(), errors, cap_recovery=True, cap_recovery_service=service + ) + assert not errors + assert stt.call_count == 2 + for call in stt.call_args_list: + if enforce: + assert call.kwargs['selected_service'] == service + else: + assert 'selected_service' not in call.kwargs + assert selection.call_count == (0 if enforce else 1) + + +@pytest.mark.parametrize('enforce', [False, True]) +async def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, monkeypatch, enforce): + from utils.sync import cap_hold + from models.conversation_enums import ConversationSource + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr( + cap_hold, + 'funded_allowance', + Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')), + ) + hold = AsyncMock() + monkeypatch.setattr(cap_hold, '_hold_capture', hold) + paths = [str(tmp_path / '1792152000.wav')] + assert await cap_hold.hold_capture_if_exhausted('synthetic', paths, ConversationSource.omi) is enforce + if enforce: + assert hold.call_args.args[2] == { + 'timestamp': 1792152000, + 'source': 'omi', + 'full_capture': True, + 'file_name': '1792152000.wav', + 'source_frame_map': None, + } + else: + hold.assert_not_called() + cap_hold.funded_allowance.assert_not_called() + + +def test_hold_manifest_fences_concurrent_recovery_and_stale_release(monkeypatch): + from contextlib import nullcontext + from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore + from utils.sync import cap_hold + + path = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({path: {'state': 'held', 'lease_until': 0}}) + ref = client.document('/'.join(path)) + monkeypatch.setattr(cap_hold, 'get_firestore_client', lambda: client) + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + first = cap_hold.claim('synthetic', ref, 100) + assert first and first['lease_token'] + assert cap_hold.claim('synthetic', ref, 101) is None + next_claim = cap_hold.claim('synthetic', ref, 2000) + monkeypatch.setattr(cap_hold.time, 'time', lambda: 2000) + assert not cap_hold.checkpoint('synthetic', ref, first['lease_token'], {'state': 'completed'}) + assert cap_hold.checkpoint('synthetic', ref, next_claim['lease_token'], {'finalized': True}) + cap_hold.release('synthetic', ref, first['lease_token']) + assert client.rows[path]['lease_token'] == next_claim['lease_token'] + cap_hold.release('synthetic', ref, next_claim['lease_token']) + assert client.rows[path]['lease_until'] == 0 + + +def test_hold_persists_encrypted_audio_before_manifest_and_never_expires(tmp_path, monkeypatch): + from contextlib import nullcontext + from utils.sync import cap_hold + from utils.encryption import decrypt_audio_file, decrypt + import json + + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as wav: + wav.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + wav.writeframes(b'\0' * 32000) + order = [] + uploaded = [] + ref, blob, bucket = Mock(), Mock(), Mock() + ref.get.return_value.exists = False + + def upload(stream, **kwargs): + uploaded.append(stream.read()) + order.append('upload') + + blob.upload_from_file.side_effect = upload + ref.create.side_effect = lambda *args, **kwargs: order.append('manifest') + monkeypatch.setattr(cap_hold, 'holds', lambda _: SimpleNamespace(document=lambda _: ref)) + monkeypatch.setattr(cap_hold, 'get_private_cloud_sync_bucket', lambda: bucket) + bucket.blob.return_value = blob + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(cap_hold, 'owner_storage_write_gate', lambda *_: nullcontext()) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted')) + meta = {'timestamp': 1792152000, 'source': 'omi', 'full_capture': True} + assert cap_hold.hold_segment('synthetic', str(path), meta) == 1 + assert order == ['upload', 'manifest'] + assert decrypt_audio_file(uploaded[0], 'synthetic') == path.read_bytes() + manifest = ref.create.call_args.args[0] + assert manifest['audio_path'].startswith('chunks/synthetic/cap-held/') + assert json.loads(decrypt(manifest['metadata'], 'synthetic')) == meta + assert 'expires_at' not in manifest + ref.create.reset_mock() + blob.upload_from_file.side_effect = RuntimeError('synthetic unavailable') + with pytest.raises(RuntimeError): + cap_hold.hold_segment('synthetic', str(path), meta) + ref.create.assert_not_called() + + +@pytest.mark.parametrize('corrupt', [False, True]) +def test_large_hold_streams_bounded_authenticated_chunks(tmp_path, monkeypatch, corrupt): + from contextlib import nullcontext + from pathlib import Path + from cryptography.exceptions import InvalidTag + + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as wav: + wav.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + wav.writeframes(b'\x01\x02' * cap_hold.AUDIO_CHUNK_BYTES) + expected = path.read_bytes() + ref, blob, bucket = Mock(), Mock(), Mock() + ref.get.return_value.exists = False + uploaded = [] + blob.upload_from_file.side_effect = lambda stream, **kw: uploaded.append(stream.read()) + bucket.blob.return_value = blob + monkeypatch.setattr(cap_hold, 'holds', lambda _: SimpleNamespace(document=lambda _: ref)) + monkeypatch.setattr(cap_hold, 'get_private_cloud_sync_bucket', lambda: bucket) + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(cap_hold, 'owner_storage_write_gate', lambda *_: nullcontext()) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted')) + monkeypatch.setattr(Path, 'read_bytes', Mock(side_effect=AssertionError('whole-file buffer'))) + encrypt_chunk = Mock(wraps=cap_hold.encrypt_audio_chunk) + monkeypatch.setattr(cap_hold, 'encrypt_audio_chunk', encrypt_chunk) + cap_hold.hold_segment('synthetic', str(path), {'timestamp': 1792152000, 'source': 'omi'}) + assert encrypt_chunk.call_count == 3 + assert max(len(call.args[0]) for call in encrypt_chunk.call_args_list) <= cap_hold.AUDIO_CHUNK_BYTES + encrypted = uploaded[0] + if corrupt: + encrypted = encrypted[:-1] + bytes([encrypted[-1] ^ 1]) + blob.download_to_file.side_effect = lambda output: output.write(encrypted) + output = tmp_path / 'recovered.wav' + manifest = ref.create.call_args.args[0] + if corrupt: + with pytest.raises(InvalidTag): + cap_hold.read_audio('synthetic', manifest, str(output)) + assert not output.exists() + else: + cap_hold.read_audio('synthetic', manifest, str(output)) + with output.open('rb') as recovered: + assert recovered.read() == expected + blob.delete.assert_not_called() + + +async def test_capture_hold_coordinates_db_and_storage_leaves(monkeypatch): + from models.conversation_enums import ConversationSource + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr( + cap_hold, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='allowance_unavailable')) + ) + monkeypatch.setattr(cap_hold, '_describe', Mock(return_value=('held', 20, {'source': 'omi'}))) + monkeypatch.setattr(cap_hold, '_exists', Mock(return_value=False)) + monkeypatch.setattr(cap_hold, 'upload_held_audio', Mock(return_value={'audio_path': 'synthetic'})) + monkeypatch.setattr(cap_hold, '_publish', Mock(return_value=True)) + calls = [] + + async def leaf(pool, fn, *args, **kwargs): + calls.append((pool, fn)) + return fn(*args, **kwargs) + + monkeypatch.setattr(cap_hold, 'run_blocking', leaf) + receipts, admission = {}, {} + assert await cap_hold.hold_capture_if_exhausted( + 'synthetic', ['1792152000.wav'], ConversationSource.omi, receipts=receipts, admission=admission + ) + assert calls == [ + (cap_hold.db_executor, cap_hold.funded_allowance), + (cap_hold.storage_executor, cap_hold._describe), + (cap_hold.db_executor, cap_hold._exists), + (cap_hold.storage_executor, cap_hold.upload_held_audio), + (cap_hold.db_executor, cap_hold._publish), + ] + assert admission == {'reason': 'allowance_unavailable'} + assert receipts['held']['reason'] == 'allowance_unavailable' diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py new file mode 100644 index 00000000000..7944946c38d --- /dev/null +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -0,0 +1,439 @@ +"""Recovery reaches real sync intake only after a current funded allowance.""" + +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, AsyncMock + +import pytest + +from utils.sync import cap_hold_recovery as recovery +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore + + +@pytest.fixture +def state(monkeypatch, tmp_path): + checkpoint, renew = recovery.cap_hold.checkpoint, recovery.cap_hold.renew + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + + async def immediate(pool, fn, *args, **kwargs): + return fn(*args, **kwargs) + + monkeypatch.setattr(recovery, 'run_blocking', immediate) + monkeypatch.setattr(recovery.cap_recovery_plan, 'run_blocking', immediate) + monkeypatch.setattr( + recovery.cap_recovery_plan, 'persist_piece', Mock(return_value={'audio_path': 'retained-piece', 'seconds': 20}) + ) + monkeypatch.setattr(recovery.sync_cap_recovery, 'ingested_piece', Mock(return_value=None)) + monkeypatch.setattr(recovery.time, 'time', lambda: 100.0) + monkeypatch.setattr(recovery.cap_hold, 'metadata', Mock(return_value={'timestamp': 1792152000, 'source': 'omi'})) + monkeypatch.setattr( + recovery.cap_hold, 'recovery_directory', Mock(return_value=SimpleNamespace(name=str(tmp_path), cleanup=Mock())) + ) + monkeypatch.setattr(recovery.cap_hold, 'read_audio', Mock()) + monkeypatch.setattr(recovery.cap_hold, 'renew', Mock(return_value=True)) + monkeypatch.setattr(recovery.cap_hold, 'checkpoint', Mock(return_value=True)) + monkeypatch.setattr(recovery.users_db, 'get_user_transcription_preferences', Mock(return_value={'language': 'en'})) + monkeypatch.setattr(recovery.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(recovery, 'sync_allowed', Mock(return_value=True)) + monkeypatch.setattr(recovery, 'get_hard_restriction_status', Mock(return_value=(False, None))) + monkeypatch.setattr(recovery, 'is_daily_audio_ceiling_exceeded', Mock(return_value=False)) + + def processed(*args, **kwargs): + kwargs['deferred_outcome'].update(provider='modulate', provider_called=True) + return True + + monkeypatch.setattr(recovery, 'process_segment', Mock(side_effect=processed)) + monkeypatch.setattr(recovery, 'build_person_embeddings_cache', Mock(return_value={'known': 'embedding'})) + monkeypatch.setattr( + recovery, + 'prepare_audio', + AsyncMock(side_effect=lambda uid, path, meta: ({path}, {}, {}, {})), + ) + monkeypatch.setattr(recovery, 'finalize_recovered_sync_segment', Mock()) + monkeypatch.setattr(recovery, 'record_usage', Mock()) + monkeypatch.setattr(recovery, 'get_prerecorded_service', Mock(return_value=('modulate', 'en', 'velma-2'))) + return SimpleNamespace( + snapshot=SimpleNamespace(id='hold', reference=Mock()), + row={'lease_token': 'owner-token', 'state': 'held'}, + checkpoint=checkpoint, + renew=renew, + ) + + +async def test_reset_or_upgrade_reuses_sync_and_settles_stable_usage_key(state): + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_called_once() + assert recovery.process_segment.call_args.kwargs['cap_recovery'] is True + assert recovery.process_segment.call_args.kwargs['cap_recovery_service'] == ('modulate', 'en', 'velma-2') + recovery.finalize_recovered_sync_segment.assert_called_once() + usage = recovery.record_usage.call_args.kwargs + assert usage['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + assert usage['transcription_seconds'] == 20 + assert usage['usage_metadata'] == {'funding': 'omi', 'provider': 'modulate', 'source': 'sync_v2'} + assert state.row['processed_paths'] == ['1792152000.wav'] + assert state.row['finalized'] + assert state.row['piece_receipts']['1792152000.wav']['has_content'] + assert recovery.process_segment.call_args.args[8] == {'known': 'embedding'} + + +async def test_execution_denial_keeps_checkpoint_and_audio_without_stt(state, monkeypatch): + recovery.sync_allowed.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + state.snapshot.reference.update.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_lost_hold_lease_cannot_dispatch_or_acknowledge(state): + recovery.cap_hold.renew.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + state.snapshot.reference.update.assert_not_called() + + +async def test_segment_recheck_refusal_does_not_consume_the_hold(state): + def denied(*args, **kwargs): + kwargs['deferred_outcome']['cap_held'] = True + return False + + recovery.process_segment.side_effect = denied + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + state.snapshot.reference.update.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_partial_recovery_resumes_only_unprocessed_pieces(state): + state.row['processed_paths'] = ['1792152000.wav'] + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_persistence_failure_keeps_audio_and_releases_local_staging(state): + recovery.process_segment.side_effect = RuntimeError('synthetic persistence failure') + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert not state.row['piece_receipts'] + state.snapshot.reference.update.assert_not_called() + recovery.cap_hold.recovery_directory.return_value.cleanup.assert_called_once() + + +async def test_usage_failure_reuses_persisted_transcription_receipt(state): + recovery.record_usage.side_effect = RuntimeError('synthetic usage unavailable') + with pytest.raises(RuntimeError): + await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav']['has_content'] + recovery.finalize_recovered_sync_segment.assert_not_called() + recovery.record_usage.side_effect = None + # Already accepted content can settle even if no new provider work is allowed. + recovery.sync_allowed.return_value = False + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_called_once() + recovery.finalize_recovered_sync_segment.assert_called_once() + assert recovery.record_usage.call_count == 2 + assert recovery.record_usage.call_args.kwargs['usage_metadata']['provider'] == 'modulate' + assert recovery.record_usage.call_args_list[0] == recovery.record_usage.call_args_list[1] + + +async def test_geolocation_and_speaker_context_survive_hold(state): + meta = recovery.cap_hold.metadata.return_value + meta['geolocation'] = {'latitude': 10.7, 'longitude': 106.7} + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + args = recovery.process_segment.call_args + assert args.args[8] == {'known': 'embedding'} + assert args.kwargs['geolocation'].latitude == 10.7 + assert args.kwargs['geolocation'].longitude == 106.7 + + +async def test_all_pieces_join_before_one_finalization(state): + recovery.prepare_audio.side_effect = lambda uid, path, meta: ( + {path, path.replace('1792152000', '1792152020')}, + {}, + {}, + {}, + ) + + def persist(path, uid, response, *args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = True + response['updated_memories'].add('joined-conversation') + response['_merged'] = {'joined-conversation': 'en'} + return True + + recovery.process_segment.side_effect = persist + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert recovery.process_segment.call_count == 2 + recovery.finalize_recovered_sync_segment.assert_called_once() + response = recovery.finalize_recovered_sync_segment.call_args.args[1] + assert response['updated_memories'] == {'joined-conversation'} + assert response['_merged'] == {'joined-conversation': 'en'} + assert len(state.row['piece_receipts']) == 2 + + +async def test_lost_checkpoint_fence_cannot_settle_or_finalize(state): + recovery.cap_hold.checkpoint.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.record_usage.assert_not_called() + recovery.finalize_recovered_sync_segment.assert_not_called() + + +@pytest.mark.parametrize('fault', ['save-error', 'lost-lease']) +async def test_atomic_ingest_proof_recovers_save_error_or_lost_lease(state, monkeypatch, fault): + clock = [100.0] + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({key: {'state': 'held', 'lease_until': 0, 'audio_path': 'retained'}}) + ref = client.document('/'.join(key)) + state.snapshot.reference = ref + monkeypatch.setattr(recovery.cap_hold, 'get_firestore_client', lambda: client) + monkeypatch.setattr(recovery.cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(recovery.cap_hold.time, 'time', lambda: clock[0]) + monkeypatch.setattr(recovery.cap_hold, 'renew', state.renew) + fail_save = [False] + + def checkpoint(*args): + if fail_save[0]: + raise RuntimeError('transcript checkpoint unavailable after ingestion') + return state.checkpoint(*args) + + monkeypatch.setattr(recovery.cap_hold, 'checkpoint', checkpoint) + ingested = Mock() + + def dispatch_and_ingest(*args, **kwargs): + intent = client.rows[key]['piece_receipts']['1792152000.wav'] + assert intent['state'] == 'dispatched' and intent['attempt'] == 1 + kwargs['deferred_outcome']['provider_called'] = True + client.rows[key]['piece_ingests'] = { + '1792152000.wav': {**intent, 'state': 'transcribed', 'has_content': True, 'response': {}} + } + assert kwargs['cap_recovery_service'] == ('modulate', 'en', 'velma-2') + ingested() + if fault == 'save-error': + fail_save[0] = True + else: + clock[0] = 2001 + assert recovery.cap_hold.claim('synthetic', ref, clock[0]) + return True + + recovery.sync_cap_recovery.ingested_piece.side_effect = ( + lambda uid, hold, piece: client.rows[key].get('piece_ingests', {}).get(piece) + ) + recovery.process_segment.side_effect = dispatch_and_ingest + row = recovery.cap_hold.claim('synthetic', ref, clock[0]) + if fault == 'save-error': + with pytest.raises(RuntimeError, match='checkpoint unavailable'): + await recovery._recover_row('synthetic', state.snapshot, row) + else: + assert not await recovery._recover_row('synthetic', state.snapshot, row) + assert client.rows[key]['piece_receipts']['1792152000.wav']['state'] == 'dispatched' + assert client.rows[key]['audio_path'] == 'retained' + fail_save[0] = False + # A real new fenced claim loads only durable state, not the prior local row. + clock[0] = 4002 + retry = recovery.cap_hold.claim('synthetic', ref, clock[0]) + assert retry['lease_token'] != row['lease_token'] + assert await recovery._recover_row('synthetic', state.snapshot, retry) + assert retry['retain_audio'] + recovery.process_segment.assert_called_once() + ingested.assert_called_once() + recovery.prepare_audio.assert_awaited_once() + recovery.record_usage.assert_called_once() + recovery.finalize_recovered_sync_segment.assert_called_once() + + +@pytest.mark.parametrize('restriction', ['hard', 'daily']) +async def test_recovery_cannot_bypass_current_abuse_restrictions(state, restriction): + if restriction == 'hard': + recovery.get_hard_restriction_status.return_value = (True, 60) + else: + recovery.is_daily_audio_ceiling_exceeded.return_value = True + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.cap_hold.read_audio.assert_not_called() + + +async def test_lost_completion_lease_cannot_settle_or_erase_audio(state, monkeypatch): + state.snapshot.to_dict = Mock(return_value={'state': 'held'}) + monkeypatch.setattr(recovery.cap_hold, 'recovery_cooldown', Mock(return_value=True)) + monkeypatch.setattr(recovery.cap_hold, 'pending_holds', Mock(return_value=[state.snapshot])) + monkeypatch.setattr(recovery.cap_hold, 'claim', Mock(return_value=state.row)) + monkeypatch.setattr(recovery.cap_hold, 'release', Mock()) + monkeypatch.setattr(recovery, '_recover_row', AsyncMock(return_value=True)) + monkeypatch.setattr(recovery, '_settle_and_erase', Mock()) + recovery.cap_hold.checkpoint.return_value = False + await recovery.recover('synthetic') + recovery._settle_and_erase.assert_not_called() + recovery.cap_hold.release.assert_called_once_with('synthetic', state.snapshot.reference, 'owner-token') + + +@pytest.mark.parametrize('reason', ['provider_select', 'cap_held']) +async def test_known_noncall_clears_intent_and_next_claim_retries(state, reason): + normal = recovery.process_segment.side_effect + + def noncall(*args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = False + if reason == 'cap_held': + kwargs['deferred_outcome']['cap_held'] = True + else: + args[4].append('provider_select') + return False + + recovery.process_segment.side_effect = noncall + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert not state.row['piece_receipts'] + recovery.process_segment.side_effect = normal + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert recovery.process_segment.call_count == 2 + recovery.prepare_audio.assert_awaited_once() # Frozen piece, no changed VAD plan. + + +async def test_ambiguous_piece_redispatches_only_once_then_retains_audio_and_finishes_sibling(state, monkeypatch): + clock = [100.0] + monkeypatch.setattr(recovery.time, 'time', lambda: clock[0]) + recovery.prepare_audio.side_effect = lambda uid, path, meta: ( + {path, path.replace('1792152000', '1792152020')}, + {}, + {}, + {}, + ) + attempts = [] + + def process(path, uid, response, lock, errors, *args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = True + if '1792152000' in path: + attempts.append(kwargs['cap_recovery_checkpoint']['attempt']) + errors.append('ambiguous_provider_result') + return False + response['updated_memories'].add('sibling') + response['_merged'] = {'sibling': 'en'} + return True + + recovery.process_segment.side_effect = process + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1] + recovery.finalize_recovered_sync_segment.assert_called_once() + assert recovery.finalize_recovered_sync_segment.call_args.args[1]['updated_memories'] == {'sibling'} + # Still cooling down: neither provider nor sibling finalization repeats. + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1] + clock[0] = 1901 + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + clock[0] = 3702 + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + failed = state.row['piece_receipts']['1792152000.wav'] + assert failed['state'] == 'failed_reconciliation' and failed['attempt'] == 2 + assert state.row['retain_audio'] and state.row['failed_reconciliation'] + recovery.finalize_recovered_sync_segment.assert_called_once() + recovery.record_usage.assert_called_once() + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + monkeypatch.setattr(recovery, '_erase', Mock()) + monkeypatch.setattr(recovery.sync_cap_holds, 'settle_hold', Mock()) + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + recovery._erase.assert_not_called() + recovery.sync_cap_holds.settle_hold.assert_not_called() # Never claim failed audio completed. + + +async def test_ingested_ambiguity_finalizes_without_provider_and_settles_once(state): + state.row['piece_plan'] = {'1792152000.wav': {'audio_path': 'retained-piece', 'seconds': 20}} + state.row['piece_receipts'] = {'1792152000.wav': {'state': 'dispatched', 'attempt': 1, 'dispatched_at': -1801}} + recovery.sync_cap_recovery.ingested_piece.return_value = { + 'state': 'transcribed', + 'has_content': True, + 'seconds': 20, + 'response': {'updated_memories': ['ingested'], '_merged': {'ingested': 'en'}}, + 'usage_metadata': {'funding': 'omi', 'provider': 'modulate', 'source': 'sync_v2'}, + } + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.cap_hold.read_audio.assert_not_called() + recovery.prepare_audio.assert_not_awaited() + recovery.finalize_recovered_sync_segment.assert_called_once() + assert state.row['retain_audio'] + assert recovery.record_usage.call_args.kwargs['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + + +async def test_second_attempt_noncall_reverts_prior_intent_without_spending_retry(state): + prior = {'state': 'dispatched', 'attempt': 1, 'dispatched_at': -1801} + state.row.update( + piece_plan={'1792152000.wav': {'audio_path': 'piece', 'seconds': 20}}, piece_receipts={'1792152000.wav': prior} + ) + + def noncall(*args, **kwargs): + kwargs['deferred_outcome'].update(provider_called=False, cap_held=True) + return False + + recovery.process_segment.side_effect = noncall + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav'] == prior + + +async def test_flag_off_recovery_has_no_io(state, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false') + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + await recovery.recover('synthetic') + recovery.cap_hold.metadata.assert_not_called() + recovery.process_segment.assert_not_called() + + +async def test_not_ingested_piece_gets_one_successful_redispatch_with_stable_usage_key(state, monkeypatch): + clock = [100.0] + monkeypatch.setattr(recovery.time, 'time', lambda: clock[0]) + attempts = [] + + def process(*args, **kwargs): + attempt = kwargs['cap_recovery_checkpoint']['attempt'] + attempts.append(attempt) + kwargs['deferred_outcome']['provider_called'] = True + if attempt == 1: + args[4].append('provider_reply_unknown') + return False + args[2]['new_memories'].add('recovered') + return True + + recovery.process_segment.side_effect = process + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + clock[0] = 1901 + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + assert state.row['piece_receipts']['1792152000.wav']['state'] == 'transcribed' + recovery.record_usage.assert_called_once() + assert recovery.record_usage.call_args.kwargs['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + + +async def test_cap_denial_after_provider_call_remains_ambiguous(state): + def called_then_denied(*args, **kwargs): + kwargs['deferred_outcome'].update(provider_called=True, cap_held=True) + return False + + recovery.process_segment.side_effect = called_then_denied + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav']['state'] == 'dispatched' + + +def test_retained_manifest_cannot_dispatch_and_does_not_starve_pending_holds(state, monkeypatch): + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({key: {'state': 'retained', 'lease_token': 'owner', 'lease_until': 0}}) + ref = client.document('/'.join(key)) + monkeypatch.setattr(recovery.cap_hold, 'get_firestore_client', lambda: client) + recovery.cap_hold.release('synthetic', ref, 'owner') + assert client.rows[key]['next_attempt_at'] == 253402300799 + monkeypatch.setattr(recovery.cap_hold, 'external_write_fence', lambda _: nullcontext()) + assert recovery.cap_hold.claim('synthetic', ref, 2000) is None + + +def test_reconciled_retention_waits_for_ledger_settlement_and_retries_failure(state, monkeypatch): + state.row.update(state='completed', retain_audio=True) + recovery.cap_hold.metadata.return_value['content_id'] = 'content' + settle = Mock(side_effect=RuntimeError('synthetic settlement failure')) + monkeypatch.setattr(recovery.sync_cap_holds, 'settle_hold', settle) + monkeypatch.setattr(recovery, '_erase', Mock()) + with pytest.raises(RuntimeError): + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + state.snapshot.reference.update.assert_not_called() + assert state.row['state'] == 'completed' + settle.side_effect = None + settle.return_value = True + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + assert state.snapshot.reference.update.call_args.args[0]['state'] == 'retained' + recovery._erase.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_reconciliation_metrics.py b/backend/tests/unit/test_sync_cap_reconciliation_metrics.py new file mode 100644 index 00000000000..93937fa04dd --- /dev/null +++ b/backend/tests/unit/test_sync_cap_reconciliation_metrics.py @@ -0,0 +1,23 @@ +import logging + +import pytest + +from utils.observability.stt_cap_dispatch import RECONCILIATION, record_reconciliation + + +@pytest.mark.parametrize('enabled', [False, True]) +@pytest.mark.parametrize('outcome', ['cleared_noncall', 'reconciled_ingested', 'redispatched', 'failed']) +def test_reconciliation_counter_and_log_are_additive_bounded_and_identifiable(monkeypatch, caplog, enabled, outcome): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enabled)) + metric = RECONCILIATION.labels(outcome=outcome) + before = metric._value.get() + with caplog.at_level(logging.INFO): + record_reconciliation(outcome, 'synthetic-hold', '1792152000.wav') + assert metric._value.get() == before + int(enabled) + if enabled: + assert f'outcome={outcome}' in caplog.text + assert 'hold_id=synthetic-hold' in caplog.text and '1792152000.wav' in caplog.text + else: + assert not caplog.text + for family in RECONCILIATION.collect(): + assert all(set(sample.labels) == {'outcome'} for sample in family.samples) diff --git a/backend/tests/unit/test_sync_cap_recovery_audio.py b/backend/tests/unit/test_sync_cap_recovery_audio.py new file mode 100644 index 00000000000..b31eafc05e1 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_audio.py @@ -0,0 +1,59 @@ +"""Held capture replay runs coverage before VAD and preserves source coordinates.""" + +from unittest.mock import AsyncMock, Mock + +from utils.sync import cap_recovery_audio as audio + + +async def test_recovery_replays_wal_and_safety_coverage_before_vad(monkeypatch): + events = [] + mapping = {'claim': {'codec': 'pcm16'}, 'offsets': [0, 400, 800]} + + async def run_blocking(pool, fn, *args, **kwargs): + return fn(*args, **kwargs) + + async def wal(*args, **kwargs): + events.append('wal') + assert args[5] == {'audio_pcm16_1792152000.bin': mapping['claim']} + assert args[7] == {'/local/audio_pcm16_1792152000.wav': [400, 400]} + return ['/local/remaining.wav'], {'/local/remaining.wav': mapping}, False + + async def safety(*args, **kwargs): + events.append('safety') + assert kwargs['protected_paths'] == {'/local/remaining.wav': mapping} + return ['/local/remaining.wav'], set() + + def vad(path, segments, errors, **kwargs): + events.append('vad') + assert path == '/local/remaining.wav' + assert kwargs['source_frame_map'] == mapping + segments.add('/local/piece.wav') + kwargs['segment_source_maps']['/local/piece.wav'] = (mapping, 400) + + monkeypatch.setattr(audio, 'run_blocking', run_blocking) + monkeypatch.setattr(audio, 'apply_sync_wal_audio_coverage', wal) + monkeypatch.setattr(audio, 'apply_safety_wal_coverage', safety) + monkeypatch.setattr(audio, 'sync_safety_wal_enabled', lambda: True) + monkeypatch.setattr(audio, 'sync_lineage_resolve_active_for', lambda uid: False) + monkeypatch.setattr(audio, 'retrieve_vad_segments', vad) + paths, maps, _, _ = await audio.prepare( + 'synthetic', + '/local/audio_pcm16_1792152000.wav', + {'source': 'omi', 'full_capture': True, 'source_frame_map': mapping, 'recording_session_id': 'origin'}, + ) + assert events == ['wal', 'safety', 'vad'] + assert paths == {'/local/piece.wav'} + assert maps['/local/piece.wav'] == (mapping, 400) + + +async def test_already_segmented_hold_preserves_position_without_repeating_vad(monkeypatch): + vad = Mock() + coverage = AsyncMock() + monkeypatch.setattr(audio, 'retrieve_vad_segments', vad) + monkeypatch.setattr(audio, 'apply_sync_wal_audio_coverage', coverage) + position = [{'claim': {'codec': 'pcm16'}, 'offsets': [0, 400]}, 10] + paths, maps, _, _ = await audio.prepare('synthetic', '/local/piece.wav', {'source_position_map': position}) + assert paths == {'/local/piece.wav'} + assert maps['/local/piece.wav'] == tuple(position) + vad.assert_not_called() + coverage.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_recovery_checkpoint.py b/backend/tests/unit/test_sync_cap_recovery_checkpoint.py new file mode 100644 index 00000000000..40f573c4b3b --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_checkpoint.py @@ -0,0 +1,68 @@ +"""Recovery proof shares conversation transaction and never bypasses its lease.""" + +from unittest.mock import Mock + +import pytest + +from database import sync_cap_recovery as checkpoint +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore +from utils.conversations import lifecycle + + +@pytest.mark.parametrize('fence', ['valid', 'expired', 'replaced', 'attempt']) +def test_piece_ingest_proof_commits_with_assignment_or_is_fenced(monkeypatch, fence): + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + row = { + 'lease_token': 'owner', + 'lease_until': 1000, + 'piece_receipts': {'piece.wav': {'state': 'dispatched', 'attempt': 1}}, + } + if fence == 'expired': + row['lease_until'] = 99 + elif fence == 'replaced': + row['lease_token'] = 'new-owner' + elif fence == 'attempt': + row['piece_receipts']['piece.wav']['attempt'] = 2 + client = StrictFirestore({key: row}) + monkeypatch.setattr(checkpoint, 'get_firestore_client', lambda: client) + monkeypatch.setattr(checkpoint.time, 'time', lambda: 100) + monkeypatch.setattr(lifecycle, 'observe_completed_conversation_shape', Mock()) + context = { + 'hold_id': 'hold', + 'piece': 'piece.wav', + 'lease_token': 'owner', + 'attempt': 1, + 'receipt': {'seconds': 20, 'attempt': 1, 'usage_metadata': {'provider': 'modulate'}}, + } + conversation = {'id': 'canonical', 'status': 'completed', 'sync_relevance': 'keep'} + tx = client.transaction() + + def assign(uid, incoming, **kwargs): + kwargs['before_assign'](tx) + tx.set(client.document('users/synthetic/conversations/canonical'), conversation) + kwargs['after_assign'](tx, (conversation, True, [])) + return conversation, True, [] + + monkeypatch.setattr(lifecycle.conversations_db, 'assign_sync_conversation', assign) + incoming = {'id': 'piece-lineage-anchor', 'status': 'completed', 'language': 'en'} + if fence != 'valid': + with pytest.raises(checkpoint.RecoveryIngestFenced): + lifecycle.ingest_sync_conversation('synthetic', incoming, recovery_checkpoint=context) + assert not tx.has_written + assert not client.rows[key].get('piece_ingests') + return + lifecycle.ingest_sync_conversation('synthetic', incoming, recovery_checkpoint=context) + assert tx.sets and tx.updates # Same transaction; strict fixture forbids late reads. + proof = checkpoint.ingested_piece('synthetic', 'hold', 'piece.wav') + assert proof['lineage_anchor'] == 'piece-lineage-anchor' + assert proof['conversation_id'] == 'canonical' + assert proof['response']['new_memories'] == ['canonical'] + assert proof['response']['_merged'] == {'canonical': 'en'} + + +def test_ordinary_ingest_preserves_main_assignment_call(monkeypatch): + assigned = {'id': 'canonical', 'status': 'completed'} + call = Mock(return_value=(assigned, False, [])) + monkeypatch.setattr(lifecycle.conversations_db, 'assign_sync_conversation', call) + lifecycle.ingest_sync_conversation('synthetic', assigned) + call.assert_called_once_with('synthetic', assigned, candidate_id=None, target_id=None) diff --git a/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py b/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py new file mode 100644 index 00000000000..4ed6b979d11 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py @@ -0,0 +1,61 @@ +"""Explicit provider-call signal covers real pre-dispatch exits in both modes.""" + +import threading +from unittest.mock import Mock, AsyncMock + +import pytest + +from utils.sync import pipeline + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('exit_at', ['url', 'keyterms', 'cap', 'provider_error', 'empty']) +def test_real_segment_reports_provider_boundary_without_changing_flag_off_calls(monkeypatch, enforce, exit_at): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + gate = Mock(return_value=exit_at != 'cap') + monkeypatch.setattr(pipeline, 'sync_allowed', gate) + monkeypatch.setattr( + pipeline, + 'get_syncing_file_temporal_signed_url', + Mock(side_effect=RuntimeError('synthetic URL failure') if exit_at == 'url' else None, return_value='url'), + ) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr( + pipeline, + 'keyterms', + AsyncMock( + side_effect=RuntimeError('synthetic keyterms failure') if exit_at == 'keyterms' else None, return_value=[] + ), + ) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', Mock(return_value=('modulate', 'en', 'velma-2'))) + stt = Mock( + side_effect=RuntimeError('synthetic provider failure') if exit_at == 'provider_error' else None, + return_value=([], 'en'), + ) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, 'record_cap_audio', Mock()) + monkeypatch.setattr(pipeline, 'get_wav_duration', Mock(return_value=20)) + monkeypatch.setattr(pipeline, 'funded_allowance', Mock(return_value=Mock(reason='available'))) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + outcome, errors = {}, [] + assert not pipeline.process_segment( + '1792152000.wav', + 'synthetic', + {}, + threading.Lock(), + errors, + cap_recovery=True, + cap_recovery_checkpoint={'attempt': 1}, + deferred_outcome=outcome, + ) + if enforce: + called = exit_at in {'provider_error', 'empty'} + assert outcome['provider_called'] is called + assert stt.call_count == int(called) + assert outcome.get('recovery_ambiguous', False) is (exit_at == 'empty') + else: + assert 'provider_called' not in outcome + assert 'recovery_ambiguous' not in outcome + assert stt.call_count == (0 if exit_at in {'url', 'keyterms'} else 1 if exit_at == 'provider_error' else 2) + gate.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_recovery_plan.py b/backend/tests/unit/test_sync_cap_recovery_plan.py new file mode 100644 index 00000000000..d01875764e1 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_plan.py @@ -0,0 +1,78 @@ +"""Immutable pieces retain exact bytes and lineage across changing VAD/coverage.""" + +import json +from types import SimpleNamespace +from unittest.mock import Mock, AsyncMock +import wave + +import pytest + +from utils.sync import cap_recovery_plan as plan + + +@pytest.mark.parametrize('upload_fails', [False, True]) +async def test_piece_plan_is_durable_before_dispatch_and_replays_frozen_audio(tmp_path, monkeypatch, upload_fails): + async def leaf(pool, fn, *args): + return fn(*args) + + monkeypatch.setattr(plan, 'run_blocking', leaf) + blobs = {} + + def upload(uid, path, identity, context): + if upload_fails: + raise RuntimeError('synthetic storage failure') + blob = f'chunks/{uid}/cap-held/{identity}' + with open(path, 'rb') as source: + blobs[blob] = source.read() + return {'audio_path': blob, 'metadata': json.dumps(context)} + + monkeypatch.setattr(plan.cap_hold, 'upload_held_audio', upload) + monkeypatch.setattr(plan.cap_hold, 'read_audio', lambda uid, row, path: None) + piece = tmp_path / '1792152000.wav' + with wave.open(str(piece), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\x01\x02' * 1600) + original = piece.read_bytes() + context = { + 'target_conversation_id': 'bound', + 'source_position_map': ({'claim': 'synthetic'}, 100), + 'lineage_binding': 'bound', + } + prepare = AsyncMock( + return_value=( + {str(piece)}, + {str(piece): context['source_position_map']}, + {str(piece): 'bound'}, + {str(piece): 'bound'}, + ) + ) + row = {'audio_path': 'original-held-audio'} + + async def save(uid, snapshot, stored, update): + stored.update(update) + return True + + directory, snapshot = SimpleNamespace(name=str(tmp_path)), SimpleNamespace(id='hold') + if upload_fails: + with pytest.raises(RuntimeError): + await plan.ensure_plan('synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save) + assert 'piece_plan' not in row and row['audio_path'] == 'original-held-audio' + return + pieces, _, _ = await plan.ensure_plan('synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save) + assert row['piece_plan'] == pieces + # Later code/config may change VAD and WAL overlap. Neither may redefine a retry. + prepare.side_effect = AssertionError('must not prepare the source again') + piece.write_bytes(b'changed-local-source') + monkeypatch.setattr( + plan.cap_hold, 'read_audio', lambda uid, item, path: piece.write_bytes(blobs[item['audio_path']]) + ) + monkeypatch.setattr(plan.cap_hold, 'metadata', lambda uid, item: json.loads(item['metadata'])) + replay, locals_, contexts = await plan.ensure_plan( + 'synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save + ) + path, restored_context = await plan.stage_piece( + 'synthetic', piece.name, replay[piece.name], directory, locals_, contexts + ) + assert path == str(piece) and piece.read_bytes() == original + assert restored_context == context + prepare.assert_awaited_once() diff --git a/backend/tests/unit/test_sync_record_usage.py b/backend/tests/unit/test_sync_record_usage.py index 6d0d55982b2..14222ea3f51 100644 --- a/backend/tests/unit/test_sync_record_usage.py +++ b/backend/tests/unit/test_sync_record_usage.py @@ -8,6 +8,7 @@ - v1 deprecation headers appear on all response paths """ +import ast import os import re @@ -39,6 +40,25 @@ def _extract_function_body(source, func_name): return source[start:end] +def _usage_has_exception_boundary(body): + """Check nesting, rather than a character window around a formatted call.""" + for node in ast.walk(ast.parse(body)): + if not isinstance(node, ast.Try): + continue + catches = any( + isinstance(handler.type, ast.Name) and handler.type.id == 'Exception' for handler in node.handlers + ) + if not catches: + continue + for statement in node.body: + for call in ast.walk(statement): + if isinstance(call, ast.Call) and any( + isinstance(argument, ast.Name) and argument.id == 'record_usage' for argument in call.args + ): + return True + return False + + # --------------------------------------------------------------------------- # record_usage import # --------------------------------------------------------------------------- @@ -102,14 +122,7 @@ def test_record_usage_not_under_fair_use_guard(self): def test_record_usage_wrapped_in_try_except(self): """record_usage must be protected by try/except.""" - body = self._get_v1_body() - record_idx = body.find('record_usage,') - # Check there's a try: before it (within ~5 lines) - preceding_chunk = body[max(0, record_idx - 200) : record_idx] - assert 'try:' in preceding_chunk, "record_usage must be inside a try block" - # Check there's an except after it - following_chunk = body[record_idx : record_idx + 200] - assert 'except Exception' in following_chunk, "record_usage must have an except handler" + assert _usage_has_exception_boundary(self._get_v1_body()), 'record_usage must be inside a protected try body' def test_record_usage_logs_error_on_failure(self): """Error from record_usage must be logged, not silenced.""" @@ -159,12 +172,7 @@ def test_static_record_usage_before_finalize_sync_job(self): assert record_pos < complete_pos, "record_usage must run before final job publication" def test_record_usage_wrapped_in_try_except(self): - body = self._get_v2_body() - record_idx = body.find('record_usage,') - preceding = body[max(0, record_idx - 1000) : record_idx] - assert 'try:' in preceding, "record_usage must be inside a try block" - following = body[record_idx : record_idx + 800] - assert 'except Exception' in following, "record_usage must have an except handler" + assert _usage_has_exception_boundary(self._get_v2_body()), 'record_usage must be inside a protected try body' # --------------------------------------------------------------------------- diff --git a/backend/tests/unit/test_transcription_allowance.py b/backend/tests/unit/test_transcription_allowance.py index 72ff2faffb1..988e4db88e4 100644 --- a/backend/tests/unit/test_transcription_allowance.py +++ b/backend/tests/unit/test_transcription_allowance.py @@ -471,3 +471,30 @@ def failed_lookup(uid): assert result.remaining_seconds == 0 assert result.limit_seconds == cap assert result.reason == ('usage_invalid' if failure == 'invalid_usage' else 'allowance_unavailable') + + +@pytest.mark.parametrize('launch', [None, 'not-a-date', '2026-99-99']) +@pytest.mark.parametrize('enforce', [False, True]) +def test_launch_config_does_not_manufacture_zero_usage_when_enforcing(sub, monkeypatch, launch, enforce): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + if launch is None: + monkeypatch.delenv('SUBSCRIPTION_LAUNCH_DATE', raising=False) + else: + monkeypatch.setenv('SUBSCRIPTION_LAUNCH_DATE', launch) + if enforce: + with pytest.raises(ValueError): + sub.get_monthly_usage_for_subscription('synthetic') + else: + assert sub.get_monthly_usage_for_subscription('synthetic') == {} + + +def test_explicit_future_launch_is_intentionally_unlaunched(sub, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setenv('SUBSCRIPTION_LAUNCH_DATE', '2099-01-01') + assert sub.get_monthly_usage_for_subscription('synthetic') == {} + + +def test_funded_request_cannot_use_deepgram_key_for_another_provider(sub, monkeypatch): + _situate(monkeypatch, sub, plan=PlanType.plus, used_seconds=PLUS_CAP, byok_enrolled=True, byok_header=True) + assert sub.resolve_transcription_allowance('uid').reason == 'byok' + assert sub.resolve_transcription_allowance('uid', omi_funded=True).reason == 'plan_allowance_exhausted' diff --git a/backend/utils/batch_pipeline/admission.py b/backend/utils/batch_pipeline/admission.py index c4844d4dabb..43fa82302fc 100644 --- a/backend/utils/batch_pipeline/admission.py +++ b/backend/utils/batch_pipeline/admission.py @@ -3,6 +3,8 @@ import math import time +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.stt_cap_dispatch import sync_allowed from database.batch_pipeline_privacy import BatchPrivacyBlocked from utils.batch_pipeline.scheduler import owns_lease from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded @@ -34,6 +36,11 @@ def read(tx): retries[segment] = 30 continue job, binding, intake = snapshot + if cap_dispatch_enabled() and not sync_allowed( + job['owner_id'], float(job.get('duration_seconds', 0)), 'parakeet' + ): + retries[segment] = 60 + continue if int(job['priority']) == 0: restricted, retry_after = get_hard_restriction_status(job['owner_id']) if restricted: diff --git a/backend/utils/batch_pipeline/contracts.py b/backend/utils/batch_pipeline/contracts.py index 93dea5081b9..885e1363bf3 100644 --- a/backend/utils/batch_pipeline/contracts.py +++ b/backend/utils/batch_pipeline/contracts.py @@ -109,8 +109,14 @@ def __post_init__(self): raise ValueError('result_uri must be a bounded durable encrypted reference') +@dataclass(frozen=True) +class HeldWork: + segment_id: str + retry_after: int = 60 + + class BatchWorker(Protocol): - def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: ... + def process_batch(self, items: Sequence[WorkItem], **kwargs) -> Sequence[WorkResult | HeldWork]: ... @dataclass(frozen=True) diff --git a/backend/utils/batch_pipeline/projection.py b/backend/utils/batch_pipeline/projection.py index a378d358257..a08e32eb0a9 100644 --- a/backend/utils/batch_pipeline/projection.py +++ b/backend/utils/batch_pipeline/projection.py @@ -4,6 +4,7 @@ No admission flag is consulted: already accepted work drains after rollback. """ +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled from datetime import datetime, timezone import time import math @@ -58,7 +59,7 @@ def build_incoming(item: WorkItem, binding, payload: dict, *, geolocation=None) transcript_segments=segments, source=ConversationSource(binding.source), language=binding.language or payload.get('language'), - is_locked=binding.is_locked, + is_locked=False if cap_dispatch_enabled() else binding.is_locked, private_cloud_sync_enabled=binding.private_cloud_sync_enabled, client_device_id=binding.client_device_id, client_platform=binding.client_platform, diff --git a/backend/utils/batch_pipeline/pump.py b/backend/utils/batch_pipeline/pump.py index 2e2f91a3048..7f5ff169258 100644 --- a/backend/utils/batch_pipeline/pump.py +++ b/backend/utils/batch_pipeline/pump.py @@ -6,7 +6,8 @@ from collections.abc import Callable from typing import Any -from utils.batch_pipeline.contracts import BatchWorker, HEARTBEAT_SECONDS, LeasedWork, PoolCapacity +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.batch_pipeline.contracts import BatchWorker, HEARTBEAT_SECONDS, LeasedWork, PoolCapacity, HeldWork from utils.batch_pipeline.merge import OrderedMerge from utils.batch_pipeline.errors import FatalGpuError from utils.batch_pipeline.compute_client import from_env as compute_client_from_env @@ -128,7 +129,7 @@ def claim(): lost: set[str] = set() renew = asyncio.create_task(self._renew(leases, done, lost)) try: - required = [v for v in leases if v.item.requires_compute_admission] + required = [v for v in leases if v.item.requires_compute_admission or cap_dispatch_enabled()] if required: receipt = {} try: @@ -159,8 +160,17 @@ def claim(): leases = [v for v in leases if v.item.segment_id not in lost] if not leases: return 0 + dispatch_kwargs = {} + if cap_dispatch_enabled(): + + def authorize_dispatch(segments): + selected = [v for v in leases if v.item.segment_id in segments] + receipt = self.authorize_compute(selected) if self.authorize_compute is not None else {} + return set(receipt.get('allowed') or []) + + dispatch_kwargs['authorize_dispatch'] = authorize_dispatch compute = asyncio.create_task( - run_blocking(sync_executor, self.worker.process_batch, [v.item for v in leases]) + run_blocking(sync_executor, self.worker.process_batch, [v.item for v in leases], **dispatch_kwargs) ) # Cancelling an asyncio wrapper does not stop CUDA. Keep renewal and # wait for the bounded model operation before leaving this process. @@ -205,6 +215,11 @@ def claim(): lost.discard(segment) try: result = result_map.get(segment) + if isinstance(result, HeldWork): + await run_blocking( + db_executor, self.scheduler.defer, segment, lease.token, time.time(), result.retry_after + ) + continue if result is None: if await run_blocking(db_executor, self.scheduler.fail, segment, lease.token, time.time()): await self.drain_owner(lease.item.owner_id) diff --git a/backend/utils/batch_pipeline/quarantine_recovery.py b/backend/utils/batch_pipeline/quarantine_recovery.py index 7c6ac63c09b..3de33ab1ac0 100644 --- a/backend/utils/batch_pipeline/quarantine_recovery.py +++ b/backend/utils/batch_pipeline/quarantine_recovery.py @@ -6,6 +6,9 @@ import time import wave +from utils.stt_cap_dispatch import sync_allowed, funded_allowance +from config.stt_cap_dispatch import enabled as cap_enabled +from utils.observability.stt_cap_dispatch import record_audio from config.batch_pipeline import recovery_owners from database.batch_pipeline_privacy import BatchPrivacyBlocked from database.batch_pipeline_projection_protocol import ProjectionBlocked @@ -87,9 +90,13 @@ def recover(self, segment_id, owner_id): if self.clock() - job['recovery']['started_at'] > 900: self.protocol.settle(segment_id, token, self.clock()) return 'held' + if not sync_allowed(owner_id, item.duration_seconds): + return 'held' audio = self.artifacts.fetch_audio(item) # Privacy and the intent token are rechecked after download and # immediately before issuing the sole application-level STT call. + if not sync_allowed(owner_id, item.duration_seconds): + return 'held' if owner_id not in recovery_owners() or not self.protocol.start(segment_id, token, self.clock()): return 'held' from database.batch_pipeline_projection_protocol import normalized_binding @@ -100,6 +107,14 @@ def recover(self, segment_id, owner_id): ) try: payload = self.stt(item, normalized_binding(job['projection_binding']), audio) + if cap_enabled(): + record_audio( + 'sync', + 'accepted', + 'unknown', + item.duration_seconds, + funded_allowance(owner_id).reason == 'plan_allowance_exhausted', + ) except Exception: self.protocol.settle(segment_id, token, self.clock()) return 'failed_held' diff --git a/backend/utils/conversations/lifecycle.py b/backend/utils/conversations/lifecycle.py index 151017461b1..260db48847f 100644 --- a/backend/utils/conversations/lifecycle.py +++ b/backend/utils/conversations/lifecycle.py @@ -21,6 +21,7 @@ from database import conversations as conversations_db from database import oversized_conversation_terminal as oversized_terminal_db from database import recording_sessions as recording_sessions_db +from database import sync_cap_recovery from database.firestore_read_metrics import FirestoreReadSite from database.firestore_transaction_retry import FirestoreContentionExhausted from models.conversation_enums import ConversationStatus @@ -134,15 +135,21 @@ def create_completed_conversation(uid: str, conversation_data: dict[str, Any], * @sync_phase('firestore') -def ingest_sync_conversation(uid: str, incoming: dict[str, Any], *, candidate_id=None, target_id=None): +def ingest_sync_conversation( + uid: str, incoming: dict[str, Any], *, candidate_id=None, target_id=None, recovery_checkpoint: dict | None = None +): """Admit a retained deterministic sync row and atomically append later chunks. Enrichment follows persistence; filler remains recoverable under Show discarded. Existing lifecycle fields are preserved by the transactional append. """ _require_status(incoming, ConversationStatus.completed) + checkpoint_hooks: dict[str, Any] = {} + if recovery_checkpoint is not None: + before, after = sync_cap_recovery.ingest_hooks(uid, recovery_checkpoint, incoming) + checkpoint_hooks = {'before_assign': before, 'after_assign': after} assigned, created, survivors = conversations_db.assign_sync_conversation( - uid, incoming, candidate_id=candidate_id, target_id=target_id + uid, incoming, candidate_id=candidate_id, target_id=target_id, **checkpoint_hooks ) record_sync_intake_outcome(created=created) if created: diff --git a/backend/utils/locked_content_recovery.py b/backend/utils/locked_content_recovery.py index 30ae626454d..880a9abcf00 100644 --- a/backend/utils/locked_content_recovery.py +++ b/backend/utils/locked_content_recovery.py @@ -7,13 +7,22 @@ import logging from datetime import datetime, timezone from uuid import uuid4 +from typing import Callable +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled from utils.locked_content_flags import reset_recovery_enabled logger = logging.getLogger(__name__) PAGE_SIZE = 100 LEASE_SECONDS = 300 RETRY_SECONDS = 60 +_cap_audio_recovery: Callable[[str], None] | None = None + + +def register_cap_audio_recovery(callback: Callable[[str], None]) -> None: + """The API sync owner supplies STT recovery; lean auth consumers do not.""" + global _cap_audio_recovery + _cap_audio_recovery = callback def _memory_page_drained(count: int, page_size: int) -> bool: @@ -31,6 +40,11 @@ def _memory_page_drained(count: int, page_size: int) -> bool: def recover_locked_content(uid: str, *, now: datetime | None = None) -> None: + if stt_cap_dispatch_enabled() and _cap_audio_recovery is not None: + try: + _cap_audio_recovery(uid) + except Exception as error: + logger.warning('event=stt_cap_recovery outcome=schedule_deferred exception_type=%s', type(error).__name__) if not reset_recovery_enabled(): return # Lazy imports keep the flag-off auth path free of additional dependencies. diff --git a/backend/utils/observability/stt_cap_dispatch.py b/backend/utils/observability/stt_cap_dispatch.py new file mode 100644 index 00000000000..cf09365799a --- /dev/null +++ b/backend/utils/observability/stt_cap_dispatch.py @@ -0,0 +1,30 @@ +"""Additive dispatch provenance, independent of root usage and unit projection.""" + +from prometheus_client import Counter +from config.stt_cap_dispatch import enabled +import logging + +AUDIO_SECONDS = Counter( + 'omi_stt_cap_provider_audio_seconds', + 'Provider-accepted or cap-suppressed audio; never the legacy monthly usage counter', + ['path', 'outcome', 'provider', 'allowance'], +) +CONFIG_ERRORS = Counter('omi_stt_cap_launch_config_errors', 'Invalid subscription launch configuration', ['reason']) +RECONCILIATION = Counter('omi_stt_cap_recovery_reconciliation', 'Held piece reconciliation transitions', ['outcome']) +logger = logging.getLogger(__name__) + + +def record_reconciliation(outcome: str, hold_id: str, piece: str) -> None: + if not enabled(): + return + if outcome not in {'cleared_noncall', 'reconciled_ingested', 'redispatched', 'failed'}: + raise ValueError('unknown reconciliation outcome') + RECONCILIATION.labels(outcome=outcome).inc() + logger.info('event=stt_cap_recovery_reconciliation outcome=%s hold_id=%s piece=%r', outcome, hold_id, piece) + + +def record_audio(path, outcome, provider, seconds, exhausted=False): + provider = getattr(provider, 'value', provider) + provider = provider if provider in {'deepgram', 'modulate', 'soniox', 'parakeet'} else 'unknown' + if seconds > 0: + AUDIO_SECONDS.labels(path, outcome, provider, 'exhausted' if exhausted else 'other').inc(seconds) diff --git a/backend/utils/stt/live_failure.py b/backend/utils/stt/live_failure.py index 3a945a4f108..7369a90ea6a 100644 --- a/backend/utils/stt/live_failure.py +++ b/backend/utils/stt/live_failure.py @@ -304,6 +304,9 @@ async def terminate_live_stt_backoff( if reason not in {'provider_unavailable', 'reconnect_budget'}: raise ValueError('unsupported live STT backoff reason') + admission = getattr(session, 'stt_cap_admission', None) + if admission is not None and admission.funded_paused is True: + return False if session.stt_terminal_failure: return False session.stt_terminal_failure = True @@ -464,6 +467,9 @@ async def terminate_live_stt_session( fields are deliberately bounded and never include provider exception text. """ + admission = getattr(session, 'stt_cap_admission', None) + if admission is not None and admission.funded_paused is True: + return False if session.stt_terminal_failure: return False diff --git a/backend/utils/stt/live_session.py b/backend/utils/stt/live_session.py index 6ca2efee6ba..6e29b48f2b4 100644 --- a/backend/utils/stt/live_session.py +++ b/backend/utils/stt/live_session.py @@ -12,6 +12,7 @@ from utils.observability.fallback import FirstTextDeadlineDiagnostics, ReplayLagDiagnostics, record_fallback from utils.observability.transcription import record_live_stt_audio_seconds from utils.observability.routing_cohort import current_routing_cohort +from utils.stt_cap_dispatch import CapSocket, session_admission, suppress_socket_audio from utils.stt import streaming as st from utils.dream_vocabulary import keyterms from utils.stt.live_failure import PendingLiveFailover, settle_terminal_socket @@ -86,6 +87,7 @@ def get_metrics(self) -> dict[str, Any]: def to_json_log(self) -> dict[str, Any]: return {'event': 'managed_live_vad_metrics', **self.get_metrics()} + @session_admission async def connect( self, sample_rate: int, @@ -303,10 +305,28 @@ def attach_then_rebase(seg_list: list[dict[str, Any]]) -> None: if is_window: from utils.stt.parakeet_window import connect_window - raw = connect_window(callback, sample_rate) - if self.no_text_rescue.enabled: - raw.allow_no_text_rescue = self.no_text_rescue.allow_window_rescue - raw.progress_deadline_enabled = True + def new_window(): + window_raw = connect_window(callback, sample_rate) + if self.no_text_rescue.enabled: + window_raw.allow_no_text_rescue = self.no_text_rescue.allow_window_rescue + window_raw.progress_deadline_enabled = True + return window_raw + + admission = getattr(self.receiver, 'cap_admission', None) + if admission is None: + raw = new_window() + else: + + async def open_window(): + return new_window() + + raw = CapSocket( + admission, + 'parakeet', + sample_rate, + open_window, + new_window() if admission.allowed('parakeet') else None, + ) elif service == st.STTService.parakeet: raw = await st.process_audio_parakeet(callback, language, sample_rate, 1, keywords=keywords) elif service == st.STTService.soniox: @@ -591,7 +611,11 @@ def __init__( if window: from utils.stt.parakeet_window import SessionPcmGain, WINDOW_INGEST_AGC, WindowedParakeetSocket - if isinstance(raw, WindowedParakeetSocket): + if isinstance(raw, CapSocket): + raw.on_open( + lambda transport: transport.set_replay_progress_callback(self._trim_window_replay_to_anchor) + ) + elif isinstance(raw, WindowedParakeetSocket): raw.set_replay_progress_callback(self._trim_window_replay_to_anchor) if WINDOW_INGEST_AGC: self._ingest_gain = SessionPcmGain() @@ -798,6 +822,8 @@ def set_health_callbacks(self, on_success: Callable[[], None], on_close: Callabl if self._routing_active: self._health_success, self._health_close = on_success, on_close + elif self.window and isinstance(self.raw, CapSocket): + self.raw.on_open(lambda transport: transport.set_health_callbacks(on_success, on_close)) elif isinstance(self.raw, WindowedParakeetSocket): self.raw.set_health_callbacks(on_success, on_close) @@ -850,6 +876,8 @@ def process_replay_capture(data, wall_time, score_pcm=None, *, start_sample=None setattr(gate, 'process_audio', process_replay_capture) def send(self, data: bytes, start_sample: int | None = None) -> bool: + if suppress_socket_audio(self.raw, data): + return True if self._closing_for_health: return False if self.is_connection_dead: @@ -1068,6 +1096,8 @@ def commit_send(self) -> None: commit() def send_admitted_audio(self, data: bytes, spans: Any) -> bool: + if suppress_socket_audio(self.raw, data): + return True from utils.stt.soniox_idle import unaccepted_onset data, spans = unaccepted_onset(data, spans, self._send_tracker) diff --git a/backend/utils/stt/live_target_connect.py b/backend/utils/stt/live_target_connect.py index 0f1df391cd9..74a04047dbd 100644 --- a/backend/utils/stt/live_target_connect.py +++ b/backend/utils/stt/live_target_connect.py @@ -2,6 +2,8 @@ from __future__ import annotations +from utils.stt_cap_dispatch import live_provider + import asyncio import os from urllib.parse import urlencode @@ -12,6 +14,7 @@ from utils.stt import streaming +@live_provider('modulate', 1) async def connect_modulate(callback, sample_rate, language): target = connecting_target.get() if target is None or target.endpoint is None: diff --git a/backend/utils/stt/pre_recorded.py b/backend/utils/stt/pre_recorded.py index c1513d166f2..411543578e2 100644 --- a/backend/utils/stt/pre_recorded.py +++ b/backend/utils/stt/pre_recorded.py @@ -1068,7 +1068,9 @@ def transcribe_bytes( ) -def get_prerecorded_provider(language: Optional[str] = 'en') -> PrerecordedSTTProvider: +def get_prerecorded_provider( + language: Optional[str] = 'en', *, selected_service: Optional[Tuple[str, Optional[str], str]] = None +) -> PrerecordedSTTProvider: """Construct exactly the language-aware provider selected for telemetry.""" # Offline harness sessions swap the provider for a deterministic stub # (selection/telemetry still resolve normally above them). Imported lazily: @@ -1077,7 +1079,7 @@ def get_prerecorded_provider(language: Optional[str] = 'en') -> PrerecordedSTTPr if prerecorded_stub_enabled(): return PrerecordedStubProvider() - service, _provider_language, model = get_prerecorded_service(language) + service, _provider_language, model = selected_service or get_prerecorded_service(language) if service == PrerecordedSTTService.MODULATE: return ModulatePrerecordedProvider() if service == PrerecordedSTTService.PARAKEET: @@ -1099,9 +1101,15 @@ def prerecorded( language: Optional[str] = None, model: str = "nova-3", keywords: Optional[Sequence[str]] = None, + *, + selected_service: Optional[Tuple[str, Optional[str], str]] = None, ) -> Union[List[Dict[str, Any]], Tuple[List[Dict[str, Any]], str]]: """Route pre-recorded URL transcription through STT_PRERECORDED_MODEL.""" - provider = get_prerecorded_provider(language) + provider = ( + get_prerecorded_provider(language, selected_service=selected_service) + if selected_service is not None + else get_prerecorded_provider(language) + ) return provider.transcribe_url( audio_url, speakers_count=speakers_count, diff --git a/backend/utils/stt/soniox.py b/backend/utils/stt/soniox.py index a6a7fe44dc0..d2b6a014d5b 100644 --- a/backend/utils/stt/soniox.py +++ b/backend/utils/stt/soniox.py @@ -5,6 +5,7 @@ its own. Imports flow one way: this module never imports from ``streaming.py``. """ +from utils.stt_cap_dispatch import live_provider import asyncio import json import logging @@ -715,6 +716,7 @@ def _handle_tokens(self, tokens: List[Any]) -> None: self._stream_transcript(ready) +@live_provider('soniox', 1) async def process_audio_soniox( stream_transcript: Callable[[List[Dict[str, Any]]], None], sample_rate: int, diff --git a/backend/utils/stt/streaming.py b/backend/utils/stt/streaming.py index 2326e3273ce..c24d145f682 100644 --- a/backend/utils/stt/streaming.py +++ b/backend/utils/stt/streaming.py @@ -1,3 +1,4 @@ +from utils.stt_cap_dispatch import live_provider import asyncio import inspect import io @@ -1099,6 +1100,7 @@ def _deepgram_is_available() -> bool: return _managed_deepgram_client() is not None or bool(get_byok_key('deepgram')) +@live_provider('deepgram', 2) async def process_audio_dg( stream_transcript: Callable[[List[Dict[str, Any]]], None], language: str, @@ -1875,6 +1877,7 @@ def _handle_utterance(self, msg: Dict[str, Any]) -> None: self._stream_transcript(segments) +@live_provider('modulate', 1) async def process_audio_modulate( stream_transcript: Callable[[List[Dict[str, Any]]], None], sample_rate: int, @@ -2352,6 +2355,7 @@ async def _receive_loop(self, ws: Any) -> None: self._mark_dead(f"parakeet ws recv: {e}") +@live_provider('parakeet', 2) async def process_audio_parakeet( stream_transcript: Callable[[List[Dict[str, Any]]], None], language: str, diff --git a/backend/utils/stt/vad_gate.py b/backend/utils/stt/vad_gate.py index 43467eae9c7..2178d0c95b9 100644 --- a/backend/utils/stt/vad_gate.py +++ b/backend/utils/stt/vad_gate.py @@ -12,6 +12,8 @@ """ import audioop +from utils.stt_cap_dispatch import suppress_socket_audio + import logging import os import threading @@ -794,6 +796,8 @@ def _counted(self, audio: bytes) -> bytes: def send(self, data: bytes, wall_time: Optional[float] = None, start_sample: Optional[int] = None) -> bool: """Send audio through VAD gate and report whether it was accepted.""" + if suppress_socket_audio(self._conn, data): + return True if self.is_connection_dead: return False if self._gate is None: @@ -876,6 +880,8 @@ def commit_send(self) -> None: commit() def send_admitted_audio(self, data: bytes, spans: Any) -> bool: + if suppress_socket_audio(self._conn, data): + return True from utils.stt.soniox_idle import unaccepted_onset data, spans = unaccepted_onset(data, spans, self._send_tracker) diff --git a/backend/utils/stt_cap_dispatch.py b/backend/utils/stt_cap_dispatch.py new file mode 100644 index 00000000000..bde1742ddcf --- /dev/null +++ b/backend/utils/stt_cap_dispatch.py @@ -0,0 +1,260 @@ +"""Funded-provider admission; legacy root usage counters deliberately stay unchanged.""" + +import logging +import os +from functools import wraps +from contextvars import ContextVar +from typing import Any +from utils.stt.socket import STTSocket +from utils.stt.live_router import connecting_target + +from config.stt_cap_dispatch import enabled +from utils.observability.stt_cap_dispatch import record_audio +from utils.byok import get_byok_key +from utils.executors import db_executor, run_blocking + +logger = logging.getLogger(__name__) + + +def customer_funded(provider): + return ( + getattr(provider, 'value', provider) == 'deepgram' + and os.getenv('DEEPGRAM_SELF_HOSTED_ENABLED', '').lower() != 'true' + and bool(get_byok_key('deepgram')) + ) + + +def funded_allowance(uid, source=None): + # Streaming imports this module even with enforcement off. Load billing + # policy only at the admission boundary, not during provider-module import. + from utils.subscription import resolve_transcription_allowance + + return resolve_transcription_allowance(uid, source=source, byok_active=False, omi_funded=True) + + +def sync_allowed(uid: str, seconds: float = 0, provider: Any = 'unknown') -> bool: + if not enabled(): + return True + allowance = funded_allowance(uid) + if not allowance.managed: + record_audio('sync', 'held', provider, seconds, allowance.reason == 'plan_allowance_exhausted') + return allowance.managed + + +class LiveAdmission: + """One balance shared by every channel, replay and replacement in a session. + + Refresh uses the existing usage flush cadence. Accepted packets debit the + finite snapshot locally, so no further packet starts after it reaches zero. + Provider queues already accepting audio remain free to drain. + """ + + def __init__(self, uid, source=None): + self.uid, self.source = uid, source + self.remaining = 0 + self.reason = 'allowance_unavailable' + self.managed = False + self.held = [] + self.omi_selected = False + + async def refresh(self): + allowance = await run_blocking(db_executor, funded_allowance, self.uid, self.source) + self.remaining, self.reason, self.managed = allowance.remaining_seconds, allowance.reason, allowance.managed + if self.managed: + for socket in list(self.held): + try: + await socket.resume() + except Exception as error: + logger.warning( + 'event=stt_cap_dispatch outcome=resume_deferred exception_type=%s', type(error).__name__ + ) + + @property + def funded_paused(self) -> bool: + return self.omi_selected and not (self.managed and (self.remaining is None or self.remaining > 0)) + + def allowed(self, provider): + return customer_funded(provider) or self.managed and (self.remaining is None or self.remaining > 0) + + def send(self, raw, data, provider, rate, *args, **kwargs): + if not self.allowed(provider): + record_audio( + 'live', 'suppressed', provider, len(data) / (2 * rate), self.reason == 'plan_allowance_exhausted' + ) + # Policy absorption is not a transport failure. The receive/archive + # paths continue, and recovery must not fail over or close capture. + return True + accepted = raw.send(data, *args, **kwargs) + if accepted is not False and not customer_funded(provider): + seconds = len(data) / (2 * rate) + record_audio('live', 'accepted', provider, seconds, self.reason == 'plan_allowance_exhausted') + if self.remaining is not None: + self.remaining = max(0, self.remaining - seconds) + if self.remaining == 0: + self.reason = 'plan_allowance_exhausted' + return accepted + + +current_live_admission = ContextVar('stt_cap_live_admission', default=None) + + +class CapSocket(STTSocket): + """Keep receiving at cap, without opening a paid provider or ending capture.""" + + def __init__(self, admission, provider, rate, factory, raw=None): + self.admission, self.provider, self.rate = admission, provider, rate + self.factory, self.raw = factory, raw + self.closed = False + self.cap_dispatch_socket = True + self._open_callbacks = [] + if not customer_funded(provider): + admission.omi_selected = True + if raw is None: + admission.held.append(self) + logger.info('event=stt_cap_dispatch path=live outcome=held provider=%s', provider) + + async def resume(self): + if self.raw is None and not self.closed and self.admission.allowed(self.provider): + raw = await self.factory() + if raw is None: + raise RuntimeError('held provider constructor returned no socket') + # Registry constructors can invoke another decorated provider. A + # resumed leg must retain exactly one debit/metric boundary, just + # like the initial construction path below. + while isinstance(raw, CapSocket): + self.provider, self.rate, self.factory = raw.provider, raw.rate, raw.factory + if not customer_funded(self.provider): + self.admission.omi_selected = True + if raw in self.admission.held: + self.admission.held.remove(raw) + if raw.raw is None: + self._open_callbacks.extend(raw._open_callbacks) + return # Actual fallback remains denied; keep this leg held. + raw = raw.raw + self.raw = raw + for callback in self._open_callbacks: + callback(raw) + self.admission.held.remove(self) + + def on_open(self, callback): + self._open_callbacks.append(callback) + if self.raw is not None: + callback(self.raw) + + def set_wire_ledger(self, ledger): + self.on_open(lambda transport: transport.set_wire_ledger(ledger)) + + def set_capture_axis_ledger(self, ledger): + self.on_open(lambda transport: transport.set_capture_axis_ledger(ledger)) + + @property + def __class__(self) -> type: + return self.raw.__class__ if self.raw is not None else type(self) + + @__class__.setter + def __class__(self, value: type) -> None: + raise TypeError('provider proxy class is read only') + + def __getattr__(self, name): + if self.raw is None: + raise AttributeError(name) + return getattr(self.raw, name) + + @property + def is_connection_dead(self): + if not self.admission.allowed(self.provider): + return False # A paused provider cannot end ongoing capture. + return self.raw.is_connection_dead if self.raw is not None else False + + @property + def death_reason(self): + return self.raw.death_reason if self.raw is not None else None + + def suppress(self, data): + if self.raw is None or not self.admission.allowed(self.provider): + record_audio( + 'live', + 'suppressed', + self.provider, + len(data) / (2 * self.rate), + self.admission.reason == 'plan_allowance_exhausted', + ) + return True + return False + + def send(self, data, *args, **kwargs): + if self.raw is None: + record_audio( + 'live', + 'suppressed', + self.provider, + len(data) / (2 * self.rate), + self.admission.reason == 'plan_allowance_exhausted', + ) + return True + return self.admission.send(self.raw, data, self.provider, self.rate, *args, **kwargs) + + def finish(self): + self.closed = True + if self.raw is not None: + return self.raw.finish() + + def finalize(self): + if self.raw is not None: + return self.raw.finalize() + + +def live_provider(provider, rate_index): + """Applied at actual provider constructors, including fallback constructors.""" + + def decorate(fn): + @wraps(fn) + async def connect(*args, **kwargs): + admission = current_live_admission.get() if enabled() else None + if admission is None: + return await fn(*args, **kwargs) + rate = kwargs.get('sample_rate') or args[rate_index] + target = connecting_target.get() + + async def factory(): + token = connecting_target.set(target) + try: + return await fn(*args, **kwargs) + finally: + connecting_target.reset(token) + + allowed = admission.allowed(provider) + try: + raw = await factory() if allowed else None + except Exception: + if admission.allowed(provider): + raise + raw = None # Permission changed while the dial was in flight. + if allowed and raw is None: + return None + if isinstance(raw, CapSocket): + return raw + return CapSocket(admission, provider, rate, factory, raw) + + return connect + + return decorate + + +def session_admission(fn): + @wraps(fn) + async def connect(self, *args, **kwargs): + admission = getattr(self.receiver, 'cap_admission', None) + token = current_live_admission.set(admission) + try: + return await fn(self, *args, **kwargs) + finally: + current_live_admission.reset(token) + + return connect + + +def suppress_socket_audio(socket, data): + # Explicit identity avoids treating test doubles or unrelated adapters as + # cap owners. Suppress before VAD/timeline adapters record provider spans. + return getattr(socket, 'cap_dispatch_socket', False) is True and socket.suppress(data) diff --git a/backend/utils/subscription.py b/backend/utils/subscription.py index c29ab72da01..268f2d5b47d 100644 --- a/backend/utils/subscription.py +++ b/backend/utils/subscription.py @@ -1,3 +1,5 @@ +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import CONFIG_ERRORS import os from database.fsr_read_cache import fresh_reader from dataclasses import dataclass @@ -1584,9 +1586,15 @@ def get_monthly_usage_for_subscription(uid: str) -> Dict[str, Any]: """ Gets the current monthly usage for subscription purposes, considering the launch date from env variables. The launch date format is expected to be YYYY-MM-DD. - If the launch date is not set, not valid, or in the future, usage is considered zero. + Legacy mode treats unset, invalid and future dates as zero usage. Dispatch + enforcement rejects missing/invalid configuration; an explicit future date + is the intentionally unlaunched policy in both modes. """ subscription_launch_date_str = os.getenv('SUBSCRIPTION_LAUNCH_DATE') + if stt_cap_dispatch_enabled() and not subscription_launch_date_str: + CONFIG_ERRORS.labels(reason='missing').inc() + logger.error('event=stt_cap_launch_config outcome=invalid reason=missing') + raise ValueError('missing subscription launch date') if not subscription_launch_date_str: # Subscription not launched, so no usage is counted against limits. return {} @@ -1595,6 +1603,10 @@ def get_monthly_usage_for_subscription(uid: str) -> Dict[str, Any]: # Use strptime to enforce YYYY-MM-DD format launch_date = datetime.strptime(subscription_launch_date_str, '%Y-%m-%d').replace(tzinfo=timezone.utc) except ValueError: + if stt_cap_dispatch_enabled(): + CONFIG_ERRORS.labels(reason='invalid').inc() + logger.error('event=stt_cap_launch_config outcome=invalid reason=invalid') + raise # Invalid date format, treat as not launched. return {} @@ -1668,6 +1680,7 @@ def resolve_transcription_allowance( subscription: Any = _UNRESOLVED, usage: Any = _UNRESOLVED, byok_active: Any = _UNRESOLVED, + omi_funded: bool = False, ) -> TranscriptionAllowance: """Resolve the managed-transcription allowance for one uid, once. Never raises. @@ -1701,10 +1714,13 @@ def resolve_transcription_allowance( # lookup failure must not open a billed socket for a paywalled account, # and only a validated key on this request (not a stored fingerprint) # exempts. - if is_trial_paywalled(uid, source, required_byok_provider='deepgram', strict=True): + paywall_kwargs = {'byok_exempt': False} if omi_funded else {} + if is_trial_paywalled(uid, source, required_byok_provider='deepgram', strict=True, **paywall_kwargs): return TranscriptionAllowance(TRANSCRIPTION_MODE_BLOCKED, 0, 'trial_paywalled') # Require the Deepgram header on this request so a user cannot activate # BYOK with fake fingerprints and then omit x-byok-deepgram to ride Omi's key. + if omi_funded: + byok_active = False if byok_active is _UNRESOLVED: byok_active = users_db.is_byok_active(uid) if byok_active and get_byok_key('deepgram'): diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py new file mode 100644 index 00000000000..86c949def28 --- /dev/null +++ b/backend/utils/sync/cap_hold.py @@ -0,0 +1,303 @@ +"""Encrypted, owner-scoped pre-STT holds. Pending audio has no application TTL. + +The manifest is written only after upload succeeds. A failed hold fails the +upload/job, so the client never receives success for audio we did not retain. +""" + +import hashlib +import json +import logging +from pathlib import Path +import time +from tempfile import TemporaryDirectory, TemporaryFile +import struct +import wave +from uuid import uuid4 +from typing import Any +from models.conversation_enums import ConversationSource +from utils.observability.stt_cap_dispatch import record_audio +from utils.stt_cap_dispatch import funded_allowance + +from google.cloud import firestore +from google.api_core.exceptions import AlreadyExists +from database import redis_db +from config.stt_cap_dispatch import enabled +from utils.sync.files import get_timestamp_from_path +from utils.executors import db_executor, storage_executor, run_blocking +from database._client import get_firestore_client +from database.legal_holds import external_write_fence +from utils.encryption import encrypt, decrypt, encrypt_audio_chunk, decrypt_audio_chunk +from utils.other.storage import get_private_cloud_sync_bucket, owner_storage_write_gate + +logger = logging.getLogger(__name__) +COLLECTION = 'stt_cap_holds' +AUDIO_CHUNK_BYTES = 1024 * 1024 +RETAINED_NEXT_ATTEMPT_AT = 253402300799 # Terminal rows sort after actionable work, without a new query/index. + + +def holds(uid: str) -> Any: + return get_firestore_client().collection('users').document(uid).collection(COLLECTION) + + +def pending_holds(uid: str, *, limit: int = 10) -> list[Any]: + return list(holds(uid).order_by('next_attempt_at').limit(limit).stream()) + + +def capture_identity(path: str, metadata: dict[str, Any]) -> str: + partition = { + key: metadata.get(key) for key in ('timestamp', 'source', 'client_device_id', 'target_conversation_id') + } + digest = hashlib.sha256(json.dumps(partition, sort_keys=True).encode()) + with open(path, 'rb') as source: + while chunk := source.read(AUDIO_CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +def _describe(path: str, metadata: dict[str, Any]) -> tuple[str, float, dict]: + metadata = dict(metadata) + geolocation = metadata.get('geolocation') + if geolocation is not None and hasattr(geolocation, 'model_dump'): + metadata['geolocation'] = geolocation.model_dump(mode='json') + identity = capture_identity(path, metadata) + with wave.open(path, 'rb') as wav: + seconds = wav.getnframes() / wav.getframerate() + return identity, seconds, metadata + + +def _exists(uid: str, identity: str) -> bool: + with external_write_fence(uid): + return bool(holds(uid).document(identity).get().exists) + + +def upload_held_audio(uid: str, path: str, identity: str, metadata: dict) -> dict: + blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' + bucket = get_private_cloud_sync_bucket() + with external_write_fence(uid), owner_storage_write_gate(uid, bucket): + blob = bucket.blob(blob_path) + blob.chunk_size = 8 * AUDIO_CHUNK_BYTES + with recovery_directory(uid) as directory, TemporaryFile(dir=directory) as encrypted: + with open(path, 'rb') as source: + while chunk := source.read(AUDIO_CHUNK_BYTES): + encrypted.write(encrypt_audio_chunk(chunk, uid)) + encrypted.seek(0) + blob.upload_from_file(encrypted, content_type='application/octet-stream') + encoded_metadata = encrypt(json.dumps(metadata), uid) + fields = {'audio_path': blob_path, 'metadata': encoded_metadata} + if len(encoded_metadata.encode()) > 512 * 1024: + metadata_digest = hashlib.sha256(encoded_metadata.encode()).hexdigest() + metadata_path = f'chunks/{uid}/cap-held/{identity}.{metadata_digest}.metadata.enc' + bucket.blob(metadata_path).upload_from_string(encoded_metadata, content_type='application/octet-stream') + fields = {'audio_path': blob_path, 'metadata_path': metadata_path} + return fields + + +def _publish(uid: str, identity: str, seconds: float, fields: dict) -> bool: + # Reacquire the account fence after upload: deletion may have started while + # the async coordinator switched pools. Never acknowledge a refused manifest. + with external_write_fence(uid): + try: + holds(uid).document(identity).create( + { + **fields, + 'seconds': seconds, + 'state': 'held', + 'created_at': time.time(), + 'next_attempt_at': 0, + 'lease_until': 0, + } + ) + except AlreadyExists: + return False # Preserve the winner's immutable progress checkpoint. + return True + + +def _finish(identity: str, seconds: float, reason: str, receipt: dict | None, created: bool) -> float: + if receipt is not None: + receipt.update(id=identity, seconds=seconds, reason=reason) + if created: + record_audio('sync', 'held', 'unknown', seconds, reason == 'plan_allowance_exhausted') + logger.info('event=stt_cap_dispatch path=sync outcome=held seconds=%.3f', seconds) + return seconds + + +def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict | None = None) -> float: + identity, seconds, metadata = _describe(path, metadata) + created = False + if not _exists(uid, identity): + fields = upload_held_audio(uid, path, identity, metadata) + created = _publish(uid, identity, seconds, fields) + return _finish(identity, seconds, funded_allowance(uid).reason, receipt, created) + + +async def _hold_capture(uid: str, path: str, metadata: dict, reason: str, receipt: dict) -> None: + # Pool workers are leaves; no sync worker waits for another executor. + identity, seconds, metadata = await run_blocking(storage_executor, _describe, path, metadata) + created = False + if not await run_blocking(db_executor, _exists, uid, identity): + fields = await run_blocking(storage_executor, upload_held_audio, uid, path, identity, metadata) + created = await run_blocking(db_executor, _publish, uid, identity, seconds, fields) + _finish(identity, seconds, reason, receipt, created) + + +def claim(uid: str, ref: Any, now: float) -> dict[str, Any] | None: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any): + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else None + if not row or row.get('state') in {'completed', 'retained'} or row.get('lease_until', 0) > now: + return None + token = uuid4().hex + tx.update(ref, {'lease_until': now + 1800, 'lease_token': token}) + row['lease_token'] = token + return row + + with external_write_fence(uid): + return commit(client.transaction()) + + +def read_audio(uid: str, row: dict[str, Any], path: str) -> None: + expected = f'chunks/{uid}/cap-held/' + if not row['audio_path'].startswith(expected): + raise ValueError('hold audio owner mismatch') + blob = get_private_cloud_sync_bucket().blob(row['audio_path']) + blob.chunk_size = 8 * AUDIO_CHUNK_BYTES + with TemporaryFile(dir=Path(path).parent) as encrypted: + blob.download_to_file(encrypted) + encrypted.seek(0) + try: + with open(path, 'wb') as output: + while prefix := encrypted.read(4): + if len(prefix) != 4: + raise ValueError('truncated held-audio chunk') + size = struct.unpack('>I', prefix)[0] + if not 28 <= size <= AUDIO_CHUNK_BYTES + 28: + raise ValueError('invalid held-audio chunk size') + payload = encrypted.read(size) + if len(payload) != size: + raise ValueError('truncated held-audio payload') + audio, _ = decrypt_audio_chunk(prefix + payload, uid) + output.write(audio) + except Exception: + Path(path).unlink(missing_ok=True) + raise + + +def metadata(uid: str, row: dict[str, Any]) -> dict[str, Any]: + if row.get('metadata_path'): + if not row['metadata_path'].startswith(f'chunks/{uid}/cap-held/'): + raise ValueError('hold metadata owner mismatch') + encoded = get_private_cloud_sync_bucket().blob(row['metadata_path']).download_as_text() + else: + encoded = row['metadata'] + return json.loads(decrypt(encoded, uid)) + + +def release(uid: str, ref: Any, token: str) -> None: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any): + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else {} + if row.get('lease_token') == token: + tx.update( + ref, + { + 'lease_until': 0, + 'lease_token': '', + 'next_attempt_at': RETAINED_NEXT_ATTEMPT_AT if row.get('state') == 'retained' else time.time(), + }, + ) + + return commit(client.transaction()) + + +async def hold_capture_if_exhausted( + uid: str, + paths: list[str], + source: ConversationSource, + *, + receipts: dict | None = None, + source_frame_maps: dict | None = None, + admission: dict | None = None, + **context: Any, +) -> bool: + # Shared by v1 intake and v2 execution, before VAD can discard silence. + if not paths or not enabled(): + return False + allowance = await run_blocking(db_executor, funded_allowance, uid) + if allowance.managed: + return False + if admission is not None: + admission['reason'] = allowance.reason + for path in paths: + receipt: dict = {} + await _hold_capture( + uid, + path, + { + **context, + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'full_capture': True, + 'file_name': Path(path).name, + 'source_frame_map': (source_frame_maps or {}).get(path), + }, + allowance.reason, + receipt, + ) + if receipts is not None: + receipts[receipt['id']] = receipt + return True + + +def defer_segment(uid: str, path: str, metadata: dict, response: dict, lock: Any) -> None: + """Report a durable late hold to both coordinators, never as silence.""" + receipt: dict = {} + hold_segment(uid, path, {**metadata, 'file_name': Path(path).name}, receipt=receipt) + with lock: + response.setdefault('_cap_holds', {})[receipt['id']] = receipt + + +def checkpoint(uid: str, ref: Any, token: str, updates: dict) -> bool: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if row.get('lease_token') != token or row.get('lease_until', 0) <= time.time(): + return False + tx.update(ref, updates) + return True + + with external_write_fence(uid): + return commit(client.transaction()) + + +def recovery_cooldown(uid: str) -> bool: + return bool(redis_db.r.set(f'users:{uid}:stt_cap_recovery:cooldown', '1', nx=True, ex=60)) + + +def renew(uid: str, ref: Any, token: str, now: float) -> bool: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any) -> bool: + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else {} + if row.get('lease_token') != token or row.get('lease_until', 0) <= now: + return False + tx.update(ref, {'lease_until': now + 1800}) + return True + + with external_write_fence(uid): + return commit(client.transaction()) + + +def recovery_directory(uid: str) -> TemporaryDirectory: + parent = Path('syncing') / uid + parent.mkdir(parents=True, exist_ok=True) + return TemporaryDirectory(prefix='cap-recovery-', dir=parent) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py new file mode 100644 index 00000000000..ff4f375514b --- /dev/null +++ b/backend/utils/sync/cap_hold_recovery.py @@ -0,0 +1,378 @@ +"""Bounded held-audio recovery from the existing locked-content request path. + +The async coordinator borrows leaf workers. Audio stays held until transcription, +content persistence and idempotent usage settlement succeed, including silence. +""" + +import asyncio +import logging +import threading +import time + +from anyio import from_thread +from typing import Any +from google.api_core.exceptions import NotFound + +from config.stt_cap_dispatch import enabled +from database import users as users_db +from database import sync_cap_holds, sync_cap_recovery +from utils.analytics import record_usage +from utils.executors import run_blocking, db_executor, sync_executor, storage_executor, start_background_task +from utils.stt_cap_dispatch import sync_allowed +from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded +from utils.stt.pre_recorded import get_prerecorded_service +from utils.sync import cap_hold, cap_recovery_plan +from utils.observability.stt_cap_dispatch import record_reconciliation +from utils.sync.cap_recovery_audio import prepare as prepare_audio +from models.geolocation import Geolocation +from utils.sync.pipeline import ( + process_segment, + build_person_embeddings_cache, + finalize_recovered_sync_segment, +) +from models.conversation import ConversationSource + +logger = logging.getLogger(__name__) +RECONCILIATION_COOLDOWN_SECONDS = 1800 + + +def _erase(uid: str, snapshot: Any, row: dict[str, Any]) -> None: + # Account deletion already purges this existing owner-scoped chunk prefix. + objects = [row, *(row.get('piece_plan') or {}).values()] + for path in {item[key] for item in objects for key in ('audio_path', 'metadata_path') if item.get(key)}: + if not path.startswith(f'chunks/{uid}/cap-held/'): + raise ValueError('hold audio owner mismatch') + try: + cap_hold.get_private_cloud_sync_bucket().blob(path).delete() + except NotFound: + pass + snapshot.reference.delete() + + +def _stored(value: Any) -> Any: + if isinstance(value, dict): + return {key: _stored(item) for key, item in value.items()} + if isinstance(value, (set, tuple, list)): + return [_stored(item) for item in value] + return value + + +def _response(receipts: dict) -> dict: + response: dict[str, Any] = {'new_memories': set(), 'updated_memories': set()} + for receipt in receipts.values(): + for key, value in receipt.get('response', {}).items(): + if key in {'new_memories', 'updated_memories', '_fenced_conversation_ids'}: + response.setdefault(key, set()).update(value) + elif isinstance(value, dict): + response.setdefault(key, {}).update(value) + return response + + +async def _save(uid: str, snapshot: Any, row: dict, updates: dict) -> bool: + saved = await run_blocking(db_executor, cap_hold.checkpoint, uid, snapshot.reference, row['lease_token'], updates) + if saved: + row.update(updates) + return saved + + +def _new_dispatch_allowed(uid: str) -> bool: + if not sync_allowed(uid): + return False + restricted, _ = get_hard_restriction_status(uid) + return not restricted and not is_daily_audio_ceiling_exceeded(uid) + + +async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: + if not enabled(): + return False + if row.get('finalized'): + return True + meta = await run_blocking(storage_executor, cap_hold.metadata, uid, row) + if meta.get('content_id'): + status = await run_blocking(db_executor, sync_cap_holds.recovery_status, uid, meta['content_id'], snapshot.id) + if status in {'completed', 'settled'}: + return await _save(uid, snapshot, row, {'finalized': True, 'ledger_already_settled': True}) + if status != 'ready': + return False + if not row.get('piece_receipts') and not await run_blocking(db_executor, _new_dispatch_allowed, uid): + return False + prefs = await run_blocking(db_executor, users_db.get_user_transcription_preferences, uid) or {} + private = await run_blocking(db_executor, users_db.get_user_private_cloud_sync_enabled, uid) + protection = await run_blocking(db_executor, users_db.get_data_protection_level, uid) if private else None + try: + embeddings = await run_blocking(db_executor, build_person_embeddings_cache, uid) + except Exception as error: + logger.warning( + 'event=stt_cap_recovery outcome=speaker_context_unavailable exception_type=%s', type(error).__name__ + ) + embeddings = {} + geolocation = Geolocation.model_validate(meta['geolocation']) if meta.get('geolocation') else None + directory = await run_blocking(storage_executor, cap_hold.recovery_directory, uid) + receipts: dict[str, dict[str, Any]] = dict(row.get('piece_receipts') or {}) + + async def save_receipt(name: str, receipt: dict[str, Any] | None) -> bool: + updated = dict(receipts) + if receipt is None: + updated.pop(name, None) + else: + updated[name] = receipt + if not await _save(uid, snapshot, row, {'piece_receipts': updated}): + return False + receipts.clear() + receipts.update(updated) + return True + + try: + try: + plan, local_paths, contexts = await cap_recovery_plan.ensure_plan( + uid, snapshot, row, meta, directory, prepare_audio, _save + ) + except cap_recovery_plan.PlanFenced: + return False + processed = set(row.get('processed_paths') or []) + pending = False + for name in sorted(set(plan) | set(receipts)): + if not await run_blocking( + db_executor, cap_hold.renew, uid, snapshot.reference, row['lease_token'], time.time() + ): + return False + if name in processed: + continue + receipt: dict[str, Any] | None = receipts.get(name) + if receipt and receipt.get('state') == 'failed_reconciliation': + continue + attempt = 1 + if receipt and receipt.get('state') == 'dispatched': + if 'dispatched_at' not in receipt: + receipt = { + **receipt, + 'attempt': receipt.get('attempt', 1), + 'dispatched_at': row.get('created_at', time.time()), + } + if not await save_receipt(name, receipt): + return False + if time.time() - receipt['dispatched_at'] < RECONCILIATION_COOLDOWN_SECONDS: + pending = True + continue + proof = await run_blocking(db_executor, sync_cap_recovery.ingested_piece, uid, snapshot.id, name) + if proof: + receipt = {**proof, 'reconciled_ingested': True} + if not await save_receipt(name, receipt): + return False + record_reconciliation('reconciled_ingested', snapshot.id, name) + elif receipt.get('attempt', 1) >= 2 or name not in plan: + if not await save_receipt( + name, {**receipt, 'state': 'failed_reconciliation', 'has_content': False} + ): + return False + record_reconciliation('failed', snapshot.id, name) + continue + else: + attempt = 2 + if receipt is None or receipt.get('state') == 'dispatched': + if not await run_blocking(db_executor, _new_dispatch_allowed, uid): + pending = True + continue + previous = receipt + try: + segment_path, context = await cap_recovery_plan.stage_piece( + uid, name, plan[name], directory, local_paths, contexts + ) + language = prefs.get('language') or '' + service = get_prerecorded_service( + language if prefs.get('single_language_mode') and language else 'multi' + ) + except Exception as error: + logger.warning( + 'event=stt_cap_recovery outcome=prepare_deferred hold_id=%s piece=%r exception_type=%s', + snapshot.id, + name, + type(error).__name__, + ) + pending = True + continue + intent: dict[str, Any] = { + 'state': 'dispatched', + 'provider': service[0], + 'model': service[2], + 'funding': 'omi', + 'attempt': attempt, + 'dispatched_at': time.time(), + 'seconds': plan[name]['seconds'], + 'usage_metadata': {'funding': 'omi', 'provider': service[0], 'source': 'sync_v2'}, + } + if not await save_receipt(name, intent): + return False + response = {'new_memories': set(), 'updated_memories': set()} + errors, outcome = [], {'provider_called': False} + try: + ok = await run_blocking( + sync_executor, + process_segment, + segment_path, + uid, + response, + threading.Lock(), + errors, + ConversationSource(meta['source']), + False, + prefs, + embeddings, + private_cloud_sync_enabled=bool(private), + data_protection_level=protection, + client_device_id=meta.get('client_device_id'), + client_platform=meta.get('client_platform'), + target_conversation_id=context.get('target_conversation_id'), + geolocation=geolocation, + source_position_map=context.get('source_position_map'), + lineage_binding=context.get('lineage_binding'), + sync_lane=meta.get('sync_lane', 'fresh'), + cap_recovery=True, + deferred_outcome=outcome, + cap_recovery_service=service, + cap_recovery_checkpoint={ + 'hold_id': snapshot.id, + 'piece': name, + 'lease_token': row['lease_token'], + 'attempt': attempt, + 'receipt': intent, + }, + ) + except Exception as error: + ok = False + errors.append(type(error).__name__) + logger.warning( + 'event=stt_cap_recovery outcome=processing_deferred hold_id=%s piece=%r exception_type=%s', + snapshot.id, + name, + type(error).__name__, + ) + if not outcome['provider_called']: + if not await save_receipt(name, previous): + return False + record_reconciliation('cleared_noncall', snapshot.id, name) + pending = True + continue + if attempt == 2: + record_reconciliation('redispatched', snapshot.id, name) + if errors or outcome.get('cap_held') or outcome.get('recovery_ambiguous'): + pending = True + continue + receipt = { + **intent, + 'state': 'transcribed', + 'response': _stored(response), + 'seconds': intent['seconds'] if ok else 0, + 'has_content': bool(ok), + } + if not await save_receipt(name, receipt): + return False + if receipt['has_content']: + seconds = int(receipt['seconds']) + await run_blocking( + db_executor, + record_usage, + uid, + transcription_seconds=seconds, + speech_seconds=seconds, + idempotency_key=f'cap-hold-{snapshot.id}-{name}', + usage_metadata=receipt.get( + 'usage_metadata', {'funding': 'omi', 'provider': 'unknown', 'source': 'sync_v2'} + ), + ) + processed.add(name) + if not await _save(uid, snapshot, row, {'processed_paths': sorted(processed)}): + return False + # Finalize completed siblings even while another piece is cooling down. + finalized = set(row.get('finalized_paths') or []) + ready = {name: receipts[name] for name in processed - finalized if name in receipts} + if ready: + await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, _response(ready), bool(private)) + finalized.update(ready) + if not await _save(uid, snapshot, row, {'finalized_paths': sorted(finalized)}): + return False + if pending: + return False + failed = any(r.get('state') == 'failed_reconciliation' for r in receipts.values()) + retained = failed or any(r.get('reconciled_ingested') for r in receipts.values()) + return await _save( + uid, + snapshot, + row, + { + 'finalized': True, + 'recovery_response': _stored(_response(receipts)), + 'failed_reconciliation': failed, + 'retain_audio': retained, + }, + ) + finally: + await run_blocking(storage_executor, directory.cleanup) + + +def _settle_and_erase(uid: str, snapshot: Any, row: dict) -> None: + if row.get('failed_reconciliation'): + snapshot.reference.update({'state': 'retained', 'next_attempt_at': cap_hold.RETAINED_NEXT_ATTEMPT_AT}) + return # Siblings finalized, but the held ledger must not claim failed audio was transcribed. + if not row.get('settled'): + meta = cap_hold.metadata(uid, row) + if meta.get('content_id') and not row.get('ledger_already_settled'): + if not sync_cap_holds.settle_hold( + uid, + meta['content_id'], + snapshot.id, + row.get('recovery_response') or {}, + len(row.get('piece_receipts') or {}), + ): + raise RuntimeError('hold content ledger ownership changed') + snapshot.reference.update({'settled': True}) + if row.get('retain_audio'): + snapshot.reference.update({'state': 'retained', 'next_attempt_at': cap_hold.RETAINED_NEXT_ATTEMPT_AT}) + else: + _erase(uid, snapshot, row) + + +async def recover(uid: str) -> None: + # One cheap distributed cooldown before querying usage or manifests. Failed + # Redis means deferred recovery, never duplicate provider dispatch. + if not enabled(): + return + acquired = await run_blocking(db_executor, cap_hold.recovery_cooldown, uid) + if not acquired: + return + rows = await run_blocking(db_executor, cap_hold.pending_holds, uid) + for snapshot in rows: + row = snapshot.to_dict() or {} + try: + if row.get('state') == 'completed': + await run_blocking(storage_executor, _settle_and_erase, uid, snapshot, row) + continue + row = await run_blocking(db_executor, cap_hold.claim, uid, snapshot.reference, time.time()) + if row is None: + continue + try: + if await _recover_row(uid, snapshot, row): + # Completed remains retryable if ledger settlement fails. + # Only successful settlement (or terminal failure) retains it. + if not await _save(uid, snapshot, row, {'state': 'completed'}): + continue + await run_blocking(storage_executor, _settle_and_erase, uid, snapshot, row) + finally: + await run_blocking(db_executor, cap_hold.release, uid, snapshot.reference, row['lease_token']) + except Exception as error: + logger.warning('event=stt_cap_recovery outcome=deferred exception_type=%s', type(error).__name__) + + +async def _schedule(uid: str) -> None: + start_background_task(recover(uid), name='stt_cap_hold_recovery') + + +def kick(uid: str) -> None: + if not enabled(): + return + try: + asyncio.get_running_loop() + except RuntimeError: + from_thread.run(_schedule, uid) + else: + start_background_task(recover(uid), name='stt_cap_hold_recovery') diff --git a/backend/utils/sync/cap_recovery_audio.py b/backend/utils/sync/cap_recovery_audio.py new file mode 100644 index 00000000000..e6cc50a9884 --- /dev/null +++ b/backend/utils/sync/cap_recovery_audio.py @@ -0,0 +1,88 @@ +"""Replay ordinary WAL coverage, VAD and lineage planning for a held capture.""" + +from pathlib import Path + +from config.sync_live_dedupe import sync_lineage_resolve_active_for +from config.sync_safety_wal import sync_safety_wal_enabled +from utils.executors import run_blocking, db_executor, storage_executor, sync_executor +from utils.sync import recording_lineage +from utils.sync.files import get_timestamp_from_path, get_wav_duration +from utils.sync.pipeline import retrieve_vad_segments, cleanup_recovered_sync_files +from utils.sync.safety_wal_coverage import apply_safety_wal_coverage +from utils.sync.wal_audio_coverage import apply_sync_wal_audio_coverage + + +async def prepare(uid: str, path: str, meta: dict) -> tuple[set, dict, dict, dict]: + if not meta.get('full_capture'): + position = meta.get('source_position_map') + return {path}, {path: tuple(position) if position else None}, {}, {} + mapping = meta.get('source_frame_map') or {} + offsets = mapping.get('offsets') or [] + frames = [end - start for start, end in zip(offsets, offsets[1:])] + claims = {Path(path).name.replace('.wav', '.bin'): mapping['claim']} if mapping else None + paths, source_maps, _ = await apply_sync_wal_audio_coverage( + uid, + meta['source'], + False, + meta.get('client_device_id'), + meta.get('recording_session_id'), + claims, + [path], + {path: frames}, + run_blocking=run_blocking, + db_executor=db_executor, + storage_executor=storage_executor, + cleanup_files=cleanup_recovered_sync_files, + ) + if sync_safety_wal_enabled(): + inputs = list(paths) + paths, _ = await apply_safety_wal_coverage( + uid, + paths, + meta.get('target_conversation_id'), + run_blocking=run_blocking, + db_executor=db_executor, + storage_executor=storage_executor, + job_ref='cap_recovery', + protected_paths=source_maps, + capture_partition={ + 'source': meta['source'], + 'client_device_id': meta.get('client_device_id'), + 'is_locked': False, + }, + ) + if inputs and not paths: + paths = inputs # Same conservative envelope fallback as ordinary sync. + segments, segment_maps, errors = set(), {}, [] + for wav_path in paths: + await run_blocking( + sync_executor, + retrieve_vad_segments, + wav_path, + segments, + errors, + source_frame_map=source_maps.get(wav_path), + segment_source_maps=segment_maps, + ) + if errors: + raise RuntimeError('held capture VAD failed') + targets, bindings = {}, {} + origin = meta.get('recording_session_id') + if origin and sync_lineage_resolve_active_for(uid): + targets = await run_blocking( + db_executor, + recording_lineage.plan_segment_targets, + sorted(segments, key=get_timestamp_from_path), + get_timestamp_from_path, + get_wav_duration, + uid, + origin, + meta.get('target_conversation_id'), + meta['source'], + meta.get('client_device_id'), + False, + meta.get('job_id'), + bindings, + segment_source_maps=segment_maps, + ) + return segments, segment_maps, targets, bindings diff --git a/backend/utils/sync/cap_recovery_plan.py b/backend/utils/sync/cap_recovery_plan.py new file mode 100644 index 00000000000..11ec4c06798 --- /dev/null +++ b/backend/utils/sync/cap_recovery_plan.py @@ -0,0 +1,71 @@ +"""Freeze encrypted VAD pieces and lineage before any held-audio dispatch.""" + +import hashlib +import json +from pathlib import Path +from typing import Any, Awaitable, Callable + +from utils.executors import run_blocking, storage_executor +from utils.sync import cap_hold +from utils.sync.files import get_wav_duration + + +class PlanFenced(RuntimeError): + pass + + +def persist_piece(uid: str, hold_id: str, path: str, context: dict[str, Any]) -> dict[str, Any]: + identity = hashlib.sha256( + (hold_id + cap_hold.capture_identity(path, context) + json.dumps(context, sort_keys=True)).encode() + ).hexdigest() + return {**cap_hold.upload_held_audio(uid, path, identity, context), 'seconds': get_wav_duration(path)} + + +async def ensure_plan( + uid: str, + snapshot: Any, + row: dict[str, Any], + meta: dict[str, Any], + directory: Any, + prepare_audio: Callable[..., Awaitable[tuple[set, dict, dict, dict]]], + save: Callable[..., Awaitable[bool]], +) -> tuple[dict[str, dict[str, Any]], dict[str, str], dict[str, dict[str, Any]]]: + if 'piece_plan' in row: + return row['piece_plan'], {}, {} + name = Path(meta.get('file_name') or f"{meta['timestamp']}.wav").name + path = str(Path(directory.name) / name) + await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) + paths, source_maps, targets, bindings = await prepare_audio(uid, path, meta) + plan, local_paths, contexts = {}, {}, {} + for path in sorted(paths): + name = Path(path).name + if name in plan: + raise ValueError('held piece identity collision') + context = { + 'target_conversation_id': targets.get(path, meta.get('target_conversation_id')), + 'source_position_map': source_maps.get(path), + 'lineage_binding': bindings.get(path, meta.get('lineage_binding')), + } + plan[name] = await run_blocking(storage_executor, persist_piece, uid, snapshot.id, path, context) + local_paths[name], contexts[name] = path, context + if not await save(uid, snapshot, row, {'piece_plan': plan}): + raise PlanFenced('held piece plan ownership changed') + return plan, local_paths, contexts + + +async def stage_piece( + uid: str, + name: str, + entry: dict[str, Any], + directory: Any, + local_paths: dict[str, str], + contexts: dict[str, dict[str, Any]], +) -> tuple[str, dict[str, Any]]: + if name in local_paths: + return local_paths[name], contexts[name] + path = str(Path(directory.name) / Path(name).name) + context = await run_blocking(storage_executor, cap_hold.metadata, uid, entry) + await run_blocking(storage_executor, cap_hold.read_audio, uid, entry, path) + position = context.get('source_position_map') + context['source_position_map'] = tuple(position) if position else None + return path, context diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 80c143c1c5b..c25dbcea3a6 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -134,6 +134,9 @@ from utils.speaker_identification import detect_speaker_from_text from utils.speaker_learning_jobs import schedule_person_voice_learning_retries from utils.stt.voiceprints import usable_person_voiceprint +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.stt_cap_dispatch import sync_allowed, funded_allowance +from utils.observability.stt_cap_dispatch import record_audio as record_cap_audio from utils.stt.pre_recorded import get_prerecorded_service, postprocess_words, prerecorded from utils.stt.outcomes import ( TranscriptionFailure, @@ -780,7 +783,14 @@ def bind_or_converge_sync_ledger_completion( ) if binding.bound: return None - if not binding.completed or not is_valid_completed_sync_content_result(binding.result): + held = getattr(binding, 'held', False) is True + if not ( + held + and isinstance(binding.result, dict) + and binding.result.get('held_audio') is True + or binding.completed + and is_valid_completed_sync_content_result(binding.result) + ): raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') finalized = _require_run_owner( @@ -1223,10 +1233,10 @@ def process_segment( is_locked: bool = False, transcription_prefs: dict = None, person_embeddings_cache: dict = None, - target_conversation_id: str = None, + target_conversation_id: Optional[str] = None, turnstile: Optional[_OrderedTurnstile] = None, private_cloud_sync_enabled: bool = False, - data_protection_level: str = None, + data_protection_level: Optional[str] = None, client_device_id: Optional[str] = None, client_platform: Optional[str] = None, sync_lane: str = SyncLane.FRESH.value, @@ -1238,15 +1248,52 @@ def process_segment( attempt_ref: str | None = None, source_position_map: tuple[dict, int] | None = None, lineage_binding: Optional[str] = None, + cap_recovery: bool = False, + cap_content_id: str | None = None, + cap_recovery_service: tuple[str, str | None, str] | None = None, + cap_recovery_checkpoint: dict | None = None, ): conversation_id = None provider = 'unknown' model = 'unknown' phase = 'provider_select' + if stt_cap_dispatch_enabled() and cap_recovery and deferred_outcome is not None: + deferred_outcome['provider_called'] = False try: if ambiguous_binding_pending(uid, lineage_binding): phase = 'persistence' raise SyncAssignmentConflict('sync lineage generation ownership remains ambiguous') + if stt_cap_dispatch_enabled(): + if not sync_allowed(uid): + if not cap_recovery: + from utils.sync.cap_hold import defer_segment + + defer_segment( + uid, + path, + { + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'target_conversation_id': target_conversation_id, + 'private_cloud_sync_enabled': private_cloud_sync_enabled, + 'data_protection_level': data_protection_level, + 'client_device_id': client_device_id, + 'client_platform': client_platform, + 'geolocation': geolocation, + 'source_position_map': source_position_map, + 'lineage_binding': lineage_binding, + 'sync_lane': sync_lane, + 'content_id': cap_content_id, + 'job_id': job_id, + }, + response, + lock, + ) + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True + return False + # Queue-time locks do not survive execution-time authorization. + is_locked = False url = get_syncing_file_temporal_signed_url(path) schedule_syncing_temporal_file_deletion(path) @@ -1258,7 +1305,14 @@ def process_segment( single_language_mode = prefs.get('single_language_mode', False) req_language = user_language if (single_language_mode and user_language) else 'multi' - provider, _, model = get_prerecorded_service(req_language) + recovery_dispatch = {} + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_service is not None: + # The durable pre-dispatch intent owns this provider/model. Pin + # actual serving too, rather than reselecting after the checkpoint. + provider, _, model = cap_recovery_service + recovery_dispatch['selected_service'] = cap_recovery_service + else: + provider, _, model = get_prerecorded_service(req_language) # When single-language mode is active, trust the user's language choice # rather than Deepgram's detection (avoids overriding explicit selection). @@ -1296,6 +1350,38 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N transcript_segments: List[TranscriptSegment] = [] while True: phase = 'provider_call' + if stt_cap_dispatch_enabled() and not sync_allowed(uid): + if not cap_recovery: + from utils.sync.cap_hold import defer_segment + + defer_segment( + uid, + path, + { + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'target_conversation_id': target_conversation_id, + 'private_cloud_sync_enabled': private_cloud_sync_enabled, + 'data_protection_level': data_protection_level, + 'client_device_id': client_device_id, + 'client_platform': client_platform, + 'geolocation': geolocation, + 'source_position_map': source_position_map, + 'lineage_binding': lineage_binding, + 'sync_lane': sync_lane, + 'content_id': cap_content_id, + 'job_id': job_id, + }, + response, + lock, + ) + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True + return False + if stt_cap_dispatch_enabled() and cap_recovery and deferred_outcome is not None: + # Explicit request-boundary signal survives later cap denial, + # parse/ingest errors, and the bounded empty-result retry. + deferred_outcome['provider_called'] = True words, detected_language = prerecorded( url, speakers_count=3, @@ -1303,7 +1389,13 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N return_language=True, language=req_language, keywords=vocabulary if vocabulary else None, + **recovery_dispatch, ) + if stt_cap_dispatch_enabled(): + allowance = funded_allowance(uid) + record_cap_audio( + 'sync', 'accepted', provider, get_wav_duration(path), allowance.reason == 'plan_allowance_exhausted' + ) language = user_language if (single_language_mode and user_language) else detected_language phase = 'parse' transcript_segments = postprocess_words(words, 0) if words else [] @@ -1315,6 +1407,12 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N _log_empty_retry('recovered', words=len(words or []), raw_empty=not words) recovered_via_retry = True break + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_checkpoint is not None: + # A held piece gets at most two persisted application dispatch + # attempts. Reconciliation owns the second, not this local retry. + if deferred_outcome is not None: + deferred_outcome['recovery_ambiguous'] = True + return False if empty_retried: # Words survived the provider but nothing survived post-processing: # again no transcribable speech, valid and empty rather than failed. @@ -1445,6 +1543,11 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N incoming, candidate_id=closest_memory['id'] if closest_memory else None, target_id=target_conversation_id, + **( + {'recovery_checkpoint': cap_recovery_checkpoint} + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_checkpoint is not None + else {} + ), ) if sync_safety_wal_enabled() and assigned.get('sync_live_target'): removed = len(transcript_segments) - len(survivors) @@ -2055,6 +2158,9 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT coordinator itself holds zero thread pool slots — only leaf operations use threads, and only for their actual duration. """ + if stt_cap_dispatch_enabled(): + # A hold is pending processing, not a locked transcript partition. + should_lock = False # LocalRecordingsProvider uploads one explicitly selected native recording # at a time. The writer's exact marker distinguishes Transcribe Later from # automatic offline fallback (omibatchphoneauto) in already-released clients. @@ -2324,6 +2430,29 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT reason='policy', outcome='degraded', ) + cap_held = False + cap_hold_receipts: dict = {} + if stt_cap_dispatch_enabled(): + from utils.sync.cap_hold import hold_capture_if_exhausted + + cap_held = await hold_capture_if_exhausted( + uid, + wav_paths, + source, + receipts=cap_hold_receipts, + source_frame_maps=source_frame_maps, + target_conversation_id=target_conversation_id, + client_device_id=client_device_id, + client_platform=client_platform, + geolocation=geolocation, + recording_session_id=recording_session_id, + sync_lane=sync_lane, + content_id=content_id, + job_id=job_id, + ) + if cap_held: + await run_blocking(storage_executor, _cleanup_files, wav_paths) + wav_paths = [] # --- Phase 2: VAD --- job_phase = 'vad' await run_blocking( @@ -2424,7 +2553,9 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT if total_segments == 0: empty_outcome = ( - TranscriptionOutcome.SUCCESS if coverage_suppressed_all else TranscriptionOutcome.EXPECTED_SILENCE + TranscriptionOutcome.SUCCESS + if coverage_suppressed_all or cap_held + else TranscriptionOutcome.EXPECTED_SILENCE ) empty_result = { 'new_memories': [], @@ -2436,17 +2567,26 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT 'provider': 'unknown', 'model': 'unknown', 'lane': sync_lane, + **({'held_audio': True} if cap_held else {}), } if content_id: + ledger_writer = mark_sync_content_completed + ledger_extra = {} + if cap_held: + from database.sync_cap_holds import mark_content_held + + ledger_writer = mark_content_held + ledger_extra = {'hold_ids': list(cap_hold_receipts)} completed_ledger = await run_blocking( db_executor, - mark_sync_content_completed, + ledger_writer, uid, content_id, job_id, empty_result, run_token=active_run_lock_token, run_epoch=active_run_lock_epoch, + **ledger_extra, ) if not completed_ledger: raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') @@ -2718,7 +2858,10 @@ def _process_one_segment(path: str): attempt_ref=attempt_ref, source_position_map=segment_source_maps.get(path), **({'lineage_binding': segment_binding_reasons.get(path)} if use_lineage else {}), + **({'cap_content_id': content_id} if stt_cap_dispatch_enabled() else {}), ) + if deferred_outcome.get('cap_held'): + return # Durable policy hold, not success/silence/provider failure. if ok: # Persist result contributions before the processed marker. # Therefore any skipped segment on a retry has its visible @@ -2916,17 +3059,23 @@ def _checkpoint_fenced_conversations(): # Record DG usage after processing. job_phase = 'usage' + cap_holds = response.get('_cap_holds', {}) if stt_cap_dispatch_enabled() else {} + cap_held_seconds = sum(item['seconds'] for item in cap_holds.values()) await _record_restricted_sync_dg_usage( enabled=fair_use_restrict_dg, uid=uid, job_id=job_id, content_id=content_id, - total_speech_seconds=total_speech_seconds, + total_speech_seconds=( + max(0, total_speech_seconds - cap_held_seconds) if cap_holds else total_speech_seconds + ), ) # Build result failed_segments = len(segment_errors) successful_segments = total_segments - failed_segments + if cap_holds: + successful_segments -= len(cap_holds) job_outcome = _job_transcription_outcome(segment_errors) result = { 'new_memories': sorted(response['new_memories']), @@ -2940,6 +3089,8 @@ def _checkpoint_fenced_conversations(): if content_segment_count[0] > 0: try: usage_seconds = int(total_speech_seconds) + if cap_holds: + usage_seconds = int(max(0, total_speech_seconds - cap_held_seconds)) should_record_usage = bool(content_id) or await run_blocking( db_executor, try_mark_once, job_id, 'usage' ) @@ -2978,6 +3129,7 @@ def _checkpoint_fenced_conversations(): 'provider': bounded_provider(job_provider), 'model': _bounded_sync_model(job_model), 'lane': sync_lane, + **({'held_audio': True} if cap_holds else {}), } repeat_failure_fingerprint = _whole_job_persistence_fingerprint( failed_segments, total_segments, persistence_failure_fingerprints @@ -2993,15 +3145,23 @@ def _checkpoint_fenced_conversations(): if content_id and failed_segments == 0: # The durable content ledger proves every successful segment # before the WAL-visible completed status is published. + ledger_writer = mark_sync_content_completed + ledger_extra = {} + if cap_holds: + from database.sync_cap_holds import mark_content_held + + ledger_writer = mark_content_held + ledger_extra = {'hold_ids': list(cap_holds)} completed_ledger = await run_blocking( db_executor, - mark_sync_content_completed, + ledger_writer, uid, content_id, job_id, final_result, run_token=active_run_lock_token, run_epoch=active_run_lock_epoch, + **ledger_extra, ) if not completed_ledger: raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') @@ -3211,3 +3371,15 @@ def _download_staged_files(blob_paths: list) -> bool: if not download_syncing_temporal_file(p): return False return True + + +def finalize_recovered_sync_segment(uid: str, response: dict, private: bool) -> None: + """Use the same enrichment/audio finalization as ordinary sync intake.""" + _reprocess_merged_conversations(uid, response) + if private: + _finalize_sync_audio_files(uid, response) + + +def cleanup_recovered_sync_files(paths: list[str]) -> None: + """Coverage replay borrows the ordinary sync cleanup boundary.""" + _cleanup_files(paths) diff --git a/config/feature-flags.yaml b/config/feature-flags.yaml index 6913a1e7912..c06758a6d69 100644 --- a/config/feature-flags.yaml +++ b/config/feature-flags.yaml @@ -323,6 +323,7 @@ flags: - {"key":"SYNC_ASSIGNMENT_CONTENTION_RETRY_ENABLED","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Bounded fresh-transaction retries for sync assignment contention","fail":"closed","owner":"dazheng","created":"2026-10-15","review_by":"2026-11-15","decision":"pending","notes":"Default false in code/dev/prod. Reuses the five-attempt jittered Firestore contention helper only for sync assignment; preserves the existing expired-transaction and size-limit backstops. No serialization or fencing changes."} - {"key":"SYNC_LEGACY_FRESH_ENABLED","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Admit recent LEGACY offline sync to fresh under an atomic per-user upload and byte budget","fail":"closed","owner":"dazheng","created":"2026-10-14","review_by":"2026-11-14","decision":"pending","notes":"Code, dev and prod default false. V2 only; budget exhaustion/unavailability demotes to backfill. Roll back by setting false; accepted jobs retain their lane. Drain qualification: backend/docs/runbooks/sync-legacy-fresh-drain.md"} - {"key":"AUDIO_CANONICAL_PACKING_ENABLED","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Default off; new release-scoped canonical capture intake and owner-only verified tar packing; requires new regional bucket","fail":"closed","owner":"dazheng","created":"2026-10-15","review_by":"2026-11-14","decision":"pending"} + - {"key":"STT_CAP_DISPATCH_ENFORCEMENT","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation","fail":"closed","owner":"dazheng","created":"2026-10-16","review_by":"2026-11-15","decision":"pending"} ignore: - {"key":"DD_TRACE_ENABLED","reason":"Datadog telemetry instrumentation switch, not a product behavior flag"} - {"key":"DD_LOGS_ENABLED","reason":"Datadog telemetry log routing switch, not a product behavior flag"} diff --git a/desktop/windows/src/renderer/src/lib/omiApi.generated.ts b/desktop/windows/src/renderer/src/lib/omiApi.generated.ts index d31a4d00ada..94af0e4f2cb 100644 --- a/desktop/windows/src/renderer/src/lib/omiApi.generated.ts +++ b/desktop/windows/src/renderer/src/lib/omiApi.generated.ts @@ -5307,6 +5307,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/docs/api-reference/app-client-openapi.json b/docs/api-reference/app-client-openapi.json index 9b14e25c131..b417237bf75 100644 --- a/docs/api-reference/app-client-openapi.json +++ b/docs/api-reference/app-client-openapi.json @@ -32310,6 +32310,17 @@ "title": "Failed Segments", "type": "integer" }, + "held_audio": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Held Audio" + }, "new_memories": { "items": { "type": "string" diff --git a/web/admin/lib/services/omi-api/omiApi.generated.ts b/web/admin/lib/services/omi-api/omiApi.generated.ts index d31a4d00ada..94af0e4f2cb 100644 --- a/web/admin/lib/services/omi-api/omiApi.generated.ts +++ b/web/admin/lib/services/omi-api/omiApi.generated.ts @@ -5307,6 +5307,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/web/app/src/lib/omiApi.generated.ts b/web/app/src/lib/omiApi.generated.ts index d31a4d00ada..94af0e4f2cb 100644 --- a/web/app/src/lib/omiApi.generated.ts +++ b/web/app/src/lib/omiApi.generated.ts @@ -5307,6 +5307,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/web/personas-open-source/src/lib/omiApi.generated.ts b/web/personas-open-source/src/lib/omiApi.generated.ts index d31a4d00ada..94af0e4f2cb 100644 --- a/web/personas-open-source/src/lib/omiApi.generated.ts +++ b/web/personas-open-source/src/lib/omiApi.generated.ts @@ -5307,6 +5307,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array;