From 1f6bb7edefd9e36a5e44bab3752c440ab1c389af Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 19:56:21 +0700 Subject: [PATCH 01/19] fix: enforce transcription cap at funded STT dispatch (flag off) Failure-Class: FC-stt-allowance-not-enforced-at-dispatch --- ...tt-allowance-not-enforced-at-dispatch.json | 10 + backend/AGENTS.md | 2 +- backend/config/stt_cap_dispatch.py | 7 + backend/diarizer/batch_pipeline.py | 41 ++- backend/diarizer/batch_pipeline_runtime.py | 4 +- backend/docs/feature-flag-registry.md | 2 + backend/docs/listen_pusher_pipeline.mdx | 9 + backend/docs/stt-cap-dispatch.md | 120 +++++++++ backend/models/sync_job_status.py | 1 + backend/routers/listen/contracts.py | 1 + backend/routers/listen/legacy_recovery.py | 7 + backend/routers/listen/receiver.py | 56 ++++- backend/routers/listen/runtime.py | 8 + backend/routers/sync.py | 26 ++ .../firestore_query_driver_registry.py | 7 + .../unit/test_batch_pipeline_admission.py | 21 ++ .../tests/unit/test_batch_pipeline_pump.py | 24 ++ ...test_batch_pipeline_quarantine_recovery.py | 30 +++ .../tests/unit/test_batch_pipeline_worker.py | 42 ++++ .../unit/test_listen_stt_death_monitor.py | 19 ++ .../test_locked_content_reset_recovery.py | 14 ++ backend/tests/unit/test_stt_cap_dispatch.py | 193 ++++++++++++++ .../tests/unit/test_stt_cap_dispatch_http.py | 98 ++++++++ backend/tests/unit/test_sync_cap_hold.py | 144 +++++++++++ .../tests/unit/test_sync_cap_hold_recovery.py | 85 +++++++ .../unit/test_transcription_allowance.py | 27 ++ backend/utils/batch_pipeline/admission.py | 4 + backend/utils/batch_pipeline/contracts.py | 8 +- backend/utils/batch_pipeline/projection.py | 3 +- backend/utils/batch_pipeline/pump.py | 21 +- .../batch_pipeline/quarantine_recovery.py | 11 + backend/utils/locked_content_recovery.py | 8 + .../utils/observability/stt_cap_dispatch.py | 17 ++ backend/utils/stt/live_failure.py | 6 + backend/utils/stt/live_session.py | 40 ++- backend/utils/stt/live_target_connect.py | 3 + backend/utils/stt/soniox.py | 2 + backend/utils/stt/streaming.py | 4 + backend/utils/stt/vad_gate.py | 6 + backend/utils/stt_cap_dispatch.py | 238 ++++++++++++++++++ backend/utils/subscription.py | 20 +- backend/utils/sync/cap_hold.py | 161 ++++++++++++ backend/utils/sync/cap_hold_recovery.py | 154 ++++++++++++ backend/utils/sync/pipeline.py | 80 +++++- config/feature-flags.yaml | 1 + 45 files changed, 1760 insertions(+), 25 deletions(-) create mode 100644 .github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json create mode 100644 backend/config/stt_cap_dispatch.py create mode 100644 backend/docs/stt-cap-dispatch.md create mode 100644 backend/tests/unit/test_stt_cap_dispatch.py create mode 100644 backend/tests/unit/test_stt_cap_dispatch_http.py create mode 100644 backend/tests/unit/test_sync_cap_hold.py create mode 100644 backend/tests/unit/test_sync_cap_hold_recovery.py create mode 100644 backend/utils/observability/stt_cap_dispatch.py create mode 100644 backend/utils/stt_cap_dispatch.py create mode 100644 backend/utils/sync/cap_hold.py create mode 100644 backend/utils/sync/cap_hold_recovery.py diff --git a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json new file mode 100644 index 00000000000..3fd646d6b5a --- /dev/null +++ b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "id": "FC-stt-allowance-not-enforced-at-dispatch", + "violated_contract": "The transcription cap is computed but not enforced where Omi-funded provider audio is dispatched; replay, fallback and delayed execution can bypass request-time allowance checks.", + "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it and exercise both flag states at provider and queue boundaries.", + "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py"], + "evidence_prs": [], + "scope_hints": ["backend/routers/listen/", "backend/utils/stt/", "backend/utils/sync/", "backend/utils/batch_pipeline/"], + "status": "open" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 91a59ada47b..ef520fb69ef 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -148,7 +148,7 @@ STT policy: `config/stt_provider_policy.py`. Windowed live rollout: `docs/operat - **monitoring** (`backend/charts/monitoring/`) — Prometheus/Grafana, Loki/Alloy, alerts and HPA adapters. Sync export flags, sizing and CPU audit: [runbook](docs/runbooks/sync-cloudrun-rightsizing.md). - **backend-secrets** (`backend/charts/backend-secrets/`) — ExternalSecret and SecretStore resources that sync backend runtime secrets into GKE namespaces. -[Reset recovery](docs/locked-content-reset-recovery.md). +[Reset recovery](docs/locked-content-reset-recovery.md). Dark funded-STT admission and held-audio recovery: [dispatch contract](docs/stt-cap-dispatch.md). Runtime env contract: keep `backend/deploy/runtime_env.yaml` aligned with GKE Helm values and Cloud Run runtime env; run `backend/scripts/pre-deploy-check.sh` after backend runtime env or deploy workflow changes. The backend deploy must validate the exact JSON state emitted by `render_backend_runtime_env.py --state-output`, and any Cloud Run export/replace editor must preserve those literal values before apply. The `llm_gateway` manifest section owns the release, ingress, and static-address identity; a reserved address alone is never an endpoint contract. Gateway-mode promotion requires the control-plane gate plus `probe-llm-gateway-from-cloud-run.sh` before Cloud Run revisions are created. diff --git a/backend/config/stt_cap_dispatch.py b/backend/config/stt_cap_dispatch.py new file mode 100644 index 00000000000..59f05f96f18 --- /dev/null +++ b/backend/config/stt_cap_dispatch.py @@ -0,0 +1,7 @@ +"""Dark dispatch policy. No environment/deployment changes enable this implicitly.""" + +import os + + +def enabled() -> bool: + return os.getenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false').lower() == 'true' diff --git a/backend/diarizer/batch_pipeline.py b/backend/diarizer/batch_pipeline.py index 405cd704f01..91054abf4ea 100644 --- a/backend/diarizer/batch_pipeline.py +++ b/backend/diarizer/batch_pipeline.py @@ -12,9 +12,11 @@ import threading from typing import Any, Callable, Mapping, Sequence +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import record_audio from config.batch_pipeline_providers import ProviderConfig from config.sync_live_dedupe import sync_live_dedupe_active_for -from utils.batch_pipeline.contracts import WorkItem, WorkResult +from utils.batch_pipeline.contracts import WorkItem, WorkResult, HeldWork from utils.batch_pipeline.errors import FatalGpuError SAMPLE_RATE = 16000 @@ -231,7 +233,7 @@ def __init__( self._unavailable = False self.counts = {'items': 0, 'reused_live': 0, 'silence': 0, 'failed': 0} - def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: + def process_batch(self, items: Sequence[WorkItem], *, authorize_dispatch=None) -> Sequence[WorkResult | HeldWork]: if len(items) > 32 or len({i.segment_id for i in items}) != len(items): raise ValueError('pull must contain at most 32 distinct conversations') if any(item.duration_seconds > self.config.max_unit_seconds for item in items): @@ -247,16 +249,17 @@ def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: # Bound decoded host memory to two audio-hours (~460 MB float32). for item in items: if group and seconds + item.duration_seconds > 7200: - results.extend(self._process(group)) + results.extend(self._process(group, authorize_dispatch=authorize_dispatch)) group, seconds = [], 0 group.append(item) seconds += item.duration_seconds - results.extend(self._process(group)) + results.extend(self._process(group, authorize_dispatch=authorize_dispatch)) return results - def _process(self, items): + def _process(self, items, *, authorize_dispatch=None): states, windows, failed, results = {}, [], set(), [] failure_records = [] + held = set() shapes = {item.segment_id: {'samples': -1, 'vad_spans': -1, 'windows': -1, 'asr_words': 0} for item in items} def fail(item, exc, stage): @@ -338,11 +341,29 @@ def fail(item, exc, stage): fail(item, exc, stage) for batch in pack_windows(windows, self.config): - batch = [w for w in batch if w.segment_id not in failed] + batch = [w for w in batch if w.segment_id not in failed and w.segment_id not in held] + if cap_dispatch_enabled(): + # Authorization belongs immediately before ASR, after potentially + # slow decode/VAD and between successive microbatches. Missing + # policy authority is a hold, never provider permission. + required = {w.segment_id for w in batch} + try: + allowed = set(authorize_dispatch(required)) if authorize_dispatch is not None else set() + except Exception: + allowed = set() + held.update(required - allowed) + batch = [w for w in batch if w.segment_id not in held] if not batch: continue try: hypotheses = self.words.transcribe_batch([w.samples for w in batch]) + if cap_dispatch_enabled(): + for window in batch: + record_audio('sync', 'accepted', self.config.words, window.duration) + state = states[window.segment_id] + state['cap_provider_audio_seconds'] = ( + state.get('cap_provider_audio_seconds', 0) + window.duration + ) if len(hypotheses) != len(batch): raise ValueError('provider result cardinality mismatch') except Exception as exc: @@ -363,6 +384,9 @@ def fail(item, exc, stage): for state in states.values(): item = state['item'] + if item.segment_id in held: + results.append(HeldWork(item.segment_id)) + continue if item.segment_id in failed: continue stage = 'assemble_words' @@ -403,6 +427,11 @@ def fail(item, exc, stage): 'speech_seconds': state['speech_seconds'], 'speech_spans': state['speech_spans'], } + if cap_dispatch_enabled(): + payload['cap_dispatch'] = { + 'omi_funded_accepted_provider_audio_seconds': state.get('cap_provider_audio_seconds', 0), + 'admission': 'authorized_compute', + } stage = 'persist' uri = self.persist(item, payload) results.append(WorkResult(item.segment_id, uri, 'silence' if state['silence'] else 'success')) diff --git a/backend/diarizer/batch_pipeline_runtime.py b/backend/diarizer/batch_pipeline_runtime.py index ed1c5bee3b8..d30fdad6632 100644 --- a/backend/diarizer/batch_pipeline_runtime.py +++ b/backend/diarizer/batch_pipeline_runtime.py @@ -57,13 +57,13 @@ def __init__( self.memory_probe = memory_probe self._unavailable = False - def process_batch(self, items): + def process_batch(self, items, **kwargs): if self._unavailable: raise FatalGpuError('GPU chain unavailable; restart required') try: baseline = self.memory_probe.begin() if self.memory_probe else None try: - return self.worker.process_batch(items) + return self.worker.process_batch(items, **kwargs) finally: # Python completion is not CUDA completion. Never publish before # quiescence, even for a dedicated GPU with no live reservation. diff --git a/backend/docs/feature-flag-registry.md b/backend/docs/feature-flag-registry.md index 4b101f63cb9..c7c5a9f89e4 100644 --- a/backend/docs/feature-flag-registry.md +++ b/backend/docs/feature-flag-registry.md @@ -255,6 +255,7 @@ and an explicit empty literal renders as `''`. | `SPEAKER_GROUPING_MODE` | incumbent\|provider_strict\|provider_cannot_link; owner_link is shadow-only and rejected; default incumbent everywhere; David gates promotion | backend | env | open | — | — | incumbent (backend-listen (chart), pusher (chart)); {value: 'incumbent', category: rollout} (cloud_run/backend, cloud_run/backend-integration, cloud_run/backend-sync, cloud_run/backend-sync-backfill, gke/backend-listen, gke/pusher) | — | pending | 2026-11-08 | dazheng | | `SPEAKER_MATCH_SCORES_ENABLED` | Persist bounded internal voice-match evidence on existing conversation writes; dev/local/offline default on; production hosts explicitly on | backend | env | closed | — | — | true (backend-listen (chart), cloud_run/backend, cloud_run/backend-integration, cloud_run/backend-sync, cloud_run/backend-sync-backfill, gke/backend-listen, gke/pusher, pusher (chart)) | — | pending | 2026-11-04 | dazheng | | `STOREFRONT_LADDER_V1_ENABLED` | Default-off versioned unified plans presentation; exact true admits v2 only; installed-client endpoints unchanged | backend | env | closed | — | — | — | — | pending | 2026-11-13 | dazheng | +| `STT_CAP_DISPATCH_ENFORCEMENT` | Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation | backend | env | closed | — | — | — | — | pending | 2026-11-15 | dazheng | | `STT_FAILOVER_RECOVERY_ENABLED` | Enable live-STT failover recovery on prod listen | backend | env | closed | false | false (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-11-03 | backend | | `STT_LEARNED_LANGUAGE_PROFILE` | Use a bounded per-user spoken-language history for live STT routing and Soniox hints | backend | env | closed | false | true (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-10-28 | backend | | `STT_LIVE_DEMAND_ENABLED` | Default off; qualified English hidden capture uses window STT; visible or active consumers require vendors | backend | env | closed | false | false (backend-listen (chart), gke/backend-listen) | true (backend-listen (chart), gke/backend-listen) | — | pending | 2026-11-13 | dazheng | @@ -531,6 +532,7 @@ their code default (`fail` tells you which way a missing value resolves). - `SONIOX_ELAPSED_AXIS` — Measure Soniox elapsed timestamps before enabling speaker windows (fail: closed) - `SPEAKER_GROUPING_CROSS_SCOPE_THRESHOLD` — G2 cross-scope cosine-distance threshold; default 0.60; invalid configuration uses 0.60 (fail: closed) - `STOREFRONT_LADDER_V1_ENABLED` — Default-off versioned unified plans presentation; exact true admits v2 only; installed-client endpoints unchanged (fail: closed) +- `STT_CAP_DISPATCH_ENFORCEMENT` — Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation (fail: closed) - `STT_NO_TEXT_RESCUE_SECONDS` — Maximum paid wall and admitted audio seconds per no-text rescue; default 60 bounded 5-120 (fail: closed) - `STT_PAID_SPILLOVER_BUDGET_ENABLED` — Default-off fleet budget for paid router promotions after Parakeet capacity refusal; denial restores static order (fail: closed) - `STT_PAID_SPILLOVER_DEEPGRAM_PER_MINUTE` — Deepgram fleet paid router promotion cap per UTC minute; default 30; zero refuses promotions (fail: closed) diff --git a/backend/docs/listen_pusher_pipeline.mdx b/backend/docs/listen_pusher_pipeline.mdx index 6a20f027874..2433b74e5c4 100644 --- a/backend/docs/listen_pusher_pipeline.mdx +++ b/backend/docs/listen_pusher_pipeline.mdx @@ -6,6 +6,15 @@ description: "Sequence diagrams and finalization policy for the /v4/listen WebSo # Listen + Pusher Pipeline — Sequence Diagrams +## Default-off funded transcription cap + +`STT_CAP_DISPATCH_ENFORCEMENT` gates provider creation and audio sends through a +shared live admission state, including multichannel, replay and replacement legs. +Denial preserves receiving and existing audio storage paths; accepted provider +work drains. Custom/on-device STT and flag-off metering retain their behavior. +The [dispatch contract](stt-cap-dispatch.md) also documents sync holds, recovery, +BYOK funding, additive metrics and the per-client activation decision. + ## Windowed live transcription deadlines `PARAKEET_WINDOW_DEADLINE_CUT_ENABLED` defaults to `false` in code and diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md new file mode 100644 index 00000000000..fad6cdff74a --- /dev/null +++ b/backend/docs/stt-cap-dispatch.md @@ -0,0 +1,120 @@ +# Transcription cap at provider dispatch + +`STT_CAP_DISPATCH_ENFORCEMENT` is a default-off backend rollout switch. This PR +adds no deployment binding and does not enable it. Enabling changes the policy +for existing capped paid plans: exhausted sync uploads wait as audio rather than +being transcribed into locked conversations. Product approval is required before +activation. No ladder-v1 code or prices change. + +## Boundaries + +Live receiver creation installs one funded admission state for the session. +Actual Deepgram, Soniox, Modulate, Parakeet and window-provider constructors use +that state, including direct registry endpoints and fallback/recovery legs. +The state debits accepted PCM seconds across channels, replay and successors. +The existing 60-second usage flush refreshes it from ordinary funded allowance; +root counters retain their received-audio semantics. Timestamp adapters suppress +before recording accepted-provider spans. Denial is a policy pause, not provider +death: receiving, pusher audio and existing private-cloud/raw archive paths +continue. Accepted provider queues drain. Nothing enables cloud audio retention +for live users who did not already opt into it. Custom/on-device STT bypasses the +new state. The existing Plus threshold event still uses its existing wire shape. + +Deepgram cloud with a request key is customer funded. Every other provider, +including self-hosted Deepgram and an Omi-funded fallback from BYOK, uses the +ordinary allowance with both BYOK quota and desktop-paywall exemptions disabled. +No key or identity is a metric label. + +Sync v1 intake and v2 execution hold full decoded WAVs before VAD when denied. +The shared segment processor rechecks immediately before each STT invocation, +including its empty-result retry. Late exhaustion holds the remaining segment. +No denied segment reaches cloud STT. A successfully persisted hold is acknowledged +without creating a locked transcript. Hold failures fail the upload/job; callers +must retain/retry their source audio. V1 adds `held_audio` and `held_files` to the +exhausted acknowledgment; v2 adds `held_audio` to its durable result. + +Batch compute authorization checks current funded allowance. The worker calls +that authority again after decode/VAD and before each ASR microbatch. Missing +worker policy authority holds work. Held work uses the scheduler's existing +`defer`, preserving raw artifacts and refunding the provider-failure attempt. +Quarantine recovery rechecks both before fetching audio and before claiming its +sole provider attempt; a denied prepared intent remains recoverable. + +## Durable sync recovery + +Encrypted source audio lives under the existing private-cloud bucket's +`chunks/{uid}/cap-held/` owner prefix; encrypted metadata and progress live under +`users/{uid}/stt_cap_holds`. The checkpoint has no application TTL. Existing +account deletion recursively removes the manifest and purges this chunk prefix. +A failed object upload cannot publish a successful manifest or acknowledgment. +Concurrent identical uploads cannot overwrite a recovery checkpoint. + +The existing `locked_content_recovery` authenticated-request path kicks an async +recovery pass independently of its once-per-month unlock receipt and legacy reset +flag. A distributed 60-second cooldown bounds requests; token-fenced renewable +leases prevent concurrent processing of a hold. At reset or upgrade, current +funded allowance must pass again before every recovered piece is sent. Recovery +reuses VAD, segment intake, enrichment and optional playback finalization. Piece +receipts and idempotent usage keys preserve progress across retries. Completed +objects are deleted only after content and usage settlement. WS-only and inactive +accounts wait for their next authenticated HTTP request; there is no new cron. +Rolling back the dispatch flag pauses this new recovery, preserving its audio. + +## Launch configuration and measurement + +With enforcement off, launch-date handling is unchanged. With it on, missing or +invalid `SUBSCRIPTION_LAUNCH_DATE` logs `event=stt_cap_launch_config` and increments +`omi_stt_cap_launch_config_errors_total{reason="missing|invalid"}`. Finite-plan +allowance fails conservatively instead of inventing zero usage. A valid explicit +future date continues to mean intentionally unlaunched. + +`omi_stt_cap_provider_audio_seconds_total` is additive provenance: + +- `path`: `live` or `sync`; +- `outcome`: `accepted`, `suppressed` (live), or `held` (sync); +- `provider`: a bounded serving family or `unknown`; +- `allowance`: `exhausted` or `available`, at the admission/acceptance observation. + +Accepted live quantities count PCM accepted by a provider adapter, including +replay; failed sends do not count. Suppressed quantities count input PCM, before +VAD. Sync successful provider calls count WAV duration, including retries; +quarantine recovery counts its accepted request duration. The GPU worker counts +actual padded microbatch audio and persists additive `cap_dispatch` provenance +with `omi_funded_accepted_provider_audio_seconds` and its compute admission origin. +Held sync quantities count durably retained audio; queue denials count attempted +held quantities and can repeat across policy retries. They are not a distinct +captured-content counter. Provider-internal retries/uncertain failures are not an +exact vendor invoice ledger. Existing root counters and unit projections do not +change, and no new provenance is emitted with the flag off. + +Admission checks are not distributed quota reservations. Concurrent already +admitted work can drain across the cap; live refresh is bounded by its existing +flush cadence. Do not interpret a zero violation metric as proof against every +cross-process admission race. + +## What users see when the flag is on + +- **Mobile, on-device overflow flag off:** capture continues as main permits, but + exhausted managed sessions stop producing new cloud transcripts. Existing Plus + threshold handling remains. Uploaded offline audio waits durably; this PR adds + no held-content card or automatic on-device processing of that backlog. +- **Mobile, on-device overflow flag on:** eligible clients keep the existing + on-device offer/mode switch. Custom/on-device sessions continue normally. + Already-held server uploads still wait for reset/upgrade recovery. +- **macOS:** the existing threshold handler can stop capture, as it does today. + The backend also refuses new funded sends if the client continues; accepted work + drains. There is no new macOS on-device overflow flow in this PR. +- **Web:** no new on-device offer is added. Exhausted funded live transcription + pauses; supported sync uploads wait for recovery. Existing content stays as it + was, and current clients do not gain a new waiting-audio view from these additive + backend fields. + +These visible gaps and the change from locked transcripts to held audio are the +activation decision for David; default-off deployment preserves today's policy. + +## Follow-ups in #21488 + +Findings #6 and #7 remain outside this PR: budget-aware live refresh when ladder +allowance returns `remaining_seconds=None`, and conservative admission when usage +unit projection has coverage gaps. Carry this funded-dispatch/held-audio contract +into that ladder cutover without changing the legacy meter here. diff --git a/backend/models/sync_job_status.py b/backend/models/sync_job_status.py index ee792da689b..14c78350723 100644 --- a/backend/models/sync_job_status.py +++ b/backend/models/sync_job_status.py @@ -12,6 +12,7 @@ class SyncLocalFilesResultResponse(BaseModel): failed_segments: int = 0 total_segments: int = 0 errors: list[str] = Field(default_factory=list) + held_audio: bool | None = None class SyncJobStartResponse(BaseModel): diff --git a/backend/routers/listen/contracts.py b/backend/routers/listen/contracts.py index 988c4227876..feae93cabba 100644 --- a/backend/routers/listen/contracts.py +++ b/backend/routers/listen/contracts.py @@ -101,6 +101,7 @@ class ListenSessionState: active: bool = True close_code: int = 1001 stt_terminal_failure: bool = False + stt_cap_admission: Any = None shutdown_event: asyncio.Event = field(default_factory=asyncio.Event) audio_ring_buffer: Any = None speaker_id_enabled: bool = False diff --git a/backend/routers/listen/legacy_recovery.py b/backend/routers/listen/legacy_recovery.py index 0fbcf4c28ec..d9c93a54b92 100644 --- a/backend/routers/listen/legacy_recovery.py +++ b/backend/routers/listen/legacy_recovery.py @@ -156,6 +156,11 @@ async def monitor_stt_death(self: Any) -> None: (``stt_failed`` + WebSocket 1011) as soon as the death latch flips (#10028). """ while self.host.state.active and not self.host.state.stt_terminal_failure: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + if await self.host.wait(owner.STT_DEATH_POLL_INTERVAL_SECONDS): + return + continue socket = self.stt_socket outcome = getattr(socket, 'leg_outcome', None) if outcome is not None and outcome.owner_closing: @@ -163,6 +168,8 @@ async def monitor_stt_death(self: Any) -> None: if socket is not None and owner.live_stt_socket_is_dead(socket): if await self._failover_stt_socket(): continue + if admission is not None and admission.funded_paused is True: + continue if outcome is not None and outcome.owner_closing: return if self.host.state.active and not self.host.state.stt_terminal_failure: diff --git a/backend/routers/listen/receiver.py b/backend/routers/listen/receiver.py index 8623412ed77..2cb23416e55 100644 --- a/backend/routers/listen/receiver.py +++ b/backend/routers/listen/receiver.py @@ -9,6 +9,9 @@ import time import uuid +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.stt_cap_dispatch import LiveAdmission, current_live_admission, suppress_socket_audio +from utils.observability.stt_cap_dispatch import record_audio as record_cap_audio from utils.manual_speaker_assignments import acknowledged_teaching from utils.other.live_audio_archive import LiveAudioArchive from collections import OrderedDict, deque @@ -201,6 +204,8 @@ def __getattr__(self, name: str) -> Any: return getattr(self._conn, name) def send(self, data: bytes, start_sample: Optional[int] = None) -> bool: + if suppress_socket_audio(self._conn, data): + return True try: accepted = ( self._conn.send(data, start_sample=start_sample) if start_sample is not None else self._conn.send(data) @@ -249,6 +254,14 @@ class ListenReceiver(ReplayFilterMixin): def __init__(self, host: Any, channel_configs: List[ChannelConfig], channel_id_to_index: Dict[int, int]): self.soniox_idle_budget = SonioxIdleBudget() self.host = host + self.cap_admission = ( + LiveAdmission(host.request.uid, host.request.source) + if stt_cap_dispatch_enabled() and not host.use_custom_stt + else None + ) + self.cap_paused = False + if self.cap_admission is not None: + host.state.stt_cap_admission = self.cap_admission self.raw_archive = LiveAudioArchive(host) self.lane_meter = LiveLaneMeter() self.live_demand = None @@ -903,6 +916,10 @@ async def _create_stt_socket( ) -> Any: if not isinstance(getattr(self, 'recovery_enabled', None), bool): self.recovery_enabled = session_recovery_enabled(self) + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.reason == 'allowance_unavailable': + await admission.refresh() + cap_token = current_live_admission.set(admission) enabled_token = current_recovery_enabled.set(self.recovery_enabled) token = current_recovery.set(getattr(self, 'recovery', None) if self.recovery_enabled else None) try: @@ -918,6 +935,7 @@ async def _create_stt_socket( finally: current_recovery.reset(token) current_recovery_enabled.reset(enabled_token) + current_live_admission.reset(cap_token) async def _connect_stt_socket( self, @@ -1219,6 +1237,14 @@ def _record_selected_epoch(self, epoch: Optional[ProviderEpochTranslator], socke ).inc() async def initialize_stt(self) -> bool: + admission = getattr(self, 'cap_admission', None) + if admission is not None: + await admission.refresh() + if not admission.allowed(self.host.stt_service): + admission.omi_selected = True + self.cap_paused = True + return True # Receive/persist without consulting provider routing. + self.cap_paused = False self.live_demand = attach_live_demand(self) if self.live_demand is not None: await self.live_demand.publish() @@ -1302,6 +1328,9 @@ def callback(segments: List[Dict[str, Any]], channel: ChannelConfig = config) -> return True except ProviderChainUnavailable as error: + if admission is not None and admission.funded_paused is True: + self.cap_paused = True + return True await self._drain_stt_sockets() await terminate_live_stt_backoff( request.websocket, @@ -1311,6 +1340,9 @@ def callback(segments: List[Dict[str, Any]], channel: ChannelConfig = config) -> ) return False except Exception as error: + if admission is not None and admission.funded_paused is True: + self.cap_paused = True + return True await self._drain_stt_sockets() await terminate_live_stt_session( request.websocket, @@ -1737,6 +1769,11 @@ async def _monitor_stt_death(self) -> None: if getattr(self, 'recovery', None) is None: return await legacy_recovery.monitor_stt_death(self) while self.host.state.active and not self.host.state.stt_terminal_failure: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + if await self.host.wait(STT_DEATH_POLL_INTERVAL_SECONDS): + return + continue socket = self.stt_socket outcome = getattr(socket, 'leg_outcome', None) if outcome is not None and outcome.owner_closing: @@ -1744,6 +1781,8 @@ async def _monitor_stt_death(self) -> None: if socket is not None and live_stt_socket_is_dead(socket): if await self._failover_stt_socket(): continue + if admission is not None and admission.funded_paused is True: + continue recovery = getattr(self, 'recovery', None) if recovery is not None: if recovery.client_has_left() or (outcome is not None and outcome.owner_closing): @@ -1813,6 +1852,18 @@ async def _handle_image_chunk(self, payload: Dict[str, Any]) -> None: self.host.spawn(self._process_photo(image, chunk.id), name='photo_process') async def _flush_stt_buffer(self, buffer: bytearray, *, force: bool = False) -> None: + admission = getattr(self, 'cap_admission', None) + if admission is not None and not admission.allowed(self.host.stt_service): + record_cap_audio( + 'live', + 'suppressed', + self.host.stt_service, + len(buffer) / (2 * self.host.request.sample_rate), + admission.reason == 'plan_allowance_exhausted', + ) + buffer.clear() + self._stt_buffer_start_sample = None + return if getattr(self, 'recovery', None) is None: return await legacy_recovery.flush_stt_buffer(self, buffer, force=force) request = self.host.request @@ -1950,6 +2001,9 @@ async def _flush_stt_buffer(self, buffer: bytearray, *, force: bool = False) -> return async def _recover_ingested_audio(self) -> None: + admission = getattr(self, 'cap_admission', None) + if admission is not None and admission.funded_paused is True: + return if ( not await self._failover_stt_socket() and self.host.state.active @@ -2027,7 +2081,7 @@ async def _handle_multi_channel_audio(self, data: bytes, now: float | None = Non pcm_len=len(pcm), fair_use_track_dg_usage=self.host.state.fair_use_track_dg_usage, ) - if should_send: + if should_send and not suppress_socket_audio(self.stt_sockets_multi[channel_index], pcm): sent = await send_live_stt_audio( request.websocket, self.host.state, diff --git a/backend/routers/listen/runtime.py b/backend/routers/listen/runtime.py index 0878e836b40..9b1a5c1f150 100644 --- a/backend/routers/listen/runtime.py +++ b/backend/routers/listen/runtime.py @@ -99,6 +99,7 @@ from .conversations import LiveConversationController from .persistence import ListenPersistence from .parity_capture import ListenParityCapture +from utils.stt_cap_dispatch import customer_funded from . import receiver as receiver_module from .receiver import ListenReceiver from .registry import register as register_listen_session @@ -774,6 +775,11 @@ async def _record_usage_periodically(self) -> None: await self._refresh_credits(transcription_seconds=transcription_seconds) async def _refresh_credits(self, *, transcription_seconds: int = 0) -> None: + admission = getattr(self.receiver, 'cap_admission', None) + if admission is not None: + await admission.refresh() + if getattr(self.receiver, 'cap_paused', False) and admission.managed: + await self.receiver.initialize_stt() now = time.time() invalidated = await self.persistence.call(check_credits_invalidation, self.request.uid) needs_refresh = ( @@ -795,6 +801,8 @@ async def _refresh_credits(self, *, transcription_seconds: int = 0) -> None: elif self.state.remaining_seconds_cache is not None and transcription_seconds > 0: self.state.remaining_seconds_cache = max(0, self.state.remaining_seconds_cache - transcription_seconds) remaining = self.state.remaining_seconds_cache + if admission is not None and (admission.omi_selected or not customer_funded(self.stt_service)): + remaining = admission.remaining subscription = await self.persistence.call(user_db.get_user_valid_subscription, self.request.uid) if remaining is not None and remaining <= FREEMIUM_THRESHOLD_SECONDS and not self.state.freemium_threshold_sent: if should_emit_plus_meter_warning(subscription): diff --git a/backend/routers/sync.py b/backend/routers/sync.py index fef96875180..333a2f0a807 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -59,6 +59,8 @@ from models.geolocation import geolocation_from_private_header from models.sync_audio import AudioPrecacheResponse, AudioUrlsResponse from routers.listen.contracts import persisted_started_seconds +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.sync.cap_hold import hold_capture_if_exhausted from utils.sync.admission_responses import sync_content_in_progress_response from utils.analytics import record_sync_v1_usage as record_usage from utils.other import endpoints as auth @@ -623,6 +625,30 @@ async def sync_local_files( except HTTPException as e: raise HTTPException(status_code=e.status_code, detail=e.detail, headers=_V1_DEPRECATION_HEADERS) + cap_held = False + if cap_dispatch_enabled(): + cap_held = await run_blocking( + sync_executor, + hold_capture_if_exhausted, + uid, + wav_paths, + source, + target_conversation_id=conversation_id, + client_device_id=client_device_context.client_device_id, + client_platform=client_device_context.platform, + ) + if cap_held: + return JSONResponse( + status_code=200, + headers=_V1_DEPRECATION_HEADERS, + content={ + 'new_memories': [], + 'updated_memories': [], + 'credits_exhausted': True, + 'held_audio': True, + 'held_files': len(wav_paths), + }, + ) vad_errors = [] vad_segment_lock = threading.Lock() diff --git a/backend/tests/support/firestore_query_driver_registry.py b/backend/tests/support/firestore_query_driver_registry.py index 6a09243b973..6b714123951 100644 --- a/backend/tests/support/firestore_query_driver_registry.py +++ b/backend/tests/support/firestore_query_driver_registry.py @@ -2894,6 +2894,13 @@ def _messaging_retention_candidate(client, combo, trial): ) +_add( + DriverEntry( + 'utils.sync.cap_hold.pending_holds', + base={'uid': UID}, + neutrals={'limit': (10, 'bounded recovery page; does not affect filters')}, + ) +) _add(DriverEntry('database.locked_content.has_recoverable_locked_content', base={'uid': UID})) _add( CoveredByEntry( diff --git a/backend/tests/unit/test_batch_pipeline_admission.py b/backend/tests/unit/test_batch_pipeline_admission.py index fb3e76e1a32..f44fb702f93 100644 --- a/backend/tests/unit/test_batch_pipeline_admission.py +++ b/backend/tests/unit/test_batch_pipeline_admission.py @@ -96,3 +96,24 @@ def test_policy_backend_failure_and_missing_duration_fail_closed(monkeypatch): reserve.reset_mock() assert admission.authorize(store, tokens(lease), now=1)['allowed'] == [] reserve.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_rechecked_after_queueing_before_gpu_authorization(monkeypatch, enforce): + from utils import stt_cap_dispatch as cap + + store, queue, lease, reserve = setup(monkeypatch, fresh=True) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + receipt = admission.authorize(store, tokens(lease), now=1) + if enforce: + assert receipt == {'allowed': [], 'retry_after': {'segment': 60}} + assert queue.defer('segment', lease.token, 1, 60) + assert store.rows['jobs', 'segment']['attempt'] == 0 + assert 'compute_admission_token' not in store.rows['jobs', 'segment'] + reserve.assert_not_called() + else: + assert receipt['allowed'] == ['segment'] + cap.funded_allowance.assert_not_called() diff --git a/backend/tests/unit/test_batch_pipeline_pump.py b/backend/tests/unit/test_batch_pipeline_pump.py index 3bd8bb26042..fbff6f64eee 100644 --- a/backend/tests/unit/test_batch_pipeline_pump.py +++ b/backend/tests/unit/test_batch_pipeline_pump.py @@ -234,3 +234,27 @@ def process_batch(self, items): assert store.rows['jobs', 'first']['state'] == 'committed' assert store.rows['jobs', 'second']['state'] == 'committed' assert store.rows['results', 'second']['previous_segment_id'] == 'first' + + +@pytest.mark.asyncio +async def test_worker_dispatch_denial_defers_without_quarantining_or_losing_audio(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + calls = [] + + def authorize(leases): + calls.append([v.item.segment_id for v in leases]) + return {'allowed': calls[-1] if len(calls) == 1 else []} + + class RecheckingWorker: + def process_batch(self, items, *, authorize_dispatch): + assert not authorize_dispatch({v.segment_id for v in items}) + return [HeldWork(v.segment_id) for v in items] + + store, _, pump = setup(RecheckingWorker(), authorize_compute=authorize) + assert await pump.run_once() == 1 + job = store.rows['jobs', 'first'] + assert job['state'] == 'retry' and job['attempt'] == 0 + assert ('results', 'first') not in store.rows + assert calls == [['first'], ['first']] diff --git a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py index a6b9cee920d..1029d7d12f7 100644 --- a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py +++ b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py @@ -420,3 +420,33 @@ def test_recovery_cas_lost_token_and_artifact_registration_are_fenced(setup): with pytest.raises(BatchPrivacyBlocked): register_batch_artifact(store, binding.owner_id, 'bad', 'gs://synthetic/' + 'b' * 32 + '.enc', 'results', 0) stt.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_exhausted_recovery_never_starts_a_provider_call(setup, monkeypatch, enforce): + from types import SimpleNamespace + from utils import stt_cap_dispatch as cap + + _, _, _, artifacts, stt, recovery = setup + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + result = recovery.recover('bad', 'synthetic-owner') + if enforce: + assert result == 'held' + stt.assert_not_called() + artifacts.fetch_audio.assert_not_called() + else: + stt.assert_called_once() + cap.funded_allowance.assert_not_called() + + +def test_recovery_rechecks_after_audio_download(setup, monkeypatch): + from utils.batch_pipeline import quarantine_recovery as module + + _, _, _, artifacts, stt, recovery = setup + monkeypatch.setattr(module, 'sync_allowed', Mock(side_effect=[True, False])) + assert recovery.recover('bad', 'synthetic-owner') == 'held' + artifacts.fetch_audio.assert_called_once() + stt.assert_not_called() diff --git a/backend/tests/unit/test_batch_pipeline_worker.py b/backend/tests/unit/test_batch_pipeline_worker.py index d4b3c849a56..107af898b31 100644 --- a/backend/tests/unit/test_batch_pipeline_worker.py +++ b/backend/tests/unit/test_batch_pipeline_worker.py @@ -597,3 +597,45 @@ def diarize(samples): assert part_payload['words'] == [{'word': ['first', 'second'][index], 'timestamp': [0.25, 0.75]}] assert part_payload['diarization']['turns'] == [{'start': 0.0, 'end': 2.0, 'speaker': ['left', 'right'][index]}] assert part_payload['speech_seconds'] == 2.0 + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_provenance_is_additive_and_uses_actual_padded_provider_audio(monkeypatch, enforce): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + worker, words, saved, _, _, _ = setup_worker() + worker.process_batch([item('a', 2)], authorize_dispatch=lambda ids: ids) + if enforce: + assert saved['a']['cap_dispatch'] == { + 'omi_funded_accepted_provider_audio_seconds': 2, + 'admission': 'authorized_compute', + } + else: + assert 'cap_dispatch' not in saved['a'] + assert saved['a']['speech_seconds'] == 2 + assert words.calls == [[2]] + + +def test_worker_rechecks_immediately_before_asr_and_holds_without_failure(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + worker, words, saved, _, diarized, failures = setup_worker() + checks = [] + + def authorize(ids): + checks.append(ids) + return set() + + result = worker.process_batch([item('a', 2)], authorize_dispatch=authorize) + assert result == [HeldWork('a')] + assert checks == [{'a'}] + assert not words.calls and not saved and not diarized and not failures + + +def test_worker_missing_authority_does_not_spend_at_cap(monkeypatch): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + worker, words, saved, _, _, _ = setup_worker() + assert worker.process_batch([item('a', 2)]) == [HeldWork('a')] + assert not words.calls and not saved diff --git a/backend/tests/unit/test_listen_stt_death_monitor.py b/backend/tests/unit/test_listen_stt_death_monitor.py index af8e059df09..b08157645d6 100644 --- a/backend/tests/unit/test_listen_stt_death_monitor.py +++ b/backend/tests/unit/test_listen_stt_death_monitor.py @@ -97,3 +97,22 @@ async def test_monitor_exits_without_terminating_once_already_terminal(): with patch.object(receiver_mod, 'terminate_live_stt_session', new=AsyncMock()) as terminate: await ListenReceiver._monitor_stt_death(monitor_self) terminate.assert_not_awaited() + + +@pytest.mark.parametrize('legacy', [False, True]) +@pytest.mark.anyio +async def test_cap_pause_keeps_receiving_when_provider_dies(legacy): + from utils.stt_cap_dispatch import LiveAdmission + + receiver = _monitor_self(dead=True) + receiver.cap_admission = LiveAdmission('synthetic') + receiver.cap_admission.omi_selected = True + receiver.host.state.stt_cap_admission = receiver.cap_admission + if legacy: + receiver.recovery = None + receiver._failover_stt_socket = AsyncMock(side_effect=AssertionError('cap pause must not dial a replacement')) + with patch.object(receiver_mod, 'terminate_live_stt_session', new=AsyncMock()) as terminate: + await ListenReceiver._monitor_stt_death(receiver) + terminate.assert_not_awaited() + receiver._failover_stt_socket.assert_not_awaited() + assert receiver.host.state.active diff --git a/backend/tests/unit/test_locked_content_reset_recovery.py b/backend/tests/unit/test_locked_content_reset_recovery.py index 2794d42ee63..2775a2f5a4e 100644 --- a/backend/tests/unit/test_locked_content_reset_recovery.py +++ b/backend/tests/unit/test_locked_content_reset_recovery.py @@ -295,3 +295,17 @@ def test_initial_enablement_drains_backlog_midmonth(env): env.module.recover_locked_content('test-user', now=midmonth) assert not any(env.locks.values()) assert env.state_data['completed_period'] == '2026-11' + + +def test_cap_audio_recovery_is_kicked_even_with_legacy_reset_flag_off(env, monkeypatch): + from unittest.mock import Mock + + fake = ModuleType('utils.sync.cap_hold_recovery') + fake.kick = Mock() + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setenv('LOCKED_CONTENT_RESET_RECOVERY_ENABLED', 'false') + with stub_modules({'utils.sync.cap_hold_recovery': fake}): + env.module.recover_locked_content('test-user', now=NOW) + env.module.recover_locked_content('test-user', now=NOW) + assert fake.kick.call_count == 2 + env.state.get.assert_not_called() diff --git a/backend/tests/unit/test_stt_cap_dispatch.py b/backend/tests/unit/test_stt_cap_dispatch.py new file mode 100644 index 00000000000..910dbbbdac3 --- /dev/null +++ b/backend/tests/unit/test_stt_cap_dispatch.py @@ -0,0 +1,193 @@ +"""Dispatch-level regression matrix: no billed provider work after denial.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from config import stt_cap_dispatch as flag +from utils import stt_cap_dispatch as cap +from routers.listen.receiver import _RecordingSTTSocket +from utils.stt.live_session import LiveLegSocket +from utils.stt.vad_gate import GatedSTTSocket + + +@pytest.fixture(autouse=True) +def isolation(monkeypatch): + monkeypatch.delenv('STT_CAP_DISPATCH_ENFORCEMENT', raising=False) + monkeypatch.setattr(cap, 'get_byok_key', lambda _: None) + monkeypatch.setattr(cap, 'record_audio', Mock()) + + +@pytest.mark.parametrize('value', [None, '', 'false', '1', 'yes']) +def test_dark_default(value, monkeypatch): + if value is not None: + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', value) + resolver = Mock(side_effect=AssertionError('flag off performed admission IO')) + monkeypatch.setattr(cap, 'funded_allowance', resolver) + assert not flag.enabled() + assert cap.sync_allowed('synthetic') + resolver.assert_not_called() + + +@pytest.mark.parametrize('provider', ['deepgram', 'soniox', 'modulate', 'parakeet']) +@pytest.mark.parametrize('enforce', [False, True]) +async def test_provider_creation_and_sends_flag_matrix(provider, enforce, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + admission = cap.LiveAdmission('synthetic') + admission.reason = 'plan_allowance_exhausted' + raw = Mock(send=Mock(return_value=True), is_connection_dead=False) + factory = AsyncMock(return_value=raw) + connect = cap.live_provider(provider, 0)(factory) + token = cap.current_live_admission.set(admission) + try: + socket = await connect(16000) + finally: + cap.current_live_admission.reset(token) + socket.send(b'\0' * 32000) + assert factory.await_count == (0 if enforce else 1) + assert raw.send.call_count == (0 if enforce else 1) + assert not socket.is_connection_dead + if enforce: + cap.record_audio.assert_called_with('live', 'suppressed', provider, 1, True) + + +async def test_one_balance_covers_multichannel_replay_and_replacement(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = cap.LiveAdmission('synthetic') + state.managed, state.remaining = True, 2 + state.reason = 'plan_within_allowance' + providers = [Mock(send=Mock(return_value=True)) for _ in range(4)] + sockets = [cap.CapSocket(state, 'soniox', 16000, AsyncMock(), raw) for raw in providers] + # Two channel sends consume the shared balance. Replay and successor sends + # cannot restart it. Policy refusal never publishes transport death. + for socket in sockets: + assert socket.send(b'\0' * 32000) + assert [raw.send.call_count for raw in providers] == [1, 1, 0, 0] + assert state.remaining == 0 + assert state.reason == 'plan_allowance_exhausted' + + +async def test_byok_exempts_only_customer_deepgram_then_reauthorizes_fallback(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(cap, 'get_byok_key', lambda _: 'synthetic-key') + state = cap.LiveAdmission('synthetic') + dg, fallback = Mock(), Mock() + state.send(dg, b'00', 'deepgram', 16000) + state.send(fallback, b'00', 'modulate', 16000) + dg.send.assert_called_once() + fallback.send.assert_not_called() + monkeypatch.setenv('DEEPGRAM_SELF_HOSTED_ENABLED', 'true') + state.send(dg, b'00', 'deepgram', 16000) + assert dg.send.call_count == 1 + + +async def test_held_constructor_resumes_after_reset_without_reopening_capture(monkeypatch): + state = cap.LiveAdmission('synthetic') + raw = Mock() + factory = AsyncMock(return_value=raw) + socket = cap.CapSocket(state, 'modulate', 16000, factory) + state.send(raw, b'00', 'modulate', 16000) + factory.assert_not_called() + state.managed, state.remaining = True, 10 + await socket.resume() + assert socket.raw is raw + assert socket.send(b'00') is raw.send.return_value + socket.finish() + raw.finish.assert_called_once() + + +@pytest.mark.parametrize('managed', [False, True]) +def test_queued_sync_always_resolves_omi_funding(managed, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + resolver = Mock(return_value=SimpleNamespace(managed=managed, reason='plan_allowance_exhausted')) + monkeypatch.setattr(cap, 'resolve_transcription_allowance', resolver) + assert cap.sync_allowed('synthetic', 60, 'parakeet') is managed + resolver.assert_called_once_with('synthetic', source=None, byok_active=False, omi_funded=True) + + +@pytest.mark.parametrize('kind', ['recording', 'gated', 'managed', 'replay', 'idle-replay']) +def test_suppression_never_fabricates_provider_timeline_spans(kind): + state = cap.LiveAdmission('synthetic') + state.reason = 'plan_allowance_exhausted' + raw = Mock() + guarded = cap.CapSocket(state, 'parakeet', 16000, AsyncMock(), raw) + tracker = Mock() + if kind == 'recording': + socket = _RecordingSTTSocket(guarded, tracker) + assert socket.send(b'00', start_sample=900) + elif kind == 'gated': + socket = object.__new__(GatedSTTSocket) + socket._conn, socket._send_tracker = guarded, tracker + assert socket.send(b'00', start_sample=900) + else: + socket = object.__new__(LiveLegSocket) + socket.raw, socket._send_tracker = guarded, tracker + socket.session = SimpleNamespace(receiver=SimpleNamespace(_window_ring=lambda: None)) + if kind == 'replay': + assert socket.replay_send(b'00', 900) + elif kind == 'idle-replay': + assert socket.send_admitted_audio(b'00', ((900, 1),)) + else: + assert socket.send(b'00', start_sample=900) + raw.send.assert_not_called() + tracker.note_accepted.assert_not_called() + tracker.note_accepted_spans.assert_not_called() + + +@pytest.mark.parametrize('provider', ['deepgram', 'soniox', 'modulate', 'parakeet']) +async def test_real_provider_constructor_wiring_refuses_before_sdk_or_wire(provider, monkeypatch): + from utils.stt import streaming + import websockets + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + sdk = Mock(side_effect=AssertionError('denied constructor reached SDK')) + wire = AsyncMock(side_effect=AssertionError('denied constructor reached network')) + monkeypatch.setattr(streaming, 'DeepgramClient', sdk) + monkeypatch.setattr(websockets, 'connect', wire) + state = cap.LiveAdmission('synthetic') + token = cap.current_live_admission.set(state) + try: + if provider == 'deepgram': + socket = await streaming.process_audio_dg(Mock(), 'en', 16000, 1) + elif provider == 'soniox': + socket = await streaming.process_audio_soniox(Mock(), 16000, 'en') + elif provider == 'modulate': + socket = await streaming.process_audio_modulate(Mock(), 16000, 'en') + else: + socket = await streaming.process_audio_parakeet(Mock(), 'en', 16000, 1) + finally: + cap.current_live_admission.reset(token) + assert isinstance(socket, cap.CapSocket) + assert socket.raw is None + sdk.assert_not_called() + wire.assert_not_called() + + +async def test_initial_exhaustion_receives_without_entering_provider_router(monkeypatch): + from routers.listen.receiver import ListenReceiver + + state = cap.LiveAdmission('synthetic') + state.refresh = AsyncMock() + receiver = object.__new__(ListenReceiver) + receiver.cap_admission = state + receiver.host = SimpleNamespace(stt_service='modulate') + assert await receiver.initialize_stt() + assert receiver.cap_paused and state.funded_paused + assert not hasattr(receiver, 'live_demand') # No selection, model or provider initialization. + + +async def test_terminal_provider_error_cannot_close_capture_during_cap_pause(): + from utils.stt.live_failure import terminate_live_stt_session, terminate_live_stt_backoff + + state = cap.LiveAdmission('synthetic') + state.omi_selected = True + session = SimpleNamespace(stt_cap_admission=state, stt_terminal_failure=False, active=True) + websocket = SimpleNamespace(send_json=AsyncMock(), close=AsyncMock()) + assert not await terminate_live_stt_session( + websocket, session, failure=Mock(), reason='connection_lost', platform='ios' + ) + assert not await terminate_live_stt_backoff(websocket, session, reason='provider_unavailable', retry_after=30) + assert session.active and not session.stt_terminal_failure + websocket.send_json.assert_not_called() + websocket.close.assert_not_called() diff --git a/backend/tests/unit/test_stt_cap_dispatch_http.py b/backend/tests/unit/test_stt_cap_dispatch_http.py new file mode 100644 index 00000000000..dd169a46900 --- /dev/null +++ b/backend/tests/unit/test_stt_cap_dispatch_http.py @@ -0,0 +1,98 @@ +"""Actual sync HTTP routing with local fake persistence/provider boundaries.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +import pytest + +from routers import sync as router +from utils.sync.lanes import SyncLane + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.setenv('OMI_ENV_STAGE', 'dev') + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(router, 'get_sync_ledger_fence_mode', lambda: router.SyncLedgerFenceMode.ACTIVE) + monkeypatch.setattr( + router, + 'classify_sync_lane', + lambda *a, **kw: SimpleNamespace( + lane=SyncLane.FRESH, + trust=SimpleNamespace(value='untrusted'), + maximum_age_seconds=0, + reason='synthetic', + automatic_recovery_allowed=True, + ), + ) + monkeypatch.setattr(router, 'get_hard_restriction_status', lambda _: (False, None)) + monkeypatch.setattr(router, 'is_daily_audio_ceiling_exceeded', lambda _: False) + monkeypatch.setattr(router, 'has_transcription_credits', lambda _: False) + monkeypatch.setattr( + router, 'resolve_transcription_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted') + ) + monkeypatch.setattr(router, 'retrieve_file_paths', lambda *_: ['synthetic.bin']) + monkeypatch.setattr(router, 'archive_mobile_batch_raw_files', lambda *_: []) + monkeypatch.setattr(router, 'decode_files_to_wav', lambda _: ['1792152000.wav']) + monkeypatch.setattr(router, '_cleanup_files', Mock()) + monkeypatch.setattr(router, 'hold_capture_if_exhausted', Mock(return_value=True)) + monkeypatch.setattr(router, 'retrieve_vad_segments', Mock(side_effect=AssertionError('denied audio reached VAD'))) + app = FastAPI() + app.include_router(router.router) + app.dependency_overrides[router.auth.get_current_user_uid] = lambda: 'synthetic' + with TestClient(app, raise_server_exceptions=False) as test_client: + yield test_client + + +def test_exhausted_upload_acknowledges_only_after_durable_hold(client): + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json() == { + 'new_memories': [], + 'updated_memories': [], + 'credits_exhausted': True, + 'held_audio': True, + 'held_files': 1, + } + router.hold_capture_if_exhausted.assert_called_once() + router.retrieve_vad_segments.assert_not_called() + + +def test_failed_hold_is_not_reported_as_an_accepted_upload(client): + router.hold_capture_if_exhausted.side_effect = RuntimeError('synthetic storage unavailable') + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 500 + router.retrieve_vad_segments.assert_not_called() + + +def test_v2_typed_result_preserves_flag_off_wire_and_exposes_only_present_hold(): + from models.sync_job_status import SyncJobStatusResponse + + body = {'job_id': 'job', 'status': 'completed', 'result': {'new_memories': [], 'updated_memories': []}} + result = SyncJobStatusResponse(**body).model_dump(exclude_none=True)['result'] + assert result == { + 'new_memories': [], + 'updated_memories': [], + 'failed_segments': 0, + 'total_segments': 0, + 'errors': [], + } + body['result']['held_audio'] = True + assert SyncJobStatusResponse(**body).model_dump(exclude_none=True)['result']['held_audio'] is True + + +def test_flag_off_http_result_retains_main_shape_without_hold_dispatch(client, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false') + monkeypatch.setattr(router, 'FAIR_USE_ENABLED', False) + monkeypatch.setattr(router, 'retrieve_vad_segments', Mock()) + monkeypatch.setattr(router.users_db, 'get_user_transcription_preferences', Mock(return_value={})) + monkeypatch.setattr(router.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(router, 'build_person_embeddings_cache', Mock(return_value={})) + monkeypatch.setattr(router, '_reprocess_merged_conversations', Mock()) + monkeypatch.setattr(router, 'schedule_person_voice_learning_retries', Mock()) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.content == b'{"new_memories":[],"updated_memories":[]}' + router.hold_capture_if_exhausted.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py new file mode 100644 index 00000000000..292234cddd6 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -0,0 +1,144 @@ +"""Both sync versions execute process_segment; prove the last pre-STT gate.""" + +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock +import wave + +import pytest + +from utils.sync import pipeline +from utils import stt_cap_dispatch as cap + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('queued', [False, True], ids=['sync-v1', 'sync-v2-worker']) +def test_exhaustion_holds_before_url_or_provider_dispatch(tmp_path, monkeypatch, enforce, queued): + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\0' * 32000) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr( + cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) + ) + hold, url, stt = Mock(), Mock(return_value='synthetic-url'), Mock(return_value=([], 'en')) + monkeypatch.setattr(pipeline, 'hold_segment', hold) + monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', url) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr(pipeline, 'keyterms', AsyncMock(return_value=[])) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', lambda _: ('modulate', None, 'velma-2')) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + monkeypatch.setattr(pipeline, 'sync_safety_wal_enabled', lambda: False) + errors = [] + assert ( + pipeline.process_segment( + str(path), + 'synthetic', + {'new_memories': set(), 'updated_memories': set()}, + threading.Lock(), + errors, + is_locked=True, + job_id='queued-job' if queued else None, + ) + is False + ) + assert not errors + if enforce: + hold.assert_called_once() + assert hold.call_args.args[2]['timestamp'] == 1792152000 + url.assert_not_called() + stt.assert_not_called() + else: + hold.assert_not_called() + assert stt.call_count == 2 # Main's bounded empty-result retry remains unchanged. + cap.funded_allowance.assert_not_called() + + +def test_hold_failure_cannot_acknowledge_audio_or_dispatch_provider(tmp_path, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr(pipeline, 'sync_allowed', lambda *_: False) + monkeypatch.setattr(pipeline, 'hold_segment', Mock(side_effect=RuntimeError('synthetic upload failed'))) + stt = Mock() + monkeypatch.setattr(pipeline, 'prerecorded', stt) + errors = [] + pipeline.process_segment(str(tmp_path / '1792152000.wav'), 'synthetic', {}, threading.Lock(), errors) + assert errors + stt.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, monkeypatch, enforce): + from utils.sync import cap_hold + from models.conversation_enums import ConversationSource + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(cap_hold, 'sync_allowed', Mock(return_value=False)) + hold = Mock() + monkeypatch.setattr(cap_hold, 'hold_segment', hold) + paths = [str(tmp_path / '1792152000.wav')] + assert cap_hold.hold_capture_if_exhausted('synthetic', paths, ConversationSource.omi) is enforce + if enforce: + assert hold.call_args.args[2] == {'timestamp': 1792152000, 'source': 'omi', 'full_capture': True} + else: + hold.assert_not_called() + cap_hold.sync_allowed.assert_not_called() + + +def test_hold_manifest_fences_concurrent_recovery_and_stale_release(monkeypatch): + from contextlib import nullcontext + from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore + from utils.sync import cap_hold + + path = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({path: {'state': 'held', 'lease_until': 0}}) + ref = client.document('/'.join(path)) + monkeypatch.setattr(cap_hold, 'get_firestore_client', lambda: client) + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + first = cap_hold.claim('synthetic', ref, 100) + assert first and first['lease_token'] + assert cap_hold.claim('synthetic', ref, 101) is None + next_claim = cap_hold.claim('synthetic', ref, 2000) + cap_hold.release('synthetic', ref, first['lease_token']) + assert client.rows[path]['lease_token'] == next_claim['lease_token'] + cap_hold.release('synthetic', ref, next_claim['lease_token']) + assert client.rows[path]['lease_until'] == 0 + + +def test_hold_persists_encrypted_audio_before_manifest_and_never_expires(tmp_path, monkeypatch): + from contextlib import nullcontext + from utils.sync import cap_hold + from utils.encryption import decrypt_audio_file, decrypt + import json + + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as wav: + wav.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + wav.writeframes(b'\0' * 32000) + order = [] + ref, blob, bucket = Mock(), Mock(), Mock() + ref.get.return_value.exists = False + blob.upload_from_string.side_effect = lambda *args, **kwargs: order.append('upload') + ref.create.side_effect = lambda *args, **kwargs: order.append('manifest') + monkeypatch.setattr(cap_hold, 'holds', lambda _: SimpleNamespace(document=lambda _: ref)) + monkeypatch.setattr(cap_hold, 'get_private_cloud_sync_bucket', lambda: bucket) + bucket.blob.return_value = blob + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(cap_hold, 'owner_storage_write_gate', lambda *_: nullcontext()) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted')) + meta = {'timestamp': 1792152000, 'source': 'omi', 'full_capture': True} + assert cap_hold.hold_segment('synthetic', str(path), meta) == 1 + assert order == ['upload', 'manifest'] + assert decrypt_audio_file(blob.upload_from_string.call_args.args[0], 'synthetic') == path.read_bytes() + manifest = ref.create.call_args.args[0] + assert manifest['audio_path'].startswith('chunks/synthetic/cap-held/') + assert json.loads(decrypt(manifest['metadata'], 'synthetic')) == meta + assert 'expires_at' not in manifest + ref.create.reset_mock() + blob.upload_from_string.side_effect = RuntimeError('synthetic unavailable') + with pytest.raises(RuntimeError): + cap_hold.hold_segment('synthetic', str(path), meta) + ref.create.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py new file mode 100644 index 00000000000..12a0c77d8dc --- /dev/null +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -0,0 +1,85 @@ +"""Recovery reaches real sync intake only after a current funded allowance.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from utils.sync import cap_hold_recovery as recovery + + +@pytest.fixture +def state(monkeypatch, tmp_path): + async def immediate(pool, fn, *args, **kwargs): + return fn(*args, **kwargs) + + monkeypatch.setattr(recovery, 'run_blocking', immediate) + monkeypatch.setattr(recovery.cap_hold, 'metadata', Mock(return_value={'timestamp': 1792152000, 'source': 'omi'})) + monkeypatch.setattr( + recovery.cap_hold, 'recovery_directory', Mock(return_value=SimpleNamespace(name=str(tmp_path), cleanup=Mock())) + ) + monkeypatch.setattr(recovery.cap_hold, 'read_audio', Mock()) + monkeypatch.setattr(recovery.cap_hold, 'renew', Mock(return_value=True)) + monkeypatch.setattr(recovery.users_db, 'get_user_transcription_preferences', Mock(return_value={'language': 'en'})) + monkeypatch.setattr(recovery.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(recovery, 'sync_allowed', Mock(return_value=True)) + monkeypatch.setattr(recovery, 'process_segment', Mock(return_value=True)) + monkeypatch.setattr(recovery, 'finalize_recovered_sync_segment', Mock()) + monkeypatch.setattr(recovery, 'get_wav_duration', Mock(return_value=20)) + monkeypatch.setattr(recovery, 'record_usage', Mock()) + return SimpleNamespace( + snapshot=SimpleNamespace(id='hold', reference=Mock()), + row={'lease_token': 'owner-token', 'state': 'held'}, + ) + + +async def test_reset_or_upgrade_reuses_sync_and_settles_stable_usage_key(state): + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_called_once() + assert recovery.process_segment.call_args.kwargs['cap_recovery'] is True + recovery.finalize_recovered_sync_segment.assert_called_once() + usage = recovery.record_usage.call_args.kwargs + assert usage['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + assert usage['transcription_seconds'] == 20 + state.snapshot.reference.update.assert_called_once() + + +async def test_execution_denial_keeps_checkpoint_and_audio_without_stt(state, monkeypatch): + recovery.sync_allowed.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + state.snapshot.reference.update.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_lost_hold_lease_cannot_dispatch_or_acknowledge(state): + recovery.cap_hold.renew.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + state.snapshot.reference.update.assert_not_called() + + +async def test_segment_recheck_refusal_does_not_consume_the_hold(state): + def denied(*args, **kwargs): + kwargs['deferred_outcome']['cap_held'] = True + return False + + recovery.process_segment.side_effect = denied + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + state.snapshot.reference.update.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_partial_recovery_resumes_only_unprocessed_pieces(state): + state.row['processed_paths'] = ['1792152000.wav'] + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.record_usage.assert_not_called() + + +async def test_persistence_failure_keeps_audio_and_releases_local_staging(state): + recovery.process_segment.side_effect = RuntimeError('synthetic persistence failure') + with pytest.raises(RuntimeError): + await recovery._recover_row('synthetic', state.snapshot, state.row) + state.snapshot.reference.update.assert_not_called() + recovery.cap_hold.recovery_directory.return_value.cleanup.assert_called_once() diff --git a/backend/tests/unit/test_transcription_allowance.py b/backend/tests/unit/test_transcription_allowance.py index 72ff2faffb1..988e4db88e4 100644 --- a/backend/tests/unit/test_transcription_allowance.py +++ b/backend/tests/unit/test_transcription_allowance.py @@ -471,3 +471,30 @@ def failed_lookup(uid): assert result.remaining_seconds == 0 assert result.limit_seconds == cap assert result.reason == ('usage_invalid' if failure == 'invalid_usage' else 'allowance_unavailable') + + +@pytest.mark.parametrize('launch', [None, 'not-a-date', '2026-99-99']) +@pytest.mark.parametrize('enforce', [False, True]) +def test_launch_config_does_not_manufacture_zero_usage_when_enforcing(sub, monkeypatch, launch, enforce): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + if launch is None: + monkeypatch.delenv('SUBSCRIPTION_LAUNCH_DATE', raising=False) + else: + monkeypatch.setenv('SUBSCRIPTION_LAUNCH_DATE', launch) + if enforce: + with pytest.raises(ValueError): + sub.get_monthly_usage_for_subscription('synthetic') + else: + assert sub.get_monthly_usage_for_subscription('synthetic') == {} + + +def test_explicit_future_launch_is_intentionally_unlaunched(sub, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setenv('SUBSCRIPTION_LAUNCH_DATE', '2099-01-01') + assert sub.get_monthly_usage_for_subscription('synthetic') == {} + + +def test_funded_request_cannot_use_deepgram_key_for_another_provider(sub, monkeypatch): + _situate(monkeypatch, sub, plan=PlanType.plus, used_seconds=PLUS_CAP, byok_enrolled=True, byok_header=True) + assert sub.resolve_transcription_allowance('uid').reason == 'byok' + assert sub.resolve_transcription_allowance('uid', omi_funded=True).reason == 'plan_allowance_exhausted' diff --git a/backend/utils/batch_pipeline/admission.py b/backend/utils/batch_pipeline/admission.py index c4844d4dabb..a27b3eb8a33 100644 --- a/backend/utils/batch_pipeline/admission.py +++ b/backend/utils/batch_pipeline/admission.py @@ -3,6 +3,7 @@ import math import time +from utils.stt_cap_dispatch import sync_allowed from database.batch_pipeline_privacy import BatchPrivacyBlocked from utils.batch_pipeline.scheduler import owns_lease from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded @@ -34,6 +35,9 @@ def read(tx): retries[segment] = 30 continue job, binding, intake = snapshot + if not sync_allowed(job['owner_id'], float(job.get('duration_seconds', 0)), 'parakeet'): + retries[segment] = 60 + continue if int(job['priority']) == 0: restricted, retry_after = get_hard_restriction_status(job['owner_id']) if restricted: diff --git a/backend/utils/batch_pipeline/contracts.py b/backend/utils/batch_pipeline/contracts.py index 93dea5081b9..885e1363bf3 100644 --- a/backend/utils/batch_pipeline/contracts.py +++ b/backend/utils/batch_pipeline/contracts.py @@ -109,8 +109,14 @@ def __post_init__(self): raise ValueError('result_uri must be a bounded durable encrypted reference') +@dataclass(frozen=True) +class HeldWork: + segment_id: str + retry_after: int = 60 + + class BatchWorker(Protocol): - def process_batch(self, items: Sequence[WorkItem]) -> Sequence[WorkResult]: ... + def process_batch(self, items: Sequence[WorkItem], **kwargs) -> Sequence[WorkResult | HeldWork]: ... @dataclass(frozen=True) diff --git a/backend/utils/batch_pipeline/projection.py b/backend/utils/batch_pipeline/projection.py index a378d358257..a08e32eb0a9 100644 --- a/backend/utils/batch_pipeline/projection.py +++ b/backend/utils/batch_pipeline/projection.py @@ -4,6 +4,7 @@ No admission flag is consulted: already accepted work drains after rollback. """ +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled from datetime import datetime, timezone import time import math @@ -58,7 +59,7 @@ def build_incoming(item: WorkItem, binding, payload: dict, *, geolocation=None) transcript_segments=segments, source=ConversationSource(binding.source), language=binding.language or payload.get('language'), - is_locked=binding.is_locked, + is_locked=False if cap_dispatch_enabled() else binding.is_locked, private_cloud_sync_enabled=binding.private_cloud_sync_enabled, client_device_id=binding.client_device_id, client_platform=binding.client_platform, diff --git a/backend/utils/batch_pipeline/pump.py b/backend/utils/batch_pipeline/pump.py index 2e2f91a3048..7f5ff169258 100644 --- a/backend/utils/batch_pipeline/pump.py +++ b/backend/utils/batch_pipeline/pump.py @@ -6,7 +6,8 @@ from collections.abc import Callable from typing import Any -from utils.batch_pipeline.contracts import BatchWorker, HEARTBEAT_SECONDS, LeasedWork, PoolCapacity +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.batch_pipeline.contracts import BatchWorker, HEARTBEAT_SECONDS, LeasedWork, PoolCapacity, HeldWork from utils.batch_pipeline.merge import OrderedMerge from utils.batch_pipeline.errors import FatalGpuError from utils.batch_pipeline.compute_client import from_env as compute_client_from_env @@ -128,7 +129,7 @@ def claim(): lost: set[str] = set() renew = asyncio.create_task(self._renew(leases, done, lost)) try: - required = [v for v in leases if v.item.requires_compute_admission] + required = [v for v in leases if v.item.requires_compute_admission or cap_dispatch_enabled()] if required: receipt = {} try: @@ -159,8 +160,17 @@ def claim(): leases = [v for v in leases if v.item.segment_id not in lost] if not leases: return 0 + dispatch_kwargs = {} + if cap_dispatch_enabled(): + + def authorize_dispatch(segments): + selected = [v for v in leases if v.item.segment_id in segments] + receipt = self.authorize_compute(selected) if self.authorize_compute is not None else {} + return set(receipt.get('allowed') or []) + + dispatch_kwargs['authorize_dispatch'] = authorize_dispatch compute = asyncio.create_task( - run_blocking(sync_executor, self.worker.process_batch, [v.item for v in leases]) + run_blocking(sync_executor, self.worker.process_batch, [v.item for v in leases], **dispatch_kwargs) ) # Cancelling an asyncio wrapper does not stop CUDA. Keep renewal and # wait for the bounded model operation before leaving this process. @@ -205,6 +215,11 @@ def claim(): lost.discard(segment) try: result = result_map.get(segment) + if isinstance(result, HeldWork): + await run_blocking( + db_executor, self.scheduler.defer, segment, lease.token, time.time(), result.retry_after + ) + continue if result is None: if await run_blocking(db_executor, self.scheduler.fail, segment, lease.token, time.time()): await self.drain_owner(lease.item.owner_id) diff --git a/backend/utils/batch_pipeline/quarantine_recovery.py b/backend/utils/batch_pipeline/quarantine_recovery.py index 7c6ac63c09b..a12da84f193 100644 --- a/backend/utils/batch_pipeline/quarantine_recovery.py +++ b/backend/utils/batch_pipeline/quarantine_recovery.py @@ -6,6 +6,9 @@ import time import wave +from utils.stt_cap_dispatch import sync_allowed, funded_allowance +from config.stt_cap_dispatch import enabled as cap_enabled +from utils.observability.stt_cap_dispatch import record_audio from config.batch_pipeline import recovery_owners from database.batch_pipeline_privacy import BatchPrivacyBlocked from database.batch_pipeline_projection_protocol import ProjectionBlocked @@ -87,9 +90,13 @@ def recover(self, segment_id, owner_id): if self.clock() - job['recovery']['started_at'] > 900: self.protocol.settle(segment_id, token, self.clock()) return 'held' + if not sync_allowed(owner_id, item.duration_seconds): + return 'held' audio = self.artifacts.fetch_audio(item) # Privacy and the intent token are rechecked after download and # immediately before issuing the sole application-level STT call. + if not sync_allowed(owner_id, item.duration_seconds): + return 'held' if owner_id not in recovery_owners() or not self.protocol.start(segment_id, token, self.clock()): return 'held' from database.batch_pipeline_projection_protocol import normalized_binding @@ -100,6 +107,10 @@ def recover(self, segment_id, owner_id): ) try: payload = self.stt(item, normalized_binding(job['projection_binding']), audio) + if cap_enabled(): + record_audio( + 'sync', 'accepted', 'unknown', item.duration_seconds, not funded_allowance(owner_id).managed + ) except Exception: self.protocol.settle(segment_id, token, self.clock()) return 'failed_held' diff --git a/backend/utils/locked_content_recovery.py b/backend/utils/locked_content_recovery.py index 30ae626454d..8b5f83f2b19 100644 --- a/backend/utils/locked_content_recovery.py +++ b/backend/utils/locked_content_recovery.py @@ -8,6 +8,7 @@ from datetime import datetime, timezone from uuid import uuid4 +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled from utils.locked_content_flags import reset_recovery_enabled logger = logging.getLogger(__name__) @@ -31,6 +32,13 @@ def _memory_page_drained(count: int, page_size: int) -> bool: def recover_locked_content(uid: str, *, now: datetime | None = None) -> None: + if stt_cap_dispatch_enabled(): + from utils.sync.cap_hold_recovery import kick + + try: + kick(uid) + except Exception as error: + logger.warning('event=stt_cap_recovery outcome=schedule_deferred exception_type=%s', type(error).__name__) if not reset_recovery_enabled(): return # Lazy imports keep the flag-off auth path free of additional dependencies. diff --git a/backend/utils/observability/stt_cap_dispatch.py b/backend/utils/observability/stt_cap_dispatch.py new file mode 100644 index 00000000000..bc84dd5b0fd --- /dev/null +++ b/backend/utils/observability/stt_cap_dispatch.py @@ -0,0 +1,17 @@ +"""Additive dispatch provenance, independent of root usage and unit projection.""" + +from prometheus_client import Counter + +AUDIO_SECONDS = Counter( + 'omi_stt_cap_provider_audio_seconds', + 'Provider-accepted or cap-suppressed audio; never the legacy monthly usage counter', + ['path', 'outcome', 'provider', 'allowance'], +) +CONFIG_ERRORS = Counter('omi_stt_cap_launch_config_errors', 'Invalid subscription launch configuration', ['reason']) + + +def record_audio(path, outcome, provider, seconds, exhausted=False): + provider = getattr(provider, 'value', provider) + provider = provider if provider in {'deepgram', 'modulate', 'soniox', 'parakeet'} else 'unknown' + if seconds > 0: + AUDIO_SECONDS.labels(path, outcome, provider, 'exhausted' if exhausted else 'available').inc(seconds) diff --git a/backend/utils/stt/live_failure.py b/backend/utils/stt/live_failure.py index 3a945a4f108..7369a90ea6a 100644 --- a/backend/utils/stt/live_failure.py +++ b/backend/utils/stt/live_failure.py @@ -304,6 +304,9 @@ async def terminate_live_stt_backoff( if reason not in {'provider_unavailable', 'reconnect_budget'}: raise ValueError('unsupported live STT backoff reason') + admission = getattr(session, 'stt_cap_admission', None) + if admission is not None and admission.funded_paused is True: + return False if session.stt_terminal_failure: return False session.stt_terminal_failure = True @@ -464,6 +467,9 @@ async def terminate_live_stt_session( fields are deliberately bounded and never include provider exception text. """ + admission = getattr(session, 'stt_cap_admission', None) + if admission is not None and admission.funded_paused is True: + return False if session.stt_terminal_failure: return False diff --git a/backend/utils/stt/live_session.py b/backend/utils/stt/live_session.py index 6ca2efee6ba..6e29b48f2b4 100644 --- a/backend/utils/stt/live_session.py +++ b/backend/utils/stt/live_session.py @@ -12,6 +12,7 @@ from utils.observability.fallback import FirstTextDeadlineDiagnostics, ReplayLagDiagnostics, record_fallback from utils.observability.transcription import record_live_stt_audio_seconds from utils.observability.routing_cohort import current_routing_cohort +from utils.stt_cap_dispatch import CapSocket, session_admission, suppress_socket_audio from utils.stt import streaming as st from utils.dream_vocabulary import keyterms from utils.stt.live_failure import PendingLiveFailover, settle_terminal_socket @@ -86,6 +87,7 @@ def get_metrics(self) -> dict[str, Any]: def to_json_log(self) -> dict[str, Any]: return {'event': 'managed_live_vad_metrics', **self.get_metrics()} + @session_admission async def connect( self, sample_rate: int, @@ -303,10 +305,28 @@ def attach_then_rebase(seg_list: list[dict[str, Any]]) -> None: if is_window: from utils.stt.parakeet_window import connect_window - raw = connect_window(callback, sample_rate) - if self.no_text_rescue.enabled: - raw.allow_no_text_rescue = self.no_text_rescue.allow_window_rescue - raw.progress_deadline_enabled = True + def new_window(): + window_raw = connect_window(callback, sample_rate) + if self.no_text_rescue.enabled: + window_raw.allow_no_text_rescue = self.no_text_rescue.allow_window_rescue + window_raw.progress_deadline_enabled = True + return window_raw + + admission = getattr(self.receiver, 'cap_admission', None) + if admission is None: + raw = new_window() + else: + + async def open_window(): + return new_window() + + raw = CapSocket( + admission, + 'parakeet', + sample_rate, + open_window, + new_window() if admission.allowed('parakeet') else None, + ) elif service == st.STTService.parakeet: raw = await st.process_audio_parakeet(callback, language, sample_rate, 1, keywords=keywords) elif service == st.STTService.soniox: @@ -591,7 +611,11 @@ def __init__( if window: from utils.stt.parakeet_window import SessionPcmGain, WINDOW_INGEST_AGC, WindowedParakeetSocket - if isinstance(raw, WindowedParakeetSocket): + if isinstance(raw, CapSocket): + raw.on_open( + lambda transport: transport.set_replay_progress_callback(self._trim_window_replay_to_anchor) + ) + elif isinstance(raw, WindowedParakeetSocket): raw.set_replay_progress_callback(self._trim_window_replay_to_anchor) if WINDOW_INGEST_AGC: self._ingest_gain = SessionPcmGain() @@ -798,6 +822,8 @@ def set_health_callbacks(self, on_success: Callable[[], None], on_close: Callabl if self._routing_active: self._health_success, self._health_close = on_success, on_close + elif self.window and isinstance(self.raw, CapSocket): + self.raw.on_open(lambda transport: transport.set_health_callbacks(on_success, on_close)) elif isinstance(self.raw, WindowedParakeetSocket): self.raw.set_health_callbacks(on_success, on_close) @@ -850,6 +876,8 @@ def process_replay_capture(data, wall_time, score_pcm=None, *, start_sample=None setattr(gate, 'process_audio', process_replay_capture) def send(self, data: bytes, start_sample: int | None = None) -> bool: + if suppress_socket_audio(self.raw, data): + return True if self._closing_for_health: return False if self.is_connection_dead: @@ -1068,6 +1096,8 @@ def commit_send(self) -> None: commit() def send_admitted_audio(self, data: bytes, spans: Any) -> bool: + if suppress_socket_audio(self.raw, data): + return True from utils.stt.soniox_idle import unaccepted_onset data, spans = unaccepted_onset(data, spans, self._send_tracker) diff --git a/backend/utils/stt/live_target_connect.py b/backend/utils/stt/live_target_connect.py index 0f1df391cd9..74a04047dbd 100644 --- a/backend/utils/stt/live_target_connect.py +++ b/backend/utils/stt/live_target_connect.py @@ -2,6 +2,8 @@ from __future__ import annotations +from utils.stt_cap_dispatch import live_provider + import asyncio import os from urllib.parse import urlencode @@ -12,6 +14,7 @@ from utils.stt import streaming +@live_provider('modulate', 1) async def connect_modulate(callback, sample_rate, language): target = connecting_target.get() if target is None or target.endpoint is None: diff --git a/backend/utils/stt/soniox.py b/backend/utils/stt/soniox.py index e6cfb703229..c276cb8461f 100644 --- a/backend/utils/stt/soniox.py +++ b/backend/utils/stt/soniox.py @@ -5,6 +5,7 @@ its own. Imports flow one way: this module never imports from ``streaming.py``. """ +from utils.stt_cap_dispatch import live_provider import asyncio import json import logging @@ -712,6 +713,7 @@ def _handle_tokens(self, tokens: List[Any]) -> None: self._stream_transcript(ready) +@live_provider('soniox', 1) async def process_audio_soniox( stream_transcript: Callable[[List[Dict[str, Any]]], None], sample_rate: int, diff --git a/backend/utils/stt/streaming.py b/backend/utils/stt/streaming.py index 312949de619..ef14ea4e7bf 100644 --- a/backend/utils/stt/streaming.py +++ b/backend/utils/stt/streaming.py @@ -1,3 +1,4 @@ +from utils.stt_cap_dispatch import live_provider import asyncio import inspect import io @@ -1098,6 +1099,7 @@ def _deepgram_is_available() -> bool: return _managed_deepgram_client() is not None or bool(get_byok_key('deepgram')) +@live_provider('deepgram', 2) async def process_audio_dg( stream_transcript: Callable[[List[Dict[str, Any]]], None], language: str, @@ -1847,6 +1849,7 @@ def _handle_utterance(self, msg: Dict[str, Any]) -> None: self._stream_transcript(segments) +@live_provider('modulate', 1) async def process_audio_modulate( stream_transcript: Callable[[List[Dict[str, Any]]], None], sample_rate: int, @@ -2320,6 +2323,7 @@ async def _receive_loop(self, ws: Any) -> None: self._mark_dead(f"parakeet ws recv: {e}") +@live_provider('parakeet', 2) async def process_audio_parakeet( stream_transcript: Callable[[List[Dict[str, Any]]], None], language: str, diff --git a/backend/utils/stt/vad_gate.py b/backend/utils/stt/vad_gate.py index 43467eae9c7..2178d0c95b9 100644 --- a/backend/utils/stt/vad_gate.py +++ b/backend/utils/stt/vad_gate.py @@ -12,6 +12,8 @@ """ import audioop +from utils.stt_cap_dispatch import suppress_socket_audio + import logging import os import threading @@ -794,6 +796,8 @@ def _counted(self, audio: bytes) -> bytes: def send(self, data: bytes, wall_time: Optional[float] = None, start_sample: Optional[int] = None) -> bool: """Send audio through VAD gate and report whether it was accepted.""" + if suppress_socket_audio(self._conn, data): + return True if self.is_connection_dead: return False if self._gate is None: @@ -876,6 +880,8 @@ def commit_send(self) -> None: commit() def send_admitted_audio(self, data: bytes, spans: Any) -> bool: + if suppress_socket_audio(self._conn, data): + return True from utils.stt.soniox_idle import unaccepted_onset data, spans = unaccepted_onset(data, spans, self._send_tracker) diff --git a/backend/utils/stt_cap_dispatch.py b/backend/utils/stt_cap_dispatch.py new file mode 100644 index 00000000000..5ead768c8f7 --- /dev/null +++ b/backend/utils/stt_cap_dispatch.py @@ -0,0 +1,238 @@ +"""Funded-provider admission; legacy root usage counters deliberately stay unchanged.""" + +import logging +import os +from functools import wraps +from contextvars import ContextVar +from typing import Any +from utils.stt.socket import STTSocket +from utils.stt.live_router import connecting_target + +from config.stt_cap_dispatch import enabled +from utils.observability.stt_cap_dispatch import record_audio +from utils.byok import get_byok_key +from utils.executors import db_executor, run_blocking +from utils.subscription import resolve_transcription_allowance + +logger = logging.getLogger(__name__) + + +def customer_funded(provider): + return ( + getattr(provider, 'value', provider) == 'deepgram' + and os.getenv('DEEPGRAM_SELF_HOSTED_ENABLED', '').lower() != 'true' + and bool(get_byok_key('deepgram')) + ) + + +def funded_allowance(uid, source=None): + return resolve_transcription_allowance(uid, source=source, byok_active=False, omi_funded=True) + + +def sync_allowed(uid: str, seconds: float = 0, provider: Any = 'unknown') -> bool: + if not enabled(): + return True + allowance = funded_allowance(uid) + if not allowance.managed: + record_audio('sync', 'held', provider, seconds, allowance.reason == 'plan_allowance_exhausted') + return allowance.managed + + +class LiveAdmission: + """One balance shared by every channel, replay and replacement in a session. + + Refresh uses the existing usage flush cadence. Accepted packets debit the + finite snapshot locally, so no further packet starts after it reaches zero. + Provider queues already accepting audio remain free to drain. + """ + + def __init__(self, uid, source=None): + self.uid, self.source = uid, source + self.remaining = 0 + self.reason = 'allowance_unavailable' + self.managed = False + self.held = [] + self.omi_selected = False + + async def refresh(self): + allowance = await run_blocking(db_executor, funded_allowance, self.uid, self.source) + self.remaining, self.reason, self.managed = allowance.remaining_seconds, allowance.reason, allowance.managed + if self.managed: + for socket in list(self.held): + try: + await socket.resume() + except Exception as error: + logger.warning( + 'event=stt_cap_dispatch outcome=resume_deferred exception_type=%s', type(error).__name__ + ) + + @property + def funded_paused(self) -> bool: + return self.omi_selected and not (self.managed and (self.remaining is None or self.remaining > 0)) + + def allowed(self, provider): + return customer_funded(provider) or self.managed and (self.remaining is None or self.remaining > 0) + + def send(self, raw, data, provider, rate, *args, **kwargs): + if not self.allowed(provider): + record_audio( + 'live', 'suppressed', provider, len(data) / (2 * rate), self.reason == 'plan_allowance_exhausted' + ) + # Policy absorption is not a transport failure. The receive/archive + # paths continue, and recovery must not fail over or close capture. + return True + accepted = raw.send(data, *args, **kwargs) + if accepted is not False and not customer_funded(provider): + seconds = len(data) / (2 * rate) + record_audio('live', 'accepted', provider, seconds, self.reason == 'plan_allowance_exhausted') + if self.remaining is not None: + self.remaining = max(0, self.remaining - seconds) + if self.remaining == 0: + self.reason = 'plan_allowance_exhausted' + return accepted + + +current_live_admission = ContextVar('stt_cap_live_admission', default=None) + + +class CapSocket(STTSocket): + """Keep receiving at cap, without opening a paid provider or ending capture.""" + + def __init__(self, admission, provider, rate, factory, raw=None): + self.admission, self.provider, self.rate = admission, provider, rate + self.factory, self.raw = factory, raw + self.closed = False + self.cap_dispatch_socket = True + self._open_callbacks = [] + if not customer_funded(provider): + admission.omi_selected = True + if raw is None: + admission.held.append(self) + logger.info('event=stt_cap_dispatch path=live outcome=held provider=%s', provider) + + async def resume(self): + if self.raw is None and not self.closed and self.admission.allowed(self.provider): + raw = await self.factory() + if raw is None: + raise RuntimeError('held provider constructor returned no socket') + self.raw = raw + for callback in self._open_callbacks: + callback(raw) + self.admission.held.remove(self) + + def on_open(self, callback): + self._open_callbacks.append(callback) + if self.raw is not None: + callback(self.raw) + + @property + def __class__(self) -> type: + return self.raw.__class__ if self.raw is not None else type(self) + + @__class__.setter + def __class__(self, value: type) -> None: + raise TypeError('provider proxy class is read only') + + def __getattr__(self, name): + if self.raw is None: + raise AttributeError(name) + return getattr(self.raw, name) + + @property + def is_connection_dead(self): + if not self.admission.allowed(self.provider): + return False # A paused provider cannot end ongoing capture. + return self.raw.is_connection_dead if self.raw is not None else False + + @property + def death_reason(self): + return self.raw.death_reason if self.raw is not None else None + + def suppress(self, data): + if self.raw is None or not self.admission.allowed(self.provider): + record_audio( + 'live', + 'suppressed', + self.provider, + len(data) / (2 * self.rate), + self.admission.reason == 'plan_allowance_exhausted', + ) + return True + return False + + def send(self, data, *args, **kwargs): + if self.raw is None: + record_audio( + 'live', + 'suppressed', + self.provider, + len(data) / (2 * self.rate), + self.admission.reason == 'plan_allowance_exhausted', + ) + return True + return self.admission.send(self.raw, data, self.provider, self.rate, *args, **kwargs) + + def finish(self): + self.closed = True + if self.raw is not None: + return self.raw.finish() + + def finalize(self): + if self.raw is not None: + return self.raw.finalize() + + +def live_provider(provider, rate_index): + """Applied at actual provider constructors, including fallback constructors.""" + + def decorate(fn): + @wraps(fn) + async def connect(*args, **kwargs): + admission = current_live_admission.get() if enabled() else None + if admission is None: + return await fn(*args, **kwargs) + rate = kwargs.get('sample_rate') or args[rate_index] + target = connecting_target.get() + + async def factory(): + token = connecting_target.set(target) + try: + return await fn(*args, **kwargs) + finally: + connecting_target.reset(token) + + allowed = admission.allowed(provider) + try: + raw = await factory() if allowed else None + except Exception: + if admission.allowed(provider): + raise + raw = None # Permission changed while the dial was in flight. + if allowed and raw is None: + return None + if isinstance(raw, CapSocket): + return raw + return CapSocket(admission, provider, rate, factory, raw) + + return connect + + return decorate + + +def session_admission(fn): + @wraps(fn) + async def connect(self, *args, **kwargs): + admission = getattr(self.receiver, 'cap_admission', None) + token = current_live_admission.set(admission) + try: + return await fn(self, *args, **kwargs) + finally: + current_live_admission.reset(token) + + return connect + + +def suppress_socket_audio(socket, data): + # Explicit identity avoids treating test doubles or unrelated adapters as + # cap owners. Suppress before VAD/timeline adapters record provider spans. + return getattr(socket, 'cap_dispatch_socket', False) is True and socket.suppress(data) diff --git a/backend/utils/subscription.py b/backend/utils/subscription.py index 4dafdc94b69..193af417aca 100644 --- a/backend/utils/subscription.py +++ b/backend/utils/subscription.py @@ -1,3 +1,5 @@ +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import CONFIG_ERRORS import os from dataclasses import dataclass import time @@ -1582,9 +1584,15 @@ def get_monthly_usage_for_subscription(uid: str) -> Dict[str, Any]: """ Gets the current monthly usage for subscription purposes, considering the launch date from env variables. The launch date format is expected to be YYYY-MM-DD. - If the launch date is not set, not valid, or in the future, usage is considered zero. + Legacy mode treats unset, invalid and future dates as zero usage. Dispatch + enforcement rejects missing/invalid configuration; an explicit future date + is the intentionally unlaunched policy in both modes. """ subscription_launch_date_str = os.getenv('SUBSCRIPTION_LAUNCH_DATE') + if stt_cap_dispatch_enabled() and not subscription_launch_date_str: + CONFIG_ERRORS.labels(reason='missing').inc() + logger.error('event=stt_cap_launch_config outcome=invalid reason=missing') + raise ValueError('missing subscription launch date') if not subscription_launch_date_str: # Subscription not launched, so no usage is counted against limits. return {} @@ -1593,6 +1601,10 @@ def get_monthly_usage_for_subscription(uid: str) -> Dict[str, Any]: # Use strptime to enforce YYYY-MM-DD format launch_date = datetime.strptime(subscription_launch_date_str, '%Y-%m-%d').replace(tzinfo=timezone.utc) except ValueError: + if stt_cap_dispatch_enabled(): + CONFIG_ERRORS.labels(reason='invalid').inc() + logger.error('event=stt_cap_launch_config outcome=invalid reason=invalid') + raise # Invalid date format, treat as not launched. return {} @@ -1666,6 +1678,7 @@ def resolve_transcription_allowance( subscription: Any = _UNRESOLVED, usage: Any = _UNRESOLVED, byok_active: Any = _UNRESOLVED, + omi_funded: bool = False, ) -> TranscriptionAllowance: """Resolve the managed-transcription allowance for one uid, once. Never raises. @@ -1699,10 +1712,13 @@ def resolve_transcription_allowance( # lookup failure must not open a billed socket for a paywalled account, # and only a validated key on this request (not a stored fingerprint) # exempts. - if is_trial_paywalled(uid, source, required_byok_provider='deepgram', strict=True): + paywall_kwargs = {'byok_exempt': False} if omi_funded else {} + if is_trial_paywalled(uid, source, required_byok_provider='deepgram', strict=True, **paywall_kwargs): return TranscriptionAllowance(TRANSCRIPTION_MODE_BLOCKED, 0, 'trial_paywalled') # Require the Deepgram header on this request so a user cannot activate # BYOK with fake fingerprints and then omit x-byok-deepgram to ride Omi's key. + if omi_funded: + byok_active = False if byok_active is _UNRESOLVED: byok_active = users_db.is_byok_active(uid) if byok_active and get_byok_key('deepgram'): diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py new file mode 100644 index 00000000000..f730b6eed70 --- /dev/null +++ b/backend/utils/sync/cap_hold.py @@ -0,0 +1,161 @@ +"""Encrypted, owner-scoped pre-STT holds. Pending audio has no application TTL. + +The manifest is written only after upload succeeds. A failed hold fails the +upload/job, so the client never receives success for audio we did not retain. +""" + +import hashlib +import json +import logging +from pathlib import Path +import time +from tempfile import TemporaryDirectory +import wave +from uuid import uuid4 +from typing import Any +from models.conversation_enums import ConversationSource +from utils.observability.stt_cap_dispatch import record_audio +from utils.stt_cap_dispatch import funded_allowance + +from google.cloud import firestore +from google.api_core.exceptions import AlreadyExists +from database import redis_db +from config.stt_cap_dispatch import enabled +from utils.sync.files import get_timestamp_from_path +from utils.stt_cap_dispatch import sync_allowed +from database._client import get_firestore_client +from database.legal_holds import external_write_fence +from utils.encryption import encrypt, decrypt, encrypt_audio_chunk, decrypt_audio_file +from utils.other.storage import get_private_cloud_sync_bucket, owner_storage_write_gate + +logger = logging.getLogger(__name__) +COLLECTION = 'stt_cap_holds' + + +def holds(uid: str) -> Any: + return get_firestore_client().collection('users').document(uid).collection(COLLECTION) + + +def pending_holds(uid: str, *, limit: int = 10) -> list[Any]: + return list(holds(uid).limit(limit).stream()) + + +def hold_segment(uid: str, path: str, metadata: dict[str, Any]) -> float: + audio = Path(path).read_bytes() + partition = { + key: metadata.get(key) for key in ('timestamp', 'source', 'client_device_id', 'target_conversation_id') + } + identity = hashlib.sha256(json.dumps(partition, sort_keys=True).encode() + audio).hexdigest() + blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' + bucket = get_private_cloud_sync_bucket() + with wave.open(path, 'rb') as wav: + seconds = wav.getnframes() / wav.getframerate() + with external_write_fence(uid), owner_storage_write_gate(uid, bucket): + ref = holds(uid).document(identity) + if ref.get().exists: + return seconds + blob = bucket.blob(blob_path) + blob.upload_from_string(encrypt_audio_chunk(audio, uid), content_type='application/octet-stream') + try: + ref.create( + { + 'audio_path': blob_path, + 'metadata': encrypt(json.dumps(metadata), uid), + 'seconds': seconds, + 'state': 'held', + 'created_at': time.time(), + 'lease_until': 0, + } + ) + except AlreadyExists: + return seconds # A concurrent upload owns the immutable recovery checkpoint. + record_audio('sync', 'held', 'unknown', seconds, funded_allowance(uid).reason == 'plan_allowance_exhausted') + logger.info('event=stt_cap_dispatch path=sync outcome=held seconds=%.3f', seconds) + return seconds + + +def claim(uid: str, ref: Any, now: float) -> dict[str, Any] | None: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any): + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else None + if not row or row.get('state') == 'completed' or row.get('lease_until', 0) > now: + return None + token = uuid4().hex + tx.update(ref, {'lease_until': now + 1800, 'lease_token': token}) + row['lease_token'] = token + return row + + with external_write_fence(uid): + return commit(client.transaction()) + + +def read_audio(uid: str, row: dict[str, Any], path: str) -> None: + expected = f'chunks/{uid}/cap-held/' + if not row['audio_path'].startswith(expected): + raise ValueError('hold audio owner mismatch') + raw = get_private_cloud_sync_bucket().blob(row['audio_path']).download_as_bytes() + Path(path).write_bytes(decrypt_audio_file(raw, uid)) + + +def metadata(uid: str, row: dict[str, Any]) -> dict[str, Any]: + return json.loads(decrypt(row['metadata'], uid)) + + +def release(uid: str, ref: Any, token: str) -> None: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any): + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else {} + if row.get('lease_token') == token: + tx.update(ref, {'lease_until': 0, 'lease_token': ''}) + + return commit(client.transaction()) + + +def hold_capture_if_exhausted(uid: str, paths: list[str], source: ConversationSource, **context: Any) -> bool: + # Shared by v1 intake and v2 execution, before VAD can discard silence. + if not enabled() or sync_allowed(uid): + return False + for path in paths: + hold_segment( + uid, + path, + { + **context, + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'full_capture': True, + }, + ) + return True + + +def recovery_cooldown(uid: str) -> bool: + return bool(redis_db.r.set(f'users:{uid}:stt_cap_recovery:cooldown', '1', nx=True, ex=60)) + + +def renew(uid: str, ref: Any, token: str, now: float) -> bool: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any) -> bool: + snapshot = ref.get(transaction=tx) + row = snapshot.to_dict() if snapshot.exists else {} + if row.get('lease_token') != token or row.get('lease_until', 0) <= now: + return False + tx.update(ref, {'lease_until': now + 1800}) + return True + + with external_write_fence(uid): + return commit(client.transaction()) + + +def recovery_directory(uid: str) -> TemporaryDirectory: + parent = Path('syncing') / uid + parent.mkdir(parents=True, exist_ok=True) + return TemporaryDirectory(prefix='cap-recovery-', dir=parent) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py new file mode 100644 index 00000000000..f599681ad69 --- /dev/null +++ b/backend/utils/sync/cap_hold_recovery.py @@ -0,0 +1,154 @@ +"""Bounded held-audio recovery from the existing locked-content request path. + +The async coordinator borrows leaf workers. Audio stays held until transcription, +content persistence and idempotent usage settlement succeed, including silence. +""" + +import asyncio +import logging +from pathlib import Path +import threading +import time + +from anyio import from_thread +from typing import Any +from google.cloud import firestore +from google.api_core.exceptions import NotFound + +from config.stt_cap_dispatch import enabled +from database import users as users_db +from utils.analytics import record_usage +from utils.executors import run_blocking, db_executor, sync_executor, storage_executor, start_background_task +from utils.stt_cap_dispatch import sync_allowed +from utils.sync import cap_hold +from utils.sync.files import get_wav_duration +from utils.sync.pipeline import ( + process_segment, + retrieve_vad_segments, + finalize_recovered_sync_segment, +) +from models.conversation import ConversationSource + +logger = logging.getLogger(__name__) + + +def _erase(uid: str, snapshot: Any, row: dict[str, Any]) -> None: + # Account deletion already purges this existing owner-scoped chunk prefix. + path = row['audio_path'] + if not path.startswith(f'chunks/{uid}/cap-held/'): + raise ValueError('hold audio owner mismatch') + try: + cap_hold.get_private_cloud_sync_bucket().blob(path).delete() + except NotFound: + pass + snapshot.reference.delete() + + +async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: + meta = await run_blocking(db_executor, cap_hold.metadata, uid, row) + prefs = await run_blocking(db_executor, users_db.get_user_transcription_preferences, uid) + private = await run_blocking(db_executor, users_db.get_user_private_cloud_sync_enabled, uid) + protection = await run_blocking(db_executor, users_db.get_data_protection_level, uid) if private else None + directory = await run_blocking(storage_executor, cap_hold.recovery_directory, uid) + try: + path = str(Path(directory.name) / f"{meta['timestamp']}.wav") + await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) + paths, errors = {path}, [] + if meta.get('full_capture'): + paths = set() + await run_blocking(sync_executor, retrieve_vad_segments, path, paths, errors) + if errors: + return False + processed = set(row.get('processed_paths') or []) + for segment_path in sorted(paths): + if not await run_blocking( + db_executor, cap_hold.renew, uid, snapshot.reference, row['lease_token'], time.time() + ): + return False + name = Path(segment_path).name + if name in processed: + continue + if not await run_blocking(db_executor, sync_allowed, uid): + return False + response = {'new_memories': set(), 'updated_memories': set()} + outcome = {} + ok = await run_blocking( + sync_executor, + process_segment, + segment_path, + uid, + response, + threading.Lock(), + errors, + ConversationSource(meta['source']), + False, + prefs, + private_cloud_sync_enabled=bool(private), + data_protection_level=protection, + client_device_id=meta.get('client_device_id'), + client_platform=meta.get('client_platform'), + target_conversation_id=meta.get('target_conversation_id'), + cap_recovery=True, + deferred_outcome=outcome, + ) + if errors or outcome.get('cap_held'): + return False + if ok: + await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, response, bool(private)) + seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) + await run_blocking( + db_executor, + record_usage, + uid, + transcription_seconds=int(seconds), + speech_seconds=int(seconds), + idempotency_key=f'cap-hold-{snapshot.id}-{name}', + usage_metadata={'funding': 'omi', 'provider': 'unknown', 'source': 'sync_v2'}, + ) + await run_blocking( + db_executor, snapshot.reference.update, {'processed_paths': firestore.ArrayUnion([name])} + ) + finally: + await run_blocking(storage_executor, directory.cleanup) + return True + + +async def recover(uid: str) -> None: + # One cheap distributed cooldown before querying usage or manifests. Failed + # Redis means deferred recovery, never duplicate provider dispatch. + acquired = await run_blocking(db_executor, cap_hold.recovery_cooldown, uid) + if not acquired or not await run_blocking(db_executor, sync_allowed, uid): + return + rows = await run_blocking(db_executor, cap_hold.pending_holds, uid) + for snapshot in rows: + row = snapshot.to_dict() or {} + try: + if row.get('state') == 'completed': + await run_blocking(storage_executor, _erase, uid, snapshot, row) + continue + row = await run_blocking(db_executor, cap_hold.claim, uid, snapshot.reference, time.time()) + if row is None: + continue + try: + if await _recover_row(uid, snapshot, row): + await run_blocking(db_executor, snapshot.reference.update, {'state': 'completed'}) + await run_blocking(storage_executor, _erase, uid, snapshot, row) + finally: + await run_blocking(db_executor, cap_hold.release, uid, snapshot.reference, row['lease_token']) + except Exception as error: + logger.warning('event=stt_cap_recovery outcome=deferred exception_type=%s', type(error).__name__) + + +async def _schedule(uid: str) -> None: + start_background_task(recover(uid), name='stt_cap_hold_recovery') + + +def kick(uid: str) -> None: + if not enabled(): + return + try: + asyncio.get_running_loop() + except RuntimeError: + from_thread.run(_schedule, uid) + else: + start_background_task(recover(uid), name='stt_cap_hold_recovery') diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 80c143c1c5b..77f9129f4b0 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -134,6 +134,10 @@ from utils.speaker_identification import detect_speaker_from_text from utils.speaker_learning_jobs import schedule_person_voice_learning_retries from utils.stt.voiceprints import usable_person_voiceprint +from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled +from utils.stt_cap_dispatch import sync_allowed, funded_allowance +from utils.observability.stt_cap_dispatch import record_audio as record_cap_audio +from utils.sync.cap_hold import hold_segment, hold_capture_if_exhausted from utils.stt.pre_recorded import get_prerecorded_service, postprocess_words, prerecorded from utils.stt.outcomes import ( TranscriptionFailure, @@ -1223,10 +1227,10 @@ def process_segment( is_locked: bool = False, transcription_prefs: dict = None, person_embeddings_cache: dict = None, - target_conversation_id: str = None, + target_conversation_id: Optional[str] = None, turnstile: Optional[_OrderedTurnstile] = None, private_cloud_sync_enabled: bool = False, - data_protection_level: str = None, + data_protection_level: Optional[str] = None, client_device_id: Optional[str] = None, client_platform: Optional[str] = None, sync_lane: str = SyncLane.FRESH.value, @@ -1238,6 +1242,7 @@ def process_segment( attempt_ref: str | None = None, source_position_map: tuple[dict, int] | None = None, lineage_binding: Optional[str] = None, + cap_recovery: bool = False, ): conversation_id = None provider = 'unknown' @@ -1247,6 +1252,27 @@ def process_segment( if ambiguous_binding_pending(uid, lineage_binding): phase = 'persistence' raise SyncAssignmentConflict('sync lineage generation ownership remains ambiguous') + if stt_cap_dispatch_enabled(): + if not sync_allowed(uid): + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True + if not cap_recovery: + hold_segment( + uid, + path, + { + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'target_conversation_id': target_conversation_id, + 'private_cloud_sync_enabled': private_cloud_sync_enabled, + 'data_protection_level': data_protection_level, + 'client_device_id': client_device_id, + 'client_platform': client_platform, + }, + ) + return False + # Queue-time locks do not survive execution-time authorization. + is_locked = False url = get_syncing_file_temporal_signed_url(path) schedule_syncing_temporal_file_deletion(path) @@ -1296,6 +1322,24 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N transcript_segments: List[TranscriptSegment] = [] while True: phase = 'provider_call' + if stt_cap_dispatch_enabled() and not sync_allowed(uid): + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True + if not cap_recovery: + hold_segment( + uid, + path, + { + 'timestamp': get_timestamp_from_path(path), + 'source': getattr(source, 'value', source), + 'target_conversation_id': target_conversation_id, + 'private_cloud_sync_enabled': private_cloud_sync_enabled, + 'data_protection_level': data_protection_level, + 'client_device_id': client_device_id, + 'client_platform': client_platform, + }, + ) + return False words, detected_language = prerecorded( url, speakers_count=3, @@ -1304,6 +1348,11 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N language=req_language, keywords=vocabulary if vocabulary else None, ) + if stt_cap_dispatch_enabled(): + allowance = funded_allowance(uid) + record_cap_audio( + 'sync', 'accepted', provider, get_wav_duration(path), allowance.reason == 'plan_allowance_exhausted' + ) language = user_language if (single_language_mode and user_language) else detected_language phase = 'parse' transcript_segments = postprocess_words(words, 0) if words else [] @@ -2261,6 +2310,21 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT failure_key='invalid_audio', ) return + cap_held = False + if stt_cap_dispatch_enabled(): + cap_held = await run_blocking( + sync_executor, + hold_capture_if_exhausted, + uid, + wav_paths, + source, + target_conversation_id=target_conversation_id, + client_device_id=client_device_id, + client_platform=client_platform, + ) + if cap_held: + await run_blocking(storage_executor, _cleanup_files, wav_paths) + wav_paths = [] try: wav_paths, source_frame_maps, coverage_suppressed_all = await apply_sync_wal_audio_coverage( uid, @@ -2424,7 +2488,9 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT if total_segments == 0: empty_outcome = ( - TranscriptionOutcome.SUCCESS if coverage_suppressed_all else TranscriptionOutcome.EXPECTED_SILENCE + TranscriptionOutcome.SUCCESS + if coverage_suppressed_all or cap_held + else TranscriptionOutcome.EXPECTED_SILENCE ) empty_result = { 'new_memories': [], @@ -2436,6 +2502,7 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT 'provider': 'unknown', 'model': 'unknown', 'lane': sync_lane, + **({'held_audio': True} if cap_held else {}), } if content_id: completed_ledger = await run_blocking( @@ -3211,3 +3278,10 @@ def _download_staged_files(blob_paths: list) -> bool: if not download_syncing_temporal_file(p): return False return True + + +def finalize_recovered_sync_segment(uid: str, response: dict, private: bool) -> None: + """Use the same enrichment/audio finalization as ordinary sync intake.""" + _reprocess_merged_conversations(uid, response) + if private: + _finalize_sync_audio_files(uid, response) diff --git a/config/feature-flags.yaml b/config/feature-flags.yaml index 0477d4d9e0e..8e333b97878 100644 --- a/config/feature-flags.yaml +++ b/config/feature-flags.yaml @@ -1,4 +1,5 @@ flags: + - {"key":"STT_CAP_DISPATCH_ENFORCEMENT","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Default-off funded STT dispatch admission for live, sync and queued batch; preserves audio before cloud STT at cap; no deployment activation","fail":"closed","owner":"dazheng","created":"2026-10-16","review_by":"2026-11-15","decision":"pending"} - {"key":"BATCH_PIPELINE_RECOVERY_MODE","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Default off; on enables fenced per-segment legacy STT recovery only for recovery allowlisted owners","fail":"closed","owner":"dazheng","created":"2026-10-16","review_by":"2026-11-15","decision":"pending","notes":"Retain the whole-upload batch claim. Failed or ambiguous STT stays held without automatic redispatch; admission and worker flags are independent."} - {"key":"BATCH_PIPELINE_RECOVERY_UID_ALLOWLIST","aliases":[],"kind":"env","lifecycle":"rollout","surfaces":["backend"],"summary":"Comma-separated recovery owners; empty denies all even with recovery mode on","fail":"closed","owner":"dazheng","created":"2026-10-16","review_by":"2026-11-15","decision":"pending","notes":"Initial owner-only rollout; no percentage admission and no production config mutation in this PR."} - {"key":"BATCH_PIPELINE_API_URL","aliases":[],"kind":"env","lifecycle":"config_switch","surfaces":["backend"],"summary":"Explicit HTTPS backend serving URL for fenced pre-compute spend admission; empty pauses required jobs without GPU inference","fail":"closed","owner":"dazheng","created":"2026-10-15","decision":"keep","notes":"David prod cohort: GPU worker enabled separately from CPU admission; backend-sync is the HTTPS compute-policy/OIDC target. CPU WORKER_ENABLED remains false. Keep ops/config lifecycle; this setting is not percentage rollout authority."} From 2a07b6b378686da0d57f3d79c923553ef8227731 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 19:59:19 +0700 Subject: [PATCH 02/19] fix: retain deferred live wire ledger binding through cap pauses Failure-Class: FC-stt-allowance-not-enforced-at-dispatch --- backend/tests/unit/test_stt_cap_dispatch.py | 14 ++++++++++++++ backend/utils/stt_cap_dispatch.py | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/backend/tests/unit/test_stt_cap_dispatch.py b/backend/tests/unit/test_stt_cap_dispatch.py index 910dbbbdac3..0a37d18a32f 100644 --- a/backend/tests/unit/test_stt_cap_dispatch.py +++ b/backend/tests/unit/test_stt_cap_dispatch.py @@ -191,3 +191,17 @@ async def test_terminal_provider_error_cannot_close_capture_during_cap_pause(): assert session.active and not session.stt_terminal_failure websocket.send_json.assert_not_called() websocket.close.assert_not_called() + + +async def test_paused_soniox_binds_wire_ledger_before_resumed_audio(): + state = cap.LiveAdmission('synthetic') + raw = Mock() + socket = cap.CapSocket(state, 'soniox', 16000, AsyncMock(return_value=raw)) + ledger = object() + socket.set_wire_ledger(ledger) + socket.set_capture_axis_ledger(ledger) + raw.set_wire_ledger.assert_not_called() + state.managed, state.remaining = True, 10 + await socket.resume() + raw.set_wire_ledger.assert_called_once_with(ledger) + raw.set_capture_axis_ledger.assert_called_once_with(ledger) diff --git a/backend/utils/stt_cap_dispatch.py b/backend/utils/stt_cap_dispatch.py index 5ead768c8f7..5dd4614d083 100644 --- a/backend/utils/stt_cap_dispatch.py +++ b/backend/utils/stt_cap_dispatch.py @@ -125,6 +125,12 @@ def on_open(self, callback): if self.raw is not None: callback(self.raw) + def set_wire_ledger(self, ledger): + self.on_open(lambda transport: transport.set_wire_ledger(ledger)) + + def set_capture_axis_ledger(self, ledger): + self.on_open(lambda transport: transport.set_capture_axis_ledger(ledger)) + @property def __class__(self) -> type: return self.raw.__class__ if self.raw is not None else type(self) From 9581c4299ab3a577eac47579bd1993df064b086e Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 20:03:34 +0700 Subject: [PATCH 03/19] fix: avoid labeling unresolved cap observations as available Failure-Class: FC-stt-allowance-not-enforced-at-dispatch --- backend/docs/stt-cap-dispatch.md | 3 ++- backend/utils/observability/stt_cap_dispatch.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index fad6cdff74a..6f046934930 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -73,7 +73,8 @@ future date continues to mean intentionally unlaunched. - `path`: `live` or `sync`; - `outcome`: `accepted`, `suppressed` (live), or `held` (sync); - `provider`: a bounded serving family or `unknown`; -- `allowance`: `exhausted` or `available`, at the admission/acceptance observation. +- `allowance`: `exhausted` or `other`, at the admission/acceptance observation; `other` + includes available allowance and untrusted lookups, without claiming availability. Accepted live quantities count PCM accepted by a provider adapter, including replay; failed sends do not count. Suppressed quantities count input PCM, before diff --git a/backend/utils/observability/stt_cap_dispatch.py b/backend/utils/observability/stt_cap_dispatch.py index bc84dd5b0fd..ea4abf33746 100644 --- a/backend/utils/observability/stt_cap_dispatch.py +++ b/backend/utils/observability/stt_cap_dispatch.py @@ -14,4 +14,4 @@ def record_audio(path, outcome, provider, seconds, exhausted=False): provider = getattr(provider, 'value', provider) provider = provider if provider in {'deepgram', 'modulate', 'soniox', 'parakeet'} else 'unknown' if seconds > 0: - AUDIO_SECONDS.labels(path, outcome, provider, 'exhausted' if exhausted else 'available').inc(seconds) + AUDIO_SECONDS.labels(path, outcome, provider, 'exhausted' if exhausted else 'other').inc(seconds) From acb09a1733555852aff2b05786cf5588f512ee27 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 20:41:57 +0700 Subject: [PATCH 04/19] fix: export GPU cap audio counters and preserve exhaustion provenance Failure-Class: FC-stt-allowance-not-enforced-at-dispatch --- backend/diarizer/batch_pipeline_runtime.py | 5 +++++ .../test_batch_pipeline_quarantine_recovery.py | 16 ++++++++++++++++ .../tests/unit/test_batch_pipeline_runtime.py | 18 ++++++++++++++++++ .../batch_pipeline/quarantine_recovery.py | 6 +++++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/backend/diarizer/batch_pipeline_runtime.py b/backend/diarizer/batch_pipeline_runtime.py index d30fdad6632..b63a033703d 100644 --- a/backend/diarizer/batch_pipeline_runtime.py +++ b/backend/diarizer/batch_pipeline_runtime.py @@ -15,6 +15,8 @@ from prometheus_client import CollectorRegistry, CONTENT_TYPE_LATEST, generate_latest +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled +from utils.observability.stt_cap_dispatch import AUDIO_SECONDS from config.batch_pipeline_providers import provider_config from database._client import get_data_plane_firestore_client from database.batch_pipeline_store import FirestoreStore @@ -86,6 +88,9 @@ def process_batch(self, items, **kwargs): def health_server(state: RuntimeState, registry: CollectorRegistry, port: int) -> ThreadingHTTPServer: + if cap_dispatch_enabled(): + registry.register(AUDIO_SECONDS) + class Handler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/metrics': diff --git a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py index 1029d7d12f7..8a8450c2e5b 100644 --- a/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py +++ b/backend/tests/unit/test_batch_pipeline_quarantine_recovery.py @@ -450,3 +450,19 @@ def test_recovery_rechecks_after_audio_download(setup, monkeypatch): assert recovery.recover('bad', 'synthetic-owner') == 'held' artifacts.fetch_audio.assert_called_once() stt.assert_not_called() + + +def test_unavailable_completion_lookup_is_not_reported_as_genuine_exhaustion(setup, monkeypatch): + from types import SimpleNamespace + from utils.batch_pipeline import quarantine_recovery as module + + _, _, _, _, _, recovery = setup + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr(module, 'sync_allowed', Mock(return_value=True)) + monkeypatch.setattr( + module, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='allowance_unavailable')) + ) + metric = Mock() + monkeypatch.setattr(module, 'record_audio', metric) + assert recovery.recover('bad', 'synthetic-owner') == 'recovered' + metric.assert_called_once_with('sync', 'accepted', 'unknown', 5, False) diff --git a/backend/tests/unit/test_batch_pipeline_runtime.py b/backend/tests/unit/test_batch_pipeline_runtime.py index ed67bf217ae..dbd8e797c90 100644 --- a/backend/tests/unit/test_batch_pipeline_runtime.py +++ b/backend/tests/unit/test_batch_pipeline_runtime.py @@ -289,3 +289,21 @@ def test_memory_probe_separates_host_and_device_peaks(): assert result['gpu_allocated_peak_bytes'] == 30 assert result['gpu_reserved_peak_bytes'] == 40 assert result['host_process_peak_bytes'] > 0 + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_cap_audio_counter_is_exported_by_the_worker_metrics_registry(monkeypatch, enforce): + from utils.observability.stt_cap_dispatch import record_audio + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + state = runtime.RuntimeState() + server = runtime.health_server(state, CollectorRegistry(), 0) + try: + record_audio('sync', 'accepted', 'parakeet', 5) + body = urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}/metrics').read().decode() + assert ('omi_stt_cap_provider_audio_seconds_total' in body) is enforce + if enforce: + assert 'path="sync"' in body and 'provider="parakeet"' in body + finally: + server.shutdown() + server.server_close() diff --git a/backend/utils/batch_pipeline/quarantine_recovery.py b/backend/utils/batch_pipeline/quarantine_recovery.py index a12da84f193..3de33ab1ac0 100644 --- a/backend/utils/batch_pipeline/quarantine_recovery.py +++ b/backend/utils/batch_pipeline/quarantine_recovery.py @@ -109,7 +109,11 @@ def recover(self, segment_id, owner_id): payload = self.stt(item, normalized_binding(job['projection_binding']), audio) if cap_enabled(): record_audio( - 'sync', 'accepted', 'unknown', item.duration_seconds, not funded_allowance(owner_id).managed + 'sync', + 'accepted', + 'unknown', + item.duration_seconds, + funded_allowance(owner_id).reason == 'plan_allowance_exhausted', ) except Exception: self.protocol.settle(segment_id, token, self.clock()) From 9dfdcfa1b2d5b8a8ec8d6778f399c57cdf33b590 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 21:01:44 +0700 Subject: [PATCH 05/19] fix: publish additive held-audio result in client API snapshot --- docs/api-reference/app-client-openapi.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/api-reference/app-client-openapi.json b/docs/api-reference/app-client-openapi.json index 4770912fdb4..5d9063f1700 100644 --- a/docs/api-reference/app-client-openapi.json +++ b/docs/api-reference/app-client-openapi.json @@ -31964,6 +31964,17 @@ "title": "Failed Segments", "type": "integer" }, + "held_audio": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Held Audio" + }, "new_memories": { "items": { "type": "string" From ca343a516d6e3a4cb237593720a9b7ae4c22cf6c Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 21:14:55 +0700 Subject: [PATCH 06/19] fix: regenerate client DTOs for additive held-audio result --- app/lib/backend/schema/gen/conversation_wire.g.dart | 4 ++++ desktop/windows/src/renderer/src/lib/omiApi.generated.ts | 1 + web/admin/lib/services/omi-api/omiApi.generated.ts | 1 + web/app/src/lib/omiApi.generated.ts | 1 + web/personas-open-source/src/lib/omiApi.generated.ts | 1 + 5 files changed, 8 insertions(+) diff --git a/app/lib/backend/schema/gen/conversation_wire.g.dart b/app/lib/backend/schema/gen/conversation_wire.g.dart index 7d95826219b..a872ee03a2f 100644 --- a/app/lib/backend/schema/gen/conversation_wire.g.dart +++ b/app/lib/backend/schema/gen/conversation_wire.g.dart @@ -1669,6 +1669,7 @@ class GeneratedSearchConversationsResponse { class GeneratedSyncLocalFilesResultResponse { final List? errors; final int failedSegments; + final bool? heldAudio; final List? newMemories; final int totalSegments; final List? updatedMemories; @@ -1676,6 +1677,7 @@ class GeneratedSyncLocalFilesResultResponse { const GeneratedSyncLocalFilesResultResponse({ this.errors, this.failedSegments = 0, + this.heldAudio, this.newMemories, this.totalSegments = 0, this.updatedMemories, @@ -1685,6 +1687,7 @@ class GeneratedSyncLocalFilesResultResponse { return GeneratedSyncLocalFilesResultResponse( errors: _readFieldValue>(_readField(json, const ["errors"]), "errors", _readStringList, requiredField: false, nullable: true), failedSegments: _required(_readFieldValue(_readField(json, const ["failed_segments"]), "failed_segments", _readInt, requiredField: false, nullable: false, defaultValue: 0), "failed_segments"), + heldAudio: _readFieldValue(_readField(json, const ["held_audio"]), "held_audio", _readBool, requiredField: false, nullable: true), newMemories: _readFieldValue>(_readField(json, const ["new_memories"]), "new_memories", _readStringList, requiredField: false, nullable: true), totalSegments: _required(_readFieldValue(_readField(json, const ["total_segments"]), "total_segments", _readInt, requiredField: false, nullable: false, defaultValue: 0), "total_segments"), updatedMemories: _readFieldValue>(_readField(json, const ["updated_memories"]), "updated_memories", _readStringList, requiredField: false, nullable: true), @@ -1695,6 +1698,7 @@ class GeneratedSyncLocalFilesResultResponse { return { 'errors': errors, 'failed_segments': failedSegments, + 'held_audio': heldAudio, 'new_memories': newMemories, 'total_segments': totalSegments, 'updated_memories': updatedMemories, diff --git a/desktop/windows/src/renderer/src/lib/omiApi.generated.ts b/desktop/windows/src/renderer/src/lib/omiApi.generated.ts index e56ded387b4..a6c89e83339 100644 --- a/desktop/windows/src/renderer/src/lib/omiApi.generated.ts +++ b/desktop/windows/src/renderer/src/lib/omiApi.generated.ts @@ -5241,6 +5241,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/web/admin/lib/services/omi-api/omiApi.generated.ts b/web/admin/lib/services/omi-api/omiApi.generated.ts index e56ded387b4..a6c89e83339 100644 --- a/web/admin/lib/services/omi-api/omiApi.generated.ts +++ b/web/admin/lib/services/omi-api/omiApi.generated.ts @@ -5241,6 +5241,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/web/app/src/lib/omiApi.generated.ts b/web/app/src/lib/omiApi.generated.ts index e56ded387b4..a6c89e83339 100644 --- a/web/app/src/lib/omiApi.generated.ts +++ b/web/app/src/lib/omiApi.generated.ts @@ -5241,6 +5241,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; diff --git a/web/personas-open-source/src/lib/omiApi.generated.ts b/web/personas-open-source/src/lib/omiApi.generated.ts index e56ded387b4..a6c89e83339 100644 --- a/web/personas-open-source/src/lib/omiApi.generated.ts +++ b/web/personas-open-source/src/lib/omiApi.generated.ts @@ -5241,6 +5241,7 @@ export interface SyncJobStatusResponse { export interface SyncLocalFilesResultResponse { errors?: Array; failed_segments?: number; + held_audio?: boolean | null; new_memories?: Array; total_segments?: number; updated_memories?: Array; From e26e11ce28a9b73df5eab507d4b232acfd02ba31 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 21:18:51 +0700 Subject: [PATCH 07/19] fix: declare generated cap response DTO as internal mobile change --- app/changelog/unreleased/20261016-stt-cap-response-dto.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/changelog/unreleased/20261016-stt-cap-response-dto.json diff --git a/app/changelog/unreleased/20261016-stt-cap-response-dto.json b/app/changelog/unreleased/20261016-stt-cap-response-dto.json new file mode 100644 index 00000000000..bb56a275667 --- /dev/null +++ b/app/changelog/unreleased/20261016-stt-cap-response-dto.json @@ -0,0 +1,3 @@ +{ + "kind": "none" +} From 89120f3d216c1b5c9c8f4261ff7c5d0542b6b050 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 22:03:39 +0700 Subject: [PATCH 08/19] fix: keep cap recovery and policy imports within their owning runtime --- backend/docs/stt-cap-dispatch.md | 4 +++- backend/routers/listen/runtime.py | 2 +- backend/routers/sync.py | 12 ++++++++++++ .../unit/test_locked_content_reset_recovery.py | 6 +++--- backend/tests/unit/test_parakeet_prerecorded.py | 7 +------ backend/tests/unit/test_stt_cap_dispatch.py | 2 +- backend/tests/unit/test_sync_cap_hold.py | 6 +++--- backend/utils/locked_content_recovery.py | 14 ++++++++++---- backend/utils/stt_cap_dispatch.py | 5 ++++- backend/utils/sync/pipeline.py | 7 ++++++- 10 files changed, 44 insertions(+), 21 deletions(-) diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index 6f046934930..dadab4dafa7 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -51,7 +51,9 @@ Concurrent identical uploads cannot overwrite a recovery checkpoint. The existing `locked_content_recovery` authenticated-request path kicks an async recovery pass independently of its once-per-month unlock receipt and legacy reset -flag. A distributed 60-second cooldown bounds requests; token-fenced renewable +flag. The flagged API sync router registers the recovery callback; auth-only and +pusher images do not load the sync pipeline. A distributed 60-second cooldown +bounds requests; token-fenced renewable leases prevent concurrent processing of a hold. At reset or upgrade, current funded allowance must pass again before every recovered piece is sent. Recovery reuses VAD, segment intake, enrichment and optional playback finalization. Piece diff --git a/backend/routers/listen/runtime.py b/backend/routers/listen/runtime.py index 9b1a5c1f150..9e79d0aaa27 100644 --- a/backend/routers/listen/runtime.py +++ b/backend/routers/listen/runtime.py @@ -775,7 +775,7 @@ async def _record_usage_periodically(self) -> None: await self._refresh_credits(transcription_seconds=transcription_seconds) async def _refresh_credits(self, *, transcription_seconds: int = 0) -> None: - admission = getattr(self.receiver, 'cap_admission', None) + admission = getattr(getattr(self, 'receiver', None), 'cap_admission', None) if admission is not None: await admission.refresh() if getattr(self.receiver, 'cap_paused', False) and admission.managed: diff --git a/backend/routers/sync.py b/backend/routers/sync.py index 333a2f0a807..c0803c8cd3e 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -185,6 +185,18 @@ from utils.sync import backfill_cutover, legacy_fresh, stage as sync_stage, uid_sequencer from utils.sync.provenance import capture_matches_server_conversation as _capture_matches_server_conversation +# Only the API process importing this router owns held-audio STT recovery. +# Pusher/auth-only images retain the lightweight locked-content recovery path. +if cap_dispatch_enabled(): + from utils.locked_content_recovery import register_cap_audio_recovery + + def _kick_cap_hold_recovery(uid: str) -> None: + from utils.sync.cap_hold_recovery import kick + + kick(uid) + + register_cap_audio_recovery(_kick_cap_hold_recovery) + logger = logging.getLogger(__name__) # Audio constants diff --git a/backend/tests/unit/test_locked_content_reset_recovery.py b/backend/tests/unit/test_locked_content_reset_recovery.py index 2775a2f5a4e..649e9fd07e6 100644 --- a/backend/tests/unit/test_locked_content_reset_recovery.py +++ b/backend/tests/unit/test_locked_content_reset_recovery.py @@ -304,8 +304,8 @@ def test_cap_audio_recovery_is_kicked_even_with_legacy_reset_flag_off(env, monke fake.kick = Mock() monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') monkeypatch.setenv('LOCKED_CONTENT_RESET_RECOVERY_ENABLED', 'false') - with stub_modules({'utils.sync.cap_hold_recovery': fake}): - env.module.recover_locked_content('test-user', now=NOW) - env.module.recover_locked_content('test-user', now=NOW) + env.module.register_cap_audio_recovery(fake.kick) + env.module.recover_locked_content('test-user', now=NOW) + env.module.recover_locked_content('test-user', now=NOW) assert fake.kick.call_count == 2 env.state.get.assert_not_called() diff --git a/backend/tests/unit/test_parakeet_prerecorded.py b/backend/tests/unit/test_parakeet_prerecorded.py index 2dd0e218484..d0825d35cf6 100644 --- a/backend/tests/unit/test_parakeet_prerecorded.py +++ b/backend/tests/unit/test_parakeet_prerecorded.py @@ -18,6 +18,7 @@ os.environ.setdefault('HOSTED_PARAKEET_API_URL', 'http://fake-parakeet:8080') import utils.stt.pre_recorded as pr # noqa: E402 +from utils.stt.streaming import STTService, get_stt_service_for_language # noqa: E402 def _cdist(a, b, metric='cosine'): @@ -522,8 +523,6 @@ def make_stream_resp(): class TestStreamingFactoryRouting: def test_parakeet_in_stt_service_models(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet']), patch.dict( os.environ, {'HOSTED_PARAKEET_API_URL': 'http://fake-parakeet:8080'} ): @@ -532,8 +531,6 @@ def test_parakeet_in_stt_service_models(self): assert model == 'parakeet' def test_parakeet_streaming_fallback_for_cjk_uses_deepgram(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet', 'dg-nova-3']), patch.dict( os.environ, {'HOSTED_PARAKEET_API_URL': 'http://fake-parakeet:8080'} ): @@ -542,8 +539,6 @@ def test_parakeet_streaming_fallback_for_cjk_uses_deepgram(self): assert model == 'nova-3' def test_parakeet_fallback_without_url_uses_policy_default(self): - from utils.stt.streaming import STTService, get_stt_service_for_language - with patch('utils.stt.streaming.stt_service_models', ['parakeet']), patch.dict(os.environ, {}, clear=False): env_backup = os.environ.pop('HOSTED_PARAKEET_API_URL', None) try: diff --git a/backend/tests/unit/test_stt_cap_dispatch.py b/backend/tests/unit/test_stt_cap_dispatch.py index 0a37d18a32f..39c5c8b097f 100644 --- a/backend/tests/unit/test_stt_cap_dispatch.py +++ b/backend/tests/unit/test_stt_cap_dispatch.py @@ -101,7 +101,7 @@ async def test_held_constructor_resumes_after_reset_without_reopening_capture(mo def test_queued_sync_always_resolves_omi_funding(managed, monkeypatch): monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') resolver = Mock(return_value=SimpleNamespace(managed=managed, reason='plan_allowance_exhausted')) - monkeypatch.setattr(cap, 'resolve_transcription_allowance', resolver) + monkeypatch.setattr('utils.subscription.resolve_transcription_allowance', resolver) assert cap.sync_allowed('synthetic', 60, 'parakeet') is managed resolver.assert_called_once_with('synthetic', source=None, byok_active=False, omi_funded=True) diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py index 292234cddd6..dc5e0fc93af 100644 --- a/backend/tests/unit/test_sync_cap_hold.py +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -7,7 +7,7 @@ import pytest -from utils.sync import pipeline +from utils.sync import cap_hold, pipeline from utils import stt_cap_dispatch as cap @@ -24,7 +24,7 @@ def test_exhaustion_holds_before_url_or_provider_dispatch(tmp_path, monkeypatch, cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) ) hold, url, stt = Mock(), Mock(return_value='synthetic-url'), Mock(return_value=([], 'en')) - monkeypatch.setattr(pipeline, 'hold_segment', hold) + monkeypatch.setattr(cap_hold, 'hold_segment', hold) monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', url) monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) monkeypatch.setattr(pipeline, 'keyterms', AsyncMock(return_value=[])) @@ -61,7 +61,7 @@ def test_hold_failure_cannot_acknowledge_audio_or_dispatch_provider(tmp_path, mo monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) monkeypatch.setattr(pipeline, 'sync_allowed', lambda *_: False) - monkeypatch.setattr(pipeline, 'hold_segment', Mock(side_effect=RuntimeError('synthetic upload failed'))) + monkeypatch.setattr(cap_hold, 'hold_segment', Mock(side_effect=RuntimeError('synthetic upload failed'))) stt = Mock() monkeypatch.setattr(pipeline, 'prerecorded', stt) errors = [] diff --git a/backend/utils/locked_content_recovery.py b/backend/utils/locked_content_recovery.py index 8b5f83f2b19..880a9abcf00 100644 --- a/backend/utils/locked_content_recovery.py +++ b/backend/utils/locked_content_recovery.py @@ -7,6 +7,7 @@ import logging from datetime import datetime, timezone from uuid import uuid4 +from typing import Callable from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled from utils.locked_content_flags import reset_recovery_enabled @@ -15,6 +16,13 @@ PAGE_SIZE = 100 LEASE_SECONDS = 300 RETRY_SECONDS = 60 +_cap_audio_recovery: Callable[[str], None] | None = None + + +def register_cap_audio_recovery(callback: Callable[[str], None]) -> None: + """The API sync owner supplies STT recovery; lean auth consumers do not.""" + global _cap_audio_recovery + _cap_audio_recovery = callback def _memory_page_drained(count: int, page_size: int) -> bool: @@ -32,11 +40,9 @@ def _memory_page_drained(count: int, page_size: int) -> bool: def recover_locked_content(uid: str, *, now: datetime | None = None) -> None: - if stt_cap_dispatch_enabled(): - from utils.sync.cap_hold_recovery import kick - + if stt_cap_dispatch_enabled() and _cap_audio_recovery is not None: try: - kick(uid) + _cap_audio_recovery(uid) except Exception as error: logger.warning('event=stt_cap_recovery outcome=schedule_deferred exception_type=%s', type(error).__name__) if not reset_recovery_enabled(): diff --git a/backend/utils/stt_cap_dispatch.py b/backend/utils/stt_cap_dispatch.py index 5dd4614d083..9a8d202b243 100644 --- a/backend/utils/stt_cap_dispatch.py +++ b/backend/utils/stt_cap_dispatch.py @@ -12,7 +12,6 @@ from utils.observability.stt_cap_dispatch import record_audio from utils.byok import get_byok_key from utils.executors import db_executor, run_blocking -from utils.subscription import resolve_transcription_allowance logger = logging.getLogger(__name__) @@ -26,6 +25,10 @@ def customer_funded(provider): def funded_allowance(uid, source=None): + # Streaming imports this module even with enforcement off. Load billing + # policy only at the admission boundary, not during provider-module import. + from utils.subscription import resolve_transcription_allowance + return resolve_transcription_allowance(uid, source=source, byok_active=False, omi_funded=True) diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 77f9129f4b0..6834ccde5c4 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -137,7 +137,6 @@ from config.stt_cap_dispatch import enabled as stt_cap_dispatch_enabled from utils.stt_cap_dispatch import sync_allowed, funded_allowance from utils.observability.stt_cap_dispatch import record_audio as record_cap_audio -from utils.sync.cap_hold import hold_segment, hold_capture_if_exhausted from utils.stt.pre_recorded import get_prerecorded_service, postprocess_words, prerecorded from utils.stt.outcomes import ( TranscriptionFailure, @@ -1257,6 +1256,8 @@ def process_segment( if deferred_outcome is not None: deferred_outcome['cap_held'] = True if not cap_recovery: + from utils.sync.cap_hold import hold_segment + hold_segment( uid, path, @@ -1326,6 +1327,8 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N if deferred_outcome is not None: deferred_outcome['cap_held'] = True if not cap_recovery: + from utils.sync.cap_hold import hold_segment + hold_segment( uid, path, @@ -2312,6 +2315,8 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT return cap_held = False if stt_cap_dispatch_enabled(): + from utils.sync.cap_hold import hold_capture_if_exhausted + cap_held = await run_blocking( sync_executor, hold_capture_if_exhausted, From 550e565fb21d5cab1ff997dc7865a24d028a8701 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 22:59:02 +0700 Subject: [PATCH 09/19] fix: preserve held sync ownership and recovery provenance --- ...tt-allowance-not-enforced-at-dispatch.json | 2 +- backend/database/sync_cap_holds.py | 106 ++++++++++ backend/database/sync_ledger.py | 33 +++ backend/docs/stt-cap-dispatch.md | 32 ++- backend/routers/sync.py | 24 ++- backend/tests/unit/test_stt_cap_dispatch.py | 19 ++ .../tests/unit/test_stt_cap_dispatch_http.py | 38 ++++ backend/tests/unit/test_sync_cap_content.py | 99 +++++++++ .../tests/unit/test_sync_cap_coordinator.py | 113 ++++++++++ backend/tests/unit/test_sync_cap_hold.py | 67 +++++- .../tests/unit/test_sync_cap_hold_recovery.py | 83 +++++++- .../unit/test_sync_cap_recovery_audio.py | 59 ++++++ backend/utils/stt_cap_dispatch.py | 13 ++ backend/utils/sync/cap_hold.py | 124 +++++++++-- backend/utils/sync/cap_hold_recovery.py | 193 +++++++++++++----- backend/utils/sync/cap_recovery_audio.py | 88 ++++++++ backend/utils/sync/pipeline.py | 124 ++++++++--- 17 files changed, 1100 insertions(+), 117 deletions(-) create mode 100644 backend/database/sync_cap_holds.py create mode 100644 backend/tests/unit/test_sync_cap_content.py create mode 100644 backend/tests/unit/test_sync_cap_coordinator.py create mode 100644 backend/tests/unit/test_sync_cap_recovery_audio.py create mode 100644 backend/utils/sync/cap_recovery_audio.py diff --git a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json index 3fd646d6b5a..1df2d5587e3 100644 --- a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json +++ b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json @@ -3,7 +3,7 @@ "id": "FC-stt-allowance-not-enforced-at-dispatch", "violated_contract": "The transcription cap is computed but not enforced where Omi-funded provider audio is dispatched; replay, fallback and delayed execution can bypass request-time allowance checks.", "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it and exercise both flag states at provider and queue boundaries.", - "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py"], + "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_sync_cap_content.py"], "evidence_prs": [], "scope_hints": ["backend/routers/listen/", "backend/utils/stt/", "backend/utils/sync/", "backend/utils/batch_pipeline/"], "status": "open" diff --git a/backend/database/sync_cap_holds.py b/backend/database/sync_cap_holds.py new file mode 100644 index 00000000000..011cd82ad15 --- /dev/null +++ b/backend/database/sync_cap_holds.py @@ -0,0 +1,106 @@ +"""Transfer fenced sync-ledger ownership to durable, recoverable audio holds. + +A held upload is acknowledged, but is not proof of completed transcription. +Only settlement of every referenced hold can publish that proof. +""" + +from datetime import datetime, timedelta, timezone +from typing import Any + +from google.cloud import firestore + +from database._client import get_firestore_client +from database.sync_ledger import ( + sync_content_reference, + sync_content_owner_matches, + is_valid_completed_sync_content_result, + is_valid_held_sync_content_result, + LEDGER_RETENTION_DAYS, +) + +PROTOCOL = 'stt_cap_hold_v1' + + +def mark_content_held( + uid: str, + content_id: str, + job_id: str, + result: dict, + hold_ids: list[str], + *, + run_token: str | None = None, + run_epoch: int | None = None, +) -> bool: + if not hold_ids or not is_valid_held_sync_content_result(result): + raise ValueError('durable held audio receipts required') + client = get_firestore_client() + ref = sync_content_reference(client, uid, content_id) + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if not sync_content_owner_matches(row, job_id, run_token, run_epoch): + return False + tx.update( + ref, + { + 'status': 'held', + 'dispatch_protocol': PROTOCOL, + 'result': result, + 'cap_hold_ids': sorted(set(hold_ids)), + 'pending_cap_hold_ids': sorted(set(hold_ids)), + 'updated_at': datetime.now(timezone.utc), + # An unprocessed capture must survive inactive accounts. + 'expires_at': firestore.DELETE_FIELD, + }, + ) + return True + + return commit(client.transaction()) + + +def recovery_status(uid: str, content_id: str, hold_id: str) -> str: + row = sync_content_reference(get_firestore_client(), uid, content_id).get().to_dict() or {} + if row.get('status') == 'completed' and is_valid_completed_sync_content_result(row.get('result')): + return 'completed' + if row.get('dispatch_protocol') == PROTOCOL and row.get('status') == 'held': + if hold_id not in row.get('cap_hold_ids', []): + return 'waiting' + return 'ready' if hold_id in row.get('pending_cap_hold_ids', []) else 'settled' + return 'waiting' # The original worker still owns preparation/retry. + + +def settle_hold(uid: str, content_id: str, hold_id: str, response: dict, pieces: int) -> bool: + """Idempotently join one recovered hold; never overwrite another owner.""" + client = get_firestore_client() + ref = sync_content_reference(client, uid, content_id) + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if row.get('status') == 'completed': + return is_valid_completed_sync_content_result(row.get('result')) + if row.get('dispatch_protocol') != PROTOCOL or row.get('status') != 'held': + return False + if hold_id not in row.get('cap_hold_ids', []): + return False + pending = set(row.get('pending_cap_hold_ids') or []) + if hold_id not in pending: + return True + pending.remove(hold_id) + result = dict(row['result']) + for key in ('new_memories', 'updated_memories'): + result[key] = sorted(set(result.get(key) or []) | set(response.get(key) or [])) + result['recovered_segments'] = int(result.get('recovered_segments', 0)) + pieces + result['held_audio'] = bool(pending) + now = datetime.now(timezone.utc) + updates: dict[str, Any] = {'result': result, 'pending_cap_hold_ids': sorted(pending), 'updated_at': now} + if not pending: + # All source audio has reached content/usage settlement; only now + # can ordinary duplicate admission converge a completed result. + result.update(failed_segments=0, errors=[], outcome='success') + updates.update(status='completed', expires_at=now + timedelta(days=LEDGER_RETENTION_DAYS)) + tx.update(ref, updates) + return True + + return commit(client.transaction()) diff --git a/backend/database/sync_ledger.py b/backend/database/sync_ledger.py index a54e57422f3..d22ad59d37e 100644 --- a/backend/database/sync_ledger.py +++ b/backend/database/sync_ledger.py @@ -73,6 +73,7 @@ class SyncContentRunBindingOutcome(str, Enum): BOUND = 'bound' COMPLETED = 'completed' + HELD = 'held' LOST = 'lost' @@ -91,6 +92,10 @@ def bound(self) -> bool: def completed(self) -> bool: return self.outcome is SyncContentRunBindingOutcome.COMPLETED + @property + def held(self) -> bool: + return self.outcome is SyncContentRunBindingOutcome.HELD + def _ledger_ref(client: Any, uid: str, content_id: str) -> Any: collection = sync_stage.collection_name('sync_content_ledger') @@ -122,10 +127,24 @@ def _ledger_owner_matches( return bound_token == run_token and bound_epoch == run_epoch +def sync_content_reference(client: Any, uid: str, content_id: str) -> Any: + """Share the canonical owner-scoped reference with a custody transfer.""" + return _ledger_ref(client, uid, content_id) + + +def sync_content_owner_matches( + existing: Dict[str, Any], job_id: str, run_token: str | None, run_epoch: int | None = None +) -> bool: + """Custody transfer uses the same token/epoch fence as normal completion.""" + return _ledger_owner_matches(existing, job_id, run_token, run_epoch) + + def is_valid_completed_sync_content_result(result: Any) -> bool: """A ledger completion may converge Redis only for an all-success result.""" if not isinstance(result, dict): return False + if result.get('held_audio') is True: + return False failed_segments = result.get('failed_segments') total_segments = result.get('total_segments') errors = result.get('errors') @@ -152,6 +171,12 @@ def is_valid_completed_sync_content_result(result: Any) -> bool: return outcome in {'success', 'expected_silence'} +def is_valid_held_sync_content_result(result: Any) -> bool: + if not isinstance(result, dict) or result.get('held_audio') is not True: + return False + return is_valid_completed_sync_content_result({**result, 'held_audio': False}) + + def _processing_claim_updates( job_id: str, lane: str, @@ -193,6 +218,10 @@ def _claim_transaction( ) -> Dict[str, Any]: snapshot = ref.get(transaction=transaction) existing = cast(Dict[str, Any], snapshot.to_dict() or {}) if getattr(snapshot, 'exists', False) else {} + if existing.get('dispatch_protocol') == 'stt_cap_hold_v1' and existing.get('status') == 'held': + if is_valid_held_sync_content_result(existing.get('result')): + return {'outcome': 'held', 'result': existing['result'], 'job_id': existing.get('job_id')} + return {'outcome': 'busy', 'job_id': existing.get('job_id')} if existing.get('dispatch_protocol') == 'batch_v1': result = existing.get('result') if existing.get('status') == 'completed' and is_valid_completed_sync_content_result(result): @@ -293,6 +322,10 @@ def _bind_run_token_transaction( """ snapshot = ref.get(transaction=transaction) existing = cast(Dict[str, Any], snapshot.to_dict() or {}) if getattr(snapshot, 'exists', False) else {} + if existing.get('dispatch_protocol') == 'stt_cap_hold_v1' and existing.get('status') == 'held': + if is_valid_held_sync_content_result(existing.get('result')): + return SyncContentRunBinding(SyncContentRunBindingOutcome.HELD, existing['result']) + return SyncContentRunBinding(SyncContentRunBindingOutcome.LOST) if existing.get('dispatch_protocol') == 'batch_v1': result = existing.get('result') if existing.get('status') == 'completed' and is_valid_completed_sync_content_result(result): diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index dadab4dafa7..9afbdae5682 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -25,7 +25,10 @@ including self-hosted Deepgram and an Omi-funded fallback from BYOK, uses the ordinary allowance with both BYOK quota and desktop-paywall exemptions disabled. No key or identity is a metric label. -Sync v1 intake and v2 execution hold full decoded WAVs before VAD when denied. +Sync v1 intake and v2 execution hold decoded WAVs before VAD when denied. V2 +applies ordinary source-frame and safety-WAL coverage first, so already committed +audio is not enrolled as new recoverable work. Source maps, recording identity +and geolocation travel with the encrypted hold. The shared segment processor rechecks immediately before each STT invocation, including its empty-result retry. Late exhaustion holds the remaining segment. No denied segment reaches cloud STT. A successfully persisted hold is acknowledged @@ -49,14 +52,33 @@ account deletion recursively removes the manifest and purges this chunk prefix. A failed object upload cannot publish a successful manifest or acknowledgment. Concurrent identical uploads cannot overwrite a recovery checkpoint. +WAV hashing, encryption, upload, download and decryption use bounded chunks; +large decoded recordings are never materialized as whole-file byte buffers. +Large source-map metadata uses an encrypted sidecar under the same owner prefix. +Late per-segment denials are aggregated into the v1/v2 held result and excluded +from post-STT subscription/DG quantities until recovered. Root meter writers and +all flag-off call semantics remain unchanged. + +V2 transfers the fenced content ledger into `held`, without a TTL or a completed +transcription proof. Duplicate intake can acknowledge custody without dispatching +another provider. A preparing/failed source job still owns its retry; its holds +cannot race it into STT. Only settlement of every referenced hold completes the +content ledger. Holds that were never acknowledged await the source job's retry. + The existing `locked_content_recovery` authenticated-request path kicks an async recovery pass independently of its once-per-month unlock receipt and legacy reset flag. The flagged API sync router registers the recovery callback; auth-only and pusher images do not load the sync pipeline. A distributed 60-second cooldown -bounds requests; token-fenced renewable -leases prevent concurrent processing of a hold. At reset or upgrade, current -funded allowance must pass again before every recovered piece is sent. Recovery -reuses VAD, segment intake, enrichment and optional playback finalization. Piece +bounds requests; token-fenced renewable leases prevent concurrent processing of +a hold. At reset or upgrade, current +funded allowance and the existing hard abuse restrictions must pass again before +every new recovered provider request. Already persisted receipts can settle at +the cap without another provider call. Recovery +replays WAL coverage and lineage planning, loads speaker embeddings, and reuses +VAD, segment intake, enrichment and optional playback finalization. Token-fenced +piece receipts checkpoint persisted transcription before usage settlement; a +usage retry cannot repeat paid STT. Finalization runs once after all pieces join, +with the retained geolocation and source-position context. Piece receipts and idempotent usage keys preserve progress across retries. Completed objects are deleted only after content and usage settlement. WS-only and inactive accounts wait for their next authenticated HTTP request; there is no new cron. diff --git a/backend/routers/sync.py b/backend/routers/sync.py index c0803c8cd3e..a8111c22270 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -815,6 +815,9 @@ def _run_vad(path): ] ) + cap_holds = response.get('_cap_holds', {}) if cap_dispatch_enabled() else {} + cap_held_seconds = sum(item['seconds'] for item in cap_holds.values()) + await run_blocking(sync_executor, _reprocess_merged_conversations, uid, response) if private_cloud_sync_enabled: await run_blocking(sync_executor, _finalize_sync_audio_files, uid, response) @@ -823,7 +826,7 @@ def _run_vad(path): # Record DG usage after successful processing (not before, to avoid charging on retries) if fair_use_restrict_dg: try: - dg_ms = int(total_speech_seconds * 1000) + dg_ms = int(max(0, total_speech_seconds - cap_held_seconds) * 1000) if dg_ms > 0: await run_blocking(db_executor, record_dg_usage_ms, uid, dg_ms) except Exception as e: @@ -843,9 +846,13 @@ def _run_vad(path): 'new_memories': sorted(response['new_memories']), 'updated_memories': sorted(response['updated_memories']), } + if cap_holds: + result.update(held_audio=True, credits_exhausted=True, held_segments=len(cap_holds)) failed_segments = len(segment_errors) successful_segments = total_segments - failed_segments + if cap_holds: + successful_segments -= len(cap_holds) if failed_segments > 0: result['failed_segments'] = failed_segments @@ -858,15 +865,18 @@ def _run_vad(path): if total_segments > 0 and successful_segments == 0: # All segments failed — return 500 (consistent with VAD error behavior) - raise HTTPException( - status_code=500, - detail=f"All {total_segments} segment(s) failed processing: {'; '.join(segment_errors[:3])}", - headers=_V1_DEPRECATION_HEADERS, - ) + if not cap_holds: + raise HTTPException( + status_code=500, + detail=f"All {total_segments} segment(s) failed processing: {'; '.join(segment_errors[:3])}", + headers=_V1_DEPRECATION_HEADERS, + ) # Record subscription usage only when at least one segment succeeded try: usage_seconds = int(total_speech_seconds) + if cap_holds: + usage_seconds = int(max(0, total_speech_seconds - cap_held_seconds)) if successful_segments > 0 else 0 if usage_seconds > 0: await run_blocking( db_executor, record_usage, uid, transcription_seconds=usage_seconds, speech_seconds=usage_seconds @@ -1193,7 +1203,7 @@ async def sync_local_files_v2( job_id, lane_decision.lane.value, ) - if claim.get('outcome') == 'completed': + if claim.get('outcome') in {'completed', 'held'}: cached_result = claim.get('result') or {} await run_blocking(db_executor, mark_job_completed, job_id, cached_result) if backfill_slot_acquired: diff --git a/backend/tests/unit/test_stt_cap_dispatch.py b/backend/tests/unit/test_stt_cap_dispatch.py index 39c5c8b097f..60d2b31bbfa 100644 --- a/backend/tests/unit/test_stt_cap_dispatch.py +++ b/backend/tests/unit/test_stt_cap_dispatch.py @@ -97,6 +97,25 @@ async def test_held_constructor_resumes_after_reset_without_reopening_capture(mo raw.finish.assert_called_once() +async def test_resumed_registry_leg_unwraps_nested_provider_and_debits_once(monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = cap.LiveAdmission('synthetic') + raw = Mock(send=Mock(return_value=True)) + inner = cap.live_provider('modulate', 0)(AsyncMock(return_value=raw)) + socket = cap.CapSocket(state, 'modulate', 16000, lambda: inner(16000)) + state.managed, state.remaining = True, 2 + token = cap.current_live_admission.set(state) + try: + await socket.resume() + assert socket.send(b'\0' * 32000) + finally: + cap.current_live_admission.reset(token) + assert socket.raw is raw + raw.send.assert_called_once() + assert state.remaining == 1 + assert cap.record_audio.call_count == 1 + + @pytest.mark.parametrize('managed', [False, True]) def test_queued_sync_always_resolves_omi_funding(managed, monkeypatch): monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') diff --git a/backend/tests/unit/test_stt_cap_dispatch_http.py b/backend/tests/unit/test_stt_cap_dispatch_http.py index dd169a46900..f3bdf2ca188 100644 --- a/backend/tests/unit/test_stt_cap_dispatch_http.py +++ b/backend/tests/unit/test_stt_cap_dispatch_http.py @@ -96,3 +96,41 @@ def test_flag_off_http_result_retains_main_shape_without_hold_dispatch(client, m assert response.status_code == 200, response.text assert response.content == b'{"new_memories":[],"updated_memories":[]}' router.hold_capture_if_exhausted.assert_not_called() + + +@pytest.mark.parametrize('accepted', [False, True]) +def test_late_v1_hold_is_reported_and_only_processed_speech_is_metered(client, monkeypatch, accepted): + monkeypatch.setattr(router, 'FAIR_USE_ENABLED', False) + router.hold_capture_if_exhausted.return_value = False + paths = ['1792152000.wav', '1792152020.wav'] if accepted else ['1792152020.wav'] + monkeypatch.setattr(router, 'retrieve_vad_segments', lambda path, segments, errors, **kw: segments.update(paths)) + monkeypatch.setattr(router, 'get_wav_duration', lambda path: 20) + monkeypatch.setattr(router.users_db, 'get_user_transcription_preferences', Mock(return_value={})) + monkeypatch.setattr(router.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) + monkeypatch.setattr(router, 'build_person_embeddings_cache', Mock(return_value={})) + monkeypatch.setattr(router, '_reprocess_merged_conversations', Mock()) + monkeypatch.setattr(router, 'schedule_person_voice_learning_retries', Mock()) + usage = Mock() + monkeypatch.setattr(router, 'record_usage', usage) + + def process(path, uid, response, *args, **kwargs): + if path == '1792152000.wav': + response['new_memories'].add('accepted') + return True + response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 20}} + return False + + monkeypatch.setattr(router, 'process_segment', process) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json() == { + 'new_memories': ['accepted'] if accepted else [], + 'updated_memories': [], + 'held_audio': True, + 'credits_exhausted': True, + 'held_segments': 1, + } + if accepted: + assert usage.call_args.kwargs == {'transcription_seconds': 20, 'speech_seconds': 20} + else: + usage.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_content.py b/backend/tests/unit/test_sync_cap_content.py new file mode 100644 index 00000000000..859533efdea --- /dev/null +++ b/backend/tests/unit/test_sync_cap_content.py @@ -0,0 +1,99 @@ +"""A held source is durable custody, never a completed-transcription proof.""" + +from copy import deepcopy + +import pytest + +from database import sync_cap_holds as holds, sync_ledger as ledger +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore + + +@pytest.fixture +def state(monkeypatch): + monkeypatch.setenv('OMI_ENV_STAGE', 'prod') + client = StrictFirestore() + ref = ledger.sync_content_reference(client, 'synthetic', 'content') + row = { + 'status': 'processing', + 'job_id': 'original', + 'ledger_run_token': 'owner-token', + 'ledger_run_epoch': 1, + 'expires_at': 'old-ttl', + } + client.rows[ref.path] = row + monkeypatch.setattr(holds, 'get_firestore_client', lambda: client) + return client, ref + + +def _hold(): + return holds.mark_content_held( + 'synthetic', + 'content', + 'original', + { + 'held_audio': True, + 'new_memories': [], + 'updated_memories': [], + 'failed_segments': 0, + 'total_segments': 0, + 'errors': [], + 'outcome': 'success', + }, + ['a', 'b'], + run_token='owner-token', + run_epoch=1, + ) + + +def test_hold_is_fenced_and_never_manufactures_completion(state): + client, ref = state + before = deepcopy(client.rows[ref.path]) + assert not holds.mark_content_held( + 'synthetic', + 'content', + 'stale', + {'held_audio': True, 'failed_segments': 0, 'total_segments': 0, 'errors': [], 'outcome': 'success'}, + ['a'], + run_token='stale', + run_epoch=0, + ) + assert client.rows[ref.path] == before + assert _hold() + row = client.rows[ref.path] + assert row['status'] == 'held' + # The strict transaction fixture records transforms rather than applying + # DELETE_FIELD; assert the removal request, not a simulated TTL deletion. + assert str(row['expires_at']) == str(holds.firestore.DELETE_FIELD) + assert not ledger.is_valid_completed_sync_content_result(row['result']) + claim = ledger.claim_sync_content('synthetic', 'content', 'retry', 'fresh', firestore_client=client) + assert claim['outcome'] == 'held' and claim['result']['held_audio'] + assert holds.recovery_status('synthetic', 'content', 'a') == 'ready' + + +def test_only_all_recovered_holds_can_complete_and_settlement_is_idempotent(state): + client, ref = state + assert _hold() + assert holds.settle_hold('synthetic', 'content', 'a', {'new_memories': ['one']}, 2) + first = deepcopy(client.rows[ref.path]) + assert first['status'] == 'held' + assert first['result']['held_audio'] + assert holds.settle_hold('synthetic', 'content', 'a', {'new_memories': ['wrong']}, 99) + assert client.rows[ref.path] == first + assert holds.settle_hold('synthetic', 'content', 'b', {'updated_memories': ['two']}, 1) + row = client.rows[ref.path] + assert row['status'] == 'completed' + assert row['result']['new_memories'] == ['one'] + assert row['result']['updated_memories'] == ['two'] + assert row['result']['recovered_segments'] == 3 + assert ledger.is_valid_completed_sync_content_result(row['result']) + assert 'expires_at' in row + + +def test_recovery_does_not_overwrite_a_different_content_owner(state): + client, ref = state + assert _hold() + client.rows[ref.path].update(status='processing', dispatch_protocol='replacement', job_id='replacement') + before = deepcopy(client.rows[ref.path]) + assert holds.recovery_status('synthetic', 'content', 'a') == 'waiting' + assert not holds.settle_hold('synthetic', 'content', 'a', {}, 1) + assert client.rows[ref.path] == before diff --git a/backend/tests/unit/test_sync_cap_coordinator.py b/backend/tests/unit/test_sync_cap_coordinator.py new file mode 100644 index 00000000000..b0f447697e7 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_coordinator.py @@ -0,0 +1,113 @@ +"""Exercise real v2 coordination: coverage precedes custody and late holds do not bill.""" + +from pathlib import Path +from unittest.mock import Mock + +from database import sync_cap_holds +from utils.sync import cap_hold +from tests.unit.test_wal_audio_coverage_pipeline import ( + coordinator, + _wire, + _coverage_env, + _committed_envelope, + _claim, + _read_payload, + _write_labeled_wav, + _frame_bytes, + FRAME_SAMPLES, + BIN_NAME, + ORIGIN, + generation, +) + + +async def _run(module, tmp_path): + await module._run_full_pipeline_background_async( + 'job-coverage', + 'uid', + [f'/tmp/{BIN_NAME}'], + 'omi', + False, + str(tmp_path / 'job'), + client_device_id='pendant', + recording_session_id=ORIGIN, + task_mode=True, + content_id='content', + capture_evidence_claims={BIN_NAME: _claim()}, + ) + + +def _ledger(pipeline, monkeypatch): + pipeline.get_sync_content_partial_result = Mock(return_value={}) + pipeline.get_processed_sync_segment_ids = Mock(return_value=set()) + pipeline.compute_sync_segment_id = lambda uid, path: Path(path).name + pipeline.checkpoint_sync_content_partial_result = Mock(return_value=True) + pipeline.mark_sync_content_completed = Mock(return_value=True) + writer = Mock(return_value=True) + monkeypatch.setattr(sync_cap_holds, 'mark_content_held', writer) + return writer + + +async def test_full_hold_contains_only_wal_audio_not_already_committed(coordinator, monkeypatch, tmp_path): + module, stubs = coordinator + pipeline = stubs['pipeline'] + _coverage_env(monkeypatch) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + state = _wire( + pipeline, + monkeypatch, + tmp_path, + lineage_rows=[generation(1, capture_evidence=_committed_envelope(range(7)))], + ) + writer = _ledger(pipeline, monkeypatch) + monkeypatch.setattr(cap_hold, 'sync_allowed', lambda uid: False) + retained = [] + + def retain(uid, path, metadata, *, receipt): + retained.append((_read_payload(path), metadata)) + receipt.update(id='held-source', seconds=1.5) + + monkeypatch.setattr(cap_hold, 'hold_segment', retain) + await _run(module, tmp_path) + assert len(retained) == 1 + assert retained[0][0] == b''.join(_frame_bytes(v, FRAME_SAMPLES) for v in (7, 8, 9)) + assert retained[0][1]['source_frame_map']['claim']['source_frame_start'] == 7 + assert retained[0][1]['recording_session_id'] == ORIGIN + assert state.vad_seen == state.processed == [] + pipeline.record_usage.assert_not_called() + pipeline.mark_sync_content_completed.assert_not_called() + assert writer.call_args.args[3]['held_audio'] is True + assert writer.call_args.kwargs['hold_ids'] == ['held-source'] + + +async def test_late_hold_reports_custody_and_charges_only_dispatched_speech(coordinator, monkeypatch, tmp_path): + module, stubs = coordinator + pipeline = stubs['pipeline'] + _coverage_env(monkeypatch) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + _wire(pipeline, monkeypatch, tmp_path, lineage_rows=[]) + writer = _ledger(pipeline, monkeypatch) + monkeypatch.setattr(cap_hold, 'sync_allowed', lambda uid: True) + paths = [tmp_path / '1760000000.wav', tmp_path / '1760000002.5.wav'] + for path in paths: + _write_labeled_wav(path, range(5)) + pipeline.retrieve_vad_segments = lambda path, segments, errors, **kw: segments.update(map(str, paths)) + + def process(path, uid, response, lock, errors, *args, **kwargs): + if path == str(paths[0]): + response['updated_memories'].add('accepted') + return True + response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 2.5}} + kwargs['deferred_outcome']['cap_held'] = True + return False + + pipeline.process_segment = process + await _run(module, tmp_path) + pipeline.mark_sync_content_completed.assert_not_called() + result = writer.call_args.args[3] + assert result['held_audio'] is True + assert result['failed_segments'] == 0 + assert result['updated_memories'] == ['accepted'] + assert writer.call_args.kwargs['hold_ids'] == ['late'] + assert pipeline.record_usage.call_args.kwargs['transcription_seconds'] == 2 + assert pipeline.record_usage.call_args.kwargs['speech_seconds'] == 2 diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py index dc5e0fc93af..0de3dc2a62f 100644 --- a/backend/tests/unit/test_sync_cap_hold.py +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -23,7 +23,8 @@ def test_exhaustion_holds_before_url_or_provider_dispatch(tmp_path, monkeypatch, monkeypatch.setattr( cap, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')) ) - hold, url, stt = Mock(), Mock(return_value='synthetic-url'), Mock(return_value=([], 'en')) + hold = Mock(side_effect=lambda *a, **kw: kw['receipt'].update(id='held', seconds=1)) + url, stt = Mock(return_value='synthetic-url'), Mock(return_value=([], 'en')) monkeypatch.setattr(cap_hold, 'hold_segment', hold) monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', url) monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) @@ -82,7 +83,13 @@ def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, mon paths = [str(tmp_path / '1792152000.wav')] assert cap_hold.hold_capture_if_exhausted('synthetic', paths, ConversationSource.omi) is enforce if enforce: - assert hold.call_args.args[2] == {'timestamp': 1792152000, 'source': 'omi', 'full_capture': True} + assert hold.call_args.args[2] == { + 'timestamp': 1792152000, + 'source': 'omi', + 'full_capture': True, + 'file_name': '1792152000.wav', + 'source_frame_map': None, + } else: hold.assert_not_called() cap_hold.sync_allowed.assert_not_called() @@ -119,9 +126,15 @@ def test_hold_persists_encrypted_audio_before_manifest_and_never_expires(tmp_pat wav.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) wav.writeframes(b'\0' * 32000) order = [] + uploaded = [] ref, blob, bucket = Mock(), Mock(), Mock() ref.get.return_value.exists = False - blob.upload_from_string.side_effect = lambda *args, **kwargs: order.append('upload') + + def upload(stream, **kwargs): + uploaded.append(stream.read()) + order.append('upload') + + blob.upload_from_file.side_effect = upload ref.create.side_effect = lambda *args, **kwargs: order.append('manifest') monkeypatch.setattr(cap_hold, 'holds', lambda _: SimpleNamespace(document=lambda _: ref)) monkeypatch.setattr(cap_hold, 'get_private_cloud_sync_bucket', lambda: bucket) @@ -132,13 +145,57 @@ def test_hold_persists_encrypted_audio_before_manifest_and_never_expires(tmp_pat meta = {'timestamp': 1792152000, 'source': 'omi', 'full_capture': True} assert cap_hold.hold_segment('synthetic', str(path), meta) == 1 assert order == ['upload', 'manifest'] - assert decrypt_audio_file(blob.upload_from_string.call_args.args[0], 'synthetic') == path.read_bytes() + assert decrypt_audio_file(uploaded[0], 'synthetic') == path.read_bytes() manifest = ref.create.call_args.args[0] assert manifest['audio_path'].startswith('chunks/synthetic/cap-held/') assert json.loads(decrypt(manifest['metadata'], 'synthetic')) == meta assert 'expires_at' not in manifest ref.create.reset_mock() - blob.upload_from_string.side_effect = RuntimeError('synthetic unavailable') + blob.upload_from_file.side_effect = RuntimeError('synthetic unavailable') with pytest.raises(RuntimeError): cap_hold.hold_segment('synthetic', str(path), meta) ref.create.assert_not_called() + + +@pytest.mark.parametrize('corrupt', [False, True]) +def test_large_hold_streams_bounded_authenticated_chunks(tmp_path, monkeypatch, corrupt): + from contextlib import nullcontext + from pathlib import Path + from cryptography.exceptions import InvalidTag + + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as wav: + wav.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + wav.writeframes(b'\x01\x02' * cap_hold.AUDIO_CHUNK_BYTES) + expected = path.read_bytes() + ref, blob, bucket = Mock(), Mock(), Mock() + ref.get.return_value.exists = False + uploaded = [] + blob.upload_from_file.side_effect = lambda stream, **kw: uploaded.append(stream.read()) + bucket.blob.return_value = blob + monkeypatch.setattr(cap_hold, 'holds', lambda _: SimpleNamespace(document=lambda _: ref)) + monkeypatch.setattr(cap_hold, 'get_private_cloud_sync_bucket', lambda: bucket) + monkeypatch.setattr(cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(cap_hold, 'owner_storage_write_gate', lambda *_: nullcontext()) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda _: SimpleNamespace(reason='plan_allowance_exhausted')) + monkeypatch.setattr(Path, 'read_bytes', Mock(side_effect=AssertionError('whole-file buffer'))) + encrypt_chunk = Mock(wraps=cap_hold.encrypt_audio_chunk) + monkeypatch.setattr(cap_hold, 'encrypt_audio_chunk', encrypt_chunk) + cap_hold.hold_segment('synthetic', str(path), {'timestamp': 1792152000, 'source': 'omi'}) + assert encrypt_chunk.call_count == 3 + assert max(len(call.args[0]) for call in encrypt_chunk.call_args_list) <= cap_hold.AUDIO_CHUNK_BYTES + encrypted = uploaded[0] + if corrupt: + encrypted = encrypted[:-1] + bytes([encrypted[-1] ^ 1]) + blob.download_to_file.side_effect = lambda output: output.write(encrypted) + output = tmp_path / 'recovered.wav' + manifest = ref.create.call_args.args[0] + if corrupt: + with pytest.raises(InvalidTag): + cap_hold.read_audio('synthetic', manifest, str(output)) + assert not output.exists() + else: + cap_hold.read_audio('synthetic', manifest, str(output)) + with output.open('rb') as recovered: + assert recovered.read() == expected + blob.delete.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py index 12a0c77d8dc..7fbbf37a38f 100644 --- a/backend/tests/unit/test_sync_cap_hold_recovery.py +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -1,7 +1,7 @@ """Recovery reaches real sync intake only after a current funded allowance.""" from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, AsyncMock import pytest @@ -20,10 +20,19 @@ async def immediate(pool, fn, *args, **kwargs): ) monkeypatch.setattr(recovery.cap_hold, 'read_audio', Mock()) monkeypatch.setattr(recovery.cap_hold, 'renew', Mock(return_value=True)) + monkeypatch.setattr(recovery.cap_hold, 'checkpoint', Mock(return_value=True)) monkeypatch.setattr(recovery.users_db, 'get_user_transcription_preferences', Mock(return_value={'language': 'en'})) monkeypatch.setattr(recovery.users_db, 'get_user_private_cloud_sync_enabled', Mock(return_value=False)) monkeypatch.setattr(recovery, 'sync_allowed', Mock(return_value=True)) + monkeypatch.setattr(recovery, 'get_hard_restriction_status', Mock(return_value=(False, None))) + monkeypatch.setattr(recovery, 'is_daily_audio_ceiling_exceeded', Mock(return_value=False)) monkeypatch.setattr(recovery, 'process_segment', Mock(return_value=True)) + monkeypatch.setattr(recovery, 'build_person_embeddings_cache', Mock(return_value={'known': 'embedding'})) + monkeypatch.setattr( + recovery, + 'prepare_audio', + AsyncMock(side_effect=lambda uid, path, meta: ({path}, {}, {}, {})), + ) monkeypatch.setattr(recovery, 'finalize_recovered_sync_segment', Mock()) monkeypatch.setattr(recovery, 'get_wav_duration', Mock(return_value=20)) monkeypatch.setattr(recovery, 'record_usage', Mock()) @@ -41,7 +50,10 @@ async def test_reset_or_upgrade_reuses_sync_and_settles_stable_usage_key(state): usage = recovery.record_usage.call_args.kwargs assert usage['idempotency_key'] == 'cap-hold-hold-1792152000.wav' assert usage['transcription_seconds'] == 20 - state.snapshot.reference.update.assert_called_once() + assert state.row['processed_paths'] == ['1792152000.wav'] + assert state.row['finalized'] + assert state.row['piece_receipts']['1792152000.wav']['has_content'] + assert recovery.process_segment.call_args.args[8] == {'known': 'embedding'} async def test_execution_denial_keeps_checkpoint_and_audio_without_stt(state, monkeypatch): @@ -83,3 +95,70 @@ async def test_persistence_failure_keeps_audio_and_releases_local_staging(state) await recovery._recover_row('synthetic', state.snapshot, state.row) state.snapshot.reference.update.assert_not_called() recovery.cap_hold.recovery_directory.return_value.cleanup.assert_called_once() + + +async def test_usage_failure_reuses_persisted_transcription_receipt(state): + recovery.record_usage.side_effect = RuntimeError('synthetic usage unavailable') + with pytest.raises(RuntimeError): + await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav']['has_content'] + recovery.finalize_recovered_sync_segment.assert_not_called() + recovery.record_usage.side_effect = None + # Already accepted content can settle even if no new provider work is allowed. + recovery.sync_allowed.return_value = False + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_called_once() + recovery.finalize_recovered_sync_segment.assert_called_once() + assert recovery.record_usage.call_count == 2 + assert recovery.record_usage.call_args_list[0] == recovery.record_usage.call_args_list[1] + + +async def test_geolocation_and_speaker_context_survive_hold(state): + meta = recovery.cap_hold.metadata.return_value + meta['geolocation'] = {'latitude': 10.7, 'longitude': 106.7} + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + args = recovery.process_segment.call_args + assert args.args[8] == {'known': 'embedding'} + assert args.kwargs['geolocation'].latitude == 10.7 + assert args.kwargs['geolocation'].longitude == 106.7 + + +async def test_all_pieces_join_before_one_finalization(state): + recovery.prepare_audio.side_effect = lambda uid, path, meta: ( + {path, path.replace('1792152000', '1792152020')}, + {}, + {}, + {}, + ) + + def persist(path, uid, response, *args, **kwargs): + response['updated_memories'].add('joined-conversation') + response['_merged'] = {'joined-conversation': 'en'} + return True + + recovery.process_segment.side_effect = persist + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert recovery.process_segment.call_count == 2 + recovery.finalize_recovered_sync_segment.assert_called_once() + response = recovery.finalize_recovered_sync_segment.call_args.args[1] + assert response['updated_memories'] == {'joined-conversation'} + assert response['_merged'] == {'joined-conversation': 'en'} + assert len(state.row['piece_receipts']) == 2 + + +async def test_lost_checkpoint_fence_cannot_settle_or_finalize(state): + recovery.cap_hold.checkpoint.return_value = False + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.record_usage.assert_not_called() + recovery.finalize_recovered_sync_segment.assert_not_called() + + +@pytest.mark.parametrize('restriction', ['hard', 'daily']) +async def test_recovery_cannot_bypass_current_abuse_restrictions(state, restriction): + if restriction == 'hard': + recovery.get_hard_restriction_status.return_value = (True, 60) + else: + recovery.is_daily_audio_ceiling_exceeded.return_value = True + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.cap_hold.read_audio.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_recovery_audio.py b/backend/tests/unit/test_sync_cap_recovery_audio.py new file mode 100644 index 00000000000..b31eafc05e1 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_audio.py @@ -0,0 +1,59 @@ +"""Held capture replay runs coverage before VAD and preserves source coordinates.""" + +from unittest.mock import AsyncMock, Mock + +from utils.sync import cap_recovery_audio as audio + + +async def test_recovery_replays_wal_and_safety_coverage_before_vad(monkeypatch): + events = [] + mapping = {'claim': {'codec': 'pcm16'}, 'offsets': [0, 400, 800]} + + async def run_blocking(pool, fn, *args, **kwargs): + return fn(*args, **kwargs) + + async def wal(*args, **kwargs): + events.append('wal') + assert args[5] == {'audio_pcm16_1792152000.bin': mapping['claim']} + assert args[7] == {'/local/audio_pcm16_1792152000.wav': [400, 400]} + return ['/local/remaining.wav'], {'/local/remaining.wav': mapping}, False + + async def safety(*args, **kwargs): + events.append('safety') + assert kwargs['protected_paths'] == {'/local/remaining.wav': mapping} + return ['/local/remaining.wav'], set() + + def vad(path, segments, errors, **kwargs): + events.append('vad') + assert path == '/local/remaining.wav' + assert kwargs['source_frame_map'] == mapping + segments.add('/local/piece.wav') + kwargs['segment_source_maps']['/local/piece.wav'] = (mapping, 400) + + monkeypatch.setattr(audio, 'run_blocking', run_blocking) + monkeypatch.setattr(audio, 'apply_sync_wal_audio_coverage', wal) + monkeypatch.setattr(audio, 'apply_safety_wal_coverage', safety) + monkeypatch.setattr(audio, 'sync_safety_wal_enabled', lambda: True) + monkeypatch.setattr(audio, 'sync_lineage_resolve_active_for', lambda uid: False) + monkeypatch.setattr(audio, 'retrieve_vad_segments', vad) + paths, maps, _, _ = await audio.prepare( + 'synthetic', + '/local/audio_pcm16_1792152000.wav', + {'source': 'omi', 'full_capture': True, 'source_frame_map': mapping, 'recording_session_id': 'origin'}, + ) + assert events == ['wal', 'safety', 'vad'] + assert paths == {'/local/piece.wav'} + assert maps['/local/piece.wav'] == (mapping, 400) + + +async def test_already_segmented_hold_preserves_position_without_repeating_vad(monkeypatch): + vad = Mock() + coverage = AsyncMock() + monkeypatch.setattr(audio, 'retrieve_vad_segments', vad) + monkeypatch.setattr(audio, 'apply_sync_wal_audio_coverage', coverage) + position = [{'claim': {'codec': 'pcm16'}, 'offsets': [0, 400]}, 10] + paths, maps, _, _ = await audio.prepare('synthetic', '/local/piece.wav', {'source_position_map': position}) + assert paths == {'/local/piece.wav'} + assert maps['/local/piece.wav'] == tuple(position) + vad.assert_not_called() + coverage.assert_not_called() diff --git a/backend/utils/stt_cap_dispatch.py b/backend/utils/stt_cap_dispatch.py index 9a8d202b243..bde1742ddcf 100644 --- a/backend/utils/stt_cap_dispatch.py +++ b/backend/utils/stt_cap_dispatch.py @@ -118,6 +118,19 @@ async def resume(self): raw = await self.factory() if raw is None: raise RuntimeError('held provider constructor returned no socket') + # Registry constructors can invoke another decorated provider. A + # resumed leg must retain exactly one debit/metric boundary, just + # like the initial construction path below. + while isinstance(raw, CapSocket): + self.provider, self.rate, self.factory = raw.provider, raw.rate, raw.factory + if not customer_funded(self.provider): + self.admission.omi_selected = True + if raw in self.admission.held: + self.admission.held.remove(raw) + if raw.raw is None: + self._open_callbacks.extend(raw._open_callbacks) + return # Actual fallback remains denied; keep this leg held. + raw = raw.raw self.raw = raw for callback in self._open_callbacks: callback(raw) diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py index f730b6eed70..6ac21793d7d 100644 --- a/backend/utils/sync/cap_hold.py +++ b/backend/utils/sync/cap_hold.py @@ -9,7 +9,8 @@ import logging from pathlib import Path import time -from tempfile import TemporaryDirectory +from tempfile import TemporaryDirectory, TemporaryFile +import struct import wave from uuid import uuid4 from typing import Any @@ -25,11 +26,12 @@ from utils.stt_cap_dispatch import sync_allowed from database._client import get_firestore_client from database.legal_holds import external_write_fence -from utils.encryption import encrypt, decrypt, encrypt_audio_chunk, decrypt_audio_file +from utils.encryption import encrypt, decrypt, encrypt_audio_chunk, decrypt_audio_chunk from utils.other.storage import get_private_cloud_sync_bucket, owner_storage_write_gate logger = logging.getLogger(__name__) COLLECTION = 'stt_cap_holds' +AUDIO_CHUNK_BYTES = 1024 * 1024 def holds(uid: str) -> Any: @@ -37,15 +39,26 @@ def holds(uid: str) -> Any: def pending_holds(uid: str, *, limit: int = 10) -> list[Any]: - return list(holds(uid).limit(limit).stream()) + return list(holds(uid).order_by('next_attempt_at').limit(limit).stream()) -def hold_segment(uid: str, path: str, metadata: dict[str, Any]) -> float: - audio = Path(path).read_bytes() +def capture_identity(path: str, metadata: dict[str, Any]) -> str: partition = { key: metadata.get(key) for key in ('timestamp', 'source', 'client_device_id', 'target_conversation_id') } - identity = hashlib.sha256(json.dumps(partition, sort_keys=True).encode() + audio).hexdigest() + digest = hashlib.sha256(json.dumps(partition, sort_keys=True).encode()) + with open(path, 'rb') as source: + while chunk := source.read(AUDIO_CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict | None = None) -> float: + metadata = dict(metadata) + geolocation = metadata.get('geolocation') + if geolocation is not None and hasattr(geolocation, 'model_dump'): + metadata['geolocation'] = geolocation.model_dump(mode='json') + identity = capture_identity(path, metadata) blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' bucket = get_private_cloud_sync_bucket() with wave.open(path, 'rb') as wav: @@ -53,22 +66,43 @@ def hold_segment(uid: str, path: str, metadata: dict[str, Any]) -> float: with external_write_fence(uid), owner_storage_write_gate(uid, bucket): ref = holds(uid).document(identity) if ref.get().exists: + if receipt is not None: + receipt.update(id=identity, seconds=seconds) return seconds blob = bucket.blob(blob_path) - blob.upload_from_string(encrypt_audio_chunk(audio, uid), content_type='application/octet-stream') + blob.chunk_size = 8 * AUDIO_CHUNK_BYTES + with recovery_directory(uid) as directory, TemporaryFile(dir=directory) as encrypted: + with open(path, 'rb') as source: + while chunk := source.read(AUDIO_CHUNK_BYTES): + encrypted.write(encrypt_audio_chunk(chunk, uid)) + encrypted.seek(0) + blob.upload_from_file(encrypted, content_type='application/octet-stream') + encoded_metadata = encrypt(json.dumps(metadata), uid) + metadata_fields = {'metadata': encoded_metadata} + if len(encoded_metadata.encode()) > 512 * 1024: + # Bounded source-frame maps can exceed Firestore's document limit. + metadata_digest = hashlib.sha256(encoded_metadata.encode()).hexdigest() + metadata_path = f'chunks/{uid}/cap-held/{identity}.{metadata_digest}.metadata.enc' + bucket.blob(metadata_path).upload_from_string(encoded_metadata, content_type='application/octet-stream') + metadata_fields = {'metadata_path': metadata_path} try: ref.create( { 'audio_path': blob_path, - 'metadata': encrypt(json.dumps(metadata), uid), + **metadata_fields, 'seconds': seconds, 'state': 'held', 'created_at': time.time(), + 'next_attempt_at': 0, 'lease_until': 0, } ) except AlreadyExists: + if receipt is not None: + receipt.update(id=identity, seconds=seconds) return seconds # A concurrent upload owns the immutable recovery checkpoint. + if receipt is not None: + receipt.update(id=identity, seconds=seconds) record_audio('sync', 'held', 'unknown', seconds, funded_allowance(uid).reason == 'plan_allowance_exhausted') logger.info('event=stt_cap_dispatch path=sync outcome=held seconds=%.3f', seconds) return seconds @@ -96,12 +130,37 @@ def read_audio(uid: str, row: dict[str, Any], path: str) -> None: expected = f'chunks/{uid}/cap-held/' if not row['audio_path'].startswith(expected): raise ValueError('hold audio owner mismatch') - raw = get_private_cloud_sync_bucket().blob(row['audio_path']).download_as_bytes() - Path(path).write_bytes(decrypt_audio_file(raw, uid)) + blob = get_private_cloud_sync_bucket().blob(row['audio_path']) + blob.chunk_size = 8 * AUDIO_CHUNK_BYTES + with TemporaryFile(dir=Path(path).parent) as encrypted: + blob.download_to_file(encrypted) + encrypted.seek(0) + try: + with open(path, 'wb') as output: + while prefix := encrypted.read(4): + if len(prefix) != 4: + raise ValueError('truncated held-audio chunk') + size = struct.unpack('>I', prefix)[0] + if not 28 <= size <= AUDIO_CHUNK_BYTES + 28: + raise ValueError('invalid held-audio chunk size') + payload = encrypted.read(size) + if len(payload) != size: + raise ValueError('truncated held-audio payload') + audio, _ = decrypt_audio_chunk(prefix + payload, uid) + output.write(audio) + except Exception: + Path(path).unlink(missing_ok=True) + raise def metadata(uid: str, row: dict[str, Any]) -> dict[str, Any]: - return json.loads(decrypt(row['metadata'], uid)) + if row.get('metadata_path'): + if not row['metadata_path'].startswith(f'chunks/{uid}/cap-held/'): + raise ValueError('hold metadata owner mismatch') + encoded = get_private_cloud_sync_bucket().blob(row['metadata_path']).download_as_text() + else: + encoded = row['metadata'] + return json.loads(decrypt(encoded, uid)) def release(uid: str, ref: Any, token: str) -> None: @@ -112,16 +171,25 @@ def commit(tx: Any): snapshot = ref.get(transaction=tx) row = snapshot.to_dict() if snapshot.exists else {} if row.get('lease_token') == token: - tx.update(ref, {'lease_until': 0, 'lease_token': ''}) + tx.update(ref, {'lease_until': 0, 'lease_token': '', 'next_attempt_at': time.time()}) return commit(client.transaction()) -def hold_capture_if_exhausted(uid: str, paths: list[str], source: ConversationSource, **context: Any) -> bool: +def hold_capture_if_exhausted( + uid: str, + paths: list[str], + source: ConversationSource, + *, + receipts: dict | None = None, + source_frame_maps: dict | None = None, + **context: Any, +) -> bool: # Shared by v1 intake and v2 execution, before VAD can discard silence. - if not enabled() or sync_allowed(uid): + if not paths or not enabled() or sync_allowed(uid): return False for path in paths: + receipt: dict = {} hold_segment( uid, path, @@ -130,11 +198,39 @@ def hold_capture_if_exhausted(uid: str, paths: list[str], source: ConversationSo 'timestamp': get_timestamp_from_path(path), 'source': getattr(source, 'value', source), 'full_capture': True, + 'file_name': Path(path).name, + 'source_frame_map': (source_frame_maps or {}).get(path), }, + receipt=receipt, ) + if receipts is not None: + receipts[receipt['id']] = receipt return True +def defer_segment(uid: str, path: str, metadata: dict, response: dict, lock: Any) -> None: + """Report a durable late hold to both coordinators, never as silence.""" + receipt: dict = {} + hold_segment(uid, path, {**metadata, 'file_name': Path(path).name}, receipt=receipt) + with lock: + response.setdefault('_cap_holds', {})[receipt['id']] = receipt + + +def checkpoint(uid: str, ref: Any, token: str, updates: dict) -> bool: + client = get_firestore_client() + + @firestore.transactional + def commit(tx: Any) -> bool: + row = ref.get(transaction=tx).to_dict() or {} + if row.get('lease_token') != token or row.get('lease_until', 0) <= time.time(): + return False + tx.update(ref, updates) + return True + + with external_write_fence(uid): + return commit(client.transaction()) + + def recovery_cooldown(uid: str) -> bool: return bool(redis_db.r.set(f'users:{uid}:stt_cap_recovery:cooldown', '1', nx=True, ex=60)) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index f599681ad69..c0216b38acd 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -12,19 +12,22 @@ from anyio import from_thread from typing import Any -from google.cloud import firestore from google.api_core.exceptions import NotFound from config.stt_cap_dispatch import enabled from database import users as users_db +from database import sync_cap_holds from utils.analytics import record_usage from utils.executors import run_blocking, db_executor, sync_executor, storage_executor, start_background_task from utils.stt_cap_dispatch import sync_allowed +from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded from utils.sync import cap_hold from utils.sync.files import get_wav_duration +from utils.sync.cap_recovery_audio import prepare as prepare_audio +from models.geolocation import Geolocation from utils.sync.pipeline import ( process_segment, - retrieve_vad_segments, + build_person_embeddings_cache, finalize_recovered_sync_segment, ) from models.conversation import ConversationSource @@ -34,32 +37,80 @@ def _erase(uid: str, snapshot: Any, row: dict[str, Any]) -> None: # Account deletion already purges this existing owner-scoped chunk prefix. - path = row['audio_path'] - if not path.startswith(f'chunks/{uid}/cap-held/'): - raise ValueError('hold audio owner mismatch') - try: - cap_hold.get_private_cloud_sync_bucket().blob(path).delete() - except NotFound: - pass + for path in filter(None, (row['audio_path'], row.get('metadata_path'))): + if not path.startswith(f'chunks/{uid}/cap-held/'): + raise ValueError('hold audio owner mismatch') + try: + cap_hold.get_private_cloud_sync_bucket().blob(path).delete() + except NotFound: + pass snapshot.reference.delete() +def _stored(value: Any) -> Any: + if isinstance(value, dict): + return {key: _stored(item) for key, item in value.items()} + if isinstance(value, (set, tuple, list)): + return [_stored(item) for item in value] + return value + + +def _response(receipts: dict) -> dict: + response: dict[str, Any] = {'new_memories': set(), 'updated_memories': set()} + for receipt in receipts.values(): + for key, value in receipt.get('response', {}).items(): + if key in {'new_memories', 'updated_memories', '_fenced_conversation_ids'}: + response.setdefault(key, set()).update(value) + elif isinstance(value, dict): + response.setdefault(key, {}).update(value) + return response + + +async def _save(uid: str, snapshot: Any, row: dict, updates: dict) -> bool: + saved = await run_blocking(db_executor, cap_hold.checkpoint, uid, snapshot.reference, row['lease_token'], updates) + if saved: + row.update(updates) + return saved + + +def _new_dispatch_allowed(uid: str) -> bool: + if not sync_allowed(uid): + return False + restricted, _ = get_hard_restriction_status(uid) + return not restricted and not is_daily_audio_ceiling_exceeded(uid) + + async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: + if row.get('finalized'): + return True meta = await run_blocking(db_executor, cap_hold.metadata, uid, row) + if meta.get('content_id'): + status = await run_blocking(db_executor, sync_cap_holds.recovery_status, uid, meta['content_id'], snapshot.id) + if status in {'completed', 'settled'}: + return await _save(uid, snapshot, row, {'finalized': True, 'ledger_already_settled': True}) + if status != 'ready': + return False + if not row.get('piece_receipts') and not await run_blocking(db_executor, _new_dispatch_allowed, uid): + return False prefs = await run_blocking(db_executor, users_db.get_user_transcription_preferences, uid) private = await run_blocking(db_executor, users_db.get_user_private_cloud_sync_enabled, uid) protection = await run_blocking(db_executor, users_db.get_data_protection_level, uid) if private else None + try: + embeddings = await run_blocking(db_executor, build_person_embeddings_cache, uid) + except Exception as error: + logger.warning( + 'event=stt_cap_recovery outcome=speaker_context_unavailable exception_type=%s', type(error).__name__ + ) + embeddings = {} + geolocation = Geolocation.model_validate(meta['geolocation']) if meta.get('geolocation') else None directory = await run_blocking(storage_executor, cap_hold.recovery_directory, uid) try: - path = str(Path(directory.name) / f"{meta['timestamp']}.wav") + name = Path(meta.get('file_name') or f"{meta['timestamp']}.wav").name + path = str(Path(directory.name) / name) await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) - paths, errors = {path}, [] - if meta.get('full_capture'): - paths = set() - await run_blocking(sync_executor, retrieve_vad_segments, path, paths, errors) - if errors: - return False + paths, source_maps, targets, bindings = await prepare_audio(uid, path, meta) processed = set(row.get('processed_paths') or []) + receipts = dict(row.get('piece_receipts') or {}) for segment_path in sorted(paths): if not await run_blocking( db_executor, cap_hold.renew, uid, snapshot.reference, row['lease_token'], time.time() @@ -68,71 +119,105 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: name = Path(segment_path).name if name in processed: continue - if not await run_blocking(db_executor, sync_allowed, uid): - return False - response = {'new_memories': set(), 'updated_memories': set()} - outcome = {} - ok = await run_blocking( - sync_executor, - process_segment, - segment_path, - uid, - response, - threading.Lock(), - errors, - ConversationSource(meta['source']), - False, - prefs, - private_cloud_sync_enabled=bool(private), - data_protection_level=protection, - client_device_id=meta.get('client_device_id'), - client_platform=meta.get('client_platform'), - target_conversation_id=meta.get('target_conversation_id'), - cap_recovery=True, - deferred_outcome=outcome, - ) - if errors or outcome.get('cap_held'): - return False - if ok: - await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, response, bool(private)) - seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) + receipt = receipts.get(name) + if receipt is None: + if not await run_blocking(db_executor, _new_dispatch_allowed, uid): + return False + response = {'new_memories': set(), 'updated_memories': set()} + errors, outcome = [], {} + ok = await run_blocking( + sync_executor, + process_segment, + segment_path, + uid, + response, + threading.Lock(), + errors, + ConversationSource(meta['source']), + False, + prefs, + embeddings, + private_cloud_sync_enabled=bool(private), + data_protection_level=protection, + client_device_id=meta.get('client_device_id'), + client_platform=meta.get('client_platform'), + target_conversation_id=targets.get(segment_path, meta.get('target_conversation_id')), + geolocation=geolocation, + source_position_map=source_maps.get(segment_path), + lineage_binding=bindings.get(segment_path, meta.get('lineage_binding')), + sync_lane=meta.get('sync_lane', 'fresh'), + cap_recovery=True, + deferred_outcome=outcome, + ) + if errors or outcome.get('cap_held'): + return False + seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) if ok else 0 + receipt = {'response': _stored(response), 'seconds': seconds, 'has_content': bool(ok)} + receipts[name] = receipt + # Paid STT/content is now durable BEFORE usage or enrichment. + # A settlement retry reuses this receipt without dispatching STT. + if not await _save(uid, snapshot, row, {'piece_receipts': dict(receipts)}): + return False + if receipt['has_content']: + seconds = int(receipt['seconds']) await run_blocking( db_executor, record_usage, uid, - transcription_seconds=int(seconds), - speech_seconds=int(seconds), + transcription_seconds=seconds, + speech_seconds=seconds, idempotency_key=f'cap-hold-{snapshot.id}-{name}', usage_metadata={'funding': 'omi', 'provider': 'unknown', 'source': 'sync_v2'}, ) - await run_blocking( - db_executor, snapshot.reference.update, {'processed_paths': firestore.ArrayUnion([name])} - ) + processed.add(name) + if not await _save(uid, snapshot, row, {'processed_paths': sorted(processed)}): + return False + response = _response(receipts) + if receipts: + # Ordinary sync finalizes the joined batch, not each VAD piece. + await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, response, bool(private)) + return await _save(uid, snapshot, row, {'finalized': True, 'recovery_response': _stored(response)}) finally: await run_blocking(storage_executor, directory.cleanup) - return True + + +def _settle_and_erase(uid: str, snapshot: Any, row: dict) -> None: + if not row.get('settled'): + meta = cap_hold.metadata(uid, row) + if meta.get('content_id') and not row.get('ledger_already_settled'): + if not sync_cap_holds.settle_hold( + uid, + meta['content_id'], + snapshot.id, + row.get('recovery_response') or {}, + len(row.get('piece_receipts') or {}), + ): + raise RuntimeError('hold content ledger ownership changed') + snapshot.reference.update({'settled': True}) + _erase(uid, snapshot, row) async def recover(uid: str) -> None: # One cheap distributed cooldown before querying usage or manifests. Failed # Redis means deferred recovery, never duplicate provider dispatch. acquired = await run_blocking(db_executor, cap_hold.recovery_cooldown, uid) - if not acquired or not await run_blocking(db_executor, sync_allowed, uid): + if not acquired: return rows = await run_blocking(db_executor, cap_hold.pending_holds, uid) for snapshot in rows: row = snapshot.to_dict() or {} try: if row.get('state') == 'completed': - await run_blocking(storage_executor, _erase, uid, snapshot, row) + await run_blocking(storage_executor, _settle_and_erase, uid, snapshot, row) continue row = await run_blocking(db_executor, cap_hold.claim, uid, snapshot.reference, time.time()) if row is None: continue try: if await _recover_row(uid, snapshot, row): - await run_blocking(db_executor, snapshot.reference.update, {'state': 'completed'}) - await run_blocking(storage_executor, _erase, uid, snapshot, row) + if not await _save(uid, snapshot, row, {'state': 'completed'}): + continue + await run_blocking(storage_executor, _settle_and_erase, uid, snapshot, row) finally: await run_blocking(db_executor, cap_hold.release, uid, snapshot.reference, row['lease_token']) except Exception as error: diff --git a/backend/utils/sync/cap_recovery_audio.py b/backend/utils/sync/cap_recovery_audio.py new file mode 100644 index 00000000000..e6cc50a9884 --- /dev/null +++ b/backend/utils/sync/cap_recovery_audio.py @@ -0,0 +1,88 @@ +"""Replay ordinary WAL coverage, VAD and lineage planning for a held capture.""" + +from pathlib import Path + +from config.sync_live_dedupe import sync_lineage_resolve_active_for +from config.sync_safety_wal import sync_safety_wal_enabled +from utils.executors import run_blocking, db_executor, storage_executor, sync_executor +from utils.sync import recording_lineage +from utils.sync.files import get_timestamp_from_path, get_wav_duration +from utils.sync.pipeline import retrieve_vad_segments, cleanup_recovered_sync_files +from utils.sync.safety_wal_coverage import apply_safety_wal_coverage +from utils.sync.wal_audio_coverage import apply_sync_wal_audio_coverage + + +async def prepare(uid: str, path: str, meta: dict) -> tuple[set, dict, dict, dict]: + if not meta.get('full_capture'): + position = meta.get('source_position_map') + return {path}, {path: tuple(position) if position else None}, {}, {} + mapping = meta.get('source_frame_map') or {} + offsets = mapping.get('offsets') or [] + frames = [end - start for start, end in zip(offsets, offsets[1:])] + claims = {Path(path).name.replace('.wav', '.bin'): mapping['claim']} if mapping else None + paths, source_maps, _ = await apply_sync_wal_audio_coverage( + uid, + meta['source'], + False, + meta.get('client_device_id'), + meta.get('recording_session_id'), + claims, + [path], + {path: frames}, + run_blocking=run_blocking, + db_executor=db_executor, + storage_executor=storage_executor, + cleanup_files=cleanup_recovered_sync_files, + ) + if sync_safety_wal_enabled(): + inputs = list(paths) + paths, _ = await apply_safety_wal_coverage( + uid, + paths, + meta.get('target_conversation_id'), + run_blocking=run_blocking, + db_executor=db_executor, + storage_executor=storage_executor, + job_ref='cap_recovery', + protected_paths=source_maps, + capture_partition={ + 'source': meta['source'], + 'client_device_id': meta.get('client_device_id'), + 'is_locked': False, + }, + ) + if inputs and not paths: + paths = inputs # Same conservative envelope fallback as ordinary sync. + segments, segment_maps, errors = set(), {}, [] + for wav_path in paths: + await run_blocking( + sync_executor, + retrieve_vad_segments, + wav_path, + segments, + errors, + source_frame_map=source_maps.get(wav_path), + segment_source_maps=segment_maps, + ) + if errors: + raise RuntimeError('held capture VAD failed') + targets, bindings = {}, {} + origin = meta.get('recording_session_id') + if origin and sync_lineage_resolve_active_for(uid): + targets = await run_blocking( + db_executor, + recording_lineage.plan_segment_targets, + sorted(segments, key=get_timestamp_from_path), + get_timestamp_from_path, + get_wav_duration, + uid, + origin, + meta.get('target_conversation_id'), + meta['source'], + meta.get('client_device_id'), + False, + meta.get('job_id'), + bindings, + segment_source_maps=segment_maps, + ) + return segments, segment_maps, targets, bindings diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 6834ccde5c4..20b4e28ae53 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -783,7 +783,14 @@ def bind_or_converge_sync_ledger_completion( ) if binding.bound: return None - if not binding.completed or not is_valid_completed_sync_content_result(binding.result): + held = getattr(binding, 'held', False) is True + if not ( + held + and isinstance(binding.result, dict) + and binding.result.get('held_audio') is True + or binding.completed + and is_valid_completed_sync_content_result(binding.result) + ): raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') finalized = _require_run_owner( @@ -1242,6 +1249,7 @@ def process_segment( source_position_map: tuple[dict, int] | None = None, lineage_binding: Optional[str] = None, cap_recovery: bool = False, + cap_content_id: str | None = None, ): conversation_id = None provider = 'unknown' @@ -1253,12 +1261,10 @@ def process_segment( raise SyncAssignmentConflict('sync lineage generation ownership remains ambiguous') if stt_cap_dispatch_enabled(): if not sync_allowed(uid): - if deferred_outcome is not None: - deferred_outcome['cap_held'] = True if not cap_recovery: - from utils.sync.cap_hold import hold_segment + from utils.sync.cap_hold import defer_segment - hold_segment( + defer_segment( uid, path, { @@ -1269,8 +1275,18 @@ def process_segment( 'data_protection_level': data_protection_level, 'client_device_id': client_device_id, 'client_platform': client_platform, + 'geolocation': geolocation, + 'source_position_map': source_position_map, + 'lineage_binding': lineage_binding, + 'sync_lane': sync_lane, + 'content_id': cap_content_id, + 'job_id': job_id, }, + response, + lock, ) + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True return False # Queue-time locks do not survive execution-time authorization. is_locked = False @@ -1324,12 +1340,10 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N while True: phase = 'provider_call' if stt_cap_dispatch_enabled() and not sync_allowed(uid): - if deferred_outcome is not None: - deferred_outcome['cap_held'] = True if not cap_recovery: - from utils.sync.cap_hold import hold_segment + from utils.sync.cap_hold import defer_segment - hold_segment( + defer_segment( uid, path, { @@ -1340,8 +1354,18 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N 'data_protection_level': data_protection_level, 'client_device_id': client_device_id, 'client_platform': client_platform, + 'geolocation': geolocation, + 'source_position_map': source_position_map, + 'lineage_binding': lineage_binding, + 'sync_lane': sync_lane, + 'content_id': cap_content_id, + 'job_id': job_id, }, + response, + lock, ) + if deferred_outcome is not None: + deferred_outcome['cap_held'] = True return False words, detected_language = prerecorded( url, @@ -2107,6 +2131,9 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT coordinator itself holds zero thread pool slots — only leaf operations use threads, and only for their actual duration. """ + if stt_cap_dispatch_enabled(): + # A hold is pending processing, not a locked transcript partition. + should_lock = False # LocalRecordingsProvider uploads one explicitly selected native recording # at a time. The writer's exact marker distinguishes Transcribe Later from # automatic offline fallback (omibatchphoneauto) in already-released clients. @@ -2313,23 +2340,6 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT failure_key='invalid_audio', ) return - cap_held = False - if stt_cap_dispatch_enabled(): - from utils.sync.cap_hold import hold_capture_if_exhausted - - cap_held = await run_blocking( - sync_executor, - hold_capture_if_exhausted, - uid, - wav_paths, - source, - target_conversation_id=target_conversation_id, - client_device_id=client_device_id, - client_platform=client_platform, - ) - if cap_held: - await run_blocking(storage_executor, _cleanup_files, wav_paths) - wav_paths = [] try: wav_paths, source_frame_maps, coverage_suppressed_all = await apply_sync_wal_audio_coverage( uid, @@ -2393,6 +2403,31 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT reason='policy', outcome='degraded', ) + cap_held = False + cap_hold_receipts: dict = {} + if stt_cap_dispatch_enabled(): + from utils.sync.cap_hold import hold_capture_if_exhausted + + cap_held = await run_blocking( + sync_executor, + hold_capture_if_exhausted, + uid, + wav_paths, + source, + receipts=cap_hold_receipts, + source_frame_maps=source_frame_maps, + target_conversation_id=target_conversation_id, + client_device_id=client_device_id, + client_platform=client_platform, + geolocation=geolocation, + recording_session_id=recording_session_id, + sync_lane=sync_lane, + content_id=content_id, + job_id=job_id, + ) + if cap_held: + await run_blocking(storage_executor, _cleanup_files, wav_paths) + wav_paths = [] # --- Phase 2: VAD --- job_phase = 'vad' await run_blocking( @@ -2510,15 +2545,23 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT **({'held_audio': True} if cap_held else {}), } if content_id: + ledger_writer = mark_sync_content_completed + ledger_extra = {} + if cap_held: + from database.sync_cap_holds import mark_content_held + + ledger_writer = mark_content_held + ledger_extra = {'hold_ids': list(cap_hold_receipts)} completed_ledger = await run_blocking( db_executor, - mark_sync_content_completed, + ledger_writer, uid, content_id, job_id, empty_result, run_token=active_run_lock_token, run_epoch=active_run_lock_epoch, + **ledger_extra, ) if not completed_ledger: raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') @@ -2790,7 +2833,10 @@ def _process_one_segment(path: str): attempt_ref=attempt_ref, source_position_map=segment_source_maps.get(path), **({'lineage_binding': segment_binding_reasons.get(path)} if use_lineage else {}), + **({'cap_content_id': content_id} if stt_cap_dispatch_enabled() else {}), ) + if deferred_outcome.get('cap_held'): + return # Durable policy hold, not success/silence/provider failure. if ok: # Persist result contributions before the processed marker. # Therefore any skipped segment on a retry has its visible @@ -2988,17 +3034,21 @@ def _checkpoint_fenced_conversations(): # Record DG usage after processing. job_phase = 'usage' + cap_holds = response.get('_cap_holds', {}) if stt_cap_dispatch_enabled() else {} + cap_held_seconds = sum(item['seconds'] for item in cap_holds.values()) await _record_restricted_sync_dg_usage( enabled=fair_use_restrict_dg, uid=uid, job_id=job_id, content_id=content_id, - total_speech_seconds=total_speech_seconds, + total_speech_seconds=max(0, total_speech_seconds - cap_held_seconds), ) # Build result failed_segments = len(segment_errors) successful_segments = total_segments - failed_segments + if cap_holds: + successful_segments -= len(cap_holds) job_outcome = _job_transcription_outcome(segment_errors) result = { 'new_memories': sorted(response['new_memories']), @@ -3012,6 +3062,8 @@ def _checkpoint_fenced_conversations(): if content_segment_count[0] > 0: try: usage_seconds = int(total_speech_seconds) + if cap_holds: + usage_seconds = int(max(0, total_speech_seconds - cap_held_seconds)) should_record_usage = bool(content_id) or await run_blocking( db_executor, try_mark_once, job_id, 'usage' ) @@ -3050,6 +3102,7 @@ def _checkpoint_fenced_conversations(): 'provider': bounded_provider(job_provider), 'model': _bounded_sync_model(job_model), 'lane': sync_lane, + **({'held_audio': True} if cap_holds else {}), } repeat_failure_fingerprint = _whole_job_persistence_fingerprint( failed_segments, total_segments, persistence_failure_fingerprints @@ -3065,15 +3118,23 @@ def _checkpoint_fenced_conversations(): if content_id and failed_segments == 0: # The durable content ledger proves every successful segment # before the WAL-visible completed status is published. + ledger_writer = mark_sync_content_completed + ledger_extra = {} + if cap_holds: + from database.sync_cap_holds import mark_content_held + + ledger_writer = mark_content_held + ledger_extra = {'hold_ids': list(cap_holds)} completed_ledger = await run_blocking( db_executor, - mark_sync_content_completed, + ledger_writer, uid, content_id, job_id, final_result, run_token=active_run_lock_token, run_epoch=active_run_lock_epoch, + **ledger_extra, ) if not completed_ledger: raise SyncJobRunLeaseLost(f'sync content ledger owner lost: job={job_id}') @@ -3290,3 +3351,8 @@ def finalize_recovered_sync_segment(uid: str, response: dict, private: bool) -> _reprocess_merged_conversations(uid, response) if private: _finalize_sync_audio_files(uid, response) + + +def cleanup_recovered_sync_files(paths: list[str]) -> None: + """Coverage replay borrows the ordinary sync cleanup boundary.""" + _cleanup_files(paths) From 2c448f45588d3aab0d5b5f006e3af8f2a5b45654 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 23:04:49 +0700 Subject: [PATCH 10/19] test: align isolated calendar window regressions with current renderer --- backend/tests/unit/test_calendar_long_range_upcoming.py | 6 ++++-- backend/tests/unit/test_calendar_search_future_start.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/tests/unit/test_calendar_long_range_upcoming.py b/backend/tests/unit/test_calendar_long_range_upcoming.py index 89ce9128b0f..db9c84a505e 100644 --- a/backend/tests/unit/test_calendar_long_range_upcoming.py +++ b/backend/tests/unit/test_calendar_long_range_upcoming.py @@ -16,6 +16,7 @@ import pytest from testing.import_isolation import AutoMockModule, load_module_fresh, stub_modules +from utils.model_text import DisplayClock, format_span BACKEND_DIR = Path(__file__).resolve().parents[2] TOOLS_DIR = BACKEND_DIR / "utils" / "retrieval" / "tools" @@ -110,6 +111,7 @@ def calendar_tools() -> Iterator[ModuleType]: with stub_modules(stubs): # The real date parsing and max_results capping the tool relies on. load_module_fresh("utils.retrieval.tools.integration_base", str(TOOLS_DIR / "integration_base.py")) + load_module_fresh("utils.retrieval.tools.calendar_render", str(TOOLS_DIR / "calendar_render.py")) module = load_module_fresh("utils.retrieval.tools.calendar_tools", str(TOOLS_DIR / "calendar_tools.py")) module.prepare_access = lambda *_args: ("uid-1", {"connected": True, "access_token": "token"}, "token", None) yield module @@ -122,7 +124,7 @@ def _use_calendar(calendar_tools: ModuleType, events: list[dict]) -> _FakeGoogle def _shown(start: datetime) -> str: - return f"Start: {start.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC" + return f"When: {format_span(start, start + timedelta(minutes=30), DisplayClock.utc())}" def _now_minute() -> datetime: @@ -141,7 +143,7 @@ def test_range_into_the_future_lists_the_soonest_events_first(calendar_tools): ) ) - assert result.startswith("Calendar Events (10 found)") + assert result.startswith("Calendar Events (10 shown;") shown = [start for start in starts if _shown(start) in result] assert shown == starts[:10] positions = [result.index(_shown(start)) for start in shown] diff --git a/backend/tests/unit/test_calendar_search_future_start.py b/backend/tests/unit/test_calendar_search_future_start.py index 10facfc58dc..a0568095e39 100644 --- a/backend/tests/unit/test_calendar_search_future_start.py +++ b/backend/tests/unit/test_calendar_search_future_start.py @@ -15,6 +15,7 @@ import pytest from testing.import_isolation import AutoMockModule, load_module_fresh, stub_modules +from utils.model_text import DisplayClock, format_span BACKEND_DIR = Path(__file__).resolve().parents[2] TOOLS_DIR = BACKEND_DIR / "utils" / "retrieval" / "tools" @@ -109,6 +110,7 @@ def calendar_tools() -> Iterator[ModuleType]: with stub_modules(stubs): # The real date parsing and max_results capping the tool relies on. load_module_fresh("utils.retrieval.tools.integration_base", str(TOOLS_DIR / "integration_base.py")) + load_module_fresh("utils.retrieval.tools.calendar_render", str(TOOLS_DIR / "calendar_render.py")) module = load_module_fresh("utils.retrieval.tools.calendar_tools", str(TOOLS_DIR / "calendar_tools.py")) module.prepare_access = lambda *_args: ("uid-1", {"connected": True, "access_token": "token"}, "token", None) yield module @@ -121,7 +123,7 @@ def _use_calendar(calendar_tools: ModuleType, events: list[dict]) -> _FakeGoogle def _shown(start: datetime) -> str: - return f"Start: {start.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC" + return f"When: {format_span(start, start + timedelta(minutes=30), DisplayClock.utc())}" def _now_minute() -> datetime: From 7c0f57fddfa641827e0c37ae1ab2ffe28a381e5f Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 16 Oct 2026 23:28:39 +0700 Subject: [PATCH 11/19] test: isolate parity inference and assert usage protection structurally --- .../tests/unit/test_listen_parity_runtime.py | 4 +++ backend/tests/unit/test_sync_record_usage.py | 36 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/backend/tests/unit/test_listen_parity_runtime.py b/backend/tests/unit/test_listen_parity_runtime.py index cee5aae05d1..e65b4e16ff2 100644 --- a/backend/tests/unit/test_listen_parity_runtime.py +++ b/backend/tests/unit/test_listen_parity_runtime.py @@ -16,6 +16,7 @@ from utils.executors import storage_executor from utils.listen_session_bootstrap import ListenConnectBase from utils.stt.streaming import STTService +from utils.stt import language_policy _PRINCIPAL = 'allowed-runtime-principal' _TRANSCRIPT = 'synthetic runtime transcript' @@ -172,6 +173,9 @@ async def run_blocking_fake(executor, function, *args, **kwargs): monkeypatch.setattr(runtime_module, 'PUSHER_ENABLED', False) monkeypatch.setattr(runtime_module, 'should_load_speech_profile', lambda **_kwargs: False) monkeypatch.setattr(runtime_module, 'should_enable_speaker_identification', lambda **_kwargs: False) + # This seam verifies capture/persistence, with synthetic STT. Language + # inference has its own suite; do not cold-load its models in a worker. + monkeypatch.setattr(language_policy, 'classify_output', lambda *_args: ('in_profile', 'en')) monkeypatch.setattr(receiver_module, 'should_initialize_vad_gate', lambda **_kwargs: False) monkeypatch.setattr(ListenReceiver, '_create_stt_socket', create_stt_socket) monkeypatch.setattr(runtime, '_build_components', build_components) diff --git a/backend/tests/unit/test_sync_record_usage.py b/backend/tests/unit/test_sync_record_usage.py index 6d0d55982b2..14222ea3f51 100644 --- a/backend/tests/unit/test_sync_record_usage.py +++ b/backend/tests/unit/test_sync_record_usage.py @@ -8,6 +8,7 @@ - v1 deprecation headers appear on all response paths """ +import ast import os import re @@ -39,6 +40,25 @@ def _extract_function_body(source, func_name): return source[start:end] +def _usage_has_exception_boundary(body): + """Check nesting, rather than a character window around a formatted call.""" + for node in ast.walk(ast.parse(body)): + if not isinstance(node, ast.Try): + continue + catches = any( + isinstance(handler.type, ast.Name) and handler.type.id == 'Exception' for handler in node.handlers + ) + if not catches: + continue + for statement in node.body: + for call in ast.walk(statement): + if isinstance(call, ast.Call) and any( + isinstance(argument, ast.Name) and argument.id == 'record_usage' for argument in call.args + ): + return True + return False + + # --------------------------------------------------------------------------- # record_usage import # --------------------------------------------------------------------------- @@ -102,14 +122,7 @@ def test_record_usage_not_under_fair_use_guard(self): def test_record_usage_wrapped_in_try_except(self): """record_usage must be protected by try/except.""" - body = self._get_v1_body() - record_idx = body.find('record_usage,') - # Check there's a try: before it (within ~5 lines) - preceding_chunk = body[max(0, record_idx - 200) : record_idx] - assert 'try:' in preceding_chunk, "record_usage must be inside a try block" - # Check there's an except after it - following_chunk = body[record_idx : record_idx + 200] - assert 'except Exception' in following_chunk, "record_usage must have an except handler" + assert _usage_has_exception_boundary(self._get_v1_body()), 'record_usage must be inside a protected try body' def test_record_usage_logs_error_on_failure(self): """Error from record_usage must be logged, not silenced.""" @@ -159,12 +172,7 @@ def test_static_record_usage_before_finalize_sync_job(self): assert record_pos < complete_pos, "record_usage must run before final job publication" def test_record_usage_wrapped_in_try_except(self): - body = self._get_v2_body() - record_idx = body.find('record_usage,') - preceding = body[max(0, record_idx - 1000) : record_idx] - assert 'try:' in preceding, "record_usage must be inside a try block" - following = body[record_idx : record_idx + 800] - assert 'except Exception' in following, "record_usage must have an except handler" + assert _usage_has_exception_boundary(self._get_v2_body()), 'record_usage must be inside a protected try body' # --------------------------------------------------------------------------- From 85e1c6e0c369dccf10cfd585cf679b601dac5900 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 00:17:48 +0700 Subject: [PATCH 12/19] fix: retain hold admission and provider provenance across recovery --- backend/docs/stt-cap-dispatch.md | 6 +- backend/routers/sync.py | 14 +-- .../tests/unit/test_stt_cap_dispatch_http.py | 22 ++++- .../tests/unit/test_sync_cap_coordinator.py | 11 ++- backend/tests/unit/test_sync_cap_hold.py | 52 +++++++++-- .../tests/unit/test_sync_cap_hold_recovery.py | 23 ++++- backend/utils/sync/cap_hold.py | 88 +++++++++++++------ backend/utils/sync/cap_hold_recovery.py | 15 +++- backend/utils/sync/pipeline.py | 4 +- .../SettingsSubscriptionClarity.test.tsx | 4 +- 10 files changed, 187 insertions(+), 52 deletions(-) diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index 9afbdae5682..2fb4f8803e9 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -31,7 +31,9 @@ audio is not enrolled as new recoverable work. Source maps, recording identity and geolocation travel with the encrypted hold. The shared segment processor rechecks immediately before each STT invocation, including its empty-result retry. Late exhaustion holds the remaining segment. -No denied segment reaches cloud STT. A successfully persisted hold is acknowledged +No denied segment reaches cloud STT. Initial holds coordinate database and storage +leaves asynchronously; GCS work does not occupy the sync pipeline pool. +Transient admission/configuration holds do not claim credits are exhausted. A successfully persisted hold is acknowledged without creating a locked transcript. Hold failures fail the upload/job; callers must retain/retry their source audio. V1 adds `held_audio` and `held_files` to the exhausted acknowledgment; v2 adds `held_audio` to its durable result. @@ -76,7 +78,7 @@ every new recovered provider request. Already persisted receipts can settle at the cap without another provider call. Recovery replays WAL coverage and lineage planning, loads speaker embeddings, and reuses VAD, segment intake, enrichment and optional playback finalization. Token-fenced -piece receipts checkpoint persisted transcription before usage settlement; a +piece receipts checkpoint persisted transcription and serving-provider provenance before usage settlement; a usage retry cannot repeat paid STT. Finalization runs once after all pieces join, with the retained geolocation and source-position context. Piece receipts and idempotent usage keys preserve progress across retries. Completed diff --git a/backend/routers/sync.py b/backend/routers/sync.py index 45c2ea7254f..cb2e2315c8f 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -639,13 +639,13 @@ async def sync_local_files( raise HTTPException(status_code=e.status_code, detail=e.detail, headers=_V1_DEPRECATION_HEADERS) cap_held = False + cap_admission: dict = {} if cap_dispatch_enabled(): - cap_held = await run_blocking( - sync_executor, - hold_capture_if_exhausted, + cap_held = await hold_capture_if_exhausted( uid, wav_paths, source, + admission=cap_admission, target_conversation_id=conversation_id, client_device_id=client_device_context.client_device_id, client_platform=client_device_context.platform, @@ -657,7 +657,7 @@ async def sync_local_files( content={ 'new_memories': [], 'updated_memories': [], - 'credits_exhausted': True, + 'credits_exhausted': cap_admission.get('reason') == 'plan_allowance_exhausted', 'held_audio': True, 'held_files': len(wav_paths), }, @@ -848,7 +848,11 @@ def _run_vad(path): 'updated_memories': sorted(response['updated_memories']), } if cap_holds: - result.update(held_audio=True, credits_exhausted=True, held_segments=len(cap_holds)) + result.update( + held_audio=True, + credits_exhausted=any(item.get('reason') == 'plan_allowance_exhausted' for item in cap_holds.values()), + held_segments=len(cap_holds), + ) failed_segments = len(segment_errors) successful_segments = total_segments - failed_segments diff --git a/backend/tests/unit/test_stt_cap_dispatch_http.py b/backend/tests/unit/test_stt_cap_dispatch_http.py index f3bdf2ca188..25ff5e2f20b 100644 --- a/backend/tests/unit/test_stt_cap_dispatch_http.py +++ b/backend/tests/unit/test_stt_cap_dispatch_http.py @@ -1,7 +1,7 @@ """Actual sync HTTP routing with local fake persistence/provider boundaries.""" from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, AsyncMock from fastapi import FastAPI from fastapi.testclient import TestClient @@ -37,7 +37,12 @@ def client(monkeypatch): monkeypatch.setattr(router, 'archive_mobile_batch_raw_files', lambda *_: []) monkeypatch.setattr(router, 'decode_files_to_wav', lambda _: ['1792152000.wav']) monkeypatch.setattr(router, '_cleanup_files', Mock()) - monkeypatch.setattr(router, 'hold_capture_if_exhausted', Mock(return_value=True)) + + async def held(*args, **kwargs): + kwargs['admission']['reason'] = router.resolve_transcription_allowance('synthetic').reason + return True + + monkeypatch.setattr(router, 'hold_capture_if_exhausted', AsyncMock(side_effect=held)) monkeypatch.setattr(router, 'retrieve_vad_segments', Mock(side_effect=AssertionError('denied audio reached VAD'))) app = FastAPI() app.include_router(router.router) @@ -101,6 +106,7 @@ def test_flag_off_http_result_retains_main_shape_without_hold_dispatch(client, m @pytest.mark.parametrize('accepted', [False, True]) def test_late_v1_hold_is_reported_and_only_processed_speech_is_metered(client, monkeypatch, accepted): monkeypatch.setattr(router, 'FAIR_USE_ENABLED', False) + router.hold_capture_if_exhausted.side_effect = None router.hold_capture_if_exhausted.return_value = False paths = ['1792152000.wav', '1792152020.wav'] if accepted else ['1792152020.wav'] monkeypatch.setattr(router, 'retrieve_vad_segments', lambda path, segments, errors, **kw: segments.update(paths)) @@ -117,7 +123,7 @@ def process(path, uid, response, *args, **kwargs): if path == '1792152000.wav': response['new_memories'].add('accepted') return True - response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 20}} + response['_cap_holds'] = {'late': {'id': 'late', 'seconds': 20, 'reason': 'plan_allowance_exhausted'}} return False monkeypatch.setattr(router, 'process_segment', process) @@ -134,3 +140,13 @@ def process(path, uid, response, *args, **kwargs): assert usage.call_args.kwargs == {'transcription_seconds': 20, 'speech_seconds': 20} else: usage.assert_not_called() + + +@pytest.mark.parametrize('reason', ['allowance_unavailable', 'launch_config_missing', 'launch_config_invalid']) +def test_transient_policy_hold_does_not_claim_credits_exhausted(client, monkeypatch, reason): + monkeypatch.setattr(router, 'resolve_transcription_allowance', lambda _: SimpleNamespace(reason=reason)) + response = client.post('/v1/sync-local-files', files={'files': ('omi_1792152000.bin', b'synthetic')}) + assert response.status_code == 200, response.text + assert response.json()['held_audio'] is True + assert response.json()['credits_exhausted'] is False + router.retrieve_vad_segments.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_coordinator.py b/backend/tests/unit/test_sync_cap_coordinator.py index b0f447697e7..c6be217c8a6 100644 --- a/backend/tests/unit/test_sync_cap_coordinator.py +++ b/backend/tests/unit/test_sync_cap_coordinator.py @@ -1,6 +1,7 @@ """Exercise real v2 coordination: coverage precedes custody and late holds do not bill.""" from pathlib import Path +from types import SimpleNamespace from unittest.mock import Mock from database import sync_cap_holds @@ -60,14 +61,16 @@ async def test_full_hold_contains_only_wal_audio_not_already_committed(coordinat lineage_rows=[generation(1, capture_evidence=_committed_envelope(range(7)))], ) writer = _ledger(pipeline, monkeypatch) - monkeypatch.setattr(cap_hold, 'sync_allowed', lambda uid: False) + monkeypatch.setattr( + cap_hold, 'funded_allowance', lambda uid: SimpleNamespace(managed=False, reason='plan_allowance_exhausted') + ) retained = [] - def retain(uid, path, metadata, *, receipt): + async def retain(uid, path, metadata, reason, receipt): retained.append((_read_payload(path), metadata)) receipt.update(id='held-source', seconds=1.5) - monkeypatch.setattr(cap_hold, 'hold_segment', retain) + monkeypatch.setattr(cap_hold, '_hold_capture', retain) await _run(module, tmp_path) assert len(retained) == 1 assert retained[0][0] == b''.join(_frame_bytes(v, FRAME_SAMPLES) for v in (7, 8, 9)) @@ -87,7 +90,7 @@ async def test_late_hold_reports_custody_and_charges_only_dispatched_speech(coor monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') _wire(pipeline, monkeypatch, tmp_path, lineage_rows=[]) writer = _ledger(pipeline, monkeypatch) - monkeypatch.setattr(cap_hold, 'sync_allowed', lambda uid: True) + monkeypatch.setattr(cap_hold, 'funded_allowance', lambda uid: SimpleNamespace(managed=True, reason='available')) paths = [tmp_path / '1760000000.wav', tmp_path / '1760000002.5.wav'] for path in paths: _write_labeled_wav(path, range(5)) diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py index 0de3dc2a62f..0416f0fdcb3 100644 --- a/backend/tests/unit/test_sync_cap_hold.py +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -72,16 +72,20 @@ def test_hold_failure_cannot_acknowledge_audio_or_dispatch_provider(tmp_path, mo @pytest.mark.parametrize('enforce', [False, True]) -def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, monkeypatch, enforce): +async def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, monkeypatch, enforce): from utils.sync import cap_hold from models.conversation_enums import ConversationSource monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) - monkeypatch.setattr(cap_hold, 'sync_allowed', Mock(return_value=False)) - hold = Mock() - monkeypatch.setattr(cap_hold, 'hold_segment', hold) + monkeypatch.setattr( + cap_hold, + 'funded_allowance', + Mock(return_value=SimpleNamespace(managed=False, reason='plan_allowance_exhausted')), + ) + hold = AsyncMock() + monkeypatch.setattr(cap_hold, '_hold_capture', hold) paths = [str(tmp_path / '1792152000.wav')] - assert cap_hold.hold_capture_if_exhausted('synthetic', paths, ConversationSource.omi) is enforce + assert await cap_hold.hold_capture_if_exhausted('synthetic', paths, ConversationSource.omi) is enforce if enforce: assert hold.call_args.args[2] == { 'timestamp': 1792152000, @@ -92,7 +96,7 @@ def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, mon } else: hold.assert_not_called() - cap_hold.sync_allowed.assert_not_called() + cap_hold.funded_allowance.assert_not_called() def test_hold_manifest_fences_concurrent_recovery_and_stale_release(monkeypatch): @@ -109,6 +113,9 @@ def test_hold_manifest_fences_concurrent_recovery_and_stale_release(monkeypatch) assert first and first['lease_token'] assert cap_hold.claim('synthetic', ref, 101) is None next_claim = cap_hold.claim('synthetic', ref, 2000) + monkeypatch.setattr(cap_hold.time, 'time', lambda: 2000) + assert not cap_hold.checkpoint('synthetic', ref, first['lease_token'], {'state': 'completed'}) + assert cap_hold.checkpoint('synthetic', ref, next_claim['lease_token'], {'finalized': True}) cap_hold.release('synthetic', ref, first['lease_token']) assert client.rows[path]['lease_token'] == next_claim['lease_token'] cap_hold.release('synthetic', ref, next_claim['lease_token']) @@ -199,3 +206,36 @@ def test_large_hold_streams_bounded_authenticated_chunks(tmp_path, monkeypatch, with output.open('rb') as recovered: assert recovered.read() == expected blob.delete.assert_not_called() + + +async def test_capture_hold_coordinates_db_and_storage_leaves(monkeypatch): + from models.conversation_enums import ConversationSource + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + monkeypatch.setattr( + cap_hold, 'funded_allowance', Mock(return_value=SimpleNamespace(managed=False, reason='allowance_unavailable')) + ) + monkeypatch.setattr(cap_hold, '_describe', Mock(return_value=('held', 20, {'source': 'omi'}))) + monkeypatch.setattr(cap_hold, '_exists', Mock(return_value=False)) + monkeypatch.setattr(cap_hold, '_upload', Mock(return_value={'audio_path': 'synthetic'})) + monkeypatch.setattr(cap_hold, '_publish', Mock(return_value=True)) + calls = [] + + async def leaf(pool, fn, *args, **kwargs): + calls.append((pool, fn)) + return fn(*args, **kwargs) + + monkeypatch.setattr(cap_hold, 'run_blocking', leaf) + receipts, admission = {}, {} + assert await cap_hold.hold_capture_if_exhausted( + 'synthetic', ['1792152000.wav'], ConversationSource.omi, receipts=receipts, admission=admission + ) + assert calls == [ + (cap_hold.db_executor, cap_hold.funded_allowance), + (cap_hold.storage_executor, cap_hold._describe), + (cap_hold.db_executor, cap_hold._exists), + (cap_hold.storage_executor, cap_hold._upload), + (cap_hold.db_executor, cap_hold._publish), + ] + assert admission == {'reason': 'allowance_unavailable'} + assert receipts['held']['reason'] == 'allowance_unavailable' diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py index 7fbbf37a38f..0eb13273c41 100644 --- a/backend/tests/unit/test_sync_cap_hold_recovery.py +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -26,7 +26,12 @@ async def immediate(pool, fn, *args, **kwargs): monkeypatch.setattr(recovery, 'sync_allowed', Mock(return_value=True)) monkeypatch.setattr(recovery, 'get_hard_restriction_status', Mock(return_value=(False, None))) monkeypatch.setattr(recovery, 'is_daily_audio_ceiling_exceeded', Mock(return_value=False)) - monkeypatch.setattr(recovery, 'process_segment', Mock(return_value=True)) + + def processed(*args, **kwargs): + kwargs['deferred_outcome']['provider'] = 'modulate' + return True + + monkeypatch.setattr(recovery, 'process_segment', Mock(side_effect=processed)) monkeypatch.setattr(recovery, 'build_person_embeddings_cache', Mock(return_value={'known': 'embedding'})) monkeypatch.setattr( recovery, @@ -50,6 +55,7 @@ async def test_reset_or_upgrade_reuses_sync_and_settles_stable_usage_key(state): usage = recovery.record_usage.call_args.kwargs assert usage['idempotency_key'] == 'cap-hold-hold-1792152000.wav' assert usage['transcription_seconds'] == 20 + assert usage['usage_metadata'] == {'funding': 'omi', 'provider': 'modulate', 'source': 'sync_v2'} assert state.row['processed_paths'] == ['1792152000.wav'] assert state.row['finalized'] assert state.row['piece_receipts']['1792152000.wav']['has_content'] @@ -110,6 +116,7 @@ async def test_usage_failure_reuses_persisted_transcription_receipt(state): recovery.process_segment.assert_called_once() recovery.finalize_recovered_sync_segment.assert_called_once() assert recovery.record_usage.call_count == 2 + assert recovery.record_usage.call_args.kwargs['usage_metadata']['provider'] == 'modulate' assert recovery.record_usage.call_args_list[0] == recovery.record_usage.call_args_list[1] @@ -162,3 +169,17 @@ async def test_recovery_cannot_bypass_current_abuse_restrictions(state, restrict assert not await recovery._recover_row('synthetic', state.snapshot, state.row) recovery.process_segment.assert_not_called() recovery.cap_hold.read_audio.assert_not_called() + + +async def test_lost_completion_lease_cannot_settle_or_erase_audio(state, monkeypatch): + state.snapshot.to_dict = Mock(return_value={'state': 'held'}) + monkeypatch.setattr(recovery.cap_hold, 'recovery_cooldown', Mock(return_value=True)) + monkeypatch.setattr(recovery.cap_hold, 'pending_holds', Mock(return_value=[state.snapshot])) + monkeypatch.setattr(recovery.cap_hold, 'claim', Mock(return_value=state.row)) + monkeypatch.setattr(recovery.cap_hold, 'release', Mock()) + monkeypatch.setattr(recovery, '_recover_row', AsyncMock(return_value=True)) + monkeypatch.setattr(recovery, '_settle_and_erase', Mock()) + recovery.cap_hold.checkpoint.return_value = False + await recovery.recover('synthetic') + recovery._settle_and_erase.assert_not_called() + recovery.cap_hold.release.assert_called_once_with('synthetic', state.snapshot.reference, 'owner-token') diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py index 6ac21793d7d..b7b06491fb9 100644 --- a/backend/utils/sync/cap_hold.py +++ b/backend/utils/sync/cap_hold.py @@ -23,7 +23,7 @@ from database import redis_db from config.stt_cap_dispatch import enabled from utils.sync.files import get_timestamp_from_path -from utils.stt_cap_dispatch import sync_allowed +from utils.executors import db_executor, storage_executor, run_blocking from database._client import get_firestore_client from database.legal_holds import external_write_fence from utils.encryption import encrypt, decrypt, encrypt_audio_chunk, decrypt_audio_chunk @@ -53,22 +53,26 @@ def capture_identity(path: str, metadata: dict[str, Any]) -> str: return digest.hexdigest() -def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict | None = None) -> float: +def _describe(path: str, metadata: dict[str, Any]) -> tuple[str, float, dict]: metadata = dict(metadata) geolocation = metadata.get('geolocation') if geolocation is not None and hasattr(geolocation, 'model_dump'): metadata['geolocation'] = geolocation.model_dump(mode='json') identity = capture_identity(path, metadata) - blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' - bucket = get_private_cloud_sync_bucket() with wave.open(path, 'rb') as wav: seconds = wav.getnframes() / wav.getframerate() + return identity, seconds, metadata + + +def _exists(uid: str, identity: str) -> bool: + with external_write_fence(uid): + return bool(holds(uid).document(identity).get().exists) + + +def _upload(uid: str, path: str, identity: str, metadata: dict) -> dict: + blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' + bucket = get_private_cloud_sync_bucket() with external_write_fence(uid), owner_storage_write_gate(uid, bucket): - ref = holds(uid).document(identity) - if ref.get().exists: - if receipt is not None: - receipt.update(id=identity, seconds=seconds) - return seconds blob = bucket.blob(blob_path) blob.chunk_size = 8 * AUDIO_CHUNK_BYTES with recovery_directory(uid) as directory, TemporaryFile(dir=directory) as encrypted: @@ -78,18 +82,23 @@ def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict encrypted.seek(0) blob.upload_from_file(encrypted, content_type='application/octet-stream') encoded_metadata = encrypt(json.dumps(metadata), uid) - metadata_fields = {'metadata': encoded_metadata} + fields = {'audio_path': blob_path, 'metadata': encoded_metadata} if len(encoded_metadata.encode()) > 512 * 1024: - # Bounded source-frame maps can exceed Firestore's document limit. metadata_digest = hashlib.sha256(encoded_metadata.encode()).hexdigest() metadata_path = f'chunks/{uid}/cap-held/{identity}.{metadata_digest}.metadata.enc' bucket.blob(metadata_path).upload_from_string(encoded_metadata, content_type='application/octet-stream') - metadata_fields = {'metadata_path': metadata_path} + fields = {'audio_path': blob_path, 'metadata_path': metadata_path} + return fields + + +def _publish(uid: str, identity: str, seconds: float, fields: dict) -> bool: + # Reacquire the account fence after upload: deletion may have started while + # the async coordinator switched pools. Never acknowledge a refused manifest. + with external_write_fence(uid): try: - ref.create( + holds(uid).document(identity).create( { - 'audio_path': blob_path, - **metadata_fields, + **fields, 'seconds': seconds, 'state': 'held', 'created_at': time.time(), @@ -98,16 +107,38 @@ def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict } ) except AlreadyExists: - if receipt is not None: - receipt.update(id=identity, seconds=seconds) - return seconds # A concurrent upload owns the immutable recovery checkpoint. + return False # Preserve the winner's immutable progress checkpoint. + return True + + +def _finish(identity: str, seconds: float, reason: str, receipt: dict | None, created: bool) -> float: if receipt is not None: - receipt.update(id=identity, seconds=seconds) - record_audio('sync', 'held', 'unknown', seconds, funded_allowance(uid).reason == 'plan_allowance_exhausted') - logger.info('event=stt_cap_dispatch path=sync outcome=held seconds=%.3f', seconds) + receipt.update(id=identity, seconds=seconds, reason=reason) + if created: + record_audio('sync', 'held', 'unknown', seconds, reason == 'plan_allowance_exhausted') + logger.info('event=stt_cap_dispatch path=sync outcome=held seconds=%.3f', seconds) return seconds +def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict | None = None) -> float: + identity, seconds, metadata = _describe(path, metadata) + created = False + if not _exists(uid, identity): + fields = _upload(uid, path, identity, metadata) + created = _publish(uid, identity, seconds, fields) + return _finish(identity, seconds, funded_allowance(uid).reason, receipt, created) + + +async def _hold_capture(uid: str, path: str, metadata: dict, reason: str, receipt: dict) -> None: + # Pool workers are leaves; no sync worker waits for another executor. + identity, seconds, metadata = await run_blocking(storage_executor, _describe, path, metadata) + created = False + if not await run_blocking(db_executor, _exists, uid, identity): + fields = await run_blocking(storage_executor, _upload, uid, path, identity, metadata) + created = await run_blocking(db_executor, _publish, uid, identity, seconds, fields) + _finish(identity, seconds, reason, receipt, created) + + def claim(uid: str, ref: Any, now: float) -> dict[str, Any] | None: client = get_firestore_client() @@ -176,21 +207,27 @@ def commit(tx: Any): return commit(client.transaction()) -def hold_capture_if_exhausted( +async def hold_capture_if_exhausted( uid: str, paths: list[str], source: ConversationSource, *, receipts: dict | None = None, source_frame_maps: dict | None = None, + admission: dict | None = None, **context: Any, ) -> bool: # Shared by v1 intake and v2 execution, before VAD can discard silence. - if not paths or not enabled() or sync_allowed(uid): + if not paths or not enabled(): + return False + allowance = await run_blocking(db_executor, funded_allowance, uid) + if allowance.managed: return False + if admission is not None: + admission['reason'] = allowance.reason for path in paths: receipt: dict = {} - hold_segment( + await _hold_capture( uid, path, { @@ -201,7 +238,8 @@ def hold_capture_if_exhausted( 'file_name': Path(path).name, 'source_frame_map': (source_frame_maps or {}).get(path), }, - receipt=receipt, + allowance.reason, + receipt, ) if receipts is not None: receipts[receipt['id']] = receipt diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index c0216b38acd..c3171aa2ff2 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -152,7 +152,16 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: if errors or outcome.get('cap_held'): return False seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) if ok else 0 - receipt = {'response': _stored(response), 'seconds': seconds, 'has_content': bool(ok)} + receipt = { + 'response': _stored(response), + 'seconds': seconds, + 'has_content': bool(ok), + 'usage_metadata': { + 'funding': 'omi', + 'provider': outcome.get('provider') or 'unknown', + 'source': 'sync_v2', + }, + } receipts[name] = receipt # Paid STT/content is now durable BEFORE usage or enrichment. # A settlement retry reuses this receipt without dispatching STT. @@ -167,7 +176,9 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: transcription_seconds=seconds, speech_seconds=seconds, idempotency_key=f'cap-hold-{snapshot.id}-{name}', - usage_metadata={'funding': 'omi', 'provider': 'unknown', 'source': 'sync_v2'}, + usage_metadata=receipt.get( + 'usage_metadata', {'funding': 'omi', 'provider': 'unknown', 'source': 'sync_v2'} + ), ) processed.add(name) if not await _save(uid, snapshot, row, {'processed_paths': sorted(processed)}): diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 20b4e28ae53..6dec2d14a0e 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -2408,9 +2408,7 @@ async def _run_full_pipeline_background_async( # pyright: ignore[reportGeneralT if stt_cap_dispatch_enabled(): from utils.sync.cap_hold import hold_capture_if_exhausted - cap_held = await run_blocking( - sync_executor, - hold_capture_if_exhausted, + cap_held = await hold_capture_if_exhausted( uid, wav_paths, source, diff --git a/web/app/src/components/settings/__tests__/SettingsSubscriptionClarity.test.tsx b/web/app/src/components/settings/__tests__/SettingsSubscriptionClarity.test.tsx index e7c3706da5c..5fac41d78e7 100644 --- a/web/app/src/components/settings/__tests__/SettingsSubscriptionClarity.test.tsx +++ b/web/app/src/components/settings/__tests__/SettingsSubscriptionClarity.test.tsx @@ -84,7 +84,9 @@ describe('Account billing keeps paid identity separate from the allowance', () = expect(await screen.findByRole('heading', { name: 'Plus' })).toBeTruthy(); expect(screen.getByText('12.5 h used of 25 h')).toBeTruthy(); expect(screen.queryByText('Unlimited conversations')).toBeNull(); - expect(mocks.opened).toHaveBeenCalledExactlyOnceWith('usage_page', 'plus'); + await waitFor(() => + expect(mocks.opened).toHaveBeenCalledExactlyOnceWith('usage_page', 'plus'), + ); fireEvent.click(screen.getByRole('button', { name: /Unlimited Annual/ })); fireEvent.click(screen.getByRole('button', { name: 'Change Plan' })); await waitFor(() => expect(mocks.upgrade).toHaveBeenCalledWith('price_unlimited')); From c54f5a8e69a79484c300d41384a3d9a1128d2f6f Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 00:32:54 +0700 Subject: [PATCH 13/19] fix: type heterogeneous held-audio recovery receipts explicitly --- backend/utils/sync/cap_hold_recovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index c3171aa2ff2..71a07d575b6 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -110,7 +110,7 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) paths, source_maps, targets, bindings = await prepare_audio(uid, path, meta) processed = set(row.get('processed_paths') or []) - receipts = dict(row.get('piece_receipts') or {}) + receipts: dict[str, dict[str, Any]] = dict(row.get('piece_receipts') or {}) for segment_path in sorted(paths): if not await run_blocking( db_executor, cap_hold.renew, uid, snapshot.reference, row['lease_token'], time.time() @@ -119,7 +119,7 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: name = Path(segment_path).name if name in processed: continue - receipt = receipts.get(name) + receipt: dict[str, Any] | None = receipts.get(name) if receipt is None: if not await run_blocking(db_executor, _new_dispatch_allowed, uid): return False From f9605a69a328224e7698cbd229c1aad0f3ef753d Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 01:17:45 +0700 Subject: [PATCH 14/19] fix: preserve legacy sync usage arguments when no audio is held --- backend/routers/sync.py | 4 +++- backend/utils/sync/pipeline.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/routers/sync.py b/backend/routers/sync.py index cb2e2315c8f..c5d73961952 100644 --- a/backend/routers/sync.py +++ b/backend/routers/sync.py @@ -827,7 +827,9 @@ def _run_vad(path): # Record DG usage after successful processing (not before, to avoid charging on retries) if fair_use_restrict_dg: try: - dg_ms = int(max(0, total_speech_seconds - cap_held_seconds) * 1000) + dg_ms = int(total_speech_seconds * 1000) + if cap_holds: + dg_ms = int(max(0, total_speech_seconds - cap_held_seconds) * 1000) if dg_ms > 0: await run_blocking(db_executor, record_dg_usage_ms, uid, dg_ms) except Exception as e: diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 6dec2d14a0e..fae2da2ccf8 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -3039,7 +3039,9 @@ def _checkpoint_fenced_conversations(): uid=uid, job_id=job_id, content_id=content_id, - total_speech_seconds=max(0, total_speech_seconds - cap_held_seconds), + total_speech_seconds=( + max(0, total_speech_seconds - cap_held_seconds) if cap_holds else total_speech_seconds + ), ) # Build result From 914beb1905587a03132175d08ca49e6d69f02b38 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 01:29:11 +0700 Subject: [PATCH 15/19] fix: validate recap envelopes through the generated response decoder --- app/lib/backend/http/api/users.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/lib/backend/http/api/users.dart b/app/lib/backend/http/api/users.dart index a63931626a6..3431fc6b660 100644 --- a/app/lib/backend/http/api/users.dart +++ b/app/lib/backend/http/api/users.dart @@ -761,11 +761,10 @@ Future<({List items, bool ok})> getDailySummaries({int limit = 30, if (response == null || response.statusCode != 200) return (items: const [], ok: false); try { - final raw = jsonDecode(response.body); - if (raw is! Map || raw['summaries'] is! List) { + final data = wire.GeneratedDailySummariesResponse.fromJson(jsonDecode(response.body) as Map); + if (data.summaries == null) { throw const FormatException('Invalid daily recap list response'); } - final data = wire.GeneratedDailySummariesResponse.fromJson(raw); return (items: data.summaries?.map(DailySummary.fromGenerated).toList() ?? [], ok: true); } catch (e) { Logger.debug('Error parsing daily summaries: $e'); From b44eff752ef455a6805586abd19e170fd36cd932 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 03:17:54 +0700 Subject: [PATCH 16/19] Fix held STT recovery dispatch fencing and oversized retry admission --- ...tt-allowance-not-enforced-at-dispatch.json | 4 +- backend/diarizer/batch_pipeline_models.py | 5 +- backend/docs/listen_pusher_pipeline.mdx | 5 ++ backend/docs/stt-cap-dispatch.md | 19 ++++-- .../tests/unit/test_batch_pipeline_worker.py | 39 +++++++++-- .../test_prerecorded_dispatch_selection.py | 34 ++++++++++ backend/tests/unit/test_sync_cap_hold.py | 33 ++++++++++ .../tests/unit/test_sync_cap_hold_recovery.py | 65 +++++++++++++++++++ backend/utils/stt/pre_recorded.py | 14 +++- backend/utils/sync/cap_hold_recovery.py | 26 +++++++- backend/utils/sync/pipeline.py | 11 +++- 11 files changed, 237 insertions(+), 18 deletions(-) create mode 100644 backend/tests/unit/test_prerecorded_dispatch_selection.py diff --git a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json index 1df2d5587e3..9bc090f93e7 100644 --- a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json +++ b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json @@ -2,8 +2,8 @@ "schema_version": 1, "id": "FC-stt-allowance-not-enforced-at-dispatch", "violated_contract": "The transcription cap is computed but not enforced where Omi-funded provider audio is dispatched; replay, fallback and delayed execution can bypass request-time allowance checks.", - "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it and exercise both flag states at provider and queue boundaries.", - "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_sync_cap_content.py"], + "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it. Fence provider-bound recovery receipts before side effects and retain ambiguous attempts for reconciliation without re-dispatch. Exercise both flag states at provider and queue boundaries, including oversized empty retries without spending failure attempts when held.", + "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_batch_pipeline_worker.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_prerecorded_dispatch_selection.py", "backend/tests/unit/test_sync_cap_content.py"], "evidence_prs": [], "scope_hints": ["backend/routers/listen/", "backend/utils/stt/", "backend/utils/sync/", "backend/utils/batch_pipeline/"], "status": "open" diff --git a/backend/diarizer/batch_pipeline_models.py b/backend/diarizer/batch_pipeline_models.py index 6debce8f6f0..4aeee9244f7 100644 --- a/backend/diarizer/batch_pipeline_models.py +++ b/backend/diarizer/batch_pipeline_models.py @@ -97,7 +97,10 @@ def recover_empty(self, samples, max_padded_seconds, *, before_dispatch=None, af right = max(2 * SAMPLE_RATE, 8 * SAMPLE_RATE - len(samples) - left) padded_seconds = (len(samples) + left + right) / SAMPLE_RATE if padded_seconds > min(120, max_padded_seconds): - return [] + # The callback identifies enforced dispatch. Main's unflagged + # adapter still returns an empty hypothesis; policy holds must not + # spend a provider-failure attempt for audio never submitted. + return None if before_dispatch is not None else [] padded = np.pad(samples, (left, right)) if before_dispatch is not None and not before_dispatch(): return None # Retain raw work without spending a provider-failure attempt. diff --git a/backend/docs/listen_pusher_pipeline.mdx b/backend/docs/listen_pusher_pipeline.mdx index a4e864eb771..ef4855c4e72 100644 --- a/backend/docs/listen_pusher_pipeline.mdx +++ b/backend/docs/listen_pusher_pipeline.mdx @@ -14,6 +14,11 @@ Denial preserves receiving and existing audio storage paths; accepted provider work drains. Custom/on-device STT and flag-off metering retain their behavior. The [dispatch contract](stt-cap-dispatch.md) also documents sync holds, recovery, BYOK funding, additive metrics and the per-client activation decision. +Sync recovery fences a provider-bound `dispatched` receipt before paid STT or +ingestion. If the transcript checkpoint cannot be saved, later claims retain +the audio for reconciliation without repeating either side effect. Flagged +oversized GPU empty-result retries become held work without a failure attempt; +the unflagged retry follows main's behavior. ## Windowed live transcription deadlines diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index b68e1d5dba9..008ffd76ac2 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -40,8 +40,10 @@ exhausted acknowledgment; v2 adds `held_audio` to its durable result. Batch compute authorization checks current funded allowance. The worker calls that authority again after decode/VAD, before each ASR microbatch and immediately -before the full-capture empty-ASR recovery dispatch. Oversized recovery does not -reauthorize or record audio it never submits; accepted recovery counts actual +before the full-capture empty-ASR recovery dispatch. With enforcement on, +oversized recovery returns held work without spending a provider-failure attempt; +the unflagged path retains main's empty-result/failure behavior. Oversized recovery +does not reauthorize or record audio it never submits; accepted recovery counts actual synthetic padding. Missing worker policy authority holds work. Held work uses the scheduler's existing `defer`, preserving raw artifacts and refunding the provider-failure attempt. @@ -81,8 +83,17 @@ every new recovered provider request. Already persisted receipts can settle at the cap without another provider call. Recovery replays WAL coverage and lineage planning, loads speaker embeddings, and reuses VAD, segment intake, enrichment and optional playback finalization. Token-fenced -piece receipts checkpoint persisted transcription and serving-provider provenance before usage settlement; a -usage retry cannot repeat paid STT. Finalization runs once after all pieces join, +piece receipts persist a `dispatched` intent with funding, provider and model +**before** STT or ingestion, using the current token/lease fence. That selection +is pinned through the actual provider factory. Successful transcription and +ingestion replace the intent with a `transcribed` result before usage settlement; +a usage retry cannot repeat paid STT. A save error or lost lease after dispatch +leaves an ambiguous `dispatched` intent. Every subsequent claim fails closed +before VAD/lineage planning, retaining the original audio for reconciliation and +never automatically repeating STT or ingestion. This deliberately also holds an +intent if execution stops before dispatch can be confirmed; no automatic +reconciliation is added here. The log event is `event=stt_cap_recovery +outcome=reconciliation_required`. Finalization runs once after all pieces join, with the retained geolocation and source-position context. Piece receipts and idempotent usage keys preserve progress across retries. Completed objects are deleted only after content and usage settlement. WS-only and inactive diff --git a/backend/tests/unit/test_batch_pipeline_worker.py b/backend/tests/unit/test_batch_pipeline_worker.py index 26acf509f74..0064e307d55 100644 --- a/backend/tests/unit/test_batch_pipeline_worker.py +++ b/backend/tests/unit/test_batch_pipeline_worker.py @@ -802,14 +802,41 @@ def authorize(ids): assert checks == ([{'a'}, {'a'}] if enforce else []) -def test_oversized_recovery_neither_rechecks_nor_counts_unsubmitted_audio(): +@pytest.mark.parametrize('enforce', [False, True]) +def test_oversized_recovery_neither_rechecks_nor_counts_unsubmitted_audio(enforce): provider = ParakeetWords(SimpleNamespace(transcribe=lambda *a, **kw: pytest.fail('oversized provider request'))) before, after = Mock(), Mock() - assert ( - provider.recover_empty( - np.ones(117 * 16000, dtype=np.float32), 272, before_dispatch=before, after_dispatch=after - ) - == [] + result = provider.recover_empty( + np.ones(117 * 16000, dtype=np.float32), + 272, + **({'before_dispatch': before, 'after_dispatch': after} if enforce else {}), ) + assert result is None if enforce else result == [] before.assert_not_called() after.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +def test_oversized_empty_retry_holds_only_when_enforced(monkeypatch, enforce): + from utils.batch_pipeline.contracts import HeldWork + + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + model = Mock() + model.transcribe.side_effect = lambda arrays, **_: [ + SimpleNamespace(text='', timestamp={'word': []}) for _ in arrays + ] + provider = ParakeetWords(model) + worker, _, saved, _, diarized, failures = setup_worker( + decode=lambda work: np.ones(int(work.duration_seconds * 16000), dtype=np.float32), + words_providers={'parakeet': provider}, + recover_words=provider.recover_empty, + ) + result = worker.process_batch([item('a', 117)], authorize_dispatch=lambda ids: ids) + assert model.transcribe.call_count == 1 # Initial ASR only; 121s padded retry cannot fit. + assert not saved and not diarized + if enforce: + assert result == [HeldWork('a')] + assert not failures + else: + # Exact main behavior: an empty retry enters ordinary provider failure. + assert not result and failures == [('a', 'BatchValidationError')] diff --git a/backend/tests/unit/test_prerecorded_dispatch_selection.py b/backend/tests/unit/test_prerecorded_dispatch_selection.py new file mode 100644 index 00000000000..f9fac9b507e --- /dev/null +++ b/backend/tests/unit/test_prerecorded_dispatch_selection.py @@ -0,0 +1,34 @@ +"""A held piece must use the provider named by its durable dispatch intent.""" + +from unittest.mock import Mock + +import pytest + +from utils.stt import pre_recorded as stt + + +@pytest.mark.parametrize('pinned', [False, True]) +def test_url_dispatch_preserves_default_selection_or_uses_checkpointed_provider(monkeypatch, pinned): + monkeypatch.delenv('OMI_STT_STUB', raising=False) + selection = Mock(return_value=('modulate', 'en', 'velma-2')) + monkeypatch.setattr(stt, 'get_prerecorded_service', selection) + modulate, parakeet = Mock(), Mock() + monkeypatch.setattr(stt, 'ModulatePrerecordedProvider', lambda: modulate) + monkeypatch.setattr(stt, 'ParakeetPrerecordedProvider', lambda: parakeet) + options = {'selected_service': ('parakeet', 'multi', 'parakeet')} if pinned else {} + selected, unused = (parakeet, modulate) if pinned else (modulate, parakeet) + assert stt.prerecorded('synthetic-url', language='multi', **options) is selected.transcribe_url.return_value + selected.transcribe_url.assert_called_once_with( + 'synthetic-url', + speakers_count=None, + attempts=0, + return_language=False, + diarize=True, + language='multi', + keywords=None, + ) + unused.transcribe_url.assert_not_called() + if pinned: + selection.assert_not_called() + else: + selection.assert_called_once_with('multi') diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py index 0416f0fdcb3..9e029faa0d9 100644 --- a/backend/tests/unit/test_sync_cap_hold.py +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -71,6 +71,39 @@ def test_hold_failure_cannot_acknowledge_audio_or_dispatch_provider(tmp_path, mo stt.assert_not_called() +@pytest.mark.parametrize('enforce', [False, True]) +def test_recovery_pins_checkpointed_provider_only_with_enforcement(tmp_path, monkeypatch, enforce): + path = tmp_path / '1792152000.wav' + with wave.open(str(path), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\0' * 32000) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + monkeypatch.setattr(pipeline, 'sync_allowed', lambda *_: True) + monkeypatch.setattr(pipeline, 'funded_allowance', lambda *_: SimpleNamespace(reason='available')) + monkeypatch.setattr(pipeline, 'get_syncing_file_temporal_signed_url', Mock(return_value='synthetic-url')) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr(pipeline, 'keyterms', AsyncMock(return_value=[])) + selection = Mock(return_value=('modulate', None, 'velma-2')) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', selection) + stt = Mock(return_value=([], 'en')) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + errors = [] + service = ('parakeet', 'multi', 'parakeet') + assert not pipeline.process_segment( + str(path), 'synthetic', {}, threading.Lock(), errors, cap_recovery=True, cap_recovery_service=service + ) + assert not errors + assert stt.call_count == 2 + for call in stt.call_args_list: + if enforce: + assert call.kwargs['selected_service'] == service + else: + assert 'selected_service' not in call.kwargs + assert selection.call_count == (0 if enforce else 1) + + @pytest.mark.parametrize('enforce', [False, True]) async def test_full_capture_hold_precedes_vad_and_has_default_off_parity(tmp_path, monkeypatch, enforce): from utils.sync import cap_hold diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py index 0eb13273c41..89500690ac2 100644 --- a/backend/tests/unit/test_sync_cap_hold_recovery.py +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -1,15 +1,20 @@ """Recovery reaches real sync intake only after a current funded allowance.""" +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import Mock, AsyncMock import pytest from utils.sync import cap_hold_recovery as recovery +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore @pytest.fixture def state(monkeypatch, tmp_path): + checkpoint, renew = recovery.cap_hold.checkpoint, recovery.cap_hold.renew + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'true') + async def immediate(pool, fn, *args, **kwargs): return fn(*args, **kwargs) @@ -41,9 +46,12 @@ def processed(*args, **kwargs): monkeypatch.setattr(recovery, 'finalize_recovered_sync_segment', Mock()) monkeypatch.setattr(recovery, 'get_wav_duration', Mock(return_value=20)) monkeypatch.setattr(recovery, 'record_usage', Mock()) + monkeypatch.setattr(recovery, 'get_prerecorded_service', Mock(return_value=('modulate', 'en', 'velma-2'))) return SimpleNamespace( snapshot=SimpleNamespace(id='hold', reference=Mock()), row={'lease_token': 'owner-token', 'state': 'held'}, + checkpoint=checkpoint, + renew=renew, ) @@ -51,6 +59,7 @@ async def test_reset_or_upgrade_reuses_sync_and_settles_stable_usage_key(state): assert await recovery._recover_row('synthetic', state.snapshot, state.row) recovery.process_segment.assert_called_once() assert recovery.process_segment.call_args.kwargs['cap_recovery'] is True + assert recovery.process_segment.call_args.kwargs['cap_recovery_service'] == ('modulate', 'en', 'velma-2') recovery.finalize_recovered_sync_segment.assert_called_once() usage = recovery.record_usage.call_args.kwargs assert usage['idempotency_key'] == 'cap-hold-hold-1792152000.wav' @@ -156,6 +165,62 @@ def persist(path, uid, response, *args, **kwargs): async def test_lost_checkpoint_fence_cannot_settle_or_finalize(state): recovery.cap_hold.checkpoint.return_value = False assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.record_usage.assert_not_called() + recovery.finalize_recovered_sync_segment.assert_not_called() + + +@pytest.mark.parametrize('fault', ['save-error', 'lost-lease']) +async def test_ambiguous_dispatch_never_repeats_stt_or_ingestion_on_new_claim(state, monkeypatch, fault): + clock = [100.0] + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({key: {'state': 'held', 'lease_until': 0, 'audio_path': 'retained'}}) + ref = client.document('/'.join(key)) + state.snapshot.reference = ref + monkeypatch.setattr(recovery.cap_hold, 'get_firestore_client', lambda: client) + monkeypatch.setattr(recovery.cap_hold, 'external_write_fence', lambda _: nullcontext()) + monkeypatch.setattr(recovery.cap_hold.time, 'time', lambda: clock[0]) + monkeypatch.setattr(recovery.cap_hold, 'renew', state.renew) + fail_save = [False] + + def checkpoint(*args): + if fail_save[0]: + raise RuntimeError('transcript checkpoint unavailable after ingestion') + return state.checkpoint(*args) + + monkeypatch.setattr(recovery.cap_hold, 'checkpoint', checkpoint) + ingested = Mock() + + def dispatch_and_ingest(*args, **kwargs): + intent = client.rows[key]['piece_receipts']['1792152000.wav'] + assert intent == {'state': 'dispatched', 'provider': 'modulate', 'model': 'velma-2', 'funding': 'omi'} + assert kwargs['cap_recovery_service'] == ('modulate', 'en', 'velma-2') + ingested() + if fault == 'save-error': + fail_save[0] = True + else: + clock[0] = 2001 + assert recovery.cap_hold.claim('synthetic', ref, clock[0]) + return True + + recovery.process_segment.side_effect = dispatch_and_ingest + row = recovery.cap_hold.claim('synthetic', ref, clock[0]) + if fault == 'save-error': + with pytest.raises(RuntimeError, match='checkpoint unavailable'): + await recovery._recover_row('synthetic', state.snapshot, row) + else: + assert not await recovery._recover_row('synthetic', state.snapshot, row) + assert client.rows[key]['piece_receipts']['1792152000.wav']['state'] == 'dispatched' + assert client.rows[key]['audio_path'] == 'retained' + fail_save[0] = False + # A real new fenced claim loads only durable state, not the prior local row. + clock[0] = 4002 + retry = recovery.cap_hold.claim('synthetic', ref, clock[0]) + assert retry['lease_token'] != row['lease_token'] + assert not await recovery._recover_row('synthetic', state.snapshot, retry) + recovery.process_segment.assert_called_once() + ingested.assert_called_once() + recovery.prepare_audio.assert_awaited_once() recovery.record_usage.assert_not_called() recovery.finalize_recovered_sync_segment.assert_not_called() diff --git a/backend/utils/stt/pre_recorded.py b/backend/utils/stt/pre_recorded.py index c1513d166f2..411543578e2 100644 --- a/backend/utils/stt/pre_recorded.py +++ b/backend/utils/stt/pre_recorded.py @@ -1068,7 +1068,9 @@ def transcribe_bytes( ) -def get_prerecorded_provider(language: Optional[str] = 'en') -> PrerecordedSTTProvider: +def get_prerecorded_provider( + language: Optional[str] = 'en', *, selected_service: Optional[Tuple[str, Optional[str], str]] = None +) -> PrerecordedSTTProvider: """Construct exactly the language-aware provider selected for telemetry.""" # Offline harness sessions swap the provider for a deterministic stub # (selection/telemetry still resolve normally above them). Imported lazily: @@ -1077,7 +1079,7 @@ def get_prerecorded_provider(language: Optional[str] = 'en') -> PrerecordedSTTPr if prerecorded_stub_enabled(): return PrerecordedStubProvider() - service, _provider_language, model = get_prerecorded_service(language) + service, _provider_language, model = selected_service or get_prerecorded_service(language) if service == PrerecordedSTTService.MODULATE: return ModulatePrerecordedProvider() if service == PrerecordedSTTService.PARAKEET: @@ -1099,9 +1101,15 @@ def prerecorded( language: Optional[str] = None, model: str = "nova-3", keywords: Optional[Sequence[str]] = None, + *, + selected_service: Optional[Tuple[str, Optional[str], str]] = None, ) -> Union[List[Dict[str, Any]], Tuple[List[Dict[str, Any]], str]]: """Route pre-recorded URL transcription through STT_PRERECORDED_MODEL.""" - provider = get_prerecorded_provider(language) + provider = ( + get_prerecorded_provider(language, selected_service=selected_service) + if selected_service is not None + else get_prerecorded_provider(language) + ) return provider.transcribe_url( audio_url, speakers_count=speakers_count, diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index 71a07d575b6..887a3d30478 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -21,6 +21,7 @@ from utils.executors import run_blocking, db_executor, sync_executor, storage_executor, start_background_task from utils.stt_cap_dispatch import sync_allowed from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded +from utils.stt.pre_recorded import get_prerecorded_service from utils.sync import cap_hold from utils.sync.files import get_wav_duration from utils.sync.cap_recovery_audio import prepare as prepare_audio @@ -83,6 +84,11 @@ def _new_dispatch_allowed(uid: str) -> bool: async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: if row.get('finalized'): return True + if any(receipt.get('state') == 'dispatched' for receipt in (row.get('piece_receipts') or {}).values()): + # Check before re-running VAD/lineage planning, whose piece names could + # change. An ambiguous attempt blocks all new work for this source. + logger.warning('event=stt_cap_recovery outcome=reconciliation_required') + return False meta = await run_blocking(db_executor, cap_hold.metadata, uid, row) if meta.get('content_id'): status = await run_blocking(db_executor, sync_cap_holds.recovery_status, uid, meta['content_id'], snapshot.id) @@ -123,6 +129,21 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: if receipt is None: if not await run_blocking(db_executor, _new_dispatch_allowed, uid): return False + language = prefs.get('language') or '' + service = get_prerecorded_service( + language if prefs.get('single_language_mode') and language else 'multi' + ) + intent = { + 'state': 'dispatched', + 'provider': service[0], + 'model': service[2], + 'funding': 'omi', + } + receipts[name] = intent + # Persist BEFORE any paid STT or ingestion. The token and lease + # fence this write; a failed/uncertain save cannot dispatch. + if not await _save(uid, snapshot, row, {'piece_receipts': dict(receipts)}): + return False response = {'new_memories': set(), 'updated_memories': set()} errors, outcome = [], {} ok = await run_blocking( @@ -148,17 +169,20 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: sync_lane=meta.get('sync_lane', 'fresh'), cap_recovery=True, deferred_outcome=outcome, + cap_recovery_service=service, ) if errors or outcome.get('cap_held'): return False seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) if ok else 0 receipt = { + **intent, + 'state': 'transcribed', 'response': _stored(response), 'seconds': seconds, 'has_content': bool(ok), 'usage_metadata': { 'funding': 'omi', - 'provider': outcome.get('provider') or 'unknown', + 'provider': outcome.get('provider') or service[0], 'source': 'sync_v2', }, } diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index fae2da2ccf8..2c8cfcec55a 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -1250,6 +1250,7 @@ def process_segment( lineage_binding: Optional[str] = None, cap_recovery: bool = False, cap_content_id: str | None = None, + cap_recovery_service: tuple[str, str | None, str] | None = None, ): conversation_id = None provider = 'unknown' @@ -1301,7 +1302,14 @@ def process_segment( single_language_mode = prefs.get('single_language_mode', False) req_language = user_language if (single_language_mode and user_language) else 'multi' - provider, _, model = get_prerecorded_service(req_language) + recovery_dispatch = {} + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_service is not None: + # The durable pre-dispatch intent owns this provider/model. Pin + # actual serving too, rather than reselecting after the checkpoint. + provider, _, model = cap_recovery_service + recovery_dispatch['selected_service'] = cap_recovery_service + else: + provider, _, model = get_prerecorded_service(req_language) # When single-language mode is active, trust the user's language choice # rather than Deepgram's detection (avoids overriding explicit selection). @@ -1374,6 +1382,7 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N return_language=True, language=req_language, keywords=vocabulary if vocabulary else None, + **recovery_dispatch, ) if stt_cap_dispatch_enabled(): allowance = funded_allowance(uid) From 99c9a9579705dab3aee1b47b2fdc94aff2f18f0f Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 03:21:01 +0700 Subject: [PATCH 17/19] Preserve flag-off batch admission before duration coercion --- .../tests/unit/test_batch_pipeline_admission.py | 15 +++++++++++++++ backend/utils/batch_pipeline/admission.py | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/test_batch_pipeline_admission.py b/backend/tests/unit/test_batch_pipeline_admission.py index f44fb702f93..8bd5f25919d 100644 --- a/backend/tests/unit/test_batch_pipeline_admission.py +++ b/backend/tests/unit/test_batch_pipeline_admission.py @@ -117,3 +117,18 @@ def test_cap_rechecked_after_queueing_before_gpu_authorization(monkeypatch, enfo else: assert receipt['allowed'] == ['segment'] cap.funded_allowance.assert_not_called() + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('duration', [None, 'invalid']) +def test_flag_off_never_parses_new_dispatch_duration(monkeypatch, enforce, duration): + store, _, lease, _ = setup(monkeypatch, fresh=True) + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + store.rows['jobs', 'segment']['duration_seconds'] = duration + gate = Mock(side_effect=AssertionError('unparseable duration must not reach dispatch policy')) + monkeypatch.setattr(admission, 'sync_allowed', gate) + receipt = admission.authorize(store, tokens(lease), now=1) + assert receipt == ( + {'allowed': [], 'retry_after': {'segment': 30}} if enforce else {'allowed': ['segment'], 'retry_after': {}} + ) + gate.assert_not_called() diff --git a/backend/utils/batch_pipeline/admission.py b/backend/utils/batch_pipeline/admission.py index a27b3eb8a33..43fa82302fc 100644 --- a/backend/utils/batch_pipeline/admission.py +++ b/backend/utils/batch_pipeline/admission.py @@ -3,6 +3,7 @@ import math import time +from config.stt_cap_dispatch import enabled as cap_dispatch_enabled from utils.stt_cap_dispatch import sync_allowed from database.batch_pipeline_privacy import BatchPrivacyBlocked from utils.batch_pipeline.scheduler import owns_lease @@ -35,7 +36,9 @@ def read(tx): retries[segment] = 30 continue job, binding, intake = snapshot - if not sync_allowed(job['owner_id'], float(job.get('duration_seconds', 0)), 'parakeet'): + if cap_dispatch_enabled() and not sync_allowed( + job['owner_id'], float(job.get('duration_seconds', 0)), 'parakeet' + ): retries[segment] = 60 continue if int(job['priority']) == 0: From f2fbef1442af007a8e6519cdfd49ba531a93c243 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 04:18:29 +0700 Subject: [PATCH 18/19] fix: reconcile held STT attempts without stranding audio or siblings --- ...tt-allowance-not-enforced-at-dispatch.json | 4 +- backend/database/sync_cap_recovery.py | 73 +++++ backend/docs/listen_pusher_pipeline.mdx | 6 +- backend/docs/stt-cap-dispatch.md | 52 +++- .../tests/unit/test_sync_cap_hold_recovery.py | 207 +++++++++++++- .../test_sync_cap_reconciliation_metrics.py | 23 ++ .../unit/test_sync_cap_recovery_checkpoint.py | 68 +++++ .../test_sync_cap_recovery_dispatch_signal.py | 61 +++++ .../tests/unit/test_sync_cap_recovery_plan.py | 78 ++++++ backend/utils/conversations/lifecycle.py | 11 +- .../utils/observability/stt_cap_dispatch.py | 13 + backend/utils/sync/cap_hold.py | 12 +- backend/utils/sync/cap_hold_recovery.py | 253 +++++++++++++----- backend/utils/sync/cap_recovery_plan.py | 55 ++++ backend/utils/sync/pipeline.py | 18 ++ 15 files changed, 829 insertions(+), 105 deletions(-) create mode 100644 backend/database/sync_cap_recovery.py create mode 100644 backend/tests/unit/test_sync_cap_reconciliation_metrics.py create mode 100644 backend/tests/unit/test_sync_cap_recovery_checkpoint.py create mode 100644 backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py create mode 100644 backend/tests/unit/test_sync_cap_recovery_plan.py create mode 100644 backend/utils/sync/cap_recovery_plan.py diff --git a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json index 9bc090f93e7..5bec7819a79 100644 --- a/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json +++ b/.github/failure-classes/FC-stt-allowance-not-enforced-at-dispatch.json @@ -2,8 +2,8 @@ "schema_version": 1, "id": "FC-stt-allowance-not-enforced-at-dispatch", "violated_contract": "The transcription cap is computed but not enforced where Omi-funded provider audio is dispatched; replay, fallback and delayed execution can bypass request-time allowance checks.", - "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it. Fence provider-bound recovery receipts before side effects and retain ambiguous attempts for reconciliation without re-dispatch. Exercise both flag states at provider and queue boundaries, including oversized empty retries without spending failure attempts when held.", - "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_batch_pipeline_worker.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_prerecorded_dispatch_selection.py", "backend/tests/unit/test_sync_cap_content.py"], + "canonical_prevention": "Authorize actual provider funding at creation and every send, share live admission across channels and successors, and recheck queued execution before dispatch. Persist denied sync audio before acknowledging it. Fence provider-bound recovery receipts before side effects and atomically prove ingestion, clear known non-calls, and reconcile ambiguous attempts with one bounded re-dispatch while retaining terminal failures and finalizing siblings. Exercise both flag states at provider and queue boundaries, including oversized empty retries without spending failure attempts when held.", + "canonical_prevention_artifact": ["backend/tests/unit/test_stt_cap_dispatch.py", "backend/tests/unit/test_sync_cap_hold.py", "backend/tests/unit/test_batch_pipeline_admission.py", "backend/tests/unit/test_batch_pipeline_worker.py", "backend/tests/unit/test_sync_cap_coordinator.py", "backend/tests/unit/test_sync_cap_hold_recovery.py", "backend/tests/unit/test_sync_cap_recovery_checkpoint.py", "backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py", "backend/tests/unit/test_sync_cap_recovery_plan.py", "backend/tests/unit/test_sync_cap_reconciliation_metrics.py", "backend/tests/unit/test_prerecorded_dispatch_selection.py", "backend/tests/unit/test_sync_cap_content.py"], "evidence_prs": [], "scope_hints": ["backend/routers/listen/", "backend/utils/stt/", "backend/utils/sync/", "backend/utils/batch_pipeline/"], "status": "open" diff --git a/backend/database/sync_cap_recovery.py b/backend/database/sync_cap_recovery.py new file mode 100644 index 00000000000..cc8d65c42bb --- /dev/null +++ b/backend/database/sync_cap_recovery.py @@ -0,0 +1,73 @@ +"""Piece-lineage proof committed in the exact conversation-ingest transaction.""" + +import time +from typing import Any + +from database._client import get_firestore_client + + +class RecoveryIngestFenced(RuntimeError): + """An expired/replaced recovery attempt cannot append conversation content.""" + + +def ingest_hooks(uid: str, context: dict, incoming: dict): + ref = ( + get_firestore_client() + .collection('users') + .document(uid) + .collection('stt_cap_holds') + .document(context['hold_id']) + ) + current = {} + piece = context['piece'] + + def before(transaction): + row = ref.get(transaction=transaction).to_dict() or {} + receipt = (row.get('piece_receipts') or {}).get(piece) or {} + if ( + row.get('lease_token') != context['lease_token'] + or row.get('lease_until', 0) <= time.time() + or receipt.get('state') != 'dispatched' + or receipt.get('attempt', 1) != context['attempt'] + ): + raise RecoveryIngestFenced('held piece ingest ownership changed') + current.clear() + current.update(row.get('piece_ingests') or {}) + + def after(transaction, assigned): + conversation, created, _ = assigned + cid = conversation['id'] + response: dict[str, Any] = {'new_memories': [], 'updated_memories': []} + repeat = conversation.get('_sync_lineage_repeat_only') + if not repeat: + response['new_memories' if created else 'updated_memories'] = [cid] + if conversation.get('sync_relevance') == 'keep' and ( + not repeat or conversation.get('_sync_lineage_completion_pending') + ): + response['_merged'] = {cid: incoming.get('language')} + current[piece] = { + **context['receipt'], + 'state': 'transcribed', + 'has_content': True, + 'response': response, + 'lineage_anchor': incoming['id'], + 'conversation_id': cid, + } + # No reads here: assignment has already staged its conversation writes. + transaction.update(ref, {'piece_ingests': dict(current)}) + + return before, after + + +def ingested_piece(uid: str, hold_id: str, piece: str) -> dict | None: + row = ( + get_firestore_client() + .collection('users') + .document(uid) + .collection('stt_cap_holds') + .document(hold_id) + .get() + .to_dict() + or {} + ) + return (row.get('piece_ingests') or {}).get(piece) diff --git a/backend/docs/listen_pusher_pipeline.mdx b/backend/docs/listen_pusher_pipeline.mdx index 42d55973228..02db56a9ef6 100644 --- a/backend/docs/listen_pusher_pipeline.mdx +++ b/backend/docs/listen_pusher_pipeline.mdx @@ -15,8 +15,10 @@ work drains. Custom/on-device STT and flag-off metering retain their behavior. The [dispatch contract](stt-cap-dispatch.md) also documents sync holds, recovery, BYOK funding, additive metrics and the per-client activation decision. Sync recovery fences a provider-bound `dispatched` receipt before paid STT or -ingestion. If the transcript checkpoint cannot be saved, later claims retain -the audio for reconciliation without repeating either side effect. Flagged +ingestion, and atomically checkpoints ingest proof alongside the conversation. +Known non-calls clear the intent. After 30 minutes, ambiguous work reconciles +from that proof or gets one second dispatch; a second ambiguity retains the audio +as failed reconciliation. Successful siblings finalize independently. Flagged oversized GPU empty-result retries become held work without a failure attempt; the unflagged retry follows main's behavior. diff --git a/backend/docs/stt-cap-dispatch.md b/backend/docs/stt-cap-dispatch.md index 008ffd76ac2..72617fb3506 100644 --- a/backend/docs/stt-cap-dispatch.md +++ b/backend/docs/stt-cap-dispatch.md @@ -83,20 +83,44 @@ every new recovered provider request. Already persisted receipts can settle at the cap without another provider call. Recovery replays WAL coverage and lineage planning, loads speaker embeddings, and reuses VAD, segment intake, enrichment and optional playback finalization. Token-fenced -piece receipts persist a `dispatched` intent with funding, provider and model -**before** STT or ingestion, using the current token/lease fence. That selection -is pinned through the actual provider factory. Successful transcription and -ingestion replace the intent with a `transcribed` result before usage settlement; -a usage retry cannot repeat paid STT. A save error or lost lease after dispatch -leaves an ambiguous `dispatched` intent. Every subsequent claim fails closed -before VAD/lineage planning, retaining the original audio for reconciliation and -never automatically repeating STT or ingestion. This deliberately also holds an -intent if execution stops before dispatch can be confirmed; no automatic -reconciliation is added here. The log event is `event=stt_cap_recovery -outcome=reconciliation_required`. Finalization runs once after all pieces join, -with the retained geolocation and source-position context. Piece -receipts and idempotent usage keys preserve progress across retries. Completed -objects are deleted only after content and usage settlement. WS-only and inactive +piece receipts persist a `dispatched` intent with funding, provider, model, +`dispatched_at` and `attempt=1` **before** STT or ingestion, under the lease fence. +Provider selection is pinned. Before dispatch, recovery freezes encrypted VAD +piece bytes and their lineage context under the existing owner prefix. A later +coverage/VAD decision cannot redefine a piece or suppress a sibling checkpoint. +The original full audio remains retained throughout. + +`process_segment` explicitly reports `provider_called`, set immediately before +`prerecorded`, never inferred from an error class. A known non-call atomically +removes its intent (or reverts an unspent second attempt). The next claim retries +normally. If a call may have happened, the piece waits 30 minutes. Then: + +1. A proof written in the **same transaction as conversation ingestion** identifies + the hold, piece and deterministic conversation lineage. Recovery marks that + piece transcribed, settles its existing idempotent usage key and finalizes it + without another provider call. +2. Without a proof, current allowance/abuse admission permits **one** additional + application dispatch, persisted as `attempt=2`. Recovery owns retries; the + ordinary segment's local empty-result retry is disabled for these checkpointed + recovery attempts. Provider-internal SDK retries remain outside this ledger. +3. A second ambiguous attempt gets the same cooldown/proof check, then becomes + `failed_reconciliation`. Its audio and manifest remain retained. Successful + siblings still settle usage and finalize independently; their paths checkpoint + finalization progress. A failed piece never publishes an all-success content + ledger result. + +A reconciled ingest also retains raw audio, since playback storage may not have +finished before the interrupted processor returned. Terminal retained manifests +sort after actionable work and cannot dispatch. Successful ledger settlement is +retryable before that terminal transition. Neither retained form has an +application TTL or is silently erased. Ordinary fully successful recovery deletes +source and frozen pieces only after content/usage settlement. + +`omi_stt_cap_recovery_reconciliation_total{outcome}` counts `cleared_noncall`, +`reconciled_ingested`, `redispatched`, and `failed`. Structured +`event=stt_cap_recovery_reconciliation` logs include hold id and piece name, never +content. Metric labels contain no uid, hold id or piece name. New recovery and +its signals are inert when the dispatch flag is off. WS-only and inactive accounts wait for their next authenticated HTTP request; there is no new cron. Rolling back the dispatch flag pauses this new recovery, preserving its audio. diff --git a/backend/tests/unit/test_sync_cap_hold_recovery.py b/backend/tests/unit/test_sync_cap_hold_recovery.py index 89500690ac2..7944946c38d 100644 --- a/backend/tests/unit/test_sync_cap_hold_recovery.py +++ b/backend/tests/unit/test_sync_cap_hold_recovery.py @@ -19,6 +19,12 @@ async def immediate(pool, fn, *args, **kwargs): return fn(*args, **kwargs) monkeypatch.setattr(recovery, 'run_blocking', immediate) + monkeypatch.setattr(recovery.cap_recovery_plan, 'run_blocking', immediate) + monkeypatch.setattr( + recovery.cap_recovery_plan, 'persist_piece', Mock(return_value={'audio_path': 'retained-piece', 'seconds': 20}) + ) + monkeypatch.setattr(recovery.sync_cap_recovery, 'ingested_piece', Mock(return_value=None)) + monkeypatch.setattr(recovery.time, 'time', lambda: 100.0) monkeypatch.setattr(recovery.cap_hold, 'metadata', Mock(return_value={'timestamp': 1792152000, 'source': 'omi'})) monkeypatch.setattr( recovery.cap_hold, 'recovery_directory', Mock(return_value=SimpleNamespace(name=str(tmp_path), cleanup=Mock())) @@ -33,7 +39,7 @@ async def immediate(pool, fn, *args, **kwargs): monkeypatch.setattr(recovery, 'is_daily_audio_ceiling_exceeded', Mock(return_value=False)) def processed(*args, **kwargs): - kwargs['deferred_outcome']['provider'] = 'modulate' + kwargs['deferred_outcome'].update(provider='modulate', provider_called=True) return True monkeypatch.setattr(recovery, 'process_segment', Mock(side_effect=processed)) @@ -44,7 +50,6 @@ def processed(*args, **kwargs): AsyncMock(side_effect=lambda uid, path, meta: ({path}, {}, {}, {})), ) monkeypatch.setattr(recovery, 'finalize_recovered_sync_segment', Mock()) - monkeypatch.setattr(recovery, 'get_wav_duration', Mock(return_value=20)) monkeypatch.setattr(recovery, 'record_usage', Mock()) monkeypatch.setattr(recovery, 'get_prerecorded_service', Mock(return_value=('modulate', 'en', 'velma-2'))) return SimpleNamespace( @@ -106,8 +111,8 @@ async def test_partial_recovery_resumes_only_unprocessed_pieces(state): async def test_persistence_failure_keeps_audio_and_releases_local_staging(state): recovery.process_segment.side_effect = RuntimeError('synthetic persistence failure') - with pytest.raises(RuntimeError): - await recovery._recover_row('synthetic', state.snapshot, state.row) + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert not state.row['piece_receipts'] state.snapshot.reference.update.assert_not_called() recovery.cap_hold.recovery_directory.return_value.cleanup.assert_called_once() @@ -148,6 +153,7 @@ async def test_all_pieces_join_before_one_finalization(state): ) def persist(path, uid, response, *args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = True response['updated_memories'].add('joined-conversation') response['_merged'] = {'joined-conversation': 'en'} return True @@ -171,7 +177,7 @@ async def test_lost_checkpoint_fence_cannot_settle_or_finalize(state): @pytest.mark.parametrize('fault', ['save-error', 'lost-lease']) -async def test_ambiguous_dispatch_never_repeats_stt_or_ingestion_on_new_claim(state, monkeypatch, fault): +async def test_atomic_ingest_proof_recovers_save_error_or_lost_lease(state, monkeypatch, fault): clock = [100.0] key = ('users', 'synthetic', 'stt_cap_holds', 'hold') client = StrictFirestore({key: {'state': 'held', 'lease_until': 0, 'audio_path': 'retained'}}) @@ -193,7 +199,11 @@ def checkpoint(*args): def dispatch_and_ingest(*args, **kwargs): intent = client.rows[key]['piece_receipts']['1792152000.wav'] - assert intent == {'state': 'dispatched', 'provider': 'modulate', 'model': 'velma-2', 'funding': 'omi'} + assert intent['state'] == 'dispatched' and intent['attempt'] == 1 + kwargs['deferred_outcome']['provider_called'] = True + client.rows[key]['piece_ingests'] = { + '1792152000.wav': {**intent, 'state': 'transcribed', 'has_content': True, 'response': {}} + } assert kwargs['cap_recovery_service'] == ('modulate', 'en', 'velma-2') ingested() if fault == 'save-error': @@ -203,6 +213,9 @@ def dispatch_and_ingest(*args, **kwargs): assert recovery.cap_hold.claim('synthetic', ref, clock[0]) return True + recovery.sync_cap_recovery.ingested_piece.side_effect = ( + lambda uid, hold, piece: client.rows[key].get('piece_ingests', {}).get(piece) + ) recovery.process_segment.side_effect = dispatch_and_ingest row = recovery.cap_hold.claim('synthetic', ref, clock[0]) if fault == 'save-error': @@ -217,12 +230,13 @@ def dispatch_and_ingest(*args, **kwargs): clock[0] = 4002 retry = recovery.cap_hold.claim('synthetic', ref, clock[0]) assert retry['lease_token'] != row['lease_token'] - assert not await recovery._recover_row('synthetic', state.snapshot, retry) + assert await recovery._recover_row('synthetic', state.snapshot, retry) + assert retry['retain_audio'] recovery.process_segment.assert_called_once() ingested.assert_called_once() recovery.prepare_audio.assert_awaited_once() - recovery.record_usage.assert_not_called() - recovery.finalize_recovered_sync_segment.assert_not_called() + recovery.record_usage.assert_called_once() + recovery.finalize_recovered_sync_segment.assert_called_once() @pytest.mark.parametrize('restriction', ['hard', 'daily']) @@ -248,3 +262,178 @@ async def test_lost_completion_lease_cannot_settle_or_erase_audio(state, monkeyp await recovery.recover('synthetic') recovery._settle_and_erase.assert_not_called() recovery.cap_hold.release.assert_called_once_with('synthetic', state.snapshot.reference, 'owner-token') + + +@pytest.mark.parametrize('reason', ['provider_select', 'cap_held']) +async def test_known_noncall_clears_intent_and_next_claim_retries(state, reason): + normal = recovery.process_segment.side_effect + + def noncall(*args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = False + if reason == 'cap_held': + kwargs['deferred_outcome']['cap_held'] = True + else: + args[4].append('provider_select') + return False + + recovery.process_segment.side_effect = noncall + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert not state.row['piece_receipts'] + recovery.process_segment.side_effect = normal + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert recovery.process_segment.call_count == 2 + recovery.prepare_audio.assert_awaited_once() # Frozen piece, no changed VAD plan. + + +async def test_ambiguous_piece_redispatches_only_once_then_retains_audio_and_finishes_sibling(state, monkeypatch): + clock = [100.0] + monkeypatch.setattr(recovery.time, 'time', lambda: clock[0]) + recovery.prepare_audio.side_effect = lambda uid, path, meta: ( + {path, path.replace('1792152000', '1792152020')}, + {}, + {}, + {}, + ) + attempts = [] + + def process(path, uid, response, lock, errors, *args, **kwargs): + kwargs['deferred_outcome']['provider_called'] = True + if '1792152000' in path: + attempts.append(kwargs['cap_recovery_checkpoint']['attempt']) + errors.append('ambiguous_provider_result') + return False + response['updated_memories'].add('sibling') + response['_merged'] = {'sibling': 'en'} + return True + + recovery.process_segment.side_effect = process + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1] + recovery.finalize_recovered_sync_segment.assert_called_once() + assert recovery.finalize_recovered_sync_segment.call_args.args[1]['updated_memories'] == {'sibling'} + # Still cooling down: neither provider nor sibling finalization repeats. + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1] + clock[0] = 1901 + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + clock[0] = 3702 + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + failed = state.row['piece_receipts']['1792152000.wav'] + assert failed['state'] == 'failed_reconciliation' and failed['attempt'] == 2 + assert state.row['retain_audio'] and state.row['failed_reconciliation'] + recovery.finalize_recovered_sync_segment.assert_called_once() + recovery.record_usage.assert_called_once() + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + monkeypatch.setattr(recovery, '_erase', Mock()) + monkeypatch.setattr(recovery.sync_cap_holds, 'settle_hold', Mock()) + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + recovery._erase.assert_not_called() + recovery.sync_cap_holds.settle_hold.assert_not_called() # Never claim failed audio completed. + + +async def test_ingested_ambiguity_finalizes_without_provider_and_settles_once(state): + state.row['piece_plan'] = {'1792152000.wav': {'audio_path': 'retained-piece', 'seconds': 20}} + state.row['piece_receipts'] = {'1792152000.wav': {'state': 'dispatched', 'attempt': 1, 'dispatched_at': -1801}} + recovery.sync_cap_recovery.ingested_piece.return_value = { + 'state': 'transcribed', + 'has_content': True, + 'seconds': 20, + 'response': {'updated_memories': ['ingested'], '_merged': {'ingested': 'en'}}, + 'usage_metadata': {'funding': 'omi', 'provider': 'modulate', 'source': 'sync_v2'}, + } + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + recovery.process_segment.assert_not_called() + recovery.cap_hold.read_audio.assert_not_called() + recovery.prepare_audio.assert_not_awaited() + recovery.finalize_recovered_sync_segment.assert_called_once() + assert state.row['retain_audio'] + assert recovery.record_usage.call_args.kwargs['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + + +async def test_second_attempt_noncall_reverts_prior_intent_without_spending_retry(state): + prior = {'state': 'dispatched', 'attempt': 1, 'dispatched_at': -1801} + state.row.update( + piece_plan={'1792152000.wav': {'audio_path': 'piece', 'seconds': 20}}, piece_receipts={'1792152000.wav': prior} + ) + + def noncall(*args, **kwargs): + kwargs['deferred_outcome'].update(provider_called=False, cap_held=True) + return False + + recovery.process_segment.side_effect = noncall + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav'] == prior + + +async def test_flag_off_recovery_has_no_io(state, monkeypatch): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', 'false') + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + await recovery.recover('synthetic') + recovery.cap_hold.metadata.assert_not_called() + recovery.process_segment.assert_not_called() + + +async def test_not_ingested_piece_gets_one_successful_redispatch_with_stable_usage_key(state, monkeypatch): + clock = [100.0] + monkeypatch.setattr(recovery.time, 'time', lambda: clock[0]) + attempts = [] + + def process(*args, **kwargs): + attempt = kwargs['cap_recovery_checkpoint']['attempt'] + attempts.append(attempt) + kwargs['deferred_outcome']['provider_called'] = True + if attempt == 1: + args[4].append('provider_reply_unknown') + return False + args[2]['new_memories'].add('recovered') + return True + + recovery.process_segment.side_effect = process + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + clock[0] = 1901 + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert await recovery._recover_row('synthetic', state.snapshot, state.row) + assert attempts == [1, 2] + assert state.row['piece_receipts']['1792152000.wav']['state'] == 'transcribed' + recovery.record_usage.assert_called_once() + assert recovery.record_usage.call_args.kwargs['idempotency_key'] == 'cap-hold-hold-1792152000.wav' + + +async def test_cap_denial_after_provider_call_remains_ambiguous(state): + def called_then_denied(*args, **kwargs): + kwargs['deferred_outcome'].update(provider_called=True, cap_held=True) + return False + + recovery.process_segment.side_effect = called_then_denied + assert not await recovery._recover_row('synthetic', state.snapshot, state.row) + assert state.row['piece_receipts']['1792152000.wav']['state'] == 'dispatched' + + +def test_retained_manifest_cannot_dispatch_and_does_not_starve_pending_holds(state, monkeypatch): + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + client = StrictFirestore({key: {'state': 'retained', 'lease_token': 'owner', 'lease_until': 0}}) + ref = client.document('/'.join(key)) + monkeypatch.setattr(recovery.cap_hold, 'get_firestore_client', lambda: client) + recovery.cap_hold.release('synthetic', ref, 'owner') + assert client.rows[key]['next_attempt_at'] == 253402300799 + monkeypatch.setattr(recovery.cap_hold, 'external_write_fence', lambda _: nullcontext()) + assert recovery.cap_hold.claim('synthetic', ref, 2000) is None + + +def test_reconciled_retention_waits_for_ledger_settlement_and_retries_failure(state, monkeypatch): + state.row.update(state='completed', retain_audio=True) + recovery.cap_hold.metadata.return_value['content_id'] = 'content' + settle = Mock(side_effect=RuntimeError('synthetic settlement failure')) + monkeypatch.setattr(recovery.sync_cap_holds, 'settle_hold', settle) + monkeypatch.setattr(recovery, '_erase', Mock()) + with pytest.raises(RuntimeError): + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + state.snapshot.reference.update.assert_not_called() + assert state.row['state'] == 'completed' + settle.side_effect = None + settle.return_value = True + recovery._settle_and_erase('synthetic', state.snapshot, state.row) + assert state.snapshot.reference.update.call_args.args[0]['state'] == 'retained' + recovery._erase.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_reconciliation_metrics.py b/backend/tests/unit/test_sync_cap_reconciliation_metrics.py new file mode 100644 index 00000000000..93937fa04dd --- /dev/null +++ b/backend/tests/unit/test_sync_cap_reconciliation_metrics.py @@ -0,0 +1,23 @@ +import logging + +import pytest + +from utils.observability.stt_cap_dispatch import RECONCILIATION, record_reconciliation + + +@pytest.mark.parametrize('enabled', [False, True]) +@pytest.mark.parametrize('outcome', ['cleared_noncall', 'reconciled_ingested', 'redispatched', 'failed']) +def test_reconciliation_counter_and_log_are_additive_bounded_and_identifiable(monkeypatch, caplog, enabled, outcome): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enabled)) + metric = RECONCILIATION.labels(outcome=outcome) + before = metric._value.get() + with caplog.at_level(logging.INFO): + record_reconciliation(outcome, 'synthetic-hold', '1792152000.wav') + assert metric._value.get() == before + int(enabled) + if enabled: + assert f'outcome={outcome}' in caplog.text + assert 'hold_id=synthetic-hold' in caplog.text and '1792152000.wav' in caplog.text + else: + assert not caplog.text + for family in RECONCILIATION.collect(): + assert all(set(sample.labels) == {'outcome'} for sample in family.samples) diff --git a/backend/tests/unit/test_sync_cap_recovery_checkpoint.py b/backend/tests/unit/test_sync_cap_recovery_checkpoint.py new file mode 100644 index 00000000000..40f573c4b3b --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_checkpoint.py @@ -0,0 +1,68 @@ +"""Recovery proof shares conversation transaction and never bypasses its lease.""" + +from unittest.mock import Mock + +import pytest + +from database import sync_cap_recovery as checkpoint +from tests.unit.fixtures.strict_firestore_transaction import StrictFirestore +from utils.conversations import lifecycle + + +@pytest.mark.parametrize('fence', ['valid', 'expired', 'replaced', 'attempt']) +def test_piece_ingest_proof_commits_with_assignment_or_is_fenced(monkeypatch, fence): + key = ('users', 'synthetic', 'stt_cap_holds', 'hold') + row = { + 'lease_token': 'owner', + 'lease_until': 1000, + 'piece_receipts': {'piece.wav': {'state': 'dispatched', 'attempt': 1}}, + } + if fence == 'expired': + row['lease_until'] = 99 + elif fence == 'replaced': + row['lease_token'] = 'new-owner' + elif fence == 'attempt': + row['piece_receipts']['piece.wav']['attempt'] = 2 + client = StrictFirestore({key: row}) + monkeypatch.setattr(checkpoint, 'get_firestore_client', lambda: client) + monkeypatch.setattr(checkpoint.time, 'time', lambda: 100) + monkeypatch.setattr(lifecycle, 'observe_completed_conversation_shape', Mock()) + context = { + 'hold_id': 'hold', + 'piece': 'piece.wav', + 'lease_token': 'owner', + 'attempt': 1, + 'receipt': {'seconds': 20, 'attempt': 1, 'usage_metadata': {'provider': 'modulate'}}, + } + conversation = {'id': 'canonical', 'status': 'completed', 'sync_relevance': 'keep'} + tx = client.transaction() + + def assign(uid, incoming, **kwargs): + kwargs['before_assign'](tx) + tx.set(client.document('users/synthetic/conversations/canonical'), conversation) + kwargs['after_assign'](tx, (conversation, True, [])) + return conversation, True, [] + + monkeypatch.setattr(lifecycle.conversations_db, 'assign_sync_conversation', assign) + incoming = {'id': 'piece-lineage-anchor', 'status': 'completed', 'language': 'en'} + if fence != 'valid': + with pytest.raises(checkpoint.RecoveryIngestFenced): + lifecycle.ingest_sync_conversation('synthetic', incoming, recovery_checkpoint=context) + assert not tx.has_written + assert not client.rows[key].get('piece_ingests') + return + lifecycle.ingest_sync_conversation('synthetic', incoming, recovery_checkpoint=context) + assert tx.sets and tx.updates # Same transaction; strict fixture forbids late reads. + proof = checkpoint.ingested_piece('synthetic', 'hold', 'piece.wav') + assert proof['lineage_anchor'] == 'piece-lineage-anchor' + assert proof['conversation_id'] == 'canonical' + assert proof['response']['new_memories'] == ['canonical'] + assert proof['response']['_merged'] == {'canonical': 'en'} + + +def test_ordinary_ingest_preserves_main_assignment_call(monkeypatch): + assigned = {'id': 'canonical', 'status': 'completed'} + call = Mock(return_value=(assigned, False, [])) + monkeypatch.setattr(lifecycle.conversations_db, 'assign_sync_conversation', call) + lifecycle.ingest_sync_conversation('synthetic', assigned) + call.assert_called_once_with('synthetic', assigned, candidate_id=None, target_id=None) diff --git a/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py b/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py new file mode 100644 index 00000000000..4ed6b979d11 --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_dispatch_signal.py @@ -0,0 +1,61 @@ +"""Explicit provider-call signal covers real pre-dispatch exits in both modes.""" + +import threading +from unittest.mock import Mock, AsyncMock + +import pytest + +from utils.sync import pipeline + + +@pytest.mark.parametrize('enforce', [False, True]) +@pytest.mark.parametrize('exit_at', ['url', 'keyterms', 'cap', 'provider_error', 'empty']) +def test_real_segment_reports_provider_boundary_without_changing_flag_off_calls(monkeypatch, enforce, exit_at): + monkeypatch.setenv('STT_CAP_DISPATCH_ENFORCEMENT', str(enforce)) + monkeypatch.setattr(pipeline, 'ambiguous_binding_pending', lambda *_: False) + gate = Mock(return_value=exit_at != 'cap') + monkeypatch.setattr(pipeline, 'sync_allowed', gate) + monkeypatch.setattr( + pipeline, + 'get_syncing_file_temporal_signed_url', + Mock(side_effect=RuntimeError('synthetic URL failure') if exit_at == 'url' else None, return_value='url'), + ) + monkeypatch.setattr(pipeline, 'schedule_syncing_temporal_file_deletion', Mock()) + monkeypatch.setattr( + pipeline, + 'keyterms', + AsyncMock( + side_effect=RuntimeError('synthetic keyterms failure') if exit_at == 'keyterms' else None, return_value=[] + ), + ) + monkeypatch.setattr(pipeline, 'get_prerecorded_service', Mock(return_value=('modulate', 'en', 'velma-2'))) + stt = Mock( + side_effect=RuntimeError('synthetic provider failure') if exit_at == 'provider_error' else None, + return_value=([], 'en'), + ) + monkeypatch.setattr(pipeline, 'prerecorded', stt) + monkeypatch.setattr(pipeline, 'record_cap_audio', Mock()) + monkeypatch.setattr(pipeline, 'get_wav_duration', Mock(return_value=20)) + monkeypatch.setattr(pipeline, 'funded_allowance', Mock(return_value=Mock(reason='available'))) + monkeypatch.setattr(pipeline, '_record_empty_segment_as_silence', Mock()) + outcome, errors = {}, [] + assert not pipeline.process_segment( + '1792152000.wav', + 'synthetic', + {}, + threading.Lock(), + errors, + cap_recovery=True, + cap_recovery_checkpoint={'attempt': 1}, + deferred_outcome=outcome, + ) + if enforce: + called = exit_at in {'provider_error', 'empty'} + assert outcome['provider_called'] is called + assert stt.call_count == int(called) + assert outcome.get('recovery_ambiguous', False) is (exit_at == 'empty') + else: + assert 'provider_called' not in outcome + assert 'recovery_ambiguous' not in outcome + assert stt.call_count == (0 if exit_at in {'url', 'keyterms'} else 1 if exit_at == 'provider_error' else 2) + gate.assert_not_called() diff --git a/backend/tests/unit/test_sync_cap_recovery_plan.py b/backend/tests/unit/test_sync_cap_recovery_plan.py new file mode 100644 index 00000000000..12050c3d10a --- /dev/null +++ b/backend/tests/unit/test_sync_cap_recovery_plan.py @@ -0,0 +1,78 @@ +"""Immutable pieces retain exact bytes and lineage across changing VAD/coverage.""" + +import json +from types import SimpleNamespace +from unittest.mock import Mock, AsyncMock +import wave + +import pytest + +from utils.sync import cap_recovery_plan as plan + + +@pytest.mark.parametrize('upload_fails', [False, True]) +async def test_piece_plan_is_durable_before_dispatch_and_replays_frozen_audio(tmp_path, monkeypatch, upload_fails): + async def leaf(pool, fn, *args): + return fn(*args) + + monkeypatch.setattr(plan, 'run_blocking', leaf) + blobs = {} + + def upload(uid, path, identity, context): + if upload_fails: + raise RuntimeError('synthetic storage failure') + blob = f'chunks/{uid}/cap-held/{identity}' + with open(path, 'rb') as source: + blobs[blob] = source.read() + return {'audio_path': blob, 'metadata': json.dumps(context)} + + monkeypatch.setattr(plan.cap_hold, '_upload', upload) + monkeypatch.setattr(plan.cap_hold, 'read_audio', lambda uid, row, path: None) + piece = tmp_path / '1792152000.wav' + with wave.open(str(piece), 'wb') as output: + output.setparams((1, 2, 16000, 0, 'NONE', 'not compressed')) + output.writeframes(b'\x01\x02' * 1600) + original = piece.read_bytes() + context = { + 'target_conversation_id': 'bound', + 'source_position_map': ({'claim': 'synthetic'}, 100), + 'lineage_binding': 'bound', + } + prepare = AsyncMock( + return_value=( + {str(piece)}, + {str(piece): context['source_position_map']}, + {str(piece): 'bound'}, + {str(piece): 'bound'}, + ) + ) + row = {'audio_path': 'original-held-audio'} + + async def save(uid, snapshot, stored, update): + stored.update(update) + return True + + directory, snapshot = SimpleNamespace(name=str(tmp_path)), SimpleNamespace(id='hold') + if upload_fails: + with pytest.raises(RuntimeError): + await plan.ensure_plan('synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save) + assert 'piece_plan' not in row and row['audio_path'] == 'original-held-audio' + return + pieces, _, _ = await plan.ensure_plan('synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save) + assert row['piece_plan'] == pieces + # Later code/config may change VAD and WAL overlap. Neither may redefine a retry. + prepare.side_effect = AssertionError('must not prepare the source again') + piece.write_bytes(b'changed-local-source') + monkeypatch.setattr( + plan.cap_hold, 'read_audio', lambda uid, item, path: piece.write_bytes(blobs[item['audio_path']]) + ) + monkeypatch.setattr(plan.cap_hold, 'metadata', lambda uid, item: json.loads(item['metadata'])) + replay, locals_, contexts = await plan.ensure_plan( + 'synthetic', snapshot, row, {'timestamp': 1}, directory, prepare, save + ) + path, restored_context = await plan.stage_piece( + 'synthetic', piece.name, replay[piece.name], directory, locals_, contexts + ) + assert path == str(piece) and piece.read_bytes() == original + assert restored_context == context + prepare.assert_awaited_once() diff --git a/backend/utils/conversations/lifecycle.py b/backend/utils/conversations/lifecycle.py index 151017461b1..260db48847f 100644 --- a/backend/utils/conversations/lifecycle.py +++ b/backend/utils/conversations/lifecycle.py @@ -21,6 +21,7 @@ from database import conversations as conversations_db from database import oversized_conversation_terminal as oversized_terminal_db from database import recording_sessions as recording_sessions_db +from database import sync_cap_recovery from database.firestore_read_metrics import FirestoreReadSite from database.firestore_transaction_retry import FirestoreContentionExhausted from models.conversation_enums import ConversationStatus @@ -134,15 +135,21 @@ def create_completed_conversation(uid: str, conversation_data: dict[str, Any], * @sync_phase('firestore') -def ingest_sync_conversation(uid: str, incoming: dict[str, Any], *, candidate_id=None, target_id=None): +def ingest_sync_conversation( + uid: str, incoming: dict[str, Any], *, candidate_id=None, target_id=None, recovery_checkpoint: dict | None = None +): """Admit a retained deterministic sync row and atomically append later chunks. Enrichment follows persistence; filler remains recoverable under Show discarded. Existing lifecycle fields are preserved by the transactional append. """ _require_status(incoming, ConversationStatus.completed) + checkpoint_hooks: dict[str, Any] = {} + if recovery_checkpoint is not None: + before, after = sync_cap_recovery.ingest_hooks(uid, recovery_checkpoint, incoming) + checkpoint_hooks = {'before_assign': before, 'after_assign': after} assigned, created, survivors = conversations_db.assign_sync_conversation( - uid, incoming, candidate_id=candidate_id, target_id=target_id + uid, incoming, candidate_id=candidate_id, target_id=target_id, **checkpoint_hooks ) record_sync_intake_outcome(created=created) if created: diff --git a/backend/utils/observability/stt_cap_dispatch.py b/backend/utils/observability/stt_cap_dispatch.py index ea4abf33746..cf09365799a 100644 --- a/backend/utils/observability/stt_cap_dispatch.py +++ b/backend/utils/observability/stt_cap_dispatch.py @@ -1,6 +1,8 @@ """Additive dispatch provenance, independent of root usage and unit projection.""" from prometheus_client import Counter +from config.stt_cap_dispatch import enabled +import logging AUDIO_SECONDS = Counter( 'omi_stt_cap_provider_audio_seconds', @@ -8,6 +10,17 @@ ['path', 'outcome', 'provider', 'allowance'], ) CONFIG_ERRORS = Counter('omi_stt_cap_launch_config_errors', 'Invalid subscription launch configuration', ['reason']) +RECONCILIATION = Counter('omi_stt_cap_recovery_reconciliation', 'Held piece reconciliation transitions', ['outcome']) +logger = logging.getLogger(__name__) + + +def record_reconciliation(outcome: str, hold_id: str, piece: str) -> None: + if not enabled(): + return + if outcome not in {'cleared_noncall', 'reconciled_ingested', 'redispatched', 'failed'}: + raise ValueError('unknown reconciliation outcome') + RECONCILIATION.labels(outcome=outcome).inc() + logger.info('event=stt_cap_recovery_reconciliation outcome=%s hold_id=%s piece=%r', outcome, hold_id, piece) def record_audio(path, outcome, provider, seconds, exhausted=False): diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py index b7b06491fb9..b7be85fb28c 100644 --- a/backend/utils/sync/cap_hold.py +++ b/backend/utils/sync/cap_hold.py @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) COLLECTION = 'stt_cap_holds' AUDIO_CHUNK_BYTES = 1024 * 1024 +RETAINED_NEXT_ATTEMPT_AT = 253402300799 # Terminal rows sort after actionable work, without a new query/index. def holds(uid: str) -> Any: @@ -146,7 +147,7 @@ def claim(uid: str, ref: Any, now: float) -> dict[str, Any] | None: def commit(tx: Any): snapshot = ref.get(transaction=tx) row = snapshot.to_dict() if snapshot.exists else None - if not row or row.get('state') == 'completed' or row.get('lease_until', 0) > now: + if not row or row.get('state') in {'completed', 'retained'} or row.get('lease_until', 0) > now: return None token = uuid4().hex tx.update(ref, {'lease_until': now + 1800, 'lease_token': token}) @@ -202,7 +203,14 @@ def commit(tx: Any): snapshot = ref.get(transaction=tx) row = snapshot.to_dict() if snapshot.exists else {} if row.get('lease_token') == token: - tx.update(ref, {'lease_until': 0, 'lease_token': '', 'next_attempt_at': time.time()}) + tx.update( + ref, + { + 'lease_until': 0, + 'lease_token': '', + 'next_attempt_at': RETAINED_NEXT_ATTEMPT_AT if row.get('state') == 'retained' else time.time(), + }, + ) return commit(client.transaction()) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index 887a3d30478..e20e30c6fec 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -16,14 +16,14 @@ from config.stt_cap_dispatch import enabled from database import users as users_db -from database import sync_cap_holds +from database import sync_cap_holds, sync_cap_recovery from utils.analytics import record_usage from utils.executors import run_blocking, db_executor, sync_executor, storage_executor, start_background_task from utils.stt_cap_dispatch import sync_allowed from utils.fair_use import get_hard_restriction_status, is_daily_audio_ceiling_exceeded from utils.stt.pre_recorded import get_prerecorded_service -from utils.sync import cap_hold -from utils.sync.files import get_wav_duration +from utils.sync import cap_hold, cap_recovery_plan +from utils.observability.stt_cap_dispatch import record_reconciliation from utils.sync.cap_recovery_audio import prepare as prepare_audio from models.geolocation import Geolocation from utils.sync.pipeline import ( @@ -34,11 +34,13 @@ from models.conversation import ConversationSource logger = logging.getLogger(__name__) +RECONCILIATION_COOLDOWN_SECONDS = 1800 def _erase(uid: str, snapshot: Any, row: dict[str, Any]) -> None: # Account deletion already purges this existing owner-scoped chunk prefix. - for path in filter(None, (row['audio_path'], row.get('metadata_path'))): + objects = [row, *(row.get('piece_plan') or {}).values()] + for path in {item[key] for item in objects for key in ('audio_path', 'metadata_path') if item.get(key)}: if not path.startswith(f'chunks/{uid}/cap-held/'): raise ValueError('hold audio owner mismatch') try: @@ -82,14 +84,11 @@ def _new_dispatch_allowed(uid: str) -> bool: async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: + if not enabled(): + return False if row.get('finalized'): return True - if any(receipt.get('state') == 'dispatched' for receipt in (row.get('piece_receipts') or {}).values()): - # Check before re-running VAD/lineage planning, whose piece names could - # change. An ambiguous attempt blocks all new work for this source. - logger.warning('event=stt_cap_recovery outcome=reconciliation_required') - return False - meta = await run_blocking(db_executor, cap_hold.metadata, uid, row) + meta = await run_blocking(storage_executor, cap_hold.metadata, uid, row) if meta.get('content_id'): status = await run_blocking(db_executor, sync_cap_holds.recovery_status, uid, meta['content_id'], snapshot.id) if status in {'completed', 'settled'}: @@ -98,7 +97,7 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: return False if not row.get('piece_receipts') and not await run_blocking(db_executor, _new_dispatch_allowed, uid): return False - prefs = await run_blocking(db_executor, users_db.get_user_transcription_preferences, uid) + prefs = await run_blocking(db_executor, users_db.get_user_transcription_preferences, uid) or {} private = await run_blocking(db_executor, users_db.get_user_private_cloud_sync_enabled, uid) protection = await run_blocking(db_executor, users_db.get_data_protection_level, uid) if private else None try: @@ -110,86 +109,164 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: embeddings = {} geolocation = Geolocation.model_validate(meta['geolocation']) if meta.get('geolocation') else None directory = await run_blocking(storage_executor, cap_hold.recovery_directory, uid) + receipts = dict(row.get('piece_receipts') or {}) + + async def save_receipt(name, receipt): + updated = dict(receipts) + if receipt is None: + updated.pop(name, None) + else: + updated[name] = receipt + if not await _save(uid, snapshot, row, {'piece_receipts': updated}): + return False + receipts.clear() + receipts.update(updated) + return True + try: - name = Path(meta.get('file_name') or f"{meta['timestamp']}.wav").name - path = str(Path(directory.name) / name) - await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) - paths, source_maps, targets, bindings = await prepare_audio(uid, path, meta) + try: + plan, local_paths, contexts = await cap_recovery_plan.ensure_plan( + uid, snapshot, row, meta, directory, prepare_audio, _save + ) + except cap_recovery_plan.PlanFenced: + return False processed = set(row.get('processed_paths') or []) - receipts: dict[str, dict[str, Any]] = dict(row.get('piece_receipts') or {}) - for segment_path in sorted(paths): + pending = False + for name in sorted(set(plan) | set(receipts)): if not await run_blocking( db_executor, cap_hold.renew, uid, snapshot.reference, row['lease_token'], time.time() ): return False - name = Path(segment_path).name if name in processed: continue - receipt: dict[str, Any] | None = receipts.get(name) - if receipt is None: + receipt = receipts.get(name) + if receipt and receipt.get('state') == 'failed_reconciliation': + continue + attempt = 1 + if receipt and receipt.get('state') == 'dispatched': + if 'dispatched_at' not in receipt: + receipt = { + **receipt, + 'attempt': receipt.get('attempt', 1), + 'dispatched_at': row.get('created_at', time.time()), + } + if not await save_receipt(name, receipt): + return False + if time.time() - receipt['dispatched_at'] < RECONCILIATION_COOLDOWN_SECONDS: + pending = True + continue + proof = await run_blocking(db_executor, sync_cap_recovery.ingested_piece, uid, snapshot.id, name) + if proof: + receipt = {**proof, 'reconciled_ingested': True} + if not await save_receipt(name, receipt): + return False + record_reconciliation('reconciled_ingested', snapshot.id, name) + elif receipt.get('attempt', 1) >= 2 or name not in plan: + if not await save_receipt( + name, {**receipt, 'state': 'failed_reconciliation', 'has_content': False} + ): + return False + record_reconciliation('failed', snapshot.id, name) + continue + else: + attempt = 2 + if receipt is None or receipt.get('state') == 'dispatched': if not await run_blocking(db_executor, _new_dispatch_allowed, uid): - return False - language = prefs.get('language') or '' - service = get_prerecorded_service( - language if prefs.get('single_language_mode') and language else 'multi' - ) + pending = True + continue + previous = receipt + try: + segment_path, context = await cap_recovery_plan.stage_piece( + uid, name, plan[name], directory, local_paths, contexts + ) + language = prefs.get('language') or '' + service = get_prerecorded_service( + language if prefs.get('single_language_mode') and language else 'multi' + ) + except Exception as error: + logger.warning( + 'event=stt_cap_recovery outcome=prepare_deferred hold_id=%s piece=%r exception_type=%s', + snapshot.id, + name, + type(error).__name__, + ) + pending = True + continue intent = { 'state': 'dispatched', 'provider': service[0], 'model': service[2], 'funding': 'omi', + 'attempt': attempt, + 'dispatched_at': time.time(), + 'seconds': plan[name]['seconds'], + 'usage_metadata': {'funding': 'omi', 'provider': service[0], 'source': 'sync_v2'}, } - receipts[name] = intent - # Persist BEFORE any paid STT or ingestion. The token and lease - # fence this write; a failed/uncertain save cannot dispatch. - if not await _save(uid, snapshot, row, {'piece_receipts': dict(receipts)}): + if not await save_receipt(name, intent): return False response = {'new_memories': set(), 'updated_memories': set()} - errors, outcome = [], {} - ok = await run_blocking( - sync_executor, - process_segment, - segment_path, - uid, - response, - threading.Lock(), - errors, - ConversationSource(meta['source']), - False, - prefs, - embeddings, - private_cloud_sync_enabled=bool(private), - data_protection_level=protection, - client_device_id=meta.get('client_device_id'), - client_platform=meta.get('client_platform'), - target_conversation_id=targets.get(segment_path, meta.get('target_conversation_id')), - geolocation=geolocation, - source_position_map=source_maps.get(segment_path), - lineage_binding=bindings.get(segment_path, meta.get('lineage_binding')), - sync_lane=meta.get('sync_lane', 'fresh'), - cap_recovery=True, - deferred_outcome=outcome, - cap_recovery_service=service, - ) - if errors or outcome.get('cap_held'): - return False - seconds = await run_blocking(sync_executor, get_wav_duration, segment_path) if ok else 0 + errors, outcome = [], {'provider_called': False} + try: + ok = await run_blocking( + sync_executor, + process_segment, + segment_path, + uid, + response, + threading.Lock(), + errors, + ConversationSource(meta['source']), + False, + prefs, + embeddings, + private_cloud_sync_enabled=bool(private), + data_protection_level=protection, + client_device_id=meta.get('client_device_id'), + client_platform=meta.get('client_platform'), + target_conversation_id=context.get('target_conversation_id'), + geolocation=geolocation, + source_position_map=context.get('source_position_map'), + lineage_binding=context.get('lineage_binding'), + sync_lane=meta.get('sync_lane', 'fresh'), + cap_recovery=True, + deferred_outcome=outcome, + cap_recovery_service=service, + cap_recovery_checkpoint={ + 'hold_id': snapshot.id, + 'piece': name, + 'lease_token': row['lease_token'], + 'attempt': attempt, + 'receipt': intent, + }, + ) + except Exception as error: + ok = False + errors.append(type(error).__name__) + logger.warning( + 'event=stt_cap_recovery outcome=processing_deferred hold_id=%s piece=%r exception_type=%s', + snapshot.id, + name, + type(error).__name__, + ) + if not outcome['provider_called']: + if not await save_receipt(name, previous): + return False + record_reconciliation('cleared_noncall', snapshot.id, name) + pending = True + continue + if attempt == 2: + record_reconciliation('redispatched', snapshot.id, name) + if errors or outcome.get('cap_held') or outcome.get('recovery_ambiguous'): + pending = True + continue receipt = { **intent, 'state': 'transcribed', 'response': _stored(response), - 'seconds': seconds, + 'seconds': intent['seconds'] if ok else 0, 'has_content': bool(ok), - 'usage_metadata': { - 'funding': 'omi', - 'provider': outcome.get('provider') or service[0], - 'source': 'sync_v2', - }, } - receipts[name] = receipt - # Paid STT/content is now durable BEFORE usage or enrichment. - # A settlement retry reuses this receipt without dispatching STT. - if not await _save(uid, snapshot, row, {'piece_receipts': dict(receipts)}): + if not await save_receipt(name, receipt): return False if receipt['has_content']: seconds = int(receipt['seconds']) @@ -207,16 +284,37 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: processed.add(name) if not await _save(uid, snapshot, row, {'processed_paths': sorted(processed)}): return False - response = _response(receipts) - if receipts: - # Ordinary sync finalizes the joined batch, not each VAD piece. - await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, response, bool(private)) - return await _save(uid, snapshot, row, {'finalized': True, 'recovery_response': _stored(response)}) + # Finalize completed siblings even while another piece is cooling down. + finalized = set(row.get('finalized_paths') or []) + ready = {name: receipts[name] for name in processed - finalized if name in receipts} + if ready: + await run_blocking(sync_executor, finalize_recovered_sync_segment, uid, _response(ready), bool(private)) + finalized.update(ready) + if not await _save(uid, snapshot, row, {'finalized_paths': sorted(finalized)}): + return False + if pending: + return False + failed = any(r.get('state') == 'failed_reconciliation' for r in receipts.values()) + retained = failed or any(r.get('reconciled_ingested') for r in receipts.values()) + return await _save( + uid, + snapshot, + row, + { + 'finalized': True, + 'recovery_response': _stored(_response(receipts)), + 'failed_reconciliation': failed, + 'retain_audio': retained, + }, + ) finally: await run_blocking(storage_executor, directory.cleanup) def _settle_and_erase(uid: str, snapshot: Any, row: dict) -> None: + if row.get('failed_reconciliation'): + snapshot.reference.update({'state': 'retained', 'next_attempt_at': cap_hold.RETAINED_NEXT_ATTEMPT_AT}) + return # Siblings finalized, but the held ledger must not claim failed audio was transcribed. if not row.get('settled'): meta = cap_hold.metadata(uid, row) if meta.get('content_id') and not row.get('ledger_already_settled'): @@ -229,12 +327,17 @@ def _settle_and_erase(uid: str, snapshot: Any, row: dict) -> None: ): raise RuntimeError('hold content ledger ownership changed') snapshot.reference.update({'settled': True}) - _erase(uid, snapshot, row) + if row.get('retain_audio'): + snapshot.reference.update({'state': 'retained', 'next_attempt_at': cap_hold.RETAINED_NEXT_ATTEMPT_AT}) + else: + _erase(uid, snapshot, row) async def recover(uid: str) -> None: # One cheap distributed cooldown before querying usage or manifests. Failed # Redis means deferred recovery, never duplicate provider dispatch. + if not enabled(): + return acquired = await run_blocking(db_executor, cap_hold.recovery_cooldown, uid) if not acquired: return @@ -250,6 +353,8 @@ async def recover(uid: str) -> None: continue try: if await _recover_row(uid, snapshot, row): + # Completed remains retryable if ledger settlement fails. + # Only successful settlement (or terminal failure) retains it. if not await _save(uid, snapshot, row, {'state': 'completed'}): continue await run_blocking(storage_executor, _settle_and_erase, uid, snapshot, row) diff --git a/backend/utils/sync/cap_recovery_plan.py b/backend/utils/sync/cap_recovery_plan.py new file mode 100644 index 00000000000..993e97209bc --- /dev/null +++ b/backend/utils/sync/cap_recovery_plan.py @@ -0,0 +1,55 @@ +"""Freeze encrypted VAD pieces and lineage before any held-audio dispatch.""" + +import hashlib +import json +from pathlib import Path + +from utils.executors import run_blocking, storage_executor +from utils.sync import cap_hold +from utils.sync.files import get_wav_duration + + +class PlanFenced(RuntimeError): + pass + + +def persist_piece(uid: str, hold_id: str, path: str, context: dict) -> dict: + identity = hashlib.sha256( + (hold_id + cap_hold.capture_identity(path, context) + json.dumps(context, sort_keys=True)).encode() + ).hexdigest() + return {**cap_hold._upload(uid, path, identity, context), 'seconds': get_wav_duration(path)} + + +async def ensure_plan(uid, snapshot, row, meta, directory, prepare_audio, save): + if 'piece_plan' in row: + return row['piece_plan'], {}, {} + name = Path(meta.get('file_name') or f"{meta['timestamp']}.wav").name + path = str(Path(directory.name) / name) + await run_blocking(storage_executor, cap_hold.read_audio, uid, row, path) + paths, source_maps, targets, bindings = await prepare_audio(uid, path, meta) + plan, local_paths, contexts = {}, {}, {} + for path in sorted(paths): + name = Path(path).name + if name in plan: + raise ValueError('held piece identity collision') + context = { + 'target_conversation_id': targets.get(path, meta.get('target_conversation_id')), + 'source_position_map': source_maps.get(path), + 'lineage_binding': bindings.get(path, meta.get('lineage_binding')), + } + plan[name] = await run_blocking(storage_executor, persist_piece, uid, snapshot.id, path, context) + local_paths[name], contexts[name] = path, context + if not await save(uid, snapshot, row, {'piece_plan': plan}): + raise PlanFenced('held piece plan ownership changed') + return plan, local_paths, contexts + + +async def stage_piece(uid, name, entry, directory, local_paths, contexts): + if name in local_paths: + return local_paths[name], contexts[name] + path = str(Path(directory.name) / Path(name).name) + context = await run_blocking(storage_executor, cap_hold.metadata, uid, entry) + await run_blocking(storage_executor, cap_hold.read_audio, uid, entry, path) + position = context.get('source_position_map') + context['source_position_map'] = tuple(position) if position else None + return path, context diff --git a/backend/utils/sync/pipeline.py b/backend/utils/sync/pipeline.py index 2c8cfcec55a..c25dbcea3a6 100644 --- a/backend/utils/sync/pipeline.py +++ b/backend/utils/sync/pipeline.py @@ -1251,11 +1251,14 @@ def process_segment( cap_recovery: bool = False, cap_content_id: str | None = None, cap_recovery_service: tuple[str, str | None, str] | None = None, + cap_recovery_checkpoint: dict | None = None, ): conversation_id = None provider = 'unknown' model = 'unknown' phase = 'provider_select' + if stt_cap_dispatch_enabled() and cap_recovery and deferred_outcome is not None: + deferred_outcome['provider_called'] = False try: if ambiguous_binding_pending(uid, lineage_binding): phase = 'persistence' @@ -1375,6 +1378,10 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N if deferred_outcome is not None: deferred_outcome['cap_held'] = True return False + if stt_cap_dispatch_enabled() and cap_recovery and deferred_outcome is not None: + # Explicit request-boundary signal survives later cap denial, + # parse/ingest errors, and the bounded empty-result retry. + deferred_outcome['provider_called'] = True words, detected_language = prerecorded( url, speakers_count=3, @@ -1400,6 +1407,12 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N _log_empty_retry('recovered', words=len(words or []), raw_empty=not words) recovered_via_retry = True break + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_checkpoint is not None: + # A held piece gets at most two persisted application dispatch + # attempts. Reconciliation owns the second, not this local retry. + if deferred_outcome is not None: + deferred_outcome['recovery_ambiguous'] = True + return False if empty_retried: # Words survived the provider but nothing survived post-processing: # again no transcribable speech, valid and empty rather than failed. @@ -1530,6 +1543,11 @@ def _log_empty_retry(outcome: str, words: int = 0, raw_empty: bool = False) -> N incoming, candidate_id=closest_memory['id'] if closest_memory else None, target_id=target_conversation_id, + **( + {'recovery_checkpoint': cap_recovery_checkpoint} + if stt_cap_dispatch_enabled() and cap_recovery and cap_recovery_checkpoint is not None + else {} + ), ) if sync_safety_wal_enabled() and assigned.get('sync_live_target'): removed = len(transcript_segments) - len(survivors) From d54c9297bd760363f564935c5792dcf869d0fbbe Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 17 Oct 2026 04:25:28 +0700 Subject: [PATCH 19/19] fix: type recovery boundaries and share held-audio uploads --- backend/tests/unit/test_sync_cap_hold.py | 4 ++-- .../tests/unit/test_sync_cap_recovery_plan.py | 2 +- backend/utils/sync/cap_hold.py | 6 ++--- backend/utils/sync/cap_hold_recovery.py | 9 ++++--- backend/utils/sync/cap_recovery_plan.py | 24 +++++++++++++++---- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/tests/unit/test_sync_cap_hold.py b/backend/tests/unit/test_sync_cap_hold.py index 9e029faa0d9..fcd8af910ac 100644 --- a/backend/tests/unit/test_sync_cap_hold.py +++ b/backend/tests/unit/test_sync_cap_hold.py @@ -250,7 +250,7 @@ async def test_capture_hold_coordinates_db_and_storage_leaves(monkeypatch): ) monkeypatch.setattr(cap_hold, '_describe', Mock(return_value=('held', 20, {'source': 'omi'}))) monkeypatch.setattr(cap_hold, '_exists', Mock(return_value=False)) - monkeypatch.setattr(cap_hold, '_upload', Mock(return_value={'audio_path': 'synthetic'})) + monkeypatch.setattr(cap_hold, 'upload_held_audio', Mock(return_value={'audio_path': 'synthetic'})) monkeypatch.setattr(cap_hold, '_publish', Mock(return_value=True)) calls = [] @@ -267,7 +267,7 @@ async def leaf(pool, fn, *args, **kwargs): (cap_hold.db_executor, cap_hold.funded_allowance), (cap_hold.storage_executor, cap_hold._describe), (cap_hold.db_executor, cap_hold._exists), - (cap_hold.storage_executor, cap_hold._upload), + (cap_hold.storage_executor, cap_hold.upload_held_audio), (cap_hold.db_executor, cap_hold._publish), ] assert admission == {'reason': 'allowance_unavailable'} diff --git a/backend/tests/unit/test_sync_cap_recovery_plan.py b/backend/tests/unit/test_sync_cap_recovery_plan.py index 12050c3d10a..d01875764e1 100644 --- a/backend/tests/unit/test_sync_cap_recovery_plan.py +++ b/backend/tests/unit/test_sync_cap_recovery_plan.py @@ -26,7 +26,7 @@ def upload(uid, path, identity, context): blobs[blob] = source.read() return {'audio_path': blob, 'metadata': json.dumps(context)} - monkeypatch.setattr(plan.cap_hold, '_upload', upload) + monkeypatch.setattr(plan.cap_hold, 'upload_held_audio', upload) monkeypatch.setattr(plan.cap_hold, 'read_audio', lambda uid, row, path: None) piece = tmp_path / '1792152000.wav' with wave.open(str(piece), 'wb') as output: diff --git a/backend/utils/sync/cap_hold.py b/backend/utils/sync/cap_hold.py index b7be85fb28c..86c949def28 100644 --- a/backend/utils/sync/cap_hold.py +++ b/backend/utils/sync/cap_hold.py @@ -70,7 +70,7 @@ def _exists(uid: str, identity: str) -> bool: return bool(holds(uid).document(identity).get().exists) -def _upload(uid: str, path: str, identity: str, metadata: dict) -> dict: +def upload_held_audio(uid: str, path: str, identity: str, metadata: dict) -> dict: blob_path = f'chunks/{uid}/cap-held/{identity}.wav.enc' bucket = get_private_cloud_sync_bucket() with external_write_fence(uid), owner_storage_write_gate(uid, bucket): @@ -125,7 +125,7 @@ def hold_segment(uid: str, path: str, metadata: dict[str, Any], *, receipt: dict identity, seconds, metadata = _describe(path, metadata) created = False if not _exists(uid, identity): - fields = _upload(uid, path, identity, metadata) + fields = upload_held_audio(uid, path, identity, metadata) created = _publish(uid, identity, seconds, fields) return _finish(identity, seconds, funded_allowance(uid).reason, receipt, created) @@ -135,7 +135,7 @@ async def _hold_capture(uid: str, path: str, metadata: dict, reason: str, receip identity, seconds, metadata = await run_blocking(storage_executor, _describe, path, metadata) created = False if not await run_blocking(db_executor, _exists, uid, identity): - fields = await run_blocking(storage_executor, _upload, uid, path, identity, metadata) + fields = await run_blocking(storage_executor, upload_held_audio, uid, path, identity, metadata) created = await run_blocking(db_executor, _publish, uid, identity, seconds, fields) _finish(identity, seconds, reason, receipt, created) diff --git a/backend/utils/sync/cap_hold_recovery.py b/backend/utils/sync/cap_hold_recovery.py index e20e30c6fec..ff4f375514b 100644 --- a/backend/utils/sync/cap_hold_recovery.py +++ b/backend/utils/sync/cap_hold_recovery.py @@ -6,7 +6,6 @@ import asyncio import logging -from pathlib import Path import threading import time @@ -109,9 +108,9 @@ async def _recover_row(uid: str, snapshot: Any, row: dict[str, Any]) -> bool: embeddings = {} geolocation = Geolocation.model_validate(meta['geolocation']) if meta.get('geolocation') else None directory = await run_blocking(storage_executor, cap_hold.recovery_directory, uid) - receipts = dict(row.get('piece_receipts') or {}) + receipts: dict[str, dict[str, Any]] = dict(row.get('piece_receipts') or {}) - async def save_receipt(name, receipt): + async def save_receipt(name: str, receipt: dict[str, Any] | None) -> bool: updated = dict(receipts) if receipt is None: updated.pop(name, None) @@ -139,7 +138,7 @@ async def save_receipt(name, receipt): return False if name in processed: continue - receipt = receipts.get(name) + receipt: dict[str, Any] | None = receipts.get(name) if receipt and receipt.get('state') == 'failed_reconciliation': continue attempt = 1 @@ -192,7 +191,7 @@ async def save_receipt(name, receipt): ) pending = True continue - intent = { + intent: dict[str, Any] = { 'state': 'dispatched', 'provider': service[0], 'model': service[2], diff --git a/backend/utils/sync/cap_recovery_plan.py b/backend/utils/sync/cap_recovery_plan.py index 993e97209bc..11ec4c06798 100644 --- a/backend/utils/sync/cap_recovery_plan.py +++ b/backend/utils/sync/cap_recovery_plan.py @@ -3,6 +3,7 @@ import hashlib import json from pathlib import Path +from typing import Any, Awaitable, Callable from utils.executors import run_blocking, storage_executor from utils.sync import cap_hold @@ -13,14 +14,22 @@ class PlanFenced(RuntimeError): pass -def persist_piece(uid: str, hold_id: str, path: str, context: dict) -> dict: +def persist_piece(uid: str, hold_id: str, path: str, context: dict[str, Any]) -> dict[str, Any]: identity = hashlib.sha256( (hold_id + cap_hold.capture_identity(path, context) + json.dumps(context, sort_keys=True)).encode() ).hexdigest() - return {**cap_hold._upload(uid, path, identity, context), 'seconds': get_wav_duration(path)} + return {**cap_hold.upload_held_audio(uid, path, identity, context), 'seconds': get_wav_duration(path)} -async def ensure_plan(uid, snapshot, row, meta, directory, prepare_audio, save): +async def ensure_plan( + uid: str, + snapshot: Any, + row: dict[str, Any], + meta: dict[str, Any], + directory: Any, + prepare_audio: Callable[..., Awaitable[tuple[set, dict, dict, dict]]], + save: Callable[..., Awaitable[bool]], +) -> tuple[dict[str, dict[str, Any]], dict[str, str], dict[str, dict[str, Any]]]: if 'piece_plan' in row: return row['piece_plan'], {}, {} name = Path(meta.get('file_name') or f"{meta['timestamp']}.wav").name @@ -44,7 +53,14 @@ async def ensure_plan(uid, snapshot, row, meta, directory, prepare_audio, save): return plan, local_paths, contexts -async def stage_piece(uid, name, entry, directory, local_paths, contexts): +async def stage_piece( + uid: str, + name: str, + entry: dict[str, Any], + directory: Any, + local_paths: dict[str, str], + contexts: dict[str, dict[str, Any]], +) -> tuple[str, dict[str, Any]]: if name in local_paths: return local_paths[name], contexts[name] path = str(Path(directory.name) / Path(name).name)