From b7f4fe1482e72a3c0196f2993a6bf432bc0abb82 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 14 Nov 2026 10:42:43 +0300 Subject: [PATCH 1/3] fix: fence timer completion against replacement and lease takeover --- CONTEXT.md | 12 +- docs/introduction/how-it-works.md | 32 +- faststream_redis_timers/configs.py | 6 + faststream_redis_timers/message.py | 5 +- faststream_redis_timers/parser/parser.py | 19 +- faststream_redis_timers/store.py | 99 +++-- faststream_redis_timers/subscriber/usecase.py | 9 +- tests/test_at_least_once.py | 24 +- tests/test_fencing.py | 365 ++++++++++++++++++ tests/test_unit.py | 82 ++-- 10 files changed, 554 insertions(+), 99 deletions(-) create mode 100644 tests/test_fencing.py diff --git a/CONTEXT.md b/CONTEXT.md index 4951978..1580673 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -20,8 +20,8 @@ time, identified by a `timer_id`. The Timer is the *record*, not the message it has a lifecycle (Scheduled → Due → Claimed → Committed) that its payload does not. **Topic**: -The named channel a Timer is Scheduled on and that a subscriber consumes from. Two Redis keys are -derived per Topic; a Timer only ever exists on one. +The named channel a Timer is Scheduled on and that a subscriber consumes from. A Timer only ever +exists on one Topic. _Avoid_: queue. A Topic is not a work queue: Timers on a Topic are ordered by Activation time, not arrival, and a Claimed Timer stays on the Topic rather than leaving it. `channel` survives only where FastStream owns the spelling: the `raw_message` field, the @@ -53,8 +53,8 @@ excludes those. **Claim**: To take a Due Timer for processing by advancing its Activation time by the Lease, in one atomic -step, so no other worker sees it as Due. A Claim is not a lock: there is no lock key, no owner -recorded, and nothing to release. A contested Claim returns nothing and is simply skipped. +step, with unique ownership that allows only the current Claim to Commit. Scheduling a replacement +or Claiming again invalidates the previous ownership; a contested Claim returns nothing. **Lease**: The time window a Claim buys, `lease_ttl` seconds wide. It expires on its own; a worker that dies @@ -64,5 +64,5 @@ exactly-once. **Commit**: To remove a Timer permanently once a handler has accounted for it, via `ack` or `reject`. `nack` is -not a Commit: it does nothing, and the Lease expiring is what causes redelivery. Cancellation -performs the identical removal, which is why the store exposes one `remove` and not two. +not a Commit: it does nothing, and the Lease expiring is what causes redelivery. A stale Commit +is a no-op; explicit cancellation removes the current Timer regardless of Claim ownership. diff --git a/docs/introduction/how-it-works.md b/docs/introduction/how-it-works.md index 42b54ed..54bf88c 100644 --- a/docs/introduction/how-it-works.md +++ b/docs/introduction/how-it-works.md @@ -2,18 +2,20 @@ ## Redis data structures -`faststream-redis-timers` uses two Redis keys **per topic** to store scheduled timers: +`faststream-redis-timers` uses three Redis keys **per topic** to store scheduled timers: | Key pattern | Type | Purpose | |-------------|------|---------| | `timers_timeline:{topic}` | Sorted set | Maps each `timer_id` to its activation Unix timestamp (score) | | `timers_payloads:{topic}` | Hash | Maps each `timer_id` to its serialized message body | +| Binary ownership key (see below) | Hash | Maps each claimed `timer_id` to its current ownership token | -When you publish a timer for topic `invoices` with `timer_id` `abc-123`, both keys are written atomically using a Redis pipeline: +When you publish a timer for topic `invoices` with `timer_id` `abc-123`, the timer is written and any previous claim is invalidated atomically using a Redis pipeline: ``` ZADD timers_timeline:invoices abc-123 HSET timers_payloads:invoices abc-123 +HDEL abc-123 ``` The envelope is FastStream's `BinaryMessageFormatV1`, a binary-safe wire format that carries the message body alongside `correlation_id`, `content-type`, `reply_to`, and any user-supplied headers. The same format is used by FastStream's built-in Redis broker. @@ -26,16 +28,16 @@ Each subscriber runs a background polling loop that: 2. For each due timer, spawns a task (bounded by the `max_concurrent` capacity limiter) that runs an atomic Lua **claim** script that: - Verifies the timer is still due (score ≤ now) - Pushes its score forward by `lease_ttl` seconds, which grants the worker a lease - - Returns the payload + - Records a fresh unique ownership token and returns the payload 3. Delivers the payload to the user handler -4. On handler success, runs an atomic Lua **commit** script (`ZREM` + `HDEL`) to remove the timer from Redis +4. On handler success, runs an atomic Lua **commit** script that removes the timer and its claim token only if the captured token still matches 5. On handler exception, leaves the timer in place; the lease eventually expires and another worker re-claims it This is the standard SQS-style **visibility-timeout** pattern. The timer's own score in the sorted set is the lease deadline; there is no separate lock primitive. ## At-least-once delivery -The lease ensures each due timer is processed by exactly one worker at a time. Because the timer is removed from Redis only after the handler completes successfully, **no timer is lost on crash**: if the worker dies mid-handler, the lease expires and the timer is re-delivered. +An atomic claim gives one worker ownership until another successful claim or a replacement invalidates it. Lease expiry permits another worker to claim while the original handler may still be running. Because the timer is removed from Redis only after the handler completes successfully, **no timer is lost on crash**: if the worker dies mid-handler, the lease expires and the timer is re-delivered. The trade-off: handlers that take longer than `lease_ttl`, or workers that crash after the handler ran but before the commit landed, may see the timer delivered more than once. Handlers must therefore be **idempotent**. @@ -43,9 +45,11 @@ The trade-off: handlers that take longer than `lease_ttl`, or workers that crash | Action | Effect | |--------|--------| -| `ack` | Atomically removes the timer from `timers_timeline` and `timers_payloads` | +| `ack` | Removes the timer and claim metadata only if this delivery still owns the claim | | `nack` | No-op; the lease expires and another worker re-claims the timer | -| `reject` | Same as `ack`: permanently removes the timer (use for poison-pill messages) | +| `reject` | Same conditional removal as `ack` (use for poison-pill messages) | + +Replacement, cancel/recreate, and lease takeover make a previous delivery's completion a successful no-op. Corrupt-payload cleanup uses the same ownership check. Explicit cancellation remains unconditional. Successful commits, cancellation, and bulk cancellation remove claim metadata. The default ack policy is `NACK_ON_ERROR`: the timer is acknowledged on success, and left for retry on any unhandled exception. @@ -69,6 +73,20 @@ The default ack policy is `NACK_ON_ERROR`: the timer is acknowledged on success, | `lease_ttl` | `30` s | How long a worker holds the lease before another worker may re-claim. Set to ~3–5× the P99 handler runtime: lower values speed up recovery from worker death, higher values tolerate handler GC pauses and clock skew | | `command_timeout` | `None` (`lease_ttl`) | Most time the subscriber waits on one Redis command (fetch, claim, ack). A primary that goes silent without closing its connections otherwise stalls the subscriber; redis-py before 8.0 sets no socket timeout by default. Must be > 0 and <= `lease_ttl` | +The supplied Redis client must use UTF-8 encoding with `encoding_errors="strict"` (the redis-py defaults). UTF-8 codec aliases such as `UTF8` are accepted, and `decode_responses` may be either `True` or `False`. Incompatible encoders raise `ValueError` when the broker is constructed, before any Redis writes. This requirement prevents public string keys from entering the binary ownership namespace. + +Ownership keys are the byte prefix `b"\xfffaststream-redis-timers:claims:"` followed by the UTF-8 bytes of the full timeline key (`{timeline_key}:{topic}`, including any topic prefix). The leading `0xff` byte makes this namespace disjoint from every public key derived from string configuration. Existing custom timeline and payload names remain valid, including names ending in `_claims`. Redis inspection tools must use byte-mode key enumeration for these binary keys; broker operations remain compatible with `decode_responses=True`. + +## Upgrading existing timers + +Existing timer payloads and scores need no rewrite. Their first successful claim records a fresh ownership token; existing leases remain unavailable until their stored deadline. + +Stop every old process that writes to the namespace before starting fenced workers. This includes consumers and their in-flight handlers, publishing-only processes, and cancellation clients. Upgrade all these writers before resuming them. Old consumers delete unconditionally; old publishers replace payloads and deadlines without invalidating claim ownership, allowing a fenced worker's stale completion to delete the replacement. Old cancellation clients also leave claim metadata behind. Stopping only old consumers is insufficient. + +Persisted records may stay in Redis during this coordinated transition; new workers adopt them on claim. If old writers must remain active, use a separate namespace for fenced writers and workers. Downgrading any writer to the unfenced protocol restores the compatibility risk. + +Fencing protects Redis timer state. It cannot retract external effects from a handler whose lease expired. Keep handlers idempotent; this provides at-least-once delivery and does not guarantee exactly-once publication to Kafka or other systems. Leases are not renewed automatically. + ## Operational requirements `faststream-redis-timers` is designed for a **single-primary Redis** (Sentinel-managed primary/replica setups are supported; Redis Cluster is not: constructing `TimersBroker` with a `RedisCluster` client raises `TypeError`). diff --git a/faststream_redis_timers/configs.py b/faststream_redis_timers/configs.py index cee220e..d48fc2c 100644 --- a/faststream_redis_timers/configs.py +++ b/faststream_redis_timers/configs.py @@ -1,3 +1,4 @@ +import codecs import typing from dataclasses import dataclass from typing import TypeAlias @@ -20,6 +21,11 @@ class ConnectionState: def __init__(self, client: "RedisClient | None" = None) -> None: + if client is not None: + encoder = client.connection_pool.get_encoder() + if codecs.lookup(encoder.encoding).name != "utf-8" or encoder.encoding_errors != "strict": + msg = "TimersBroker requires UTF-8 Redis encoding with strict encoding errors." + raise ValueError(msg) self._client = client @property diff --git a/faststream_redis_timers/message.py b/faststream_redis_timers/message.py index 6972787..fccdb8f 100644 --- a/faststream_redis_timers/message.py +++ b/faststream_redis_timers/message.py @@ -12,6 +12,7 @@ class TimerMessage(typing.TypedDict): channel: str timer_id: str data: bytes + claim_token: typing.NotRequired[str] class TimerStreamMessage(StreamMessage["TimerMessage"]): @@ -19,10 +20,10 @@ class TimerStreamMessage(StreamMessage["TimerMessage"]): Lease-based at-least-once delivery: the timer remains in the timeline (with its score pushed forward by `lease_ttl`) while the handler runs. - `ack()` and `reject()` atomically remove it; `nack()` is a no-op so the + `ack()` and `reject()` remove it only while this delivery owns the claim; `nack()` is a no-op so the lease expires and another worker re-claims the timer. - A timer whose payload cannot be parsed is removed and raises on `decode()`, + A delivery whose payload cannot be parsed conditionally commits and raises on `decode()`, so the handler never runs. """ diff --git a/faststream_redis_timers/parser/parser.py b/faststream_redis_timers/parser/parser.py index 4cfc88b..01d1df5 100644 --- a/faststream_redis_timers/parser/parser.py +++ b/faststream_redis_timers/parser/parser.py @@ -20,12 +20,13 @@ def __init__(self, config: "TimersSubscriberConfig") -> None: async def parse_message(self, msg: "TimerMessage") -> TimerStreamMessage: timer_id = msg["timer_id"] + token = msg.get("claim_token") discard_corrupt = None try: body, headers = TimerMessageFormat.parse(msg["data"]) except ValueError as e: body, headers = msg["data"], {} - discard_corrupt = partial(self._discard_corrupt, msg, e) + discard_corrupt = partial(self._discard_corrupt, timer_id, token, len(msg["data"]), e) return TimerStreamMessage( raw_message=msg, body=body, @@ -34,21 +35,23 @@ async def parse_message(self, msg: "TimerMessage") -> TimerStreamMessage: message_id=timer_id, correlation_id=headers.get("correlation_id", timer_id), reply_to=headers.get("reply_to", ""), - _remove=partial(self._remove, timer_id), + _remove=partial(self._remove, timer_id, token), _discard_corrupt=discard_corrupt, ) - async def _remove(self, timer_id: str) -> None: + async def _remove(self, timer_id: str, token: str | None) -> None: + if token is None: + return store = self._config._outer_config.store # noqa: SLF001 with anyio.fail_after(self._config.timer_sub.effective_command_timeout): - await store.remove(self._config.full_topic, timer_id) + await store.commit(self._config.full_topic, timer_id, token) - async def _discard_corrupt(self, msg: "TimerMessage", error: ValueError) -> typing.NoReturn: + async def _discard_corrupt(self, timer_id: str, token: str | None, size: int, error: ValueError) -> typing.NoReturn: outer_config = self._config._outer_config # noqa: SLF001 - await self._remove(msg["timer_id"]) + await self._remove(timer_id, token) outer_config.logger.log( - f"Timer {msg['timer_id']!r} on {self._config.full_topic!r} removed: " - f"its {len(msg['data'])}-byte payload cannot be parsed", + f"Timer {timer_id!r} on {self._config.full_topic!r} removed if still owned: " + f"its {size}-byte payload cannot be parsed", logging.ERROR, exc_info=error, ) diff --git a/faststream_redis_timers/store.py b/faststream_redis_timers/store.py index aefb2d7..7a5c333 100644 --- a/faststream_redis_timers/store.py +++ b/faststream_redis_timers/store.py @@ -2,23 +2,12 @@ Every caller (producer, subscriber, message, broker inspection) crosses this interface; the raw Redis key derivation and Lua dispatch stay behind it. - -Lua scripts for atomic timer claim/commit ------------------------------------------ -``_CLAIM_LUA``: lease a due timer. Atomically checks the timer is still due -(score <= now) and pushes its score forward by ``lease_ttl``. Returns the -payload bytes, or nil if the timer is not due (already leased, canceled, -or scheduled for the future). - -``_COMMIT_LUA``: remove a timer after successful processing. Always succeeds. - -Both scripts are dispatched via :func:`_eval_cached`, which uses ``EVALSHA`` -with a ``NOSCRIPT`` fallback to ``SCRIPT LOAD`` so the script body only -crosses the wire once per Redis instance lifetime. """ import hashlib import typing +import uuid +from dataclasses import dataclass from redis.client import NEVER_DECODE from redis.exceptions import NoScriptError @@ -34,16 +23,36 @@ local payload = redis.call('HGET', KEYS[2], ARGV[1]) if not payload then redis.call('ZREM', KEYS[1], ARGV[1]) + redis.call('HDEL', KEYS[3], ARGV[1]) return nil end redis.call('ZADD', KEYS[1], ARGV[3], ARGV[1]) +redis.call('HSET', KEYS[3], ARGV[1], ARGV[4]) return payload """ -_COMMIT_LUA = """\ +_REMOVE_LUA = """\ redis.call('ZREM', KEYS[1], ARGV[1]) redis.call('HDEL', KEYS[2], ARGV[1]) +redis.call('HDEL', KEYS[3], ARGV[1]) +""" + +_COMMIT_LUA = ( + """\ +if redis.call('HGET', KEYS[3], ARGV[1]) ~= ARGV[2] then return 0 end """ + + _REMOVE_LUA + + "return 1\n" +) + +_REMOVE_SHA = hashlib.sha1(_REMOVE_LUA.encode(), usedforsecurity=False).hexdigest() + + +@dataclass(frozen=True, slots=True) +class TimerClaim: + payload: bytes + token: str + _CLAIM_SHA = hashlib.sha1(_CLAIM_LUA.encode(), usedforsecurity=False).hexdigest() _COMMIT_SHA = hashlib.sha1(_COMMIT_LUA.encode(), usedforsecurity=False).hexdigest() @@ -96,6 +105,11 @@ def _keys(self, full_topic: str) -> tuple[str, str]: """Return (timeline_key, payloads_key) for the given topic.""" return f"{self._timeline_key}:{full_topic}", f"{self._payloads_key}:{full_topic}" + def _claims_key(self, full_topic: str) -> bytes: + """Keep ownership keys outside the UTF-8 namespace of configured public keys.""" + timeline, _ = self._keys(full_topic) + return b"\xfffaststream-redis-timers:claims:" + timeline.encode() + async def schedule(self, full_topic: str, timer_id: str, payload: bytes, activation_ts: float) -> None: """Add a timer to the timeline and store its payload atomically.""" tl_key, pl_key = self._keys(full_topic) @@ -103,6 +117,7 @@ async def schedule(self, full_topic: str, timer_id: str, payload: bytes, activat async with client.pipeline(transaction=True) as pipe: pipe.zadd(tl_key, {timer_id: activation_ts}) pipe.hset(pl_key, timer_id, payload) + pipe.hdel(self._claims_key(full_topic), timer_id) await pipe.execute() async def due(self, full_topic: str, now: float, limit: int) -> list[str]: @@ -112,7 +127,7 @@ async def due(self, full_topic: str, now: float, limit: int) -> list[str]: bytes ID cannot be decoded as UTF-8 is atomically removed (self-heal) and excluded from the result. """ - tl_key, pl_key = self._keys(full_topic) + tl_key, _ = self._keys(full_topic) client = self._connection.client raw_ids: list[bytes] | list[str] = await client.zrangebyscore(tl_key, "-inf", now, start=0, num=limit) result: list[str] = [] @@ -121,13 +136,15 @@ async def due(self, full_topic: str, now: float, limit: int) -> list[str]: try: result.append(raw_id.decode()) except UnicodeDecodeError: - await _eval_cached(client, _COMMIT_LUA, _COMMIT_SHA, 2, tl_key, pl_key, raw_id) + claim = await self.claim(full_topic, raw_id, now, 1) + if claim is not None: + await self.commit(full_topic, raw_id, claim.token) else: result.append(raw_id) return result - async def claim(self, full_topic: str, timer_id: str, now: float, lease_ttl: int) -> bytes | None: - """Atomically lease a due timer and return its payload, or None if contested. + async def claim(self, full_topic: str, timer_id: str | bytes, now: float, lease_ttl: int) -> TimerClaim | None: + """Atomically lease a due timer and return its payload and ownership token. The timer's score is pushed forward by *lease_ttl* so other workers skip it while the handler runs. Returns ``None`` if the timer is already @@ -136,20 +153,53 @@ async def claim(self, full_topic: str, timer_id: str, now: float, lease_ttl: int tl_key, pl_key = self._keys(full_topic) client = self._connection.client claim_score = now + lease_ttl + token = uuid.uuid4().hex result: bytes | None = await _eval_cached( - client, _CLAIM_LUA, _CLAIM_SHA, 2, tl_key, pl_key, timer_id, now, claim_score + client, + _CLAIM_LUA, + _CLAIM_SHA, + 3, + tl_key, + pl_key, + self._claims_key(full_topic), + timer_id, + now, + claim_score, + token, + ) + return TimerClaim(result, token) if result is not None else None + + async def commit(self, full_topic: str, timer_id: str | bytes, token: str) -> None: + """Remove only the currently owned claim; stale completion is a no-op.""" + tl_key, pl_key = self._keys(full_topic) + await _eval_cached( + self._connection.client, + _COMMIT_LUA, + _COMMIT_SHA, + 3, + tl_key, + pl_key, + self._claims_key(full_topic), + timer_id, + token, ) - return result async def remove(self, full_topic: str, timer_id: str) -> None: """Remove a timer from both the timeline and the payloads hash atomically. - Covers commit (after ack/reject), cancel, and orphan cleanup — all three - paths produce the same effect via a single EVALSHA round-trip. + Explicit cancellation is unconditional, unlike claim completion. """ tl_key, pl_key = self._keys(full_topic) - client = self._connection.client - await _eval_cached(client, _COMMIT_LUA, _COMMIT_SHA, 2, tl_key, pl_key, timer_id) + await _eval_cached( + self._connection.client, + _REMOVE_LUA, + _REMOVE_SHA, + 3, + tl_key, + pl_key, + self._claims_key(full_topic), + timer_id, + ) async def pending(self, full_topic: str, before: float | None = None) -> list[str]: """Return all pending timer IDs, optionally restricted to those due by *before*. @@ -179,5 +229,6 @@ async def cancel_all(self, full_topic: str) -> int: pipe.zcard(tl_key) pipe.unlink(tl_key) pipe.unlink(pl_key) + pipe.unlink(self._claims_key(full_topic)) results = await pipe.execute() return int(results[0]) diff --git a/faststream_redis_timers/subscriber/usecase.py b/faststream_redis_timers/subscriber/usecase.py index 2f2c4ef..eb39c32 100644 --- a/faststream_redis_timers/subscriber/usecase.py +++ b/faststream_redis_timers/subscriber/usecase.py @@ -156,10 +156,8 @@ async def _claim_and_consume( async with limiter: now = time.time() with anyio.fail_after(self._config.timer_sub.effective_command_timeout): - raw_payload: bytes | None = await self._outer_config.store.claim( - self._config.full_topic, timer_id, now, lease_ttl - ) - if raw_payload is None: + claim = await self._outer_config.store.claim(self._config.full_topic, timer_id, now, lease_ttl) + if claim is None: self._log( log_level=logging.DEBUG, message=f"Timer {timer_id!r} claim contested (already leased or canceled)", @@ -169,7 +167,8 @@ async def _claim_and_consume( type="timer", channel=self._config.full_topic, timer_id=timer_id, - data=raw_payload, + data=claim.payload, + claim_token=claim.token, ) self._log(log_level=logging.DEBUG, message=f"Timer {timer_id!r} delivered to handler") await self.consume(msg) diff --git a/tests/test_at_least_once.py b/tests/test_at_least_once.py index db12025..e246773 100644 --- a/tests/test_at_least_once.py +++ b/tests/test_at_least_once.py @@ -6,7 +6,8 @@ from redis.asyncio import Redis from faststream_redis_timers import TimersBroker -from faststream_redis_timers.store import _CLAIM_LUA +from faststream_redis_timers.configs import ConnectionState +from faststream_redis_timers.store import TimerStore async def test_handler_success_removes_timer(broker: TimersBroker, redis_client: Redis) -> None: @@ -75,15 +76,17 @@ async def test_concurrent_claim_only_one_succeeds(redis_client: Redis) -> None: await redis_client.zadd(timeline_key, {"id-1": now - 1}) await redis_client.hset(payloads_key, "id-1", b"payload") - async def claim() -> object: - return await redis_client.eval(_CLAIM_LUA, 2, timeline_key, payloads_key, "id-1", now, now + 30) + store = TimerStore(ConnectionState(redis_client), f"cc_tl_{suffix}", f"cc_pl_{suffix}") + + async def claim(): + return await store.claim("topic", "id-1", now, 30) a, b = await asyncio.gather(claim(), claim()) results = [r for r in (a, b) if r is not None] assert len(results) == 1 - assert results[0] == b"payload" + assert results[0].payload == b"payload" - await redis_client.delete(timeline_key, payloads_key) + await store.cancel_all("topic") async def test_lease_expiry_allows_re_pickup(redis_client: Redis) -> None: @@ -95,15 +98,17 @@ async def test_lease_expiry_allows_re_pickup(redis_client: Redis) -> None: # Score is in the past — lease expired await redis_client.zadd(timeline_key, {"id-1": now - 5}) await redis_client.hset(payloads_key, "id-1", b"payload") - result = await redis_client.eval(_CLAIM_LUA, 2, timeline_key, payloads_key, "id-1", now, now + 30) - assert result == b"payload" + store = TimerStore(ConnectionState(redis_client), f"le_tl_{suffix}", f"le_pl_{suffix}") + result = await store.claim("topic", "id-1", now, 30) + assert result is not None + assert result.payload == b"payload" # Score should now be in the future (lease pushed forward) score = await redis_client.zscore(timeline_key, "id-1") assert score is not None assert score > now - await redis_client.delete(timeline_key, payloads_key) + await store.cancel_all("topic") async def test_explicit_reject_drops_timer(broker: TimersBroker, redis_client: Redis) -> None: @@ -135,7 +140,8 @@ async def test_orphan_zset_entry_is_cleaned_up(redis_client: Redis) -> None: now = time.time() await redis_client.zadd(timeline_key, {"ghost": now - 1}) - result = await redis_client.eval(_CLAIM_LUA, 2, timeline_key, payloads_key, "ghost", now, now + 30) + store = TimerStore(ConnectionState(redis_client), f"orph_tl_{suffix}", f"orph_pl_{suffix}") + result = await store.claim("topic", "ghost", now, 30) assert result is None assert await redis_client.zscore(timeline_key, "ghost") is None diff --git a/tests/test_fencing.py b/tests/test_fencing.py new file mode 100644 index 0000000..f58418c --- /dev/null +++ b/tests/test_fencing.py @@ -0,0 +1,365 @@ +import asyncio +from datetime import UTC, datetime +from typing import Annotated + +import pytest +from faststream import Context +from redis.asyncio import Redis + +from faststream_redis_timers import TimersBroker +from faststream_redis_timers.configs import ConnectionState +from faststream_redis_timers.message import TimerMessage, TimerStreamMessage +from faststream_redis_timers.parser.parser import TimerParser +from faststream_redis_timers.store import TimerStore +from .conftest import REDIS_URL + + +@pytest.mark.parametrize("completion", ["ack", "reject"]) +async def test_claimed_timer_completion_preserves_replacement(broker: TimersBroker, completion: str) -> None: + entered = asyncio.Event() + release = asyncio.Event() + completed = asyncio.Event() + deadline = datetime(2100, 1, 1, tzinfo=UTC) + + @broker.subscriber("topic") + async def handler(body: str, message: Annotated[TimerStreamMessage, Context("message")]) -> None: + assert body == "same payload" + entered.set() + await release.wait() + await getattr(message, completion)() + completed.set() + + try: + async with broker: + await broker.publish("same payload", topic="topic", timer_id="K") + await asyncio.wait_for(entered.wait(), 5) + await broker.publish("same payload", topic="topic", timer_id="K", activate_at=deadline) + release.set() + await asyncio.wait_for(completed.wait(), 5) + assert await broker.has_pending("topic", "K") + store = broker.config.broker_config.store + assert await store.pending("topic", before=deadline.timestamp() - 1) == [] + assert await store.pending("topic", before=deadline.timestamp()) == ["K"] + finally: + release.set() + await broker.cancel_all("topic") + + +@pytest.mark.parametrize("completion", ["ack", "reject", "decode"]) +@pytest.mark.parametrize("change", ["replacement", "takeover", "cancel_recreate", "cancel_all_recreate"]) +async def test_stale_completion_preserves_current_timer( + broker: TimersBroker, + redis_client: Redis, + completion: str, + change: str, +) -> None: + sub = broker.subscriber("topic") + await broker.connect() + store = broker.config.broker_config.store + payload = b'{"b": "zz"}' if completion == "decode" else b"\x00\xffsame" + tl_key = f"{store.timeline_key}:topic" + pl_key = f"{store.payloads_key}:topic" + try: + await store.schedule("topic", "K", payload, 100) + first = await store.claim("topic", "K", 100, 10) + assert first is not None + message = await TimerParser(sub._config).parse_message( # noqa: SLF001 + TimerMessage(type="timer", channel="topic", timer_id="K", data=first.payload, claim_token=first.token) + ) + if change == "takeover": + current = await store.claim("topic", "K", 111, 10) + assert current is not None + deadline = 121 + else: + if change == "cancel_recreate": + await store.remove("topic", "K") + elif change == "cancel_all_recreate": + assert await store.cancel_all("topic") == 1 + await store.schedule("topic", "K", payload, 200) + deadline = 200 + if completion == "decode": + with pytest.raises(ValueError, match="legacy timer envelope"): + await message.decode() + else: + await getattr(message, completion)() + assert await redis_client.zscore(tl_key, "K") == deadline + assert await redis_client.hget(pl_key, "K") == payload + assert await store.claim("topic", "K", deadline - 1, 10) is None + retry = await store.claim("topic", "K", deadline, 10) + assert retry is not None + assert retry.payload == payload + await store.commit("topic", "K", first.token) + assert await store.is_pending("topic", "K") + await store.commit("topic", "K", retry.token) + assert not await store.is_pending("topic", "K") + await store.commit("topic", "K", retry.token) + assert await redis_client.hget(pl_key, "K") is None + finally: + await store.cancel_all("topic") + await broker.stop() + + +async def test_claims_survive_store_restart_and_adopt_legacy_records(broker: TimersBroker, redis_client: Redis) -> None: + store = broker.config.broker_config.store + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + try: + await redis_client.zadd(tl, {"K": 100, "orphan": 100}) + await redis_client.hset(pl, "K", b"legacy") + assert await store.claim("topic", "orphan", 100, 10) is None + first, contested = await asyncio.gather( + store.claim("topic", "K", 100, 10), + store.claim("topic", "K", 100, 10), + ) + owner = first or contested + assert owner is not None + assert (first is None) != (contested is None) + restarted = TimerStore(ConnectionState(redis_client), store.timeline_key, store.payloads_key) + await restarted.commit("topic", "K", owner.token) + assert await redis_client.exists(tl, pl, claims) == 0 + assert await restarted.claim("topic", "missing", 100, 10) is None + await restarted.schedule("topic", "K", b"new", 100) + current = await restarted.claim("topic", "K", 100, 10) + assert current is not None + await restarted.remove("topic", "K") + await restarted.commit("topic", "K", current.token) + assert await redis_client.exists(tl, pl, claims) == 0 + await restarted.schedule("topic", "K", b"new", 100) + assert await restarted.claim("topic", "K", 100, 10) is not None + assert await restarted.cancel_all("topic") == 1 + assert await redis_client.exists(tl, pl, claims) == 0 + finally: + await store.cancel_all("topic") + + +@pytest.mark.parametrize("replace_after_read", [False, True]) +async def test_malformed_id_cleanup_preserves_newer_schedule( + broker: TimersBroker, + redis_client: Redis, + monkeypatch: pytest.MonkeyPatch, + replace_after_read: bool, +) -> None: + store = broker.config.broker_config.store + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + bad_id = b"\xff" + original = redis_client.zrangebyscore + + async def read_then_replace(*args, **kwargs): + result = await original(*args, **kwargs) + if replace_after_read: + async with redis_client.pipeline(transaction=True) as pipe: + pipe.zadd(tl, {bad_id: 200}) + pipe.hset(pl, bad_id, b"replacement") + await pipe.execute() + return result + + try: + await redis_client.zadd(tl, {bad_id: 100}) + await redis_client.hset(pl, bad_id, b"invalid ID") + monkeypatch.setattr(redis_client, "zrangebyscore", read_then_replace) + assert await store.due("topic", 100, 1) == [] + if replace_after_read: + assert await redis_client.zscore(tl, bad_id) == 200 + assert await redis_client.hget(pl, bad_id) == b"replacement" + else: + assert await redis_client.exists(tl, pl, claims) == 0 + finally: + await store.cancel_all("topic") + + +async def test_payload_prefix_can_end_with_claims(broker: TimersBroker, redis_client: Redis) -> None: + timeline = broker.config.broker_config.store.timeline_key + configured = TimersBroker(redis_client, timeline_key=timeline, payloads_key=f"{timeline}_claims") + store = configured.config.broker_config.store + try: + await store.schedule("app:topic", "K", b"\x00\xffpayload", 100) + claim = await store.claim("app:topic", "K", 100, 10) + assert claim is not None + assert claim.payload == b"\x00\xffpayload" + await store.commit("app:topic", "K", claim.token) + assert not await store.is_pending("app:topic", "K") + finally: + await store.cancel_all("app:topic") + + +async def test_claim_metadata_cannot_alias_another_broker_timeline(broker: TimersBroker, redis_client: Redis) -> None: + first = broker.config.broker_config.store + second = TimerStore(ConnectionState(redis_client), f"{first.timeline_key}_claims", f"{first.payloads_key}_other") + try: + await second.schedule("app:topic", "K", b"other broker", 100) + await first.schedule("app:topic", "K", b"first broker", 100) + owner = await first.claim("app:topic", "K", 100, 10) + assert owner is not None + await first.commit("app:topic", "K", owner.token) + await first.cancel_all("app:topic") + other = await second.claim("app:topic", "K", 100, 10) + assert other is not None + assert other.payload == b"other broker" + await second.commit("app:topic", "K", other.token) + finally: + await first.cancel_all("app:topic") + await second.cancel_all("app:topic") + + +@pytest.mark.parametrize("completion", ["ack", "reject", "decode"]) +async def test_valid_replacement_is_delivered_after_stale_completion(broker: TimersBroker, completion: str) -> None: + delivered = asyncio.Event() + received: list[bytes] = [] + sub = broker.subscriber("topic") + + @sub + async def handler(body: bytes) -> None: + received.append(body) + delivered.set() + + await broker.connect() + store = broker.config.broker_config.store + try: + await store.schedule("topic", "K", b'{"b": "zz"}' if completion == "decode" else b"old", 100) + claim = await store.claim("topic", "K", 100, 10) + assert claim is not None + stale = await TimerParser(sub._config).parse_message( # noqa: SLF001 + TimerMessage(type="timer", channel="topic", timer_id="K", data=claim.payload, claim_token=claim.token) + ) + replacement = b"\x00\xffvalid replacement" + await broker.publish(replacement, topic="topic", timer_id="K") + if completion == "decode": + with pytest.raises(ValueError, match="legacy timer envelope"): + await stale.decode() + else: + await getattr(stale, completion)() + async with broker: + await asyncio.wait_for(delivered.wait(), 2) + assert received == [replacement] + assert not await store.is_pending("topic", "K") + finally: + await store.cancel_all("topic") + await broker.stop() + + +async def test_upgrade_preserves_legacy_lease_until_expiry(broker: TimersBroker, redis_client: Redis) -> None: + store = broker.config.broker_config.store + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + try: + await redis_client.zadd(tl, {"K": 200}) + await redis_client.hset(pl, "K", b"legacy leased payload") + restarted = TimerStore(ConnectionState(redis_client), store.timeline_key, store.payloads_key) + assert await restarted.due("topic", 199, 1) == [] + assert await restarted.claim("topic", "K", 199, 10) is None + assert await redis_client.zscore(tl, "K") == 200 + assert await redis_client.hget(pl, "K") == b"legacy leased payload" + assert await restarted.due("topic", 200, 1) == ["K"] + claim = await restarted.claim("topic", "K", 200, 10) + assert claim is not None + assert claim.payload == b"legacy leased payload" + assert await redis_client.zscore(tl, "K") == 210 + await restarted.commit("topic", "K", claim.token) + assert not await restarted.is_pending("topic", "K") + assert await redis_client.hget(pl, "K") is None + finally: + await store.cancel_all("topic") + + +@pytest.mark.parametrize(("writer", "survives"), [("legacy", False), ("fenced", True)]) +async def test_only_fenced_publishers_protect_replacements( + broker: TimersBroker, + redis_client: Redis, + writer: str, + survives: bool, +) -> None: + """Compatibility probe: legacy writers are unsafe during fenced processing.""" + store = broker.config.broker_config.store + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + try: + await store.schedule("topic", "K", b"old", 100) + claim = await store.claim("topic", "K", 100, 10) + assert claim is not None + if writer == "legacy": + async with redis_client.pipeline(transaction=True) as pipe: + pipe.zadd(tl, {"K": 200}) + pipe.hset(pl, "K", b"replacement") + await pipe.execute() + else: + await store.schedule("topic", "K", b"replacement", 200) + assert await redis_client.zscore(tl, "K") == 200 + assert await redis_client.hget(pl, "K") == b"replacement" + await store.commit("topic", "K", claim.token) + assert await store.is_pending("topic", "K") is survives + assert await redis_client.hget(pl, "K") == (b"replacement" if survives else None) + finally: + await store.cancel_all("topic") + + +@pytest.mark.parametrize( + ("encoding", "encoding_errors", "leading_character"), + [("latin-1", "strict", "\xff"), ("utf-8", "surrogateescape", "\udcff")], +) +async def test_incompatible_encoder_is_rejected_before_writing( + broker: TimersBroker, + redis_client: Redis, + encoding: str, + encoding_errors: str, + leading_character: str, +) -> None: + timeline = broker.config.broker_config.store.timeline_key + payloads = leading_character + "faststream-redis-timers:claims:" + timeline + client = Redis.from_url(REDIS_URL, encoding=encoding, encoding_errors=encoding_errors) + try: + await client.ping() + with pytest.raises(ValueError, match=r"UTF-8.*strict"): + TimersBroker(client, timeline_key=timeline, payloads_key=payloads) + assert await redis_client.exists(f"{timeline}:topic") == 0 + assert await redis_client.exists(b"\xfffaststream-redis-timers:claims:" + f"{timeline}:topic".encode()) == 0 + finally: + await client.delete(f"{timeline}:topic", b"\xfffaststream-redis-timers:claims:" + f"{timeline}:topic".encode()) + await client.aclose() # ty: ignore[missing-attribute] + + +async def test_orphan_cleanup_removes_existing_ownership(broker: TimersBroker, redis_client: Redis) -> None: + store = broker.config.broker_config.store + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + try: + await store.schedule("topic", "K", b"payload", 100) + owner = await store.claim("topic", "K", 100, 10) + assert owner is not None + assert await redis_client.hget(claims, "K") == owner.token.encode() + assert await redis_client.hdel(pl, "K") == 1 + assert await store.claim("topic", "K", 109, 10) is None + assert await redis_client.hget(claims, "K") == owner.token.encode() + assert await store.claim("topic", "K", 110, 10) is None + assert await redis_client.exists(tl, pl, claims) == 0 + await store.commit("topic", "K", owner.token) + assert await redis_client.exists(tl, pl, claims) == 0 + finally: + await store.cancel_all("topic") + + +@pytest.mark.parametrize("encoding", ["utf-8", "UTF8", "utf_8"]) +@pytest.mark.parametrize("decode_responses", [False, True]) +async def test_utf8_encoder_aliases_support_fenced_binary_payloads( + broker: TimersBroker, + encoding: str, + decode_responses: bool, +) -> None: + client = Redis.from_url(REDIS_URL, encoding=encoding, decode_responses=decode_responses) + original = broker.config.broker_config.store + try: + await client.ping() + configured = TimersBroker(client, timeline_key=original.timeline_key, payloads_key=original.payloads_key) + store = configured.config.broker_config.store + await store.schedule("topic", "K", b"\x00\xffpayload", 100) + claim = await store.claim("topic", "K", 100, 10) + assert claim is not None + assert claim.payload == b"\x00\xffpayload" + await store.commit("topic", "K", claim.token) + assert not await store.is_pending("topic", "K") + finally: + await original.cancel_all("topic") + await client.aclose() # ty: ignore[missing-attribute] diff --git a/tests/test_unit.py b/tests/test_unit.py index 2e887f0..12ca270 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -30,6 +30,10 @@ ) +def _redis_client() -> AsyncMock: + return AsyncMock(connection_pool=Redis().connection_pool) + + # --- AsyncAPI try_it_out registry --- @@ -54,7 +58,7 @@ def test_connection_state_client_not_set_raises() -> None: async def test_connection_state_connect_returns_client() -> None: - client = AsyncMock() + client = _redis_client() state = ConnectionState(client) result = await state.connect() assert result is client @@ -69,7 +73,7 @@ async def test_connection_state_disconnect_is_noop() -> None: async def test_broker_config_connect_and_disconnect() -> None: - client = AsyncMock() + client = _redis_client() broker = TimersBroker(client) config = broker.config.broker_config await config.connect() @@ -91,28 +95,28 @@ async def test_broker_ping_without_client_returns_false() -> None: async def test_broker_ping_when_redis_raises_returns_false() -> None: - client = AsyncMock() + client = _redis_client() client.ping.side_effect = ConnectionError("timeout") broker = TimersBroker(client) assert await broker.ping() is False async def test_broker_ping_when_redis_returns_false() -> None: - client = AsyncMock() + client = _redis_client() client.ping.return_value = False broker = TimersBroker(client) assert await broker.ping() is False async def test_broker_ping_success() -> None: - client = AsyncMock() + client = _redis_client() client.ping.return_value = True broker = TimersBroker(client) assert await broker.ping() is True async def test_broker_ping_dead_task_returns_false() -> None: - client = AsyncMock() + client = _redis_client() client.ping.return_value = True broker = TimersBroker(client) sub = broker.subscriber("topic") @@ -123,7 +127,7 @@ async def test_broker_ping_dead_task_returns_false() -> None: async def test_broker_ping_live_task_returns_true() -> None: - client = AsyncMock() + client = _redis_client() client.ping.return_value = True broker = TimersBroker(client) sub = broker.subscriber("topic") @@ -350,7 +354,7 @@ async def handler(body: str) -> None: ... async def test_publisher_fetch_redis_timers_returns_matching() -> None: - client = AsyncMock() + client = _redis_client() client.zrangebyscore.return_value = [b"timer-1", b"timer-2"] broker = TimersBroker(client) pub = broker.publisher("my-topic") @@ -363,7 +367,7 @@ async def test_publisher_fetch_redis_timers_returns_matching() -> None: async def test_publisher_fetch_redis_timers_empty() -> None: - client = AsyncMock() + client = _redis_client() client.zrangebyscore.return_value = [] broker = TimersBroker(client) pub = broker.publisher("my-topic") @@ -375,7 +379,7 @@ async def test_publisher_fetch_redis_timers_empty() -> None: async def test_publisher_fetch_redis_timers_uses_correct_key() -> None: - client = AsyncMock() + client = _redis_client() client.zrangebyscore.return_value = [] broker = TimersBroker(client, timeline_key="custom_timeline") pub = broker.publisher("orders") @@ -391,12 +395,12 @@ async def test_publisher_fetch_redis_timers_uses_correct_key() -> None: def test_broker_start_timeout_propagates_to_config() -> None: - broker = TimersBroker(AsyncMock(), start_timeout=7.5) + broker = TimersBroker(_redis_client(), start_timeout=7.5) assert broker.config.broker_config.start_timeout == 7.5 def test_broker_start_timeout_default_is_three_seconds() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) assert broker.config.broker_config.start_timeout == 3.0 @@ -404,7 +408,7 @@ def test_broker_start_timeout_default_is_three_seconds() -> None: def test_duplicate_subscriber_warns() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) @broker.subscriber("same") async def first(body: str) -> None: ... @@ -420,7 +424,7 @@ async def second(body: str) -> None: ... async def test_consume_sets_start_signal_when_ping_returns_false() -> None: """If client.ping() returns falsy, start_signal is set by the finally branch on first poll.""" - client = AsyncMock() + client = _redis_client() client.ping.return_value = False client.zrangebyscore.return_value = [] broker = TimersBroker(client, start_timeout=2.0) @@ -436,31 +440,31 @@ async def handler(body: str) -> None: ... async def test_publish_rejects_empty_topic() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with pytest.raises(ValueError, match="topic must be a non-empty string"): await broker.publish("msg", topic="") async def test_cancel_timer_rejects_empty_topic() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with pytest.raises(ValueError, match="topic must be a non-empty string"): await broker.cancel_timer("", "id") async def test_has_pending_rejects_empty_topic() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with pytest.raises(ValueError, match="topic must be a non-empty string"): await broker.has_pending("", "id") async def test_get_pending_timers_rejects_empty_topic() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with pytest.raises(ValueError, match="topic must be a non-empty string"): await broker.get_pending_timers("") async def test_cancel_all_rejects_empty_topic() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with pytest.raises(ValueError, match="topic must be a non-empty string"): await broker.cancel_all("") @@ -470,7 +474,7 @@ async def test_cancel_all_rejects_empty_topic() -> None: async def test_consume_logs_get_msgs_error_with_repr() -> None: """A Redis error during _get_msgs is logged at ERROR with `{e!r}` so the type and msg survive Sentry truncation.""" - client = AsyncMock() + client = _redis_client() client.ping.return_value = True client.zrangebyscore.side_effect = ConnectionError("redis down") broker = TimersBroker(client, start_timeout=2.0) @@ -495,7 +499,7 @@ async def handler(body: str) -> None: ... async def test_claim_and_consume_logs_unhandled_error_with_repr() -> None: """An unhandled error inside the limiter block is logged with `{timer_id!r}` and `{e!r}`.""" - client = AsyncMock() + client = _redis_client() broker = TimersBroker(client) sub = broker.subscriber("topic") await broker.connect() @@ -521,7 +525,7 @@ async def test_claim_and_consume_logs_unhandled_error_with_repr() -> None: async def test_eval_cached_falls_back_on_noscript() -> None: - client = AsyncMock() + client = _redis_client() client.execute_command.side_effect = [NoScriptError("NOSCRIPT"), b"ok"] client.script_load.return_value = "abc123" @@ -533,7 +537,7 @@ async def test_eval_cached_falls_back_on_noscript() -> None: def test_distinct_topics_do_not_warn() -> None: - broker = TimersBroker(AsyncMock()) + broker = TimersBroker(_redis_client()) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") @@ -615,7 +619,7 @@ def test_timer_store_key_derivation() -> None: async def test_timer_store_schedule() -> None: """schedule() writes timer_id→activation_ts in ZADD and timer_id→payload in HSET via pipeline.""" - client = AsyncMock() + client = _redis_client() pipe = MagicMock() pipe.__aenter__ = AsyncMock(return_value=pipe) pipe.__aexit__ = AsyncMock(return_value=None) @@ -632,7 +636,7 @@ async def test_timer_store_schedule() -> None: async def test_timer_store_due_bytes_client() -> None: """due() decodes bytes IDs for a bytes-mode Redis client.""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=[b"timer-1", b"timer-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -644,7 +648,7 @@ async def test_timer_store_due_bytes_client() -> None: async def test_timer_store_due_str_client() -> None: """due() passes str IDs through unchanged for a str-mode Redis client.""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=["timer-1", "timer-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -655,7 +659,7 @@ async def test_timer_store_due_str_client() -> None: async def test_timer_store_due_empty() -> None: """due() returns an empty list when no timers are due.""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=[]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -667,7 +671,7 @@ async def test_timer_store_due_empty() -> None: async def test_timer_store_due_non_utf8_self_heal() -> None: """due() removes non-UTF-8 members via COMMIT_LUA and excludes them from the result.""" bad_id = b"\xff\xfe-broken" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=[bad_id, b"good-id"]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -682,7 +686,7 @@ async def test_timer_store_due_non_utf8_self_heal() -> None: async def test_timer_store_claim_found() -> None: """claim() returns the payload bytes when the timer is successfully claimed.""" - client = AsyncMock() + client = _redis_client() payload = b"timer-payload" store = TimerStore(ConnectionState(client), "tl", "pl") @@ -690,14 +694,16 @@ async def test_timer_store_claim_found() -> None: with patch("faststream_redis_timers.store._eval_cached", new=AsyncMock(return_value=payload)) as mock_eval: result = await store.claim("topic", "id-1", 1000.0, 30) - assert result == payload + assert result is not None + assert result.payload == payload + assert result.token # claim_score = now + lease_ttl = 1000.0 + 30 = 1030.0 assert 1030.0 in mock_eval.call_args.args async def test_timer_store_claim_contested() -> None: """claim() returns None when the timer is already leased or canceled.""" - client = AsyncMock() + client = _redis_client() store = TimerStore(ConnectionState(client), "tl", "pl") @@ -709,7 +715,7 @@ async def test_timer_store_claim_contested() -> None: async def test_timer_store_remove() -> None: """remove() calls COMMIT_LUA for both the timeline and payloads keys.""" - client = AsyncMock() + client = _redis_client() store = TimerStore(ConnectionState(client), "tl", "pl") @@ -725,7 +731,7 @@ async def test_timer_store_remove() -> None: async def test_timer_store_pending_no_before() -> None: """pending() with no before= uses '+inf' as the score ceiling (all pending timers).""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=["id-1", "id-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -737,7 +743,7 @@ async def test_timer_store_pending_no_before() -> None: async def test_timer_store_pending_with_before() -> None: """pending() with before=float restricts results to timers due by that timestamp.""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=[]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -749,7 +755,7 @@ async def test_timer_store_pending_with_before() -> None: async def test_timer_store_pending_bytes_client() -> None: """pending() decodes bytes IDs for a bytes-mode Redis client.""" - client = AsyncMock() + client = _redis_client() client.zrangebyscore = AsyncMock(return_value=[b"id-1", b"id-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -760,7 +766,7 @@ async def test_timer_store_pending_bytes_client() -> None: async def test_timer_store_is_pending_true() -> None: """is_pending() returns True when ZSCORE returns a score (timer exists).""" - client = AsyncMock() + client = _redis_client() client.zscore = AsyncMock(return_value=1000.0) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -772,7 +778,7 @@ async def test_timer_store_is_pending_true() -> None: async def test_timer_store_is_pending_false() -> None: """is_pending() returns False when ZSCORE returns None (timer absent).""" - client = AsyncMock() + client = _redis_client() client.zscore = AsyncMock(return_value=None) store = TimerStore(ConnectionState(client), "tl", "pl") @@ -783,7 +789,7 @@ async def test_timer_store_is_pending_false() -> None: async def test_timer_store_cancel_all() -> None: """cancel_all() UNLINKs both keys and returns the pre-removal ZCARD count.""" - client = AsyncMock() + client = _redis_client() pipe = MagicMock() pipe.__aenter__ = AsyncMock(return_value=pipe) pipe.__aexit__ = AsyncMock(return_value=None) From 3fcb06e10292661f0a87052157396f191afca8da Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 14 Nov 2026 11:22:00 +0300 Subject: [PATCH 2/3] fix: preserve Redis client encodings in fenced timer storage --- docs/introduction/how-it-works.md | 9 ++- docs/usage/basic.md | 1 + faststream_redis_timers/broker.py | 3 +- faststream_redis_timers/configs.py | 6 -- faststream_redis_timers/store.py | 56 +++++++++----- faststream_redis_timers/testing.py | 3 +- tests/test_fencing.py | 117 +++++++++++++++++++++++++---- 7 files changed, 150 insertions(+), 45 deletions(-) diff --git a/docs/introduction/how-it-works.md b/docs/introduction/how-it-works.md index 54bf88c..1dea290 100644 --- a/docs/introduction/how-it-works.md +++ b/docs/introduction/how-it-works.md @@ -8,7 +8,7 @@ |-------------|------|---------| | `timers_timeline:{topic}` | Sorted set | Maps each `timer_id` to its activation Unix timestamp (score) | | `timers_payloads:{topic}` | Hash | Maps each `timer_id` to its serialized message body | -| Binary ownership key (see below) | Hash | Maps each claimed `timer_id` to its current ownership token | +| `timers_claims:{digest}` by default | Hash | Maps each claimed `timer_id` to its current ownership token | When you publish a timer for topic `invoices` with `timer_id` `abc-123`, the timer is written and any previous claim is invalidated atomically using a Redis pipeline: @@ -61,6 +61,7 @@ The default ack policy is `NACK_ON_ERROR`: the timer is acknowledged on success, |-----------|---------|-------------| | `timeline_key` | `timers_timeline` | Prefix for sorted set keys (`{timeline_key}:{topic}`) | | `payloads_key` | `timers_payloads` | Prefix for hash keys (`{payloads_key}:{topic}`) | +| `claims_key` | `None` | Optional ownership hash prefix (`{claims_key}:{topic}`); by default the key is derived from the timeline and payload keys | | `start_timeout` | `3.0` s | Max wait for the first Redis ping during startup | ### Per-subscriber (`@broker.subscriber("topic", ...)`) @@ -73,9 +74,11 @@ The default ack policy is `NACK_ON_ERROR`: the timer is acknowledged on success, | `lease_ttl` | `30` s | How long a worker holds the lease before another worker may re-claim. Set to ~3–5× the P99 handler runtime: lower values speed up recovery from worker death, higher values tolerate handler GC pauses and clock skew | | `command_timeout` | `None` (`lease_ttl`) | Most time the subscriber waits on one Redis command (fetch, claim, ack). A primary that goes silent without closing its connections otherwise stalls the subscriber; redis-py before 8.0 sets no socket timeout by default. Must be > 0 and <= `lease_ttl` | -The supplied Redis client must use UTF-8 encoding with `encoding_errors="strict"` (the redis-py defaults). UTF-8 codec aliases such as `UTF8` are accepted, and `decode_responses` may be either `True` or `False`. Incompatible encoders raise `ValueError` when the broker is constructed, before any Redis writes. This requirement prevents public string keys from entering the binary ownership namespace. +Redis client encoding, encoding error handling, and `decode_responses` settings are preserved. Timer IDs use the client's encoder for storage and inspection; binary message envelopes remain raw bytes during claims. -Ownership keys are the byte prefix `b"\xfffaststream-redis-timers:claims:"` followed by the UTF-8 bytes of the full timeline key (`{timeline_key}:{topic}`, including any topic prefix). The leading `0xff` byte makes this namespace disjoint from every public key derived from string configuration. Existing custom timeline and payload names remain valid, including names ending in `_claims`. Redis inspection tools must use byte-mode key enumeration for these binary keys; broker operations remain compatible with `decode_responses=True`. +By default, ownership keys are `timers_claims:{digest}`. The digest is SHA-256 of the full timeline and payload key bytes as encoded by the client, each preceded by its eight-byte big-endian length. This gives independently configured stores separate ownership metadata, remains stable across restarts, and requires no reserved binary namespace. An explicit `claims_key` instead uses `{claims_key}:{topic}`, including any topic prefix. + +Timeline, payload, and ownership keys for a topic must be distinct after Redis encoding. Mutating operations reject overlapping keys with `ValueError` before issuing Redis commands. Use `claims_key` to select another namespace when needed. Across independently configured brokers, keep all key namespaces non-overlapping; workers sharing timers must use the same ownership configuration. Changing ownership configuration while workers are active is unsafe. ## Upgrading existing timers diff --git a/docs/usage/basic.md b/docs/usage/basic.md index 9032380..e85b2d7 100644 --- a/docs/usage/basic.md +++ b/docs/usage/basic.md @@ -83,6 +83,7 @@ async def schedule_reminder() -> None: | `client` | _required for production_ | `redis.asyncio.Redis` client instance; the caller owns its lifecycle (the broker does not close it). The constructor accepts `None`, but a broker without a client raises `IncorrectState` on the first operation; `None` is the test-broker shape (see [Testing](./testing.md)). | | `timeline_key` | `timers_timeline` | Sorted set key prefix; each topic uses `{timeline_key}:{topic}` | | `payloads_key` | `timers_payloads` | Hash key prefix; each topic uses `{payloads_key}:{topic}` | +| `claims_key` | `None` | Optional ownership hash prefix; by default derived from the encoded timeline and payload key identities. An explicit prefix uses `{claims_key}:{topic}`. Keep namespaces distinct. | | `start_timeout` | `3.0` | Seconds to wait for the subscriber's first Redis ping during startup | | `graceful_timeout` | `15.0` | Seconds to wait for in-flight timers on shutdown | diff --git a/faststream_redis_timers/broker.py b/faststream_redis_timers/broker.py index c97e836..a2a56c7 100644 --- a/faststream_redis_timers/broker.py +++ b/faststream_redis_timers/broker.py @@ -117,6 +117,7 @@ def __init__( # noqa: PLR0913 *, timeline_key: str = "timers_timeline", payloads_key: str = "timers_payloads", + claims_key: str | None = None, start_timeout: float = 3.0, decoder: CustomCallable | None = None, parser: CustomCallable | None = None, @@ -142,7 +143,7 @@ def __init__( # noqa: PLR0913 raise TypeError(msg) fd_config = FastDependsConfig(use_fastdepends=apply_types) connection = ConnectionState(client) - store = TimerStore(connection, timeline_key, payloads_key) + store = TimerStore(connection, timeline_key, payloads_key, claims_key) broker_config = TimersBrokerConfig( connection=connection, start_timeout=start_timeout, diff --git a/faststream_redis_timers/configs.py b/faststream_redis_timers/configs.py index d48fc2c..cee220e 100644 --- a/faststream_redis_timers/configs.py +++ b/faststream_redis_timers/configs.py @@ -1,4 +1,3 @@ -import codecs import typing from dataclasses import dataclass from typing import TypeAlias @@ -21,11 +20,6 @@ class ConnectionState: def __init__(self, client: "RedisClient | None" = None) -> None: - if client is not None: - encoder = client.connection_pool.get_encoder() - if codecs.lookup(encoder.encoding).name != "utf-8" or encoder.encoding_errors != "strict": - msg = "TimersBroker requires UTF-8 Redis encoding with strict encoding errors." - raise ValueError(msg) self._client = client @property diff --git a/faststream_redis_timers/store.py b/faststream_redis_timers/store.py index 7a5c333..0218bd4 100644 --- a/faststream_redis_timers/store.py +++ b/faststream_redis_timers/store.py @@ -88,10 +88,17 @@ class TimerStore: registry. """ - def __init__(self, connection: "ConnectionState", timeline_key: str, payloads_key: str) -> None: + def __init__( + self, + connection: "ConnectionState", + timeline_key: str, + payloads_key: str, + claims_key: str | None = None, + ) -> None: self._connection = connection self._timeline_key = timeline_key self._payloads_key = payloads_key + self._claims_key_prefix = claims_key @property def timeline_key(self) -> str: @@ -105,26 +112,37 @@ def _keys(self, full_topic: str) -> tuple[str, str]: """Return (timeline_key, payloads_key) for the given topic.""" return f"{self._timeline_key}:{full_topic}", f"{self._payloads_key}:{full_topic}" - def _claims_key(self, full_topic: str) -> bytes: - """Keep ownership keys outside the UTF-8 namespace of configured public keys.""" - timeline, _ = self._keys(full_topic) - return b"\xfffaststream-redis-timers:claims:" + timeline.encode() + def _claims_key(self, full_topic: str) -> str: + if self._claims_key_prefix is not None: + return f"{self._claims_key_prefix}:{full_topic}" + encoder = self._connection.client.connection_pool.get_encoder() + keys = [encoder.encode(key) for key in self._keys(full_topic)] + identity = b"".join(len(key).to_bytes(8, "big") + key for key in keys) + return f"timers_claims:{hashlib.sha256(identity).hexdigest()}" + + def _protocol_keys(self, full_topic: str) -> tuple[str, str, str]: + keys = (*self._keys(full_topic), self._claims_key(full_topic)) + encoder = self._connection.client.connection_pool.get_encoder() + if len({encoder.encode(key) for key in keys}) != len(keys): + msg = "Timeline, payloads, and claims keys must be distinct after Redis encoding." + raise ValueError(msg) + return keys async def schedule(self, full_topic: str, timer_id: str, payload: bytes, activation_ts: float) -> None: """Add a timer to the timeline and store its payload atomically.""" - tl_key, pl_key = self._keys(full_topic) + tl_key, pl_key, cl_key = self._protocol_keys(full_topic) client = self._connection.client async with client.pipeline(transaction=True) as pipe: pipe.zadd(tl_key, {timer_id: activation_ts}) pipe.hset(pl_key, timer_id, payload) - pipe.hdel(self._claims_key(full_topic), timer_id) + pipe.hdel(cl_key, timer_id) await pipe.execute() async def due(self, full_topic: str, now: float, limit: int) -> list[str]: """Return up to *limit* timer IDs whose activation_ts <= *now*. Handles both bytes-mode and str-mode Redis clients. Any member whose - bytes ID cannot be decoded as UTF-8 is atomically removed (self-heal) + bytes ID cannot be decoded with the client's encoder is atomically removed (self-heal) and excluded from the result. """ tl_key, _ = self._keys(full_topic) @@ -134,7 +152,8 @@ async def due(self, full_topic: str, now: float, limit: int) -> list[str]: for raw_id in raw_ids: if isinstance(raw_id, bytes): try: - result.append(raw_id.decode()) + encoder = client.connection_pool.get_encoder() + result.append(raw_id.decode(encoder.encoding, encoder.encoding_errors)) except UnicodeDecodeError: claim = await self.claim(full_topic, raw_id, now, 1) if claim is not None: @@ -150,7 +169,7 @@ async def claim(self, full_topic: str, timer_id: str | bytes, now: float, lease_ it while the handler runs. Returns ``None`` if the timer is already leased, canceled, or scheduled for the future. """ - tl_key, pl_key = self._keys(full_topic) + tl_key, pl_key, cl_key = self._protocol_keys(full_topic) client = self._connection.client claim_score = now + lease_ttl token = uuid.uuid4().hex @@ -161,7 +180,7 @@ async def claim(self, full_topic: str, timer_id: str | bytes, now: float, lease_ 3, tl_key, pl_key, - self._claims_key(full_topic), + cl_key, timer_id, now, claim_score, @@ -171,7 +190,7 @@ async def claim(self, full_topic: str, timer_id: str | bytes, now: float, lease_ async def commit(self, full_topic: str, timer_id: str | bytes, token: str) -> None: """Remove only the currently owned claim; stale completion is a no-op.""" - tl_key, pl_key = self._keys(full_topic) + tl_key, pl_key, cl_key = self._protocol_keys(full_topic) await _eval_cached( self._connection.client, _COMMIT_LUA, @@ -179,7 +198,7 @@ async def commit(self, full_topic: str, timer_id: str | bytes, token: str) -> No 3, tl_key, pl_key, - self._claims_key(full_topic), + cl_key, timer_id, token, ) @@ -189,7 +208,7 @@ async def remove(self, full_topic: str, timer_id: str) -> None: Explicit cancellation is unconditional, unlike claim completion. """ - tl_key, pl_key = self._keys(full_topic) + tl_key, pl_key, cl_key = self._protocol_keys(full_topic) await _eval_cached( self._connection.client, _REMOVE_LUA, @@ -197,7 +216,7 @@ async def remove(self, full_topic: str, timer_id: str) -> None: 3, tl_key, pl_key, - self._claims_key(full_topic), + cl_key, timer_id, ) @@ -212,7 +231,8 @@ async def pending(self, full_topic: str, before: float | None = None) -> list[st client = self._connection.client score_max: str | float = before if before is not None else "+inf" raw_ids: list[bytes] | list[str] = await client.zrangebyscore(tl_key, "-inf", score_max) - return [r.decode() if isinstance(r, bytes) else r for r in raw_ids] + encoder = client.connection_pool.get_encoder() + return [r.decode(encoder.encoding, encoder.encoding_errors) if isinstance(r, bytes) else r for r in raw_ids] async def is_pending(self, full_topic: str, timer_id: str) -> bool: """Return True if a timer with this ID is still in the timeline.""" @@ -223,12 +243,12 @@ async def is_pending(self, full_topic: str, timer_id: str) -> bool: async def cancel_all(self, full_topic: str) -> int: """Atomically remove every timer on this topic. Returns the count removed.""" - tl_key, pl_key = self._keys(full_topic) + tl_key, pl_key, cl_key = self._protocol_keys(full_topic) client = self._connection.client async with client.pipeline(transaction=True) as pipe: pipe.zcard(tl_key) pipe.unlink(tl_key) pipe.unlink(pl_key) - pipe.unlink(self._claims_key(full_topic)) + pipe.unlink(cl_key) results = await pipe.execute() return int(results[0]) diff --git a/faststream_redis_timers/testing.py b/faststream_redis_timers/testing.py index c7d7b84..ebccc32 100644 --- a/faststream_redis_timers/testing.py +++ b/faststream_redis_timers/testing.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock, MagicMock from faststream._internal.testing.broker import TestBroker, change_producer +from redis.asyncio import Redis from faststream_redis_timers.broker import TimersBroker from faststream_redis_timers.envelope import TimerMessageFormat @@ -66,7 +67,7 @@ def _patch_producer(self, broker: TimersBroker) -> "Generator[None]": def _patch_broker(self, broker: TimersBroker) -> "Generator[None]": # Test-broker contract: messages deliver immediately, so there are # never any pending timers. Stub the inspection paths to return that. - mock_client = AsyncMock() + mock_client = AsyncMock(connection_pool=Redis().connection_pool) mock_client.zrangebyscore.return_value = [] # get_pending_timers -> [] mock_client.zscore.return_value = None # has_pending -> False # cancel_all uses `async with client.pipeline(...) as pipe`. AsyncMock's diff --git a/tests/test_fencing.py b/tests/test_fencing.py index f58418c..4454bad 100644 --- a/tests/test_fencing.py +++ b/tests/test_fencing.py @@ -103,7 +103,7 @@ async def test_claims_survive_store_restart_and_adopt_legacy_records(broker: Tim store = broker.config.broker_config.store tl = f"{store.timeline_key}:topic" pl = f"{store.payloads_key}:topic" - claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + claims = store._claims_key("topic") # noqa: SLF001 try: await redis_client.zadd(tl, {"K": 100, "orphan": 100}) await redis_client.hset(pl, "K", b"legacy") @@ -143,7 +143,7 @@ async def test_malformed_id_cleanup_preserves_newer_schedule( store = broker.config.broker_config.store tl = f"{store.timeline_key}:topic" pl = f"{store.payloads_key}:topic" - claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + claims = store._claims_key("topic") # noqa: SLF001 bad_id = b"\xff" original = redis_client.zrangebyscore @@ -296,27 +296,49 @@ async def test_only_fenced_publishers_protect_replacements( @pytest.mark.parametrize( - ("encoding", "encoding_errors", "leading_character"), - [("latin-1", "strict", "\xff"), ("utf-8", "surrogateescape", "\udcff")], + "case", + [ + ("latin-1", "strict", "\xff", "caf\xe9", "caf\xe9"), + ("utf-8", "surrogateescape", "\udcff", "id\udcff", "id\udcff"), + ("ascii", "replace", "?", "caf\xe9", "caf?"), + ], ) -async def test_incompatible_encoder_is_rejected_before_writing( +@pytest.mark.parametrize("decode_responses", [False, True]) +async def test_client_encodings_preserve_fenced_timers( broker: TimersBroker, - redis_client: Redis, - encoding: str, - encoding_errors: str, - leading_character: str, + case: tuple[str, str, str, str, str], + decode_responses: bool, ) -> None: + encoding, encoding_errors, leading_character, timer_id, stored_id = case timeline = broker.config.broker_config.store.timeline_key payloads = leading_character + "faststream-redis-timers:claims:" + timeline - client = Redis.from_url(REDIS_URL, encoding=encoding, encoding_errors=encoding_errors) + client = Redis.from_url( + REDIS_URL, + encoding=encoding, + encoding_errors=encoding_errors, + decode_responses=decode_responses, + ) try: await client.ping() - with pytest.raises(ValueError, match=r"UTF-8.*strict"): - TimersBroker(client, timeline_key=timeline, payloads_key=payloads) - assert await redis_client.exists(f"{timeline}:topic") == 0 - assert await redis_client.exists(b"\xfffaststream-redis-timers:claims:" + f"{timeline}:topic".encode()) == 0 + configured = TimersBroker(client, timeline_key=timeline, payloads_key=payloads) + store = configured.config.broker_config.store + await store.schedule("topic", timer_id, b"\x00\xfforiginal", 100) + assert await store.due("topic", 100, 1) == [stored_id] + assert await store.pending("topic") == [stored_id] + owner = await store.claim("topic", timer_id, 100, 10) + assert owner is not None + assert owner.payload == b"\x00\xfforiginal" + await store.schedule("topic", timer_id, b"\xffreplacement", 200) + await store.commit("topic", timer_id, owner.token) + assert await store.is_pending("topic", timer_id) + replacement = await store.claim("topic", timer_id, 200, 10) + assert replacement is not None + assert replacement.payload == b"\xffreplacement" + await store.commit("topic", timer_id, replacement.token) + assert not await store.is_pending("topic", timer_id) + await store.cancel_all("topic") finally: - await client.delete(f"{timeline}:topic", b"\xfffaststream-redis-timers:claims:" + f"{timeline}:topic".encode()) + await client.delete(f"{timeline}:topic", f"{payloads}:topic") await client.aclose() # ty: ignore[missing-attribute] @@ -324,7 +346,7 @@ async def test_orphan_cleanup_removes_existing_ownership(broker: TimersBroker, r store = broker.config.broker_config.store tl = f"{store.timeline_key}:topic" pl = f"{store.payloads_key}:topic" - claims = b"\xfffaststream-redis-timers:claims:" + tl.encode() + claims = store._claims_key("topic") # noqa: SLF001 try: await store.schedule("topic", "K", b"payload", 100) owner = await store.claim("topic", "K", 100, 10) @@ -363,3 +385,66 @@ async def test_utf8_encoder_aliases_support_fenced_binary_payloads( finally: await original.cancel_all("topic") await client.aclose() # ty: ignore[missing-attribute] + + +@pytest.mark.parametrize("operation", ["schedule", "claim", "commit", "remove", "cancel_all"]) +@pytest.mark.parametrize("collision", ["timeline", "payloads", "encoded_payloads"]) +async def test_explicit_claim_namespace_collision_preserves_existing_data( + broker: TimersBroker, + redis_client: Redis, + operation: str, + collision: str, +) -> None: + original = broker.config.broker_config.store + timeline = original.timeline_key + payloads = original.payloads_key + claims = timeline if collision == "timeline" else payloads + client = Redis.from_url(REDIS_URL, encoding="ascii", encoding_errors="ignore") + try: + await original.schedule("topic", "K", b"existing", 100) + configured = TimersBroker( + client, + timeline_key=timeline, + payloads_key=payloads, + claims_key=claims + ("\u20ac" if collision == "encoded_payloads" else ""), + ) + store = configured.config.broker_config.store + args = { + "schedule": ("topic", "K", b"replacement", 200), + "claim": ("topic", "K", 100, 10), + "commit": ("topic", "K", "old-token"), + "remove": ("topic", "K"), + "cancel_all": ("topic",), + } + with pytest.raises(ValueError, match="distinct after Redis encoding"): + await getattr(store, operation)(*args[operation]) + assert await redis_client.zscore(f"{timeline}:topic", "K") == 100 + assert await redis_client.hget(f"{payloads}:topic", "K") == b"existing" + finally: + await original.cancel_all("topic") + await client.aclose() # ty: ignore[missing-attribute] + + +async def test_explicit_claim_namespace_is_used_and_cleaned(broker: TimersBroker, redis_client: Redis) -> None: + original = broker.config.broker_config.store + prefix = f"{original.timeline_key}_ownership" + configured = TimersBroker( + redis_client, + timeline_key=original.timeline_key, + payloads_key=original.payloads_key, + claims_key=prefix, + ) + store = configured.config.broker_config.store + try: + await store.schedule("app:topic", "K", b"payload", 100) + owner = await store.claim("app:topic", "K", 100, 10) + assert owner is not None + assert await redis_client.hget(f"{prefix}:app:topic", "K") == owner.token.encode() + await store.commit("app:topic", "K", owner.token) + assert await redis_client.exists(f"{prefix}:app:topic") == 0 + await store.schedule("app:topic", "K", b"payload", 100) + assert await store.claim("app:topic", "K", 100, 10) is not None + assert await store.cancel_all("app:topic") == 1 + assert await redis_client.exists(f"{prefix}:app:topic") == 0 + finally: + await store.cancel_all("app:topic") From 6151f87757694c076ab4849b7f9ef49c01af7260 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 14 Nov 2026 11:46:13 +0300 Subject: [PATCH 3/3] fix: retain raw timer identity during polling --- docs/introduction/how-it-works.md | 2 +- faststream_redis_timers/store.py | 35 +++++--- tests/test_fencing.py | 129 +++++++++++++++++++++++++++++- tests/test_unit.py | 20 +++-- 4 files changed, 161 insertions(+), 25 deletions(-) diff --git a/docs/introduction/how-it-works.md b/docs/introduction/how-it-works.md index 1dea290..a290383 100644 --- a/docs/introduction/how-it-works.md +++ b/docs/introduction/how-it-works.md @@ -74,7 +74,7 @@ The default ack policy is `NACK_ON_ERROR`: the timer is acknowledged on success, | `lease_ttl` | `30` s | How long a worker holds the lease before another worker may re-claim. Set to ~3–5× the P99 handler runtime: lower values speed up recovery from worker death, higher values tolerate handler GC pauses and clock skew | | `command_timeout` | `None` (`lease_ttl`) | Most time the subscriber waits on one Redis command (fetch, claim, ack). A primary that goes silent without closing its connections otherwise stalls the subscriber; redis-py before 8.0 sets no socket timeout by default. Must be > 0 and <= `lease_ttl` | -Redis client encoding, encoding error handling, and `decode_responses` settings are preserved. Timer IDs use the client's encoder for storage and inspection; binary message envelopes remain raw bytes during claims. +Redis client encoding, encoding error handling, and `decode_responses` settings are preserved. Timer IDs use the client's encoder for storage and inspection; binary message envelopes remain raw bytes during claims. During polling, IDs are fetched as raw bytes regardless of response decoding. IDs that cannot decode and re-encode to the same bytes are conditionally cleaned using their original byte identity, so lossy decoding cannot redirect claims or block following valid timers. By default, ownership keys are `timers_claims:{digest}`. The digest is SHA-256 of the full timeline and payload key bytes as encoded by the client, each preceded by its eight-byte big-endian length. This gives independently configured stores separate ownership metadata, remains stable across restarts, and requires no reserved binary namespace. An explicit `claims_key` instead uses `{claims_key}:{topic}`, including any topic prefix. diff --git a/faststream_redis_timers/store.py b/faststream_redis_timers/store.py index 0218bd4..29fe5e6 100644 --- a/faststream_redis_timers/store.py +++ b/faststream_redis_timers/store.py @@ -141,25 +141,34 @@ async def schedule(self, full_topic: str, timer_id: str, payload: bytes, activat async def due(self, full_topic: str, now: float, limit: int) -> list[str]: """Return up to *limit* timer IDs whose activation_ts <= *now*. - Handles both bytes-mode and str-mode Redis clients. Any member whose - bytes ID cannot be decoded with the client's encoder is atomically removed (self-heal) - and excluded from the result. + Fetches raw IDs regardless of response decoding. IDs that cannot round-trip + through the client's encoder are conditionally removed using their original bytes. """ tl_key, _ = self._keys(full_topic) client = self._connection.client - raw_ids: list[bytes] | list[str] = await client.zrangebyscore(tl_key, "-inf", now, start=0, num=limit) + raw_ids: list[bytes] = await client.execute_command( + "ZRANGEBYSCORE", + tl_key, + "-inf", + now, + "LIMIT", + 0, + limit, + **{NEVER_DECODE: []}, + ) + encoder = client.connection_pool.get_encoder() result: list[str] = [] for raw_id in raw_ids: - if isinstance(raw_id, bytes): - try: - encoder = client.connection_pool.get_encoder() - result.append(raw_id.decode(encoder.encoding, encoder.encoding_errors)) - except UnicodeDecodeError: - claim = await self.claim(full_topic, raw_id, now, 1) - if claim is not None: - await self.commit(full_topic, raw_id, claim.token) + try: + timer_id = raw_id.decode(encoder.encoding, encoder.encoding_errors) + except UnicodeDecodeError: + timer_id = None + if timer_id is not None and encoder.encode(timer_id) == raw_id: + result.append(timer_id) else: - result.append(raw_id) + claim = await self.claim(full_topic, raw_id, now, 1) + if claim is not None: + await self.commit(full_topic, raw_id, claim.token) return result async def claim(self, full_topic: str, timer_id: str | bytes, now: float, lease_ttl: int) -> TimerClaim | None: diff --git a/tests/test_fencing.py b/tests/test_fencing.py index 4454bad..58c0f2b 100644 --- a/tests/test_fencing.py +++ b/tests/test_fencing.py @@ -145,11 +145,11 @@ async def test_malformed_id_cleanup_preserves_newer_schedule( pl = f"{store.payloads_key}:topic" claims = store._claims_key("topic") # noqa: SLF001 bad_id = b"\xff" - original = redis_client.zrangebyscore + original = redis_client.execute_command async def read_then_replace(*args, **kwargs): result = await original(*args, **kwargs) - if replace_after_read: + if args[0] == "ZRANGEBYSCORE" and replace_after_read: async with redis_client.pipeline(transaction=True) as pipe: pipe.zadd(tl, {bad_id: 200}) pipe.hset(pl, bad_id, b"replacement") @@ -159,7 +159,7 @@ async def read_then_replace(*args, **kwargs): try: await redis_client.zadd(tl, {bad_id: 100}) await redis_client.hset(pl, bad_id, b"invalid ID") - monkeypatch.setattr(redis_client, "zrangebyscore", read_then_replace) + monkeypatch.setattr(redis_client, "execute_command", read_then_replace) assert await store.due("topic", 100, 1) == [] if replace_after_read: assert await redis_client.zscore(tl, bad_id) == 200 @@ -448,3 +448,126 @@ async def test_explicit_claim_namespace_is_used_and_cleaned(broker: TimersBroker assert await redis_client.exists(f"{prefix}:app:topic") == 0 finally: await store.cancel_all("app:topic") + + +@pytest.mark.parametrize("encoding_errors", ["ignore", "replace"]) +@pytest.mark.parametrize("decode_responses", [False, True]) +async def test_lossy_id_decoding_does_not_block_valid_delivery( + broker: TimersBroker, + redis_client: Redis, + encoding_errors: str, + decode_responses: bool, +) -> None: + original = broker.config.broker_config.store + client = Redis.from_url(REDIS_URL, encoding_errors=encoding_errors, decode_responses=decode_responses) + configured = TimersBroker(client, timeline_key=original.timeline_key, payloads_key=original.payloads_key) + store = configured.config.broker_config.store + delivered = asyncio.Event() + seen: list[bytes] = [] + + @configured.subscriber("topic", max_concurrent=1) + async def handler(body: bytes) -> None: + seen.append(body) + delivered.set() + + tl = f"{store.timeline_key}:topic" + pl = f"{store.payloads_key}:topic" + bad_id = b"\xffK" + alias = "K" if encoding_errors == "ignore" else "\ufffdK" + try: + await configured.connect() + await store.schedule("topic", alias, b"future alias payload", 10_000_000_000) + await redis_client.zadd(tl, {bad_id: 1}) + await redis_client.hset(pl, bad_id, b"malformed ID payload") + await configured.publish(b"\x00\xffgood", topic="topic", timer_id="good") + assert await store.due("topic", 100, 1) == [] + assert await redis_client.zscore(tl, bad_id) is None + assert await redis_client.hget(pl, bad_id) is None + assert await redis_client.zscore(tl, alias) == 10_000_000_000 + assert await redis_client.hget(pl, alias) == b"future alias payload" + async with configured: + await asyncio.wait_for(delivered.wait(), 5) + assert seen == [b"\x00\xffgood"] + assert not await store.is_pending("topic", "good") + assert await store.is_pending("topic", alias) + finally: + await store.cancel_all("topic") + await configured.stop() + await client.aclose() # ty: ignore[missing-attribute] + + +@pytest.mark.parametrize("operation", ["schedule", "commit", "cancel_all", "remove"]) +async def test_independent_stores_preserve_simultaneous_claim_owners( + broker: TimersBroker, + redis_client: Redis, + operation: str, +) -> None: + first = broker.config.broker_config.store + second = TimerStore(ConnectionState(redis_client), f"{first.timeline_key}_other", f"{first.payloads_key}_other") + try: + await first.schedule("topic", "K", b"first", 100) + await second.schedule("topic", "K", b"second", 100) + a = await first.claim("topic", "K", 100, 10) + b = await second.claim("topic", "K", 100, 10) + assert a is not None + assert b is not None + if operation == "schedule": + await first.schedule("topic", "K", b"replacement", 200) + assert await first.is_pending("topic", "K") + elif operation == "commit": + await first.commit("topic", "K", a.token) + assert not await first.is_pending("topic", "K") + elif operation == "cancel_all": + assert await first.cancel_all("topic") == 1 + else: + await first.remove("topic", "K") + assert await redis_client.zscore(f"{second.timeline_key}:topic", "K") == 110 + assert await redis_client.hget(f"{second.payloads_key}:topic", "K") == b"second" + await second.commit("topic", "K", b.token) + assert not await second.is_pending("topic", "K") + assert await redis_client.hget(f"{second.payloads_key}:topic", "K") is None + finally: + await first.cancel_all("topic") + await second.cancel_all("topic") + + +@pytest.mark.parametrize("change", ["replacement", "takeover"]) +async def test_clients_with_same_physical_keys_share_claim_ownership( + broker: TimersBroker, + redis_client: Redis, + change: str, +) -> None: + original = broker.config.broker_config.store + timeline = f"{original.timeline_key}_\u00e9" + payloads = f"{original.payloads_key}_\u00e9" + first = TimerStore(ConnectionState(redis_client), timeline, payloads) + client = Redis.from_url(REDIS_URL, encoding="latin-1", decode_responses=True) + second = TimerStore( + ConnectionState(client), + timeline.encode().decode("latin-1"), + payloads.encode().decode("latin-1"), + ) + try: + await first.schedule("topic", "K", b"original", 100) + old = await first.claim("topic", "K", 100, 10) + assert old is not None + assert await second.claim("topic", "K", 109, 10) is None + if change == "replacement": + await second.schedule("topic", "K", b"replacement", 200) + await first.commit("topic", "K", old.token) + assert await first.is_pending("topic", "K") + current = await second.claim("topic", "K", 200, 10) + assert current is not None + assert current.payload == b"replacement" + else: + current = await second.claim("topic", "K", 110, 10) + assert current is not None + assert current.payload == b"original" + await first.commit("topic", "K", old.token) + assert await first.is_pending("topic", "K") + await first.commit("topic", "K", current.token) + assert not await second.is_pending("topic", "K") + finally: + await first.cancel_all("topic") + await second.cancel_all("topic") + await client.aclose() # ty: ignore[missing-attribute] diff --git a/tests/test_unit.py b/tests/test_unit.py index 12ca270..bb94f2f 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -13,6 +13,7 @@ from faststream.specification import AsyncAPI from redis.asyncio import Redis from redis.asyncio.cluster import RedisCluster +from redis.client import NEVER_DECODE from redis.exceptions import NoScriptError from faststream_redis_timers import TestTimersBroker, TimersBroker @@ -476,7 +477,7 @@ async def test_consume_logs_get_msgs_error_with_repr() -> None: """A Redis error during _get_msgs is logged at ERROR with `{e!r}` so the type and msg survive Sentry truncation.""" client = _redis_client() client.ping.return_value = True - client.zrangebyscore.side_effect = ConnectionError("redis down") + client.execute_command.side_effect = ConnectionError("redis down") broker = TimersBroker(client, start_timeout=2.0) @broker.subscriber("topic", polling_interval=0.05) @@ -637,19 +638,22 @@ async def test_timer_store_schedule() -> None: async def test_timer_store_due_bytes_client() -> None: """due() decodes bytes IDs for a bytes-mode Redis client.""" client = _redis_client() - client.zrangebyscore = AsyncMock(return_value=[b"timer-1", b"timer-2"]) + client.execute_command = AsyncMock(return_value=[b"timer-1", b"timer-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") result = await store.due("topic", 1000.0, 10) assert result == ["timer-1", "timer-2"] - client.zrangebyscore.assert_awaited_once_with("tl:topic", "-inf", 1000.0, start=0, num=10) + client.execute_command.assert_awaited_once_with( + "ZRANGEBYSCORE", "tl:topic", "-inf", 1000.0, "LIMIT", 0, 10, **{NEVER_DECODE: []} + ) -async def test_timer_store_due_str_client() -> None: - """due() passes str IDs through unchanged for a str-mode Redis client.""" +async def test_timer_store_due_decoding_client() -> None: + """due() requests raw IDs even when the client normally decodes replies.""" client = _redis_client() - client.zrangebyscore = AsyncMock(return_value=["timer-1", "timer-2"]) + client.connection_pool = Redis(decode_responses=True).connection_pool + client.execute_command = AsyncMock(return_value=[b"timer-1", b"timer-2"]) store = TimerStore(ConnectionState(client), "tl", "pl") result = await store.due("topic", 1000.0, 10) @@ -660,7 +664,7 @@ async def test_timer_store_due_str_client() -> None: async def test_timer_store_due_empty() -> None: """due() returns an empty list when no timers are due.""" client = _redis_client() - client.zrangebyscore = AsyncMock(return_value=[]) + client.execute_command = AsyncMock(return_value=[]) store = TimerStore(ConnectionState(client), "tl", "pl") result = await store.due("topic", 1000.0, 10) @@ -672,7 +676,7 @@ async def test_timer_store_due_non_utf8_self_heal() -> None: """due() removes non-UTF-8 members via COMMIT_LUA and excludes them from the result.""" bad_id = b"\xff\xfe-broken" client = _redis_client() - client.zrangebyscore = AsyncMock(return_value=[bad_id, b"good-id"]) + client.execute_command = AsyncMock(return_value=[bad_id, b"good-id"]) store = TimerStore(ConnectionState(client), "tl", "pl")