From f395d44a0b759615d9957a64b7245768df54f823 Mon Sep 17 00:00:00 2001 From: livesite Date: Thu, 27 Aug 2026 19:33:32 +0800 Subject: [PATCH] release: add topic TTL discard and prepare 0.9.0 --- Cargo.lock | 22 +- Cargo.toml | 2 +- README.md | 90 +++-- console-ui/package.json | 2 +- console-ui/src/api/types.ts | 10 +- .../src/components/ConfigureTopicDialog.tsx | 55 +++ .../src/components/CreateResourceDialog.tsx | 35 +- .../src/components/ManagementActionDialog.tsx | 5 +- .../src/components/ResourceActionMenu.tsx | 17 +- console-ui/src/i18n.tsx | 18 + console-ui/src/layout/AppShell.tsx | 2 +- console-ui/src/pages/TopicsPage.tsx | 12 +- console-ui/src/styles.scss | 1 + crates/console/src/collector.rs | 15 + crates/console/src/managed_view.rs | 8 +- crates/console/src/management/apply.rs | 78 ++++- crates/console/src/management/backend.rs | 4 + .../console/src/management/operation_state.rs | 15 + crates/console/src/management/preview.rs | 216 +++++++++++- crates/console/src/management/reconcile.rs | 56 +++ .../console/src/management/resource_state.rs | 19 +- crates/console/src/model.rs | 6 + crates/console/src/resources.rs | 172 ++++++++- crates/console/src/session.rs | 6 + crates/operator/src/lib.rs | 2 +- crates/operator/src/management_crd.rs | 21 +- .../queue/src/bin/rustqueue-crash-worker.rs | 14 +- crates/queue/src/broker.rs | 15 +- crates/queue/src/broker/delivery.rs | 8 + crates/queue/src/broker/expiration.rs | 265 ++++++++++++++ crates/queue/src/broker/group_commit.rs | 11 + crates/queue/src/broker/io.rs | 13 + crates/queue/src/broker/management.rs | 256 +++++++++----- crates/queue/src/broker/stats.rs | 23 +- crates/queue/src/broker/topics.rs | 13 + crates/queue/src/broker_tests.rs | 331 +++++++++++++++++- crates/queue/src/channel.rs | 22 ++ crates/queue/src/lib.rs | 3 +- crates/queue/src/management.rs | 1 + crates/queue/src/metadata.rs | 27 ++ crates/queue/src/model.rs | 51 +++ crates/queue/src/outbox.rs | 63 ++-- crates/queue/src/telemetry.rs | 2 + crates/queue/src/topic.rs | 108 +++++- crates/queue/src/topic/delivery.rs | 50 ++- crates/queue/src/topic/index.rs | 125 +++++++ crates/queue/src/topic/maintenance.rs | 81 ++++- crates/queue/tests/crash_boundaries.rs | 49 ++- crates/server/src/http.rs | 5 + crates/server/src/http/manage.rs | 45 ++- crates/server/src/main.rs | 47 +++ crates/server/src/metrics.rs | 23 +- crates/server/src/tcp.rs | 1 + crates/server/src/tcp/dead_letter.rs | 5 + crates/server/src/tcp/session.rs | 146 +++++++- crates/server/src/tcp/time.rs | 7 + crates/server/src/tcp/writer.rs | 153 +++++++- deploy/helm/rustqueue/Chart.yaml | 4 +- .../crds/rustqueue.io_rustqueuetopics.yaml | 10 +- deploy/helm/rustqueue/values.yaml | 4 +- docs/architecture/share-nothing-v7.md | 23 +- docs/operations/console.md | 6 + docs/operations/kubernetes.md | 29 +- docs/releases/v0.9.0.md | 100 ++++++ fuzz/Cargo.lock | 12 +- scripts/acceptance-multi-broker-k8s.sh | 4 +- site/index.html | 2 +- 67 files changed, 2760 insertions(+), 286 deletions(-) create mode 100644 console-ui/src/components/ConfigureTopicDialog.tsx create mode 100644 crates/queue/src/broker/expiration.rs create mode 100644 docs/releases/v0.9.0.md diff --git a/Cargo.lock b/Cargo.lock index 6e4eeff..022585e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2283,7 +2283,7 @@ dependencies = [ [[package]] name = "rustqueue-bench" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "clap", @@ -2295,7 +2295,7 @@ dependencies = [ [[package]] name = "rustqueue-console" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "rustqueue-discovery" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2346,7 +2346,7 @@ dependencies = [ [[package]] name = "rustqueue-operator" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2373,7 +2373,7 @@ dependencies = [ [[package]] name = "rustqueue-protocol" -version = "0.8.4" +version = "0.9.0" dependencies = [ "bytes", "serde", @@ -2383,7 +2383,7 @@ dependencies = [ [[package]] name = "rustqueue-proxy" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2404,7 +2404,7 @@ dependencies = [ [[package]] name = "rustqueue-queue" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "bytes", @@ -2426,7 +2426,7 @@ dependencies = [ [[package]] name = "rustqueue-server" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "async-compression", @@ -2462,7 +2462,7 @@ dependencies = [ [[package]] name = "rustqueue-storage" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "crc32c", @@ -2475,14 +2475,14 @@ dependencies = [ [[package]] name = "rustqueue-telemetry" -version = "0.8.4" +version = "0.9.0" dependencies = [ "serde", ] [[package]] name = "rustqueuectl" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 363b4d1..65cb892 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ exclude = ["fuzz"] [workspace.package] -version = "0.8.4" +version = "0.9.0" edition = "2021" license = "Apache-2.0" rust-version = "1.88" diff --git a/README.md b/README.md index c3c4027..c4faa7d 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,14 @@ [NSQ performance boundaries](docs/architecture/nsq-performance.md) · [Kubernetes operations](docs/operations/kubernetes.md) · [Console operations](docs/operations/console.md) · -[v0.8.4 release](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.8.4) +[v0.9.0 release](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.9.0) -RustQueue 0.8.4 is a Kubernetes-native, NSQ V2-compatible message queue for +RustQueue 0.9.0 is a Kubernetes-native, NSQ V2-compatible message queue for trusted internal networks. It is written in Rust and uses a deliberately simple share-nothing model: each Broker owns one durable RWO PVC, while Kubernetes provides scheduling, rollout and discovery. -> Current release: [v0.8.4](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.8.4). +> Current release: [v0.9.0](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.9.0). > RustQueue is a production candidate for workloads that accept single-PVC > durability and at-least-once delivery. It does not replicate messages between > Brokers and is not an HA replacement for a replicated log. @@ -44,45 +44,54 @@ messages stored on that Broker are lost. Configure disk pressure protection, monitor the exported metrics, and choose PVC/storage failure policies that fit your workload before deploying to production. -## What's new in 0.8.4 - -- **Larger durable Channel groups.** `FIN` and `REQ` requests now share a - bounded 1 ms Channel WAL group commit of up to 1,024 requests. Every success - still crosses the affected WAL `fsync` before the client sees completion. -- **More delivery concurrency.** Channel WAL `fsync` runs after releasing the - Topic state lock and uses an independent commit gate, so delivery reservations - can continue while the durable boundary is being written. -- **Lower hot-path allocation cost.** WAL commands use stack encoding, TCP and - Channel operations share `Arc` channel identities, Channel futures avoid - an extra `Box`, and each session bounds active Channel operations at 256. -- **Stricter benchmark completion.** The benchmark consumer waits for EOF and - verifies a complete final drain before accepting a result. -- **No compatibility change.** The v7 on-disk format and the durable `FIN`/`REQ` - fsync contract remain unchanged. - -See the [v0.8.4 release notes](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.8.4) +## What's new in 0.9.0 + +- **Topic-level TTL discard.** Managed Topics can choose `TTL_DISCARD` with a + positive whole-second `messageTtlSeconds`; `RELIABLE` remains the default and + requires no TTL. The TTL starts at the Broker write timestamp and is an + absolute lifetime. +- **One final expiration boundary.** Expiry applies to unrouted, deferred, + requeued and in-flight messages. It evicts the message from every Channel, + never writes a DLQ entry, and lets `TTL_DISCARD` retry normally until the TTL + rather than ending through retention or maximum-attempt DLQ rules. +- **Safe online policy changes.** Topic creation can set the policy, and + `/v1/manage/topics/configure` supports the `CONFIGURE` operation for existing + Topics. Enabling or shortening TTL previews possible backlog loss and + requires the exact Topic name; visible multi-owner migrations block policy + changes. Extending or disabling TTL cannot revive an already expired message. +- **Deadline-bound delivery.** Reservation and socket handoff recheck the + absolute deadline. If a frame cannot finish before expiry, the consumer + connection closes instead of leaving a partially parseable NSQ frame; a + complete frame that is not `FIN`ed still expires at the same boundary. +- **Wire and storage compatibility.** `PUB`, `MPUB` and `DPUB` keep their + existing `publish_ack_mode` acknowledgement behavior, and NSQ TCP/HTTP + publishing remains unchanged. Existing and auto-created Topics stay + `RELIABLE`; old manifests default to that mode, and disk format v7 needs no + migration. + +See the [v0.9.0 release notes](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.9.0) and [NSQ performance boundaries](docs/architecture/nsq-performance.md) for the contract and benchmark interpretation. -## Download 0.8.4 +## Download 0.9.0 Every release contains native Linux binaries, the Console UI, source, the Helm Chart and a checksum manifest: | Asset | Contents | | --- | --- | -| `rustqueue-0.8.4-linux-x86_64.tar.gz` | Linux x86_64 binaries, Console UI and example configuration | -| `rustqueue-0.8.4-linux-aarch64.tar.gz` | Linux ARM64 binaries, Console UI and example configuration | -| `rustqueue-0.8.4-source.tar.gz` | Source archive for the tagged commit | -| `rustqueue-0.8.4.tgz` | Helm Chart | -| `SHA256SUMS-0.8.4` | SHA-256 checksums for every downloadable artifact | +| `rustqueue-0.9.0-linux-x86_64.tar.gz` | Linux x86_64 binaries, Console UI and example configuration | +| `rustqueue-0.9.0-linux-aarch64.tar.gz` | Linux ARM64 binaries, Console UI and example configuration | +| `rustqueue-0.9.0-source.tar.gz` | Source archive for the tagged commit | +| `rustqueue-0.9.0.tgz` | Helm Chart | +| `SHA256SUMS-0.9.0` | SHA-256 checksums for every downloadable artifact | ```sh arch="$(uname -m)" -curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.8.4/rustqueue-0.8.4-linux-${arch}.tar.gz" -curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.8.4/SHA256SUMS-0.8.4" -sha256sum --check --ignore-missing SHA256SUMS-0.8.4 -tar -xzf "rustqueue-0.8.4-linux-${arch}.tar.gz" +curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.9.0/rustqueue-0.9.0-linux-${arch}.tar.gz" +curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.9.0/SHA256SUMS-0.9.0" +sha256sum --check --ignore-missing SHA256SUMS-0.9.0 +tar -xzf "rustqueue-0.9.0-linux-${arch}.tar.gz" ``` ## Architecture @@ -127,6 +136,12 @@ operator -> eligible nodes -> StatefulSet ordinal + retained RWO PVC starts at that persisted unrouted boundary. Deleting the last durable Channel starts a new boundary at the current Topic tail; ephemeral Channels do not clear it. +- Topics default to `RELIABLE`. Managed Topics may opt into `TTL_DISCARD` with + a positive whole-second TTL measured from the Broker write timestamp. TTL is + an absolute lifetime across unrouted, deferred, requeued, and in-flight + states: expiry evicts the message from every Channel without writing a DLQ. + Logical expiry is deadline-bound; immutable Segment reclamation remains + asynchronous and reader-safe. - Once a durable Channel exists, a later Channel can still bootstrap from the last 90 seconds. This covers one official Go client default 60-second lookup poll plus its 30% jitter. The Kodo profile forces 180 seconds so one failed @@ -167,7 +182,7 @@ operator -> eligible nodes -> StatefulSet ordinal + retained RWO PVC sent is returned as ambiguous and is never retried automatically. - `rustqueue-operator`: creates the StatefulSet, retained PVCs, discovery, proxy, RBAC, disruption budgets, PVC expansion and drain-aware one-at-a-time - rolling updates. The 0.8 Kodo profile adds atomic Discovery cutover, + rolling updates. The 0.9 Kodo profile adds atomic Discovery cutover, producer-restart fencing and fail-closed decommissioning. - `rustqueue-console`: Kubernetes and broker observability backend serving the bilingual Carbon UI, with default-off native Topic/Channel management. @@ -225,7 +240,7 @@ kubectl label node worker-1 rustqueue.io/eligible=true helm upgrade --install rustqueue deploy/helm/rustqueue \ --namespace rustqueue --create-namespace \ - --set queue.image=registry.example/rustqueue:0.8.4 \ + --set queue.image=registry.example/rustqueue:0.9.0 \ --set queue.storageClassName=ssd-rwo ``` @@ -239,7 +254,7 @@ scrub, upgrade or the NSQ-compatible admin API. Client TLS is optional and always supplied through an existing Kubernetes Secret; the operator does not run a CA. -### Kodo compatibility in 0.8 +### Kodo compatibility in 0.9 Kodo compatibility is implemented entirely by RustQueue as a separate, default-off deployment profile. Kodo continues to use RustQueue Discovery: @@ -462,7 +477,7 @@ test-only direct Pod placement; production anti-affinity is unchanged. A unit fixture covers discovery indexing for 500 brokers. No 500-broker deployment or load test is part of the functional gate. -The v0.8.4 CI/CD workflow publishes a Release only after the non-Kubernetes +The v0.9.0 CI/CD workflow publishes a Release only after the non-Kubernetes release gate, both native Linux builds, packaging and checksum verification succeed. The v0.8.0 Kodo compatibility baseline additionally passed the unmodified Kodo source replay, an exact 104,857,500-byte `PUB`/`DPUB` with one @@ -549,6 +564,9 @@ by default; `[metrics].detailed_queue_metrics` enables bounded per-topic/channel series up to `max_detailed_series`. Delivery-budget bytes, waiters and cumulative waits are exported as bounded aggregate gauges/counters. +`rustqueue_ttl_discarded_messages_total` is label-free. When detailed queue +metrics are enabled, `rustqueue_topic_ttl_discarded_messages_total` uses the +same global series budget as the other per-Topic metrics. `[queue].publish_ack_mode` defaults to `"durable"`: successful publish commands follow local segment `fsync`. `"write_ack"` returns after append while consumers @@ -562,7 +580,7 @@ writes. Keep both relaxed profiles separate from durable-PUB results. ## Storage and upgrades -RustQueue 0.8 keeps disk format v7. Format v7 is a clean break: a v6 or older +RustQueue 0.9 keeps disk format v7. Format v7 is a clean break: a v6 or older directory is refused and there is no in-place migration. Within v7, record tags and existing fields are append-only. Every binary declares its reader/writer feature range and protocol message/body @@ -635,6 +653,6 @@ non-Kubernetes gate runs in GitHub Actions. ## Non-goals -RustQueue 0.8 does not provide message replication, backups, exactly-once +RustQueue 0.9 does not provide message replication, backups, exactly-once delivery, a global channel catalog, online data migration, cross-region replication, or Broker/PVC lifecycle controls in Console. diff --git a/console-ui/package.json b/console-ui/package.json index d80ae0a..2f126e4 100644 --- a/console-ui/package.json +++ b/console-ui/package.json @@ -1,7 +1,7 @@ { "name": "rustqueue-console-ui", "private": true, - "version": "0.8.4", + "version": "0.9.0", "packageManager": "pnpm@11.9.0", "type": "module", "scripts": { diff --git a/console-ui/src/api/types.ts b/console-ui/src/api/types.ts index 457350b..0fd199d 100644 --- a/console-ui/src/api/types.ts +++ b/console-ui/src/api/types.ts @@ -34,6 +34,9 @@ export interface ChannelStats { export interface TopicStats { name: string; paused: boolean; + delivery_mode: 'RELIABLE' | 'TTL_DISCARD'; + message_ttl_seconds?: number; + ttl_discarded_messages: number; message_count: number; segment_count: number; segment_bytes: number; @@ -133,6 +136,9 @@ export interface Topic { name: string; owners: string[]; paused: boolean; + delivery_mode: 'RELIABLE' | 'TTL_DISCARD'; + message_ttl_seconds?: number; + ttl_discarded_messages: number; stored_messages: number; segment_count: number; segment_bytes: number; @@ -257,9 +263,11 @@ export interface ManagementStatus { export interface ManagementAction { kind: 'topic' | 'channel'; - action: 'create' | 'pause' | 'unpause' | 'empty' | 'delete' | 'retry'; + action: 'create' | 'configure' | 'pause' | 'unpause' | 'empty' | 'delete' | 'retry'; topic: string; channel?: string; + delivery_mode?: 'RELIABLE' | 'TTL_DISCARD'; + message_ttl_seconds?: number; } export interface ActionPreview { diff --git a/console-ui/src/components/ConfigureTopicDialog.tsx b/console-ui/src/components/ConfigureTopicDialog.tsx new file mode 100644 index 0000000..4ea2b89 --- /dev/null +++ b/console-ui/src/components/ConfigureTopicDialog.tsx @@ -0,0 +1,55 @@ +import { Modal, Select, SelectItem, TextInput } from '@carbon/react'; +import { useEffect, useState } from 'react'; +import type { ManagementAction, Topic } from '../api/types'; +import { useI18n } from '../i18n'; + +interface Props { + topic?: Topic; + onClose: () => void; + onContinue: (action: ManagementAction) => void; +} + +export function ConfigureTopicDialog({ topic, onClose, onContinue }: Props) { + const { t } = useI18n(); + const [deliveryMode, setDeliveryMode] = useState<'RELIABLE' | 'TTL_DISCARD'>('RELIABLE'); + const [ttlSeconds, setTtlSeconds] = useState(''); + + useEffect(() => { + setDeliveryMode(topic?.delivery_mode || 'RELIABLE'); + setTtlSeconds(topic?.message_ttl_seconds?.toString() || ''); + }, [topic]); + + if (!topic) return null; + const ttl = Number(ttlSeconds); + const valid = deliveryMode === 'RELIABLE' || (Number.isInteger(ttl) && ttl > 0); + const submit = () => onContinue({ + kind: 'topic', + action: 'configure', + topic: topic.name, + delivery_mode: deliveryMode, + message_ttl_seconds: deliveryMode === 'TTL_DISCARD' ? ttl : undefined, + }); + + return ( + +

{topic.name}

+
+ + {deliveryMode === 'TTL_DISCARD' && ( + setTtlSeconds(event.target.value)} /> + )} +
+ + ); +} diff --git a/console-ui/src/components/CreateResourceDialog.tsx b/console-ui/src/components/CreateResourceDialog.tsx index 3a5502d..c2c3383 100644 --- a/console-ui/src/components/CreateResourceDialog.tsx +++ b/console-ui/src/components/CreateResourceDialog.tsx @@ -1,4 +1,4 @@ -import { Modal, TextInput } from '@carbon/react'; +import { Modal, Select, SelectItem, TextInput } from '@carbon/react'; import { useEffect, useState } from 'react'; import type { ManagementAction } from '../api/types'; import { useI18n } from '../i18n'; @@ -14,11 +14,27 @@ interface Props { export function CreateResourceDialog({ open, kind, topic, onClose, onContinue }: Props) { const { t } = useI18n(); const [name, setName] = useState(''); - useEffect(() => { if (!open) setName(''); }, [open]); - const valid = /^[.A-Za-z0-9_-]{1,64}$/.test(name) && !name.endsWith('#ephemeral'); + const [deliveryMode, setDeliveryMode] = useState<'RELIABLE' | 'TTL_DISCARD'>('RELIABLE'); + const [ttlSeconds, setTtlSeconds] = useState(''); + useEffect(() => { + if (!open) { + setName(''); + setDeliveryMode('RELIABLE'); + setTtlSeconds(''); + } + }, [open]); + const ttl = Number(ttlSeconds); + const policyValid = kind !== 'topic' || deliveryMode === 'RELIABLE' || (Number.isInteger(ttl) && ttl > 0); + const valid = /^[.A-Za-z0-9_-]{1,64}$/.test(name) && !name.endsWith('#ephemeral') && policyValid; const submit = () => { onContinue(kind === 'topic' - ? { kind, action: 'create', topic: name } + ? { + kind, + action: 'create', + topic: name, + delivery_mode: deliveryMode, + message_ttl_seconds: deliveryMode === 'TTL_DISCARD' ? ttl : undefined, + } : { kind, action: 'create', topic: topic || '', channel: name }); }; return ( @@ -39,6 +55,17 @@ export function CreateResourceDialog({ open, kind, topic, onClose, onContinue }: autoComplete="off" onChange={(event) => setName(event.target.value)} /> + {kind === 'topic' && ( +
+ + {deliveryMode === 'TTL_DISCARD' && ( + setTtlSeconds(event.target.value)} /> + )} +
+ )} ); } diff --git a/console-ui/src/components/ManagementActionDialog.tsx b/console-ui/src/components/ManagementActionDialog.tsx index 69f7cde..f09a770 100644 --- a/console-ui/src/components/ManagementActionDialog.tsx +++ b/console-ui/src/components/ManagementActionDialog.tsx @@ -32,6 +32,7 @@ export function ManagementActionDialog({ action, busy, onPreview, onApply, onClo if (!action) return null; const destructive = action.action === 'empty' || action.action === 'delete' || action.action === 'retry'; const expected = preview?.confirmation_required || ''; + const caution = destructive || Boolean(expected); const submit = async () => { if (!preview) return; try { @@ -44,7 +45,7 @@ export function ManagementActionDialog({ action, busy, onPreview, onApply, onClo return (
{t('topics.depth')}
{number(preview.impact.depth)}
{t('topics.inFlight')}
{number(preview.impact.in_flight)}
- {preview.impact.warnings.map((warning) => )} + {preview.impact.warnings.map((warning) => )} {expected && ( void; onAction: (action: ManagementAction) => void; } -export function ResourceActionMenu({ kind, topic, channel, paused, phase, disabled, ephemeral, onAction }: Props) { +export function ResourceActionMenu({ kind, topic, channel, paused, phase, disabled, ephemeral, onConfigure, onAction }: Props) { const { t } = useI18n(); const action = (name: ManagementAction['action']) => () => onAction({ kind, action: name, topic, channel }); const blocked = disabled || Boolean(ephemeral) || !['ACTIVE', 'FAILED'].includes(phase); return ( - {phase === 'FAILED' ? ( - - ) : ( - <> - - - - - )} + {phase === 'FAILED' && } + {phase !== 'FAILED' && kind === 'topic' && } + {phase !== 'FAILED' && } + {phase !== 'FAILED' && } + {phase !== 'FAILED' && } ); } diff --git a/console-ui/src/i18n.tsx b/console-ui/src/i18n.tsx index 58ef0d8..e2a0c9e 100644 --- a/console-ui/src/i18n.tsx +++ b/console-ui/src/i18n.tsx @@ -107,6 +107,12 @@ const messages: Record> = { 'topics.channelDetail': 'Channel 明细', 'topics.retryTotal': '集群累计重试', 'topics.deadLetterTotal': '集群累计死信', + 'topics.deliveryPolicy': '投递策略', + 'topics.deliveryMode': '投递模式', + 'topics.messageTtlSeconds': '消息 TTL（秒）', + 'topics.ttlHint': '从 Broker 写入时间起计算；到期后直接丢弃且不进入 DLQ。', + 'topics.mode.reliable': '可靠', + 'topics.mode.ttlDiscard': 'TTL 丢弃', 'management.readOnly': '只读观测模式', 'management.disabledHint': 'Helm 默认关闭管理能力。Broker 与 Console 写入路由均未启用。', 'management.locked': '管理已锁定', @@ -126,6 +132,7 @@ const messages: Record> = { 'management.retry': '重试失败操作', 'management.createTopic': '创建 Topic', 'management.createChannel': '创建 Channel', + 'management.configureTtl': '配置 TTL', 'management.channelName': 'Channel 名称', 'management.nameHint': '支持字母、数字、点、下划线和连字符，最多 64 个字符。', 'management.previewing': '正在核对最新状态和影响范围', @@ -140,6 +147,7 @@ const messages: Record> = { 'management.topicCreatedHint': 'Topic 已创建，但尚无 Channel。请先创建 Channel，再连接消费者。', 'management.warning.topic_requires_channel': '创建 Topic 不会自动创建 Channel。请先创建 Channel，再连接消费者。', 'management.warning.immediate_irreversible': '此操作立即生效，且无法撤销。', + 'management.warning.ttl_policy_may_discard_existing_messages': '启用或缩短 TTL 可能立即丢弃现有积压；已失效消息无法恢复。', 'management.tombstoneUntil': '删除保护截止时间', 'management.until': '至', 'management.phase.preparing': '准备中', @@ -147,6 +155,7 @@ const messages: Record> = { 'management.phase.failed': '失败', 'management.phase.tombstoned': '删除保护中', 'management.action.create': '创建', + 'management.action.configure': '配置 TTL', 'management.action.pause': '暂停', 'management.action.unpause': '恢复', 'management.action.empty': '立即清空', @@ -325,6 +334,12 @@ const messages: Record> = { 'topics.channelDetail': 'Channel details', 'topics.retryTotal': 'Cluster retries', 'topics.deadLetterTotal': 'Cluster dead letters', + 'topics.deliveryPolicy': 'Delivery policy', + 'topics.deliveryMode': 'Delivery mode', + 'topics.messageTtlSeconds': 'Message TTL (seconds)', + 'topics.ttlHint': 'Measured from Broker write time. Expired messages are discarded without entering the DLQ.', + 'topics.mode.reliable': 'Reliable', + 'topics.mode.ttlDiscard': 'TTL discard', 'management.readOnly': 'Read-only observation', 'management.disabledHint': 'Helm keeps management off by default. Broker and Console write routes are disabled.', 'management.locked': 'Management locked', @@ -344,6 +359,7 @@ const messages: Record> = { 'management.retry': 'Retry failed action', 'management.createTopic': 'Create topic', 'management.createChannel': 'Create channel', + 'management.configureTtl': 'Configure TTL', 'management.channelName': 'Channel name', 'management.nameHint': 'Use letters, numbers, dots, underscores, or hyphens, up to 64 characters.', 'management.previewing': 'Checking current state and impact', @@ -358,6 +374,7 @@ const messages: Record> = { 'management.topicCreatedHint': 'The topic was created without a channel. Create a channel before connecting consumers.', 'management.warning.topic_requires_channel': 'Creating a topic does not create a channel. Create one before connecting consumers.', 'management.warning.immediate_irreversible': 'This operation takes effect immediately and cannot be undone.', + 'management.warning.ttl_policy_may_discard_existing_messages': 'Enabling or shortening TTL may immediately discard existing backlog. Expired messages cannot be recovered.', 'management.tombstoneUntil': 'Deletion fence until', 'management.until': 'until', 'management.phase.preparing': 'Preparing', @@ -365,6 +382,7 @@ const messages: Record> = { 'management.phase.failed': 'Failed', 'management.phase.tombstoned': 'Deletion fence', 'management.action.create': 'Create', + 'management.action.configure': 'Configure TTL', 'management.action.pause': 'Pause', 'management.action.unpause': 'Unpause', 'management.action.empty': 'Empty now', diff --git a/console-ui/src/layout/AppShell.tsx b/console-ui/src/layout/AppShell.tsx index 7b620fc..04654c8 100644 --- a/console-ui/src/layout/AppShell.tsx +++ b/console-ui/src/layout/AppShell.tsx @@ -57,7 +57,7 @@ export function AppShell({ page, onPage, dark, onTheme, onRefresh, children }: { onClick={(event) => { event.preventDefault(); onPage(item.id); }} >{t(`nav.${item.id}`)})} -
{t('app.readOnly')}v0.7
+
{t('app.readOnly')}v0.9.0
{children}
diff --git a/console-ui/src/pages/TopicsPage.tsx b/console-ui/src/pages/TopicsPage.tsx index 3a09113..2c167f3 100644 --- a/console-ui/src/pages/TopicsPage.tsx +++ b/console-ui/src/pages/TopicsPage.tsx @@ -3,6 +3,7 @@ import { Button, Column, Grid, InlineLoading, InlineNotification, Search, Table, import { useMemo, useState } from 'react'; import { useManagement } from '../api/useManagement'; import type { ManagementAction, Snapshot, Topic } from '../api/types'; +import { ConfigureTopicDialog } from '../components/ConfigureTopicDialog'; import { CreateResourceDialog } from '../components/CreateResourceDialog'; import { EmptyState } from '../components/EmptyState'; import { ManagementAccess } from '../components/ManagementAccess'; @@ -20,6 +21,7 @@ export function TopicsPage({ snapshot, refresh }: { snapshot: Snapshot; refresh: const [query, setQuery] = useState(''); const [createKind, setCreateKind] = useState<'topic' | 'channel'>(); const [pending, setPending] = useState(); + const [configuring, setConfiguring] = useState(); const [notice, setNotice] = useState(); const [settling, setSettling] = useState(false); const filtered = useMemo(() => snapshot.topics.filter((topic) => topic.name.toLowerCase().includes(query.toLowerCase())), [query, snapshot.topics]); @@ -67,8 +69,8 @@ export function TopicsPage({ snapshot, refresh }: { snapshot: Snapshot; refresh: - {[t('topics.name'), t('common.status'), t('topics.messages'), t('topics.channels'), t('topics.segments'), t('common.owner'), ...(enabled ? [t('management.actions')] : [])].map((value) => {value})} - {filtered.map((item) => setSelected(item.name)} />)} + {[t('topics.name'), t('common.status'), t('topics.deliveryPolicy'), t('topics.messages'), t('topics.channels'), t('topics.segments'), t('common.owner'), ...(enabled ? [t('management.actions')] : [])].map((value) => {value})} + {filtered.map((item) => setConfiguring(item)} onAction={setPending} onSelect={() => setSelected(item.name)} />)}
 @@ -76,22 +78,24 @@ export function TopicsPage({ snapshot, refresh }: { snapshot: Snapshot; refresh: )} setCreateKind(undefined)} onContinue={(action) => { setCreateKind(undefined); setPending(action); }} /> + setConfiguring(undefined)} onContinue={(action) => { setConfiguring(undefined); setPending(action); }} /> setPending(undefined)} onComplete={complete} /> ); } -function TopicRow({ topic, selected, managementEnabled, managementUnlocked, onAction, onSelect }: { topic: Topic; selected: boolean; managementEnabled: boolean; managementUnlocked: boolean; onAction: (action: ManagementAction) => void; onSelect: () => void }) { +function TopicRow({ topic, selected, managementEnabled, managementUnlocked, onConfigure, onAction, onSelect }: { topic: Topic; selected: boolean; managementEnabled: boolean; managementUnlocked: boolean; onConfigure: () => void; onAction: (action: ManagementAction) => void; onSelect: () => void }) { const { t } = useI18n(); return ( +
{t(topic.delivery_mode === 'TTL_DISCARD' ? 'topics.mode.ttlDiscard' : 'topics.mode.reliable')}{topic.message_ttl_seconds && {number(topic.message_ttl_seconds)}s}
 {number(topic.stored_messages)} {topic.channels.length} {topic.segment_count} / {bytes(topic.segment_bytes)}
{topic.owners.map((owner) => {owner})}
 - {managementEnabled &&
event.stopPropagation()}>
} + {managementEnabled &&
event.stopPropagation()}>
} ); } diff --git a/console-ui/src/styles.scss b/console-ui/src/styles.scss index 89db787..c037740 100644 --- a/console-ui/src/styles.scss +++ b/console-ui/src/styles.scss @@ -80,6 +80,7 @@ button, input { font: inherit; } .management-access__copy span { color: var(--cds-text-secondary); font-size: .75rem; margin-top: .2rem; } .panel__actions { display: flex; align-items: center; gap: .5rem; } .modal-copy { margin: 0 0 1.25rem; color: var(--cds-text-secondary); line-height: 1.5; } +.topic-policy-fields { display: grid; gap: 1rem; margin-top: 1rem; } .impact-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); margin: 0 0 1rem; border: 1px solid var(--cds-border-subtle); border-radius: 8px; overflow: hidden; } .impact-grid > div { padding: .75rem; min-height: 4.25rem; background: var(--cds-layer-01); } .impact-grid > div:nth-child(odd) { border-right: 1px solid var(--cds-border-subtle); } diff --git a/crates/console/src/collector.rs b/crates/console/src/collector.rs index 0622805..c59becf 100644 --- a/crates/console/src/collector.rs +++ b/crates/console/src/collector.rs @@ -281,6 +281,9 @@ struct TopicAggregate { name: String, owners: BTreeSet, paused: bool, + delivery_mode: String, + message_ttl_seconds: Option, + ttl_discarded_messages: u64, stored_messages: u64, segment_count: u64, segment_bytes: u64, @@ -292,6 +295,15 @@ impl TopicAggregate { self.name = topic.name.clone(); self.owners.insert(owner.into()); self.paused |= topic.paused; + self.delivery_mode = match topic.delivery_mode { + rustqueue_queue::DeliveryMode::Reliable => "RELIABLE", + rustqueue_queue::DeliveryMode::TtlDiscard => "TTL_DISCARD", + } + .into(); + self.message_ttl_seconds = topic.message_ttl_seconds; + self.ttl_discarded_messages = self + .ttl_discarded_messages + .saturating_add(topic.ttl_discarded_messages); self.stored_messages = self.stored_messages.saturating_add(topic.message_count); self.segment_count = self.segment_count.saturating_add(topic.segment_count); self.segment_bytes = self.segment_bytes.saturating_add(topic.segment_bytes); @@ -308,6 +320,9 @@ impl TopicAggregate { name: self.name, owners: self.owners.into_iter().collect(), paused: self.paused, + delivery_mode: self.delivery_mode, + message_ttl_seconds: self.message_ttl_seconds, + ttl_discarded_messages: self.ttl_discarded_messages, stored_messages: self.stored_messages, segment_count: self.segment_count, segment_bytes: self.segment_bytes, diff --git a/crates/console/src/managed_view.rs b/crates/console/src/managed_view.rs index f6a368e..a4b4e1d 100644 --- a/crates/console/src/managed_view.rs +++ b/crates/console/src/managed_view.rs @@ -1,7 +1,7 @@ use crate::model::{ChannelView, TopicView}; use crate::resources::ManagedResources; use kube::ResourceExt; -use rustqueue_operator::{ManagedResourcePhase, RustQueueTopic}; +use rustqueue_operator::{ManagedResourcePhase, RustQueueTopic, TopicDeliveryMode}; use std::collections::BTreeMap; pub fn merge(topics: &mut Vec, managed: &ManagedResources) { @@ -63,6 +63,12 @@ pub fn merge(topics: &mut Vec, managed: &ManagedResources) { } fn apply_topic(topic: &mut TopicView, resource: &RustQueueTopic) { + topic.delivery_mode = match resource.spec.delivery_mode { + TopicDeliveryMode::Reliable => "RELIABLE", + TopicDeliveryMode::TtlDiscard => "TTL_DISCARD", + } + .into(); + topic.message_ttl_seconds = resource.spec.message_ttl_seconds; topic.managed_phase = phase_name(&resource.spec.phase).into(); topic.management_revision = resource.spec.revision; topic.tombstone_until_ms = resource.spec.tombstone_until_ms; diff --git a/crates/console/src/management/apply.rs b/crates/console/src/management/apply.rs index 70b35fe..d4daf25 100644 --- a/crates/console/src/management/apply.rs +++ b/crates/console/src/management/apply.rs @@ -20,6 +20,10 @@ pub struct ApplyRequest { pub(super) action: String, pub(super) topic: String, pub(super) channel: Option, + #[serde(default, alias = "deliveryMode")] + pub(super) delivery_mode: Option, + #[serde(default, alias = "messageTtlSeconds")] + pub(super) message_ttl_seconds: Option, pub(super) action_token: String, #[serde(default)] pub(super) confirmation: String, @@ -38,6 +42,8 @@ pub async fn apply( action: request.action.clone(), topic: request.topic.clone(), channel: request.channel.clone(), + delivery_mode: request.delivery_mode.clone(), + message_ttl_seconds: request.message_ttl_seconds, }; validate_request(&preview_request)?; let challenge = state @@ -142,6 +148,8 @@ fn verify_request( || request.action != challenge.action || request.topic != challenge.topic || request.channel != challenge.channel + || request.delivery_mode != challenge.delivery_mode + || request.message_ttl_seconds != challenge.message_ttl_seconds { return Err(ManagementError::conflict( "E_ACTION_TOKEN_MISMATCH", @@ -170,8 +178,20 @@ fn verify_current_owners( .topics .iter() .find(|topic| topic.name == request.topic); + if request.kind == "topic" + && request.action == "create" + && challenge.subject_kind == "catalog" + && topic.is_some() + { + return Err(ManagementError::conflict( + "E_RESOURCE_CHANGED", + "topic appeared after preview", + )); + } let mut current = if request.kind == "topic" && request.action == "create" { - challenge.owners.clone() + topic + .map(|topic| topic.owners.clone()) + .unwrap_or_else(|| challenge.owners.clone()) } else if request.kind == "channel" && request.action == "create" { topic.map(|topic| topic.owners.clone()).unwrap_or_default() } else if request.kind == "topic" { @@ -196,11 +216,63 @@ fn verify_current_owners( "resource ownership changed after preview", )); } - if is_destructive(&request.action) && current.len() > 1 { + let topic_policy_operation = + request.kind == "topic" && matches!(request.action.as_str(), "create" | "configure"); + if (is_destructive(&request.action) || topic_policy_operation) && current.len() > 1 { return Err(ManagementError::conflict( "E_MIGRATION_IN_PROGRESS", - "destructive operations are blocked while multiple owners are visible", + "destructive and topic policy operations are blocked while multiple owners are visible", )); } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn multi_owner_guard_only_blocks_topic_policy_operations() { + let mut snapshot = crate::model::Snapshot::default(); + snapshot.topics.push(crate::model::TopicView { + name: "events".into(), + owners: vec!["one".into(), "two".into()], + ..Default::default() + }); + let mut request = ApplyRequest { + kind: "channel".into(), + action: "create".into(), + topic: "events".into(), + channel: Some("workers".into()), + delivery_mode: None, + message_ttl_seconds: None, + action_token: "token".into(), + confirmation: String::new(), + }; + let mut challenge = ActionChallenge { + token: "token".into(), + kind: "channel".into(), + action: "create".into(), + topic: "events".into(), + channel: Some("workers".into()), + delivery_mode: None, + message_ttl_seconds: None, + subject_uid: "uid".into(), + resource_version: "1".into(), + subject_kind: "topic".into(), + owners: vec!["one".into(), "two".into()], + confirmation: None, + expires_at_ms: u64::MAX, + }; + + assert!(verify_current_owners(&snapshot, &request, &challenge).is_ok()); + + request.kind = "topic".into(); + request.action = "configure".into(); + request.channel = None; + challenge.kind = "topic".into(); + challenge.action = "configure".into(); + challenge.channel = None; + assert!(verify_current_owners(&snapshot, &request, &challenge).is_err()); + } +} diff --git a/crates/console/src/management/backend.rs b/crates/console/src/management/backend.rs index 2c3de45..0352bf5 100644 --- a/crates/console/src/management/backend.rs +++ b/crates/console/src/management/backend.rs @@ -15,6 +15,8 @@ pub struct OwnerOperation<'a> { pub channel: Option<&'a str>, pub action: &'a str, pub tombstone_until_ms: Option, + pub delivery_mode: Option<&'a str>, + pub message_ttl_seconds: Option, } struct BackendError { @@ -47,6 +49,8 @@ pub async fn apply_owner( "channel": operation.channel, "expected_revision": revision, "tombstone_until_ms": operation.tombstone_until_ms, + "delivery_mode": operation.delivery_mode, + "message_ttl_seconds": operation.message_ttl_seconds, }); post(state, &token, &broker, &path, &body).await } diff --git a/crates/console/src/management/operation_state.rs b/crates/console/src/management/operation_state.rs index 43c2508..81ece29 100644 --- a/crates/console/src/management/operation_state.rs +++ b/crates/console/src/management/operation_state.rs @@ -23,6 +23,19 @@ pub(super) fn new_operation( digest.update(request.kind.as_bytes()); digest.update(request.topic.as_bytes()); digest.update(request.channel.as_deref().unwrap_or_default().as_bytes()); + digest.update( + request + .delivery_mode + .as_deref() + .unwrap_or_default() + .as_bytes(), + ); + digest.update( + request + .message_ttl_seconds + .unwrap_or_default() + .to_le_bytes(), + ); Ok(ManagedResourceOperation { id: format!("op-{}", &hex::encode(digest.finalize())[..32]), action, @@ -72,6 +85,7 @@ pub(super) fn started_channel(resource: &RustQueueChannel) -> StartedOperation { fn parse_action(action: &str) -> Result { match action { "create" => Ok(ManagedResourceAction::Create), + "configure" => Ok(ManagedResourceAction::Configure), "pause" => Ok(ManagedResourceAction::Pause), "unpause" => Ok(ManagedResourceAction::Unpause), "empty" => Ok(ManagedResourceAction::Empty), @@ -86,6 +100,7 @@ fn parse_action(action: &str) -> Result pub fn action_name(action: ManagedResourceAction) -> &'static str { match action { ManagedResourceAction::Create => "create", + ManagedResourceAction::Configure => "configure", ManagedResourceAction::Pause => "pause", ManagedResourceAction::Unpause => "unpause", ManagedResourceAction::Empty => "empty", diff --git a/crates/console/src/management/preview.rs b/crates/console/src/management/preview.rs index afc20f2..1271980 100644 --- a/crates/console/src/management/preview.rs +++ b/crates/console/src/management/preview.rs @@ -20,6 +20,10 @@ pub struct PreviewRequest { pub action: String, pub topic: String, pub channel: Option, + #[serde(default, alias = "deliveryMode")] + pub delivery_mode: Option, + #[serde(default, alias = "messageTtlSeconds")] + pub message_ttl_seconds: Option, } #[derive(Clone, Debug, Serialize)] @@ -45,7 +49,9 @@ pub async fn preview( let target = resolve_target(&snapshot, &request)?; ensure_owners_healthy(&snapshot, &target.owners)?; let subject = subject(&state, &request).await?; - let confirmation = requires_confirmation(&request.action).then(|| { + let confirmation = (requires_confirmation(&request.action) + || policy_can_expire_existing(&snapshot, &request)) + .then(|| { request .channel .clone() @@ -61,6 +67,8 @@ pub async fn preview( action: request.action.clone(), topic: request.topic.clone(), channel: request.channel.clone(), + delivery_mode: request.delivery_mode.clone(), + message_ttl_seconds: request.message_ttl_seconds, subject_uid: subject.uid, resource_version: subject.resource_version, subject_kind: subject.kind, @@ -147,7 +155,7 @@ pub fn validate_request(request: &PreviewRequest) -> Result<(), ManagementError> } if !matches!( request.action.as_str(), - "create" | "pause" | "unpause" | "empty" | "delete" | "retry" + "create" | "configure" | "pause" | "unpause" | "empty" | "delete" | "retry" ) { return Err(ManagementError::bad_request( "E_BAD_ACTION", @@ -174,9 +182,54 @@ pub fn validate_request(request: &PreviewRequest) -> Result<(), ManagementError> "topic actions cannot include a channel", )); } + validate_topic_policy(request)?; Ok(()) } +fn validate_topic_policy(request: &PreviewRequest) -> Result<(), ManagementError> { + if request.kind == "channel" { + if request.delivery_mode.is_some() || request.message_ttl_seconds.is_some() { + return Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "channel actions cannot include a topic delivery policy", + )); + } + return Ok(()); + } + if !matches!(request.action.as_str(), "create" | "configure") { + if request.delivery_mode.is_some() || request.message_ttl_seconds.is_some() { + return Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "topic policy is only accepted for create or configure", + )); + } + return Ok(()); + } + let mode = request.delivery_mode.as_deref().unwrap_or("RELIABLE"); + if request.action == "configure" && request.delivery_mode.is_none() { + return Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "delivery_mode is required when configuring a topic", + )); + } + match (mode, request.message_ttl_seconds) { + ("RELIABLE", None) => Ok(()), + ("TTL_DISCARD", Some(seconds)) if seconds > 0 => Ok(()), + ("RELIABLE", Some(_)) => Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "RELIABLE topics cannot define a message TTL", + )), + ("TTL_DISCARD", _) => Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "TTL_DISCARD requires a positive message TTL", + )), + _ => Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "delivery_mode must be RELIABLE or TTL_DISCARD", + )), + } +} + fn resolve_target( snapshot: &Snapshot, request: &PreviewRequest, @@ -193,14 +246,40 @@ fn resolve_target( "topic already exists", )); } - let owner = choose_broker(snapshot)?; + let visible_owners = topic.map(|topic| topic.owners.clone()).unwrap_or_default(); + if visible_owners.len() > 1 { + return Err(ManagementError::conflict( + "E_MIGRATION_IN_PROGRESS", + "topic delivery policy cannot change while ownership is migrating", + )); + } + let owner = match visible_owners.into_iter().next() { + Some(owner) => owner, + None => choose_broker(snapshot)?, + }; + let mut warnings = vec!["topic_requires_channel".into()]; + if policy_can_expire_existing(snapshot, request) { + warnings.push("ttl_policy_may_discard_existing_messages".into()); + } + let (stored_messages, depth, in_flight, deferred) = + topic.map_or((0, 0, 0, 0), |topic| { + ( + topic.stored_messages, + topic.channels.iter().map(|channel| channel.depth).sum(), + topic.channels.iter().map(|channel| channel.in_flight).sum(), + topic.channels.iter().map(|channel| channel.deferred).sum(), + ) + }); return Ok(ResolvedTarget { owners: vec![owner.clone()], impact: Impact { owners: vec![owner], + stored_messages, + depth, + in_flight, + deferred, connections: snapshot.summary.connections, - warnings: vec!["topic_requires_channel".into()], - ..Impact::empty() + warnings, }, }); } @@ -229,7 +308,7 @@ fn resolve_target( )); } let destructive = is_destructive(&request.action); - if destructive && topic.owners.len() > 1 { + if (destructive || request.action == "configure") && topic.owners.len() > 1 { return Err(ManagementError::conflict( "E_MIGRATION_IN_PROGRESS", "destructive topic operations are blocked while multiple owners are visible", @@ -247,7 +326,9 @@ fn resolve_target( in_flight, deferred, connections: snapshot.summary.connections, - warnings: if destructive { + warnings: if policy_can_expire_existing(snapshot, request) { + vec!["ttl_policy_may_discard_existing_messages".into()] + } else if destructive { vec!["immediate_irreversible".into()] } else { Vec::new() @@ -430,6 +511,33 @@ fn requires_confirmation(action: &str) -> bool { is_destructive(action) || action == "retry" } +fn policy_can_expire_existing(snapshot: &Snapshot, request: &PreviewRequest) -> bool { + if request.kind != "topic" || !matches!(request.action.as_str(), "create" | "configure") { + return false; + } + let Some(topic) = snapshot + .topics + .iter() + .find(|topic| topic.name == request.topic) + else { + return false; + }; + if request.action == "create" { + return topic.stored_messages > 0 + && request.delivery_mode.as_deref() == Some("TTL_DISCARD"); + } + match ( + topic.delivery_mode.as_str(), + topic.message_ttl_seconds, + request.delivery_mode.as_deref(), + request.message_ttl_seconds, + ) { + ("RELIABLE", _, Some("TTL_DISCARD"), Some(_)) => true, + ("TTL_DISCARD", Some(current), Some("TTL_DISCARD"), Some(next)) => next < current, + _ => false, + } +} + struct ResolvedTarget { owners: Vec, impact: Impact, @@ -454,3 +562,97 @@ struct Subject { uid: String, resource_version: String, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::TopicView; + + fn topic_request(action: &str, mode: Option<&str>, ttl: Option) -> PreviewRequest { + PreviewRequest { + kind: "topic".into(), + action: action.into(), + topic: "events".into(), + channel: None, + delivery_mode: mode.map(str::to_owned), + message_ttl_seconds: ttl, + } + } + + #[test] + fn topic_policy_validation_enforces_mode_and_ttl_pairing() { + assert!(validate_request(&topic_request("create", None, None)).is_ok()); + assert!(validate_request(&topic_request("create", Some("TTL_DISCARD"), Some(30))).is_ok()); + assert!(validate_request(&topic_request("configure", None, None)).is_err()); + assert!(validate_request(&topic_request("configure", Some("RELIABLE"), Some(30))).is_err()); + assert!( + validate_request(&topic_request("configure", Some("TTL_DISCARD"), Some(0))).is_err() + ); + } + + #[test] + fn enabling_or_shortening_ttl_requires_destructive_confirmation() { + let mut snapshot = Snapshot::default(); + snapshot.topics.push(TopicView { + name: "events".into(), + delivery_mode: "RELIABLE".into(), + ..TopicView::default() + }); + assert!(policy_can_expire_existing( + &snapshot, + &topic_request("configure", Some("TTL_DISCARD"), Some(60)) + )); + + snapshot.topics[0].delivery_mode = "TTL_DISCARD".into(); + snapshot.topics[0].message_ttl_seconds = Some(60); + assert!(policy_can_expire_existing( + &snapshot, + &topic_request("configure", Some("TTL_DISCARD"), Some(30)) + )); + assert!(!policy_can_expire_existing( + &snapshot, + &topic_request("configure", Some("TTL_DISCARD"), Some(120)) + )); + assert!(!policy_can_expire_existing( + &snapshot, + &topic_request("configure", Some("RELIABLE"), None) + )); + + snapshot.topics[0].managed_phase = "TOMBSTONED".into(); + snapshot.topics[0].stored_messages = 1; + assert!(policy_can_expire_existing( + &snapshot, + &topic_request("create", Some("TTL_DISCARD"), Some(120)) + )); + } + + #[test] + fn configure_is_blocked_while_multiple_topic_owners_are_visible() { + let mut snapshot = Snapshot::default(); + snapshot.topics.push(TopicView { + name: "events".into(), + owners: vec!["one".into(), "two".into()], + managed_phase: "ACTIVE".into(), + ..TopicView::default() + }); + + let result = resolve_target( + &snapshot, + &topic_request("configure", Some("TTL_DISCARD"), Some(60)), + ); + assert!(result.is_err()); + } + + #[test] + fn recreate_is_blocked_while_multiple_topic_owners_are_visible() { + let mut snapshot = Snapshot::default(); + snapshot.topics.push(TopicView { + name: "events".into(), + owners: vec!["one".into(), "two".into()], + managed_phase: "TOMBSTONED".into(), + ..TopicView::default() + }); + + assert!(resolve_target(&snapshot, &topic_request("create", None, None)).is_err()); + } +} diff --git a/crates/console/src/management/reconcile.rs b/crates/console/src/management/reconcile.rs index 2f2cca3..c95e648 100644 --- a/crates/console/src/management/reconcile.rs +++ b/crates/console/src/management/reconcile.rs @@ -70,6 +70,12 @@ impl Reconciler { let action = resource_state::action_name(operation.action); let target = resource.spec.topic.clone(); let result = async { + if matches!( + operation.action, + ManagedResourceAction::Create | ManagedResourceAction::Configure + ) { + ensure_policy_owner_stable(snapshot, &resource.spec.topic, &resource.spec.owners)?; + } ensure_owners_healthy(snapshot, &resource.spec.owners)?; if matches!( operation.action, @@ -88,6 +94,20 @@ impl Reconciler { channel: None, action, tombstone_until_ms: resource.spec.tombstone_until_ms, + delivery_mode: matches!( + operation.action, + ManagedResourceAction::Create | ManagedResourceAction::Configure + ) + .then(|| match resource.spec.delivery_mode { + rustqueue_operator::TopicDeliveryMode::Reliable => "RELIABLE", + rustqueue_operator::TopicDeliveryMode::TtlDiscard => "TTL_DISCARD", + }), + message_ttl_seconds: matches!( + operation.action, + ManagedResourceAction::Create | ManagedResourceAction::Configure + ) + .then_some(resource.spec.message_ttl_seconds) + .flatten(), }, ) .await?; @@ -137,6 +157,8 @@ impl Reconciler { channel: Some(&resource.spec.channel), action, tombstone_until_ms: resource.spec.tombstone_until_ms, + delivery_mode: None, + message_ttl_seconds: None, }, ) .await?; @@ -245,6 +267,25 @@ fn next_owner<'a>(owners: &'a [String], completed: &[String]) -> Option<&'a str> .map(String::as_str) } +fn ensure_policy_owner_stable( + snapshot: &crate::model::Snapshot, + topic: &str, + expected_owners: &[String], +) -> Result<(), ManagementError> { + let observed = snapshot + .topics + .iter() + .find(|candidate| candidate.name == topic) + .ok_or_else(|| ManagementError::unavailable("policy target topic is not observable"))?; + if expected_owners.len() != 1 || observed.owners != expected_owners { + return Err(ManagementError::conflict( + "E_MIGRATION_IN_PROGRESS", + "topic delivery policy cannot change while ownership is migrating", + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -255,4 +296,19 @@ mod tests { let completed = vec!["one".into(), "three".into()]; assert_eq!(next_owner(&owners, &completed), Some("two")); } + + #[test] + fn policy_operations_require_the_same_single_observed_owner() { + let mut snapshot = crate::model::Snapshot::default(); + snapshot.topics.push(crate::model::TopicView { + name: "events".into(), + owners: vec!["one".into()], + ..Default::default() + }); + assert!(ensure_policy_owner_stable(&snapshot, "events", &["one".into()]).is_ok()); + + snapshot.topics[0].owners.push("two".into()); + assert!(ensure_policy_owner_stable(&snapshot, "events", &["one".into()]).is_err()); + assert!(ensure_policy_owner_stable(&snapshot, "events", &["two".into()]).is_err()); + } } diff --git a/crates/console/src/management/resource_state.rs b/crates/console/src/management/resource_state.rs index 0ca26a0..349c2bf 100644 --- a/crates/console/src/management/resource_state.rs +++ b/crates/console/src/management/resource_state.rs @@ -12,7 +12,7 @@ use kube::api::{Api, ObjectMeta, PostParams}; use kube::{Resource, ResourceExt}; use rustqueue_operator::{ ManagedResourcePhase, RustQueue, RustQueueChannel, RustQueueChannelSpec, RustQueueTopic, - RustQueueTopicSpec, + RustQueueTopicSpec, TopicDeliveryMode, }; use std::collections::BTreeMap; @@ -121,6 +121,8 @@ pub async fn begin_topic( phase: phase_for(operation.action), revision: 1, paused: false, + delivery_mode: request_topic_mode(request)?, + message_ttl_seconds: request.message_ttl_seconds, tombstone_until_ms: None, last_error: None, operation: Some(operation), @@ -137,6 +139,10 @@ pub async fn begin_topic( }; ensure_topic_children_idle(state, &request.topic).await?; resource.spec.owners = challenge.owners.clone(); + if request.action == "configure" { + resource.spec.delivery_mode = request_topic_mode(request)?; + resource.spec.message_ttl_seconds = request.message_ttl_seconds; + } if request.action == "retry" { retry_operation(&mut resource.spec.operation, &resource.spec.phase, now)?; } else { @@ -163,6 +169,17 @@ pub async fn begin_topic( Ok(started_topic(&replaced)) } +fn request_topic_mode(request: &ApplyRequest) -> Result { + match request.delivery_mode.as_deref().unwrap_or("RELIABLE") { + "RELIABLE" => Ok(TopicDeliveryMode::Reliable), + "TTL_DISCARD" => Ok(TopicDeliveryMode::TtlDiscard), + _ => Err(ManagementError::bad_request( + "E_BAD_TOPIC_POLICY", + "delivery_mode must be RELIABLE or TTL_DISCARD", + )), + } +} + pub async fn begin_channel( state: &AppState, request: &ApplyRequest, diff --git a/crates/console/src/model.rs b/crates/console/src/model.rs index 6e030c3..e170bfd 100644 --- a/crates/console/src/model.rs +++ b/crates/console/src/model.rs @@ -211,6 +211,9 @@ pub struct TopicView { pub name: String, pub owners: Vec, pub paused: bool, + pub delivery_mode: String, + pub message_ttl_seconds: Option, + pub ttl_discarded_messages: u64, pub stored_messages: u64, pub segment_count: u64, pub segment_bytes: u64, @@ -318,6 +321,9 @@ mod tests { .push(rustqueue_queue::TopicStats { name: "events".into(), paused: false, + delivery_mode: Default::default(), + message_ttl_seconds: None, + ttl_discarded_messages: 0, published_count: 0, message_count: 1, segment_count: 1, diff --git a/crates/console/src/resources.rs b/crates/console/src/resources.rs index 7d2b53f..9a72a17 100644 --- a/crates/console/src/resources.rs +++ b/crates/console/src/resources.rs @@ -5,7 +5,7 @@ use kube::api::{Api, ListParams, ObjectMeta, PostParams}; use kube::{Resource, ResourceExt}; use rustqueue_operator::{ ManagedResourcePhase, RustQueue, RustQueueChannel, RustQueueChannelSpec, RustQueueTopic, - RustQueueTopicSpec, + RustQueueTopicSpec, TopicDeliveryMode, }; use rustqueue_queue::{ChannelFence, ManagementFenceSnapshot}; use sha2::{Digest, Sha256}; @@ -111,13 +111,7 @@ pub async fn reconcile( Some(mut resource) if may_reconcile(&resource.spec.phase, resource.spec.tombstone_until_ms) => { - if resource.spec.owners != topic.owners || resource.spec.paused != topic.paused { - resource.spec.owners = topic.owners.clone(); - resource.spec.paused = topic.paused; - resource.spec.phase = ManagedResourcePhase::Active; - resource.spec.tombstone_until_ms = None; - resource.spec.last_error = None; - resource.spec.revision = resource.spec.revision.saturating_add(1); + if apply_observed_topic(&mut resource, topic) { topic_api .replace(&resource.name_any(), &PostParams::default(), &resource) .await?; @@ -134,6 +128,8 @@ pub async fn reconcile( phase: ManagedResourcePhase::Active, revision: 1, paused: topic.paused, + delivery_mode: topic_delivery_mode(&topic.delivery_mode), + message_ttl_seconds: topic.message_ttl_seconds, tombstone_until_ms: None, last_error: None, operation: None, @@ -152,17 +148,7 @@ pub async fn reconcile( Some(mut resource) if may_reconcile(&resource.spec.phase, resource.spec.tombstone_until_ms) => { - if resource.spec.owners != channel.owners - || resource.spec.paused != channel.paused - || resource.spec.ephemeral != channel.ephemeral - { - resource.spec.owners = channel.owners.clone(); - resource.spec.paused = channel.paused; - resource.spec.ephemeral = channel.ephemeral; - resource.spec.phase = ManagedResourcePhase::Active; - resource.spec.tombstone_until_ms = None; - resource.spec.last_error = None; - resource.spec.revision = resource.spec.revision.saturating_add(1); + if apply_observed_channel(&mut resource, channel) { channel_api .replace(&resource.name_any(), &PostParams::default(), &resource) .await?; @@ -209,6 +195,63 @@ pub async fn reconcile( list(client, &namespace, &queue).await } +fn topic_delivery_mode(value: &str) -> TopicDeliveryMode { + if value == "TTL_DISCARD" { + TopicDeliveryMode::TtlDiscard + } else { + TopicDeliveryMode::Reliable + } +} + +fn apply_observed_topic(resource: &mut RustQueueTopic, topic: &TopicView) -> bool { + let stable_owner = topic.owners.len() == 1; + let delivery_mode = topic_delivery_mode(&topic.delivery_mode); + let policy_changed = stable_owner + && (resource.spec.delivery_mode != delivery_mode + || resource.spec.message_ttl_seconds != topic.message_ttl_seconds); + let changed = resource.spec.owners != topic.owners + || resource.spec.paused != topic.paused + || resource.spec.phase != ManagedResourcePhase::Active + || resource.spec.tombstone_until_ms.is_some() + || policy_changed; + if !changed { + return false; + } + resource.spec.owners = topic.owners.clone(); + resource.spec.paused = topic.paused; + if stable_owner { + resource.spec.delivery_mode = delivery_mode; + resource.spec.message_ttl_seconds = topic.message_ttl_seconds; + } + resource.spec.phase = ManagedResourcePhase::Active; + resource.spec.tombstone_until_ms = None; + resource.spec.last_error = None; + resource.spec.revision = resource.spec.revision.saturating_add(1); + true +} + +fn apply_observed_channel( + resource: &mut RustQueueChannel, + channel: &crate::model::ChannelView, +) -> bool { + let changed = resource.spec.owners != channel.owners + || resource.spec.paused != channel.paused + || resource.spec.ephemeral != channel.ephemeral + || resource.spec.phase != ManagedResourcePhase::Active + || resource.spec.tombstone_until_ms.is_some(); + if !changed { + return false; + } + resource.spec.owners = channel.owners.clone(); + resource.spec.paused = channel.paused; + resource.spec.ephemeral = channel.ephemeral; + resource.spec.phase = ManagedResourcePhase::Active; + resource.spec.tombstone_until_ms = None; + resource.spec.last_error = None; + resource.spec.revision = resource.spec.revision.saturating_add(1); + true +} + pub async fn get_topic( client: &kube::Client, namespace: &str, @@ -298,4 +341,95 @@ mod tests { channel_resource_name("q", "a", "b:c") ); } + + #[test] + fn multi_owner_observation_does_not_rewrite_topic_policy() { + let mut resource = RustQueueTopic::new( + "topic", + RustQueueTopicSpec { + queue: "queue".into(), + topic: "orders".into(), + owners: vec!["broker-a".into()], + phase: ManagedResourcePhase::Active, + revision: 1, + paused: false, + delivery_mode: TopicDeliveryMode::TtlDiscard, + message_ttl_seconds: Some(60), + tombstone_until_ms: None, + last_error: None, + operation: None, + }, + ); + let mut observed = TopicView { + name: "orders".into(), + owners: vec!["broker-a".into(), "broker-b".into()], + delivery_mode: "RELIABLE".into(), + ..Default::default() + }; + + assert!(apply_observed_topic(&mut resource, &observed)); + assert_eq!(resource.spec.delivery_mode, TopicDeliveryMode::TtlDiscard); + assert_eq!(resource.spec.message_ttl_seconds, Some(60)); + + observed.owners.pop(); + assert!(apply_observed_topic(&mut resource, &observed)); + assert_eq!(resource.spec.delivery_mode, TopicDeliveryMode::Reliable); + assert_eq!(resource.spec.message_ttl_seconds, None); + } + + #[test] + fn observed_resources_reactivate_expired_tombstones_without_field_drift() { + let mut topic = RustQueueTopic::new( + "topic", + RustQueueTopicSpec { + queue: "queue".into(), + topic: "orders".into(), + owners: vec!["broker-a".into()], + phase: ManagedResourcePhase::Tombstoned, + revision: 1, + paused: false, + delivery_mode: TopicDeliveryMode::Reliable, + message_ttl_seconds: None, + tombstone_until_ms: Some(1), + last_error: None, + operation: None, + }, + ); + let observed = TopicView { + name: "orders".into(), + owners: vec!["broker-a".into()], + delivery_mode: "RELIABLE".into(), + ..Default::default() + }; + + assert!(apply_observed_topic(&mut topic, &observed)); + assert_eq!(topic.spec.phase, ManagedResourcePhase::Active); + assert_eq!(topic.spec.tombstone_until_ms, None); + + let mut channel = RustQueueChannel::new( + "channel", + RustQueueChannelSpec { + queue: "queue".into(), + topic: "orders".into(), + channel: "workers".into(), + owners: vec!["broker-a".into()], + phase: ManagedResourcePhase::Tombstoned, + revision: 1, + paused: false, + ephemeral: false, + tombstone_until_ms: Some(1), + last_error: None, + operation: None, + }, + ); + let observed = crate::model::ChannelView { + name: "workers".into(), + owners: vec!["broker-a".into()], + ..Default::default() + }; + + assert!(apply_observed_channel(&mut channel, &observed)); + assert_eq!(channel.spec.phase, ManagedResourcePhase::Active); + assert_eq!(channel.spec.tombstone_until_ms, None); + } } diff --git a/crates/console/src/session.rs b/crates/console/src/session.rs index 5b2c87e..15565cf 100644 --- a/crates/console/src/session.rs +++ b/crates/console/src/session.rs @@ -19,6 +19,8 @@ pub struct ActionChallenge { pub action: String, pub topic: String, pub channel: Option, + pub delivery_mode: Option, + pub message_ttl_seconds: Option, pub subject_uid: String, pub resource_version: String, pub subject_kind: String, @@ -188,6 +190,8 @@ mod tests { action: "delete".into(), topic: "orders".into(), channel: None, + delivery_mode: None, + message_ttl_seconds: None, subject_uid: "uid".into(), resource_version: "1".into(), subject_kind: "topic".into(), @@ -223,6 +227,8 @@ mod tests { action: "pause".into(), topic: format!("topic-{ordinal}"), channel: None, + delivery_mode: None, + message_ttl_seconds: None, subject_uid: "uid".into(), resource_version: "1".into(), subject_kind: "topic".into(), diff --git a/crates/operator/src/lib.rs b/crates/operator/src/lib.rs index e0dc0bb..08d3a3b 100644 --- a/crates/operator/src/lib.rs +++ b/crates/operator/src/lib.rs @@ -10,5 +10,5 @@ pub use crd::{ }; pub use management_crd::{ ManagedResourceAction, ManagedResourceOperation, ManagedResourcePhase, RustQueueChannel, - RustQueueChannelSpec, RustQueueTopic, RustQueueTopicSpec, + RustQueueChannelSpec, RustQueueTopic, RustQueueTopicSpec, TopicDeliveryMode, }; diff --git a/crates/operator/src/management_crd.rs b/crates/operator/src/management_crd.rs index 45b1a7d..627e330 100644 --- a/crates/operator/src/management_crd.rs +++ b/crates/operator/src/management_crd.rs @@ -16,12 +16,21 @@ pub enum ManagedResourcePhase { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum ManagedResourceAction { Create, + Configure, Pause, Unpause, Empty, Delete, } +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TopicDeliveryMode { + #[default] + Reliable, + TtlDiscard, +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ManagedResourceOperation { @@ -55,6 +64,11 @@ pub struct RustQueueTopicSpec { #[serde(default)] pub paused: bool, #[serde(default)] + pub delivery_mode: TopicDeliveryMode, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub message_ttl_seconds: Option, + #[serde(default)] pub tombstone_until_ms: Option, #[serde(default)] pub last_error: Option, @@ -105,6 +119,11 @@ mod tests { assert!(spec.get("tombstoneUntilMs").is_some()); assert!(spec.get("tombstone_until_ms").is_none()); assert!(spec.get("operation").is_some()); + assert_eq!( + spec["deliveryMode"]["enum"], + serde_json::json!(["RELIABLE", "TTL_DISCARD"]) + ); + assert_eq!(spec["messageTtlSeconds"]["minimum"], 1.0); assert_eq!( spec["phase"]["enum"], serde_json::json!(["PREPARING", "ACTIVE", "APPLYING", "FAILED", "TOMBSTONED"]) @@ -116,7 +135,7 @@ mod tests { assert!(channel_spec.get("ephemeral").is_some()); assert!(channel_spec.get("lastError").is_some()); let schema = channel.to_string(); - for action in ["CREATE", "PAUSE", "UNPAUSE", "EMPTY", "DELETE"] { + for action in ["CREATE", "CONFIGURE", "PAUSE", "UNPAUSE", "EMPTY", "DELETE"] { assert!(schema.contains(action)); } } diff --git a/crates/queue/src/bin/rustqueue-crash-worker.rs b/crates/queue/src/bin/rustqueue-crash-worker.rs index dcb3cf4..3159026 100644 --- a/crates/queue/src/bin/rustqueue-crash-worker.rs +++ b/crates/queue/src/bin/rustqueue-crash-worker.rs @@ -1,4 +1,4 @@ -use rustqueue_queue::{Broker, BrokerConfig}; +use rustqueue_queue::{Broker, BrokerConfig, DeliveryMode, TopicPolicy}; use std::path::Path; use std::time::Duration; @@ -40,6 +40,18 @@ async fn main() -> anyhow::Result<()> { "gc" => { broker.compact().await?; } + "ttl" => { + broker + .configure_topic_policy( + "events", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }, + ) + .await?; + broker.expire_topic_if_due("events").await?; + } _ => anyhow::bail!("unknown crash scenario {scenario}"), } anyhow::bail!("crash failpoint did not trigger for scenario {scenario}") diff --git a/crates/queue/src/broker.rs b/crates/queue/src/broker.rs index 6669880..bd9eafa 100644 --- a/crates/queue/src/broker.rs +++ b/crates/queue/src/broker.rs @@ -2,6 +2,8 @@ mod channel_commit; #[path = "broker/delivery.rs"] mod delivery; +#[path = "broker/expiration.rs"] +mod expiration; #[path = "broker/group_commit.rs"] mod group_commit; #[path = "broker/io.rs"] @@ -32,7 +34,7 @@ use rustqueue_storage::{ CompatibilityState, StorageError, BASE_STORAGE_FEATURE_LEVEL, }; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; @@ -175,6 +177,8 @@ pub enum BrokerError { OperationConflict, #[error("active tombstone deadline is required")] InvalidTombstone, + #[error("invalid topic delivery policy: {0}")] + InvalidTopicPolicy(String), #[error("message does not exist")] MessageNotFound, #[error("message is not in flight")] @@ -230,6 +234,7 @@ struct BrokerInner { registry_revision: AtomicU64, storage_healthy: Arc, gc_cursor: AtomicUsize, + expiration_schedule: Mutex, publish_groups: group_commit::PublishGroups, channel_groups: channel_commit::ChannelGroups, metrics: QueueMetrics, @@ -241,6 +246,12 @@ struct SequenceState { reserved_exclusive: u64, } +#[derive(Default)] +struct ExpirationSchedule { + by_deadline: BTreeMap>, + by_topic: HashMap, +} + impl Broker { pub fn open(config: BrokerConfig) -> Result { if config.node_id == 0 || config.node_id > u16::MAX as u64 { @@ -409,6 +420,7 @@ impl Broker { registry_revision: AtomicU64::new(revision), storage_healthy, gc_cursor: AtomicUsize::new(0), + expiration_schedule: Mutex::new(ExpirationSchedule::default()), publish_groups, channel_groups, metrics, @@ -417,6 +429,7 @@ impl Broker { }; broker.reserve_message_metadata(0)?; broker.recover_outbox()?; + broker.rebuild_expiration_schedule()?; Ok(broker) } diff --git a/crates/queue/src/broker/delivery.rs b/crates/queue/src/broker/delivery.rs index cd7d540..10fab53 100644 --- a/crates/queue/src/broker/delivery.rs +++ b/crates/queue/src/broker/delivery.rs @@ -31,6 +31,7 @@ impl Broker { ) -> Result { self.ensure_storage_healthy()?; self.ensure_management_access(topic, Some(channel))?; + self.expire_topic_if_due(topic).await?; self.expire_channel_in_flight(topic, channel).await?; let handle = self.topic(topic)?; let mut wake = handle.wake.subscribe(); @@ -80,6 +81,13 @@ impl Broker { Err(error) => return self.observe_storage_result(Err(error.into())), }; let hold = hold.expect("delivery payload read returns its byte-budget hold"); + self.expire_topic_if_due(topic).await?; + if batch.items.iter().any(|reservation| { + self.delivery_expiration_ns(topic, reservation.timestamp_ns) + .is_ok_and(|deadline| deadline.is_some_and(|deadline| deadline <= now_ns())) + }) { + return Ok(DeliveryBatch::new(Vec::new(), DeliveryGuard::empty())); + } let handle = Arc::clone(&batch.handle); let channel = batch.channel.clone(); let reservations = batch.disarm(); diff --git a/crates/queue/src/broker/expiration.rs b/crates/queue/src/broker/expiration.rs new file mode 100644 index 0000000..3358b2e --- /dev/null +++ b/crates/queue/src/broker/expiration.rs @@ -0,0 +1,265 @@ +use super::*; +use crate::model::{TopicPolicy, TtlDiscardReport}; +use std::sync::atomic::Ordering; + +const TTL_DISCARD_MAX_MESSAGES_PER_BATCH: u64 = 65_536; + +impl ExpirationSchedule { + pub(super) fn update(&mut self, topic: &str, deadline_ns: Option) { + if let Some(previous) = self.by_topic.remove(topic) { + if let Some(topics) = self.by_deadline.get_mut(&previous) { + topics.remove(topic); + if topics.is_empty() { + self.by_deadline.remove(&previous); + } + } + } + if let Some(deadline_ns) = deadline_ns { + self.by_topic.insert(topic.to_owned(), deadline_ns); + self.by_deadline + .entry(deadline_ns) + .or_default() + .insert(topic.to_owned()); + } + } + + fn contains(&self, topic: &str) -> bool { + self.by_topic.contains_key(topic) + } + + fn is_due(&self, topic: &str, now_ns: i64) -> bool { + self.by_topic + .get(topic) + .is_some_and(|deadline_ns| *deadline_ns <= now_ns) + } + + fn take_due(&mut self, now_ns: i64, limit: usize) -> Vec { + let mut due = Vec::new(); + while due.len() < limit { + let Some((&deadline, _)) = self.by_deadline.first_key_value() else { + break; + }; + if deadline > now_ns { + break; + } + let Some(mut topics) = self.by_deadline.remove(&deadline) else { + continue; + }; + while due.len() < limit { + let Some(topic) = topics.pop_first() else { + break; + }; + self.by_topic.remove(&topic); + due.push(topic); + } + if !topics.is_empty() { + for topic in &topics { + self.by_topic.insert(topic.clone(), deadline); + } + self.by_deadline.insert(deadline, topics); + } + } + due + } +} + +impl Broker { + pub(super) fn rebuild_expiration_schedule(&self) -> Result<(), BrokerError> { + let topics: Vec<_> = self + .inner + .topics + .read() + .iter() + .map(|(name, handle)| (name.clone(), Arc::clone(handle))) + .collect(); + let mut schedule = self.inner.expiration_schedule.lock(); + *schedule = ExpirationSchedule::default(); + for (name, handle) in topics { + let deadline = handle.state.lock().next_expiration_ns()?; + schedule.update(&name, deadline); + } + Ok(()) + } + + pub(super) fn refresh_expiration_schedule_locked( + &self, + topic: &str, + state: &crate::topic::Topic, + ) -> Result<(), BrokerError> { + let mut schedule = self.inner.expiration_schedule.lock(); + if schedule.contains(topic) && state.policy().ttl_seconds().is_some() { + return Ok(()); + } + let deadline = state.next_expiration_ns()?; + schedule.update(topic, deadline); + Ok(()) + } + + pub(super) fn replace_expiration_schedule_locked( + &self, + topic: &str, + state: &crate::topic::Topic, + ) -> Result<(), BrokerError> { + let deadline = state.next_expiration_ns()?; + self.inner + .expiration_schedule + .lock() + .update(topic, deadline); + Ok(()) + } + + pub fn topic_policy(&self, topic: &str) -> Result { + Ok(self.topic(topic)?.state.lock().policy()) + } + + pub fn delivery_expiration_ns( + &self, + topic: &str, + timestamp_ns: i64, + ) -> Result, BrokerError> { + Ok(self + .topic(topic)? + .state + .lock() + .expiration_ns_for_timestamp(timestamp_ns)) + } + + pub fn subscribe_topic_policy( + &self, + topic: &str, + ) -> Result, BrokerError> { + Ok(self.topic(topic)?.subscribe_policy()) + } + + pub async fn configure_topic_policy( + &self, + topic: &str, + policy: TopicPolicy, + ) -> Result { + let policy = policy + .validate() + .map_err(|error| BrokerError::InvalidTopicPolicy(error.into()))?; + let _outbox_guard = self.inner.outbox_moves.lock().await; + let broker = self.clone(); + let topic = topic.to_owned(); + let topic_for_update = topic.clone(); + let changed = self + .storage_task(move || { + let handle = broker.topic(&topic_for_update)?; + let _commit_gate = handle.commit_gate.lock(); + let _channel_commit_gate = handle.channel_commit_gate.lock(); + let mut state = handle.state.lock(); + let changed = handle.set_policy(&mut state, policy)?; + broker.replace_expiration_schedule_locked(&topic_for_update, &state)?; + Ok(changed) + }) + .await?; + if policy.delivery_mode == crate::model::DeliveryMode::TtlDiscard { + let directory = self.inner.config.data_path.join("dlq-outbox"); + let source_topic = topic.clone(); + self.storage_task(move || { + crate::outbox::remove_source_topic(&directory, &source_topic) + }) + .await?; + } + if changed { + let _ = self.expire_topic_if_due(&topic).await?; + } + Ok(changed) + } + + pub async fn expire_due_topics( + &self, + max_topics: usize, + ) -> Result, BrokerError> { + let due = self + .inner + .expiration_schedule + .lock() + .take_due(now_ns(), max_topics.max(1)); + let mut reports = Vec::new(); + for topic in due { + if let Some(report) = self.expire_topic_at(&topic, now_ns()).await? { + reports.push(report); + } + tokio::task::yield_now().await; + } + Ok(reports) + } + + pub async fn expire_topic_if_due( + &self, + topic: &str, + ) -> Result, BrokerError> { + let now_ns = now_ns(); + if !self.inner.expiration_schedule.lock().is_due(topic, now_ns) { + return Ok(None); + } + self.expire_topic_at(topic, now_ns).await + } + + async fn expire_topic_at( + &self, + topic: &str, + now_ns: i64, + ) -> Result, BrokerError> { + let broker = self.clone(); + let topic_name = topic.to_owned(); + self.storage_task(move || { + let handle = match broker.topic(&topic_name) { + Ok(handle) => handle, + Err(BrokerError::TopicNotFound) => { + broker + .inner + .expiration_schedule + .lock() + .update(&topic_name, None); + return Ok(None); + } + Err(error) => return Err(error), + }; + let _commit_gate = handle.commit_gate.lock(); + let _channel_commit_gate = handle.channel_commit_gate.lock(); + let mut state = handle.state.lock(); + let Some(prepared) = + state.prepare_ttl_discard(now_ns, TTL_DISCARD_MAX_MESSAGES_PER_BATCH)? + else { + broker.replace_expiration_schedule_locked(&topic_name, &state)?; + return Ok(None); + }; + rustqueue_storage::crash_failpoint("ttl_discard_after_wal_append_before_fsync"); + let syncs = state.prepare_channel_wal_syncs(prepared.channels.iter())?; + drop(state); + let sync_result = syncs + .into_iter() + .try_for_each(|wal| wal.sync_data().map_err(BrokerError::from)); + if let Err(error) = sync_result { + handle + .state + .lock() + .mark_channel_wal_sync_failed(prepared.channels.iter()); + return broker.observe_storage_result(Err(error)); + } + rustqueue_storage::crash_failpoint("ttl_discard_after_wal_fsync_before_manifest"); + let mut state = handle.state.lock(); + state.checkpoint_channels_if_needed(prepared.channels.iter())?; + state.commit_ttl_discard(prepared.through_position, prepared.messages)?; + rustqueue_storage::crash_failpoint("ttl_discard_after_manifest_before_metric"); + broker.replace_expiration_schedule_locked(&topic_name, &state)?; + broker + .inner + .metrics + .ttl_discarded_messages + .fetch_add(prepared.messages, Ordering::Relaxed); + let report = TtlDiscardReport { + topic: topic_name, + through_position: prepared.through_position, + messages: prepared.messages, + }; + drop(state); + handle.signal(); + Ok(Some(report)) + }) + .await + } +} diff --git a/crates/queue/src/broker/group_commit.rs b/crates/queue/src/broker/group_commit.rs index b4741e3..1df969d 100644 --- a/crates/queue/src/broker/group_commit.rs +++ b/crates/queue/src/broker/group_commit.rs @@ -335,6 +335,10 @@ impl Broker { let topic_lock_hold = self.inner.metrics.publish_topic_lock_hold.timer(); topic_state.mark_durable_through(durable_through); visibility_advanced = topic_state.deliverable_position() > deliverable_before; + if let Err(error) = self.refresh_expiration_schedule_locked(topic, &topic_state) { + fail_pending(pending); + return self.worker_result(error); + } drop(topic_state); drop(topic_lock_hold); rustqueue_storage::crash_failpoint("publish_after_fsync_before_reply"); @@ -344,6 +348,10 @@ impl Broker { topic_state.mark_deliverable_through(deliverable_through); } visibility_advanced = topic_state.deliverable_position() > deliverable_before; + if let Err(error) = self.refresh_expiration_schedule_locked(topic, &topic_state) { + fail_pending(pending); + return self.worker_result(error); + } drop(topic_state); drop(topic_lock_hold); } @@ -635,6 +643,9 @@ pub(super) fn copy_error(error: &BrokerError) -> BrokerError { }, BrokerError::OperationConflict => BrokerError::OperationConflict, BrokerError::InvalidTombstone => BrokerError::InvalidTombstone, + BrokerError::InvalidTopicPolicy(message) => { + BrokerError::InvalidTopicPolicy(message.clone()) + } BrokerError::MessageNotFound => BrokerError::MessageNotFound, BrokerError::MessageNotInFlight => BrokerError::MessageNotInFlight, BrokerError::MessageTooLarge => BrokerError::MessageTooLarge, diff --git a/crates/queue/src/broker/io.rs b/crates/queue/src/broker/io.rs index 0413f83..0a0bb81 100644 --- a/crates/queue/src/broker/io.rs +++ b/crates/queue/src/broker/io.rs @@ -283,6 +283,10 @@ impl Broker { target_topic: &str, body: Bytes, ) -> Result { + if self.topic_policy(source_topic)?.delivery_mode == crate::model::DeliveryMode::TtlDiscard + { + return Ok(false); + } if !self .source_message_is_unacknowledged(source_topic, source_channel, message_id)? .unwrap_or(false) @@ -329,6 +333,7 @@ impl Broker { let mut state = handle.state.lock(); self.ensure_management_access(topic, None)?; let ids = self.append_publish_to_topic(&mut state, bodies, delay, true, &mut metadata)?; + self.refresh_expiration_schedule_locked(topic, &state)?; if self.inner.message_index_cache.over_budget() { state.spill_message_metadata()?; } @@ -432,6 +437,14 @@ impl Broker { pub(super) fn recover_outbox(&self) -> Result<(), BrokerError> { for path in crate::outbox::paths(&self.inner.config.data_path.join("dlq-outbox"))? { let entry = crate::outbox::load(&path)?; + match self.topic_policy(&entry.source_topic) { + Ok(policy) if policy.delivery_mode == crate::model::DeliveryMode::TtlDiscard => { + crate::outbox::remove(&path)?; + continue; + } + Ok(_) | Err(BrokerError::TopicNotFound) => {} + Err(error) => return Err(error), + } if self.source_message_is_unacknowledged( &entry.source_topic, &entry.source_channel, diff --git a/crates/queue/src/broker/management.rs b/crates/queue/src/broker/management.rs index 178e589..de0a4bc 100644 --- a/crates/queue/src/broker/management.rs +++ b/crates/queue/src/broker/management.rs @@ -4,6 +4,7 @@ use crate::management::{ TopicManagementAction, }; use crate::management_ops::OperationLookup; +use crate::model::TopicPolicy; impl Broker { pub fn management_fences_ready(&self) -> bool { @@ -36,110 +37,183 @@ impl Broker { action: TopicManagementAction, expected_revision: u64, tombstone_until_ms: Option, + policy: Option, ) -> Result { validate_name(topic).map_err(|_| BrokerError::InvalidTopic)?; + let policy = match action { + TopicManagementAction::Create => Some( + policy + .unwrap_or_default() + .validate() + .map_err(|error| BrokerError::InvalidTopicPolicy(error.into()))?, + ), + TopicManagementAction::Configure => Some( + policy + .ok_or_else(|| { + BrokerError::InvalidTopicPolicy( + "CONFIGURE requires a complete topic policy".into(), + ) + })? + .validate() + .map_err(|error| BrokerError::InvalidTopicPolicy(error.into()))?, + ), + _ if policy.is_some() => { + return Err(BrokerError::InvalidTopicPolicy( + "topic policy is only accepted for CREATE or CONFIGURE".into(), + )); + } + _ => None, + }; + let _outbox_guard = if matches!( + action, + TopicManagementAction::Create | TopicManagementAction::Configure + ) { + Some(self.inner.outbox_moves.lock().await) + } else { + None + }; let broker = self.clone(); let operation_id = operation_id.to_owned(); let topic = topic.to_owned(); - self.storage_task(move || { - let _lifecycle = broker.inner.topic_lifecycle.lock(); - let fingerprint = serde_json::to_string(&("topic", &topic, action)) - .map_err(|error| BrokerError::InvalidRecord(error.to_string()))?; - let pending_operation = { - let operations = broker.inner.management_ops.lock(); - match operations - .lookup(&operation_id, &fingerprint) - .map_err(operation_catalog_error)? + let topic_after = topic.clone(); + let result = self + .storage_task(move || { + let _lifecycle = broker.inner.topic_lifecycle.lock(); + let fingerprint = serde_json::to_string(&("topic", &topic, action, policy)) + .map_err(|error| BrokerError::InvalidRecord(error.to_string()))?; + let pending_operation = { + let operations = broker.inner.management_ops.lock(); + match operations + .lookup(&operation_id, &fingerprint) + .map_err(operation_catalog_error)? + { + OperationLookup::Completed(result) => return Ok(result), + OperationLookup::Pending => Some(operation_id.clone()), + OperationLookup::New => operations.pending_id(&topic, &fingerprint), + } + }; + if pending_operation.is_none() + && matches!( + action, + TopicManagementAction::Delete | TopicManagementAction::Tombstone + ) { - OperationLookup::Completed(result) => return Ok(result), - OperationLookup::Pending => Some(operation_id.clone()), - OperationLookup::New => operations.pending_id(&topic, &fingerprint), + valid_tombstone(tombstone_until_ms, false)?; } - }; - if pending_operation.is_none() - && matches!( - action, - TopicManagementAction::Delete | TopicManagementAction::Tombstone - ) - { - valid_tombstone(tombstone_until_ms, false)?; - } - if pending_operation.is_none() - && matches!( - action, - TopicManagementAction::Pause - | TopicManagementAction::Unpause - | TopicManagementAction::Empty - ) - { - broker.topic(&topic)?; - } - let (replayed, operation_id) = match pending_operation { - Some(id) => (true, id), - None => match broker.prepare_management_operation( - &operation_id, - &fingerprint, - &topic, - expected_revision, - )? { - PreparedOperation::Completed(result) => return Ok(result), - PreparedOperation::New(id) => (false, id), - PreparedOperation::Pending(id) => (true, id), - }, - }; - let mut changed = false; - match action { - TopicManagementAction::Create => { - let existed = broker.inner.topics.read().contains_key(&topic); - broker.get_or_create_topic_locked(&topic)?; - changed = !existed; - let mut fences = broker.inner.fences.lock(); - if fences.clear_topic(&topic) { - fences.store(&broker.inner.fences_path)?; + if pending_operation.is_none() + && matches!( + action, + TopicManagementAction::Pause + | TopicManagementAction::Unpause + | TopicManagementAction::Empty + | TopicManagementAction::Configure + ) + { + broker.topic(&topic)?; + } + let (replayed, operation_id) = match pending_operation { + Some(id) => (true, id), + None => match broker.prepare_management_operation( + &operation_id, + &fingerprint, + &topic, + expected_revision, + )? { + PreparedOperation::Completed(result) => return Ok(result), + PreparedOperation::New(id) => (false, id), + PreparedOperation::Pending(id) => (true, id), + }, + }; + let mut changed = false; + match action { + TopicManagementAction::Create => { + let existed = broker.inner.topics.read().contains_key(&topic); + let handle = broker.get_or_create_topic_locked_with_policy( + &topic, + (!existed).then_some(policy.expect("create policy validated")), + )?; + changed = !existed; + let _commit_gate = handle.commit_gate.lock(); + let _channel_commit_gate = handle.channel_commit_gate.lock(); + let mut state = handle.state.lock(); + changed |= handle + .set_policy(&mut state, policy.expect("create policy validated"))?; + broker.replace_expiration_schedule_locked(&topic, &state)?; + drop(state); + let mut fences = broker.inner.fences.lock(); + if fences.clear_topic(&topic) { + fences.store(&broker.inner.fences_path)?; + changed = true; + } + } + TopicManagementAction::Configure => { + let handle = broker.topic(&topic)?; + let _commit_gate = handle.commit_gate.lock(); + let _channel_commit_gate = handle.channel_commit_gate.lock(); + let mut state = handle.state.lock(); + changed = handle + .set_policy(&mut state, policy.expect("configure policy validated"))?; + broker.replace_expiration_schedule_locked(&topic, &state)?; + } + TopicManagementAction::Pause | TopicManagementAction::Unpause => { + let handle = broker.topic(&topic)?; + let _commit_gate = handle.commit_gate.lock(); + handle + .state + .lock() + .set_paused(action == TopicManagementAction::Pause)?; changed = true; } - } - TopicManagementAction::Pause | TopicManagementAction::Unpause => { - let handle = broker.topic(&topic)?; - let _commit_gate = handle.commit_gate.lock(); - handle - .state - .lock() - .set_paused(action == TopicManagementAction::Pause)?; - changed = true; - } - TopicManagementAction::Empty => { - let handle = broker.topic(&topic)?; - let _commit_gate = handle.commit_gate.lock(); - let _channel_commit_gate = handle.channel_commit_gate.lock(); - handle.state.lock().empty_topic()?; - changed = true; - } - TopicManagementAction::Delete | TopicManagementAction::Tombstone => { - let until = valid_tombstone(tombstone_until_ms, replayed)?; - let mut fences = broker.inner.fences.lock(); - if fences.set_topic(&topic, until) { - fences.store(&broker.inner.fences_path)?; + TopicManagementAction::Empty => { + let handle = broker.topic(&topic)?; + let _commit_gate = handle.commit_gate.lock(); + let _channel_commit_gate = handle.channel_commit_gate.lock(); + handle.state.lock().empty_topic()?; changed = true; } - drop(fences); - if action == TopicManagementAction::Delete { - changed |= broker.delete_topic_locked(&topic)?; + TopicManagementAction::Delete | TopicManagementAction::Tombstone => { + let until = valid_tombstone(tombstone_until_ms, replayed)?; + let mut fences = broker.inner.fences.lock(); + if fences.set_topic(&topic, until) { + fences.store(&broker.inner.fences_path)?; + changed = true; + } + drop(fences); + if action == TopicManagementAction::Delete { + changed |= broker.delete_topic_locked(&topic)?; + } } } - } - if changed || replayed { - broker.bump_registry()?; - } - let result = ManagementResult { - revision: broker.registry_revision(), - changed, - }; - rustqueue_storage::crash_failpoint("management_after_action_before_complete"); - broker.complete_management_operation(&operation_id, result.clone())?; - Ok(result) - }) - .await + if changed || replayed { + broker.bump_registry()?; + } + let result = ManagementResult { + revision: broker.registry_revision(), + changed, + }; + rustqueue_storage::crash_failpoint("management_after_action_before_complete"); + broker.complete_management_operation(&operation_id, result.clone())?; + Ok(result) + }) + .await?; + if self + .topic_policy(&topic_after) + .is_ok_and(|value| value.delivery_mode == crate::model::DeliveryMode::TtlDiscard) + { + let directory = self.inner.config.data_path.join("dlq-outbox"); + let source_topic = topic_after.clone(); + self.storage_task(move || { + crate::outbox::remove_source_topic(&directory, &source_topic) + }) + .await?; + } + if matches!( + action, + TopicManagementAction::Create | TopicManagementAction::Configure + ) { + let _ = self.expire_topic_if_due(&topic_after).await?; + } + Ok(result) } pub async fn manage_channel( diff --git a/crates/queue/src/broker/stats.rs b/crates/queue/src/broker/stats.rs index c8ee19e..8cc81c0 100644 --- a/crates/queue/src/broker/stats.rs +++ b/crates/queue/src/broker/stats.rs @@ -26,9 +26,9 @@ impl Broker { let _commit_gate = handle.commit_gate.lock(); let mut topic = handle.state.lock(); topic.add_aggregate_stats(&mut aggregate); - if detailed && remaining >= 5 { + if detailed && remaining >= 6 { let mut stats = topic.stats(); - remaining = remaining.saturating_sub(5); + remaining = remaining.saturating_sub(6); let channel_limit = remaining / 4; stats.channels.truncate(channel_limit); remaining = remaining.saturating_sub(stats.channels.len().saturating_mul(4)); @@ -73,6 +73,11 @@ impl Broker { channel_group_commit: self.inner.channel_groups.stats(), latency: self.inner.metrics.snapshot(), delivery_budget: self.inner.delivery_budget.snapshot(), + ttl_discarded_messages: self + .inner + .metrics + .ttl_discarded_messages + .load(Ordering::Relaxed), aggregate, topics, } @@ -109,4 +114,18 @@ mod tests { .topics .is_empty()); } + + #[test] + fn detailed_topic_metrics_reserve_all_six_topic_series() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + broker.get_or_create_topic("events").unwrap(); + + assert!(broker.metrics_stats(true, 5).topics.is_empty()); + assert_eq!(broker.metrics_stats(true, 6).topics.len(), 1); + } } diff --git a/crates/queue/src/broker/topics.rs b/crates/queue/src/broker/topics.rs index 6e802ab..38e4f83 100644 --- a/crates/queue/src/broker/topics.rs +++ b/crates/queue/src/broker/topics.rs @@ -15,6 +15,14 @@ impl Broker { pub(super) fn get_or_create_topic_locked( &self, name: &str, + ) -> Result, BrokerError> { + self.get_or_create_topic_locked_with_policy(name, None) + } + + pub(super) fn get_or_create_topic_locked_with_policy( + &self, + name: &str, + initial_policy: Option, ) -> Result, BrokerError> { self.cleanup_retired_topic(name)?; let mut topics = self.inner.topics.write(); @@ -33,6 +41,10 @@ impl Broker { self.inner.compatibility.active_writer_feature_level, Arc::clone(&self.inner.message_index_cache), )?; + if let Some(policy) = initial_policy { + let mut state = topic.state.lock(); + topic.set_policy(&mut state, policy)?; + } topics.insert(name.into(), Arc::clone(&topic)); drop(topics); self.bump_registry()?; @@ -90,6 +102,7 @@ impl Broker { drop(channel_commit_gate); drop(commit_gate); self.inner.topics.write().remove(name); + self.inner.expiration_schedule.lock().update(name, None); let directory = topic_directory(&self.inner.config.data_path, name); if Arc::strong_count(&handle) == 1 && !self.inner.payload_reader.has_active_under(&directory) diff --git a/crates/queue/src/broker_tests.rs b/crates/queue/src/broker_tests.rs index d7c991e..e8ffa2a 100644 --- a/crates/queue/src/broker_tests.rs +++ b/crates/queue/src/broker_tests.rs @@ -2,8 +2,8 @@ use super::io::SEQUENCE_RESERVATION; use super::*; use crate::outbox::OutboxEntry; use crate::{ - ChannelManagementAction, ChannelManagementCommand, ManagementFenceSnapshot, - TopicManagementAction, + ChannelManagementAction, ChannelManagementCommand, DeliveryMode, ManagementFenceSnapshot, + TopicManagementAction, TopicPolicy, }; use futures::future::join_all; use std::collections::{BTreeMap, HashSet}; @@ -2085,6 +2085,7 @@ async fn failed_management_preconditions_do_not_leave_durable_blockers() { TopicManagementAction::Pause, broker.registry_revision(), None, + None, ) .await, Err(BrokerError::TopicNotFound) @@ -2116,6 +2117,7 @@ async fn failed_management_preconditions_do_not_leave_durable_blockers() { TopicManagementAction::Delete, broker.registry_revision(), None, + None, ) .await, Err(BrokerError::InvalidTombstone) @@ -2154,8 +2156,13 @@ async fn a_different_management_action_cannot_cross_a_pending_operation() { }) .unwrap(); broker.create_topic("events").await.unwrap(); - let fingerprint = - serde_json::to_string(&("topic", "events", TopicManagementAction::Pause)).unwrap(); + let fingerprint = serde_json::to_string(&( + "topic", + "events", + TopicManagementAction::Pause, + None::, + )) + .unwrap(); broker .inner .management_ops @@ -2176,6 +2183,7 @@ async fn a_different_management_action_cannot_cross_a_pending_operation() { TopicManagementAction::Delete, broker.registry_revision(), Some(now_ms() + 60_000), + None, ) .await, Err(BrokerError::OperationConflict) @@ -2215,6 +2223,7 @@ async fn management_fences_fail_closed_and_survive_restart() { TopicManagementAction::Delete, broker.registry_revision(), Some(deadline), + None, ) .await .unwrap(); @@ -2254,6 +2263,7 @@ async fn management_fences_fail_closed_and_survive_restart() { TopicManagementAction::Create, broker.registry_revision(), None, + None, ) .await .unwrap(); @@ -2279,6 +2289,7 @@ async fn management_rejects_stale_registry_revisions() { TopicManagementAction::Create, stale, None, + None, ) .await .unwrap(); @@ -2290,6 +2301,7 @@ async fn management_rejects_stale_registry_revisions() { TopicManagementAction::Pause, stale, None, + None, ) .await .unwrap_err(); @@ -2434,6 +2446,7 @@ async fn management_operation_results_are_idempotent_across_restart() { TopicManagementAction::Create, expected, None, + None, ) .await .unwrap(); @@ -2444,6 +2457,7 @@ async fn management_operation_results_are_idempotent_across_restart() { TopicManagementAction::Create, expected, None, + None, ) .await .unwrap(); @@ -2458,6 +2472,7 @@ async fn management_operation_results_are_idempotent_across_restart() { TopicManagementAction::Create, 0, None, + None, ) .await .unwrap(); @@ -2470,6 +2485,7 @@ async fn management_operation_results_are_idempotent_across_restart() { TopicManagementAction::Pause, broker.registry_revision(), None, + None, ) .await, Err(BrokerError::OperationConflict) @@ -2477,6 +2493,62 @@ async fn management_operation_results_are_idempotent_across_restart() { assert!(broker.storage_healthy()); } +#[tokio::test] +async fn management_topic_policy_is_validated_and_bound_to_operation_identity() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + let ttl_policy = TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(30), + }; + broker + .manage_topic( + "create-ttl-orders-0001", + "ttl-orders", + TopicManagementAction::Create, + broker.registry_revision(), + None, + Some(ttl_policy), + ) + .await + .unwrap(); + assert_eq!(broker.topic_policy("ttl-orders").unwrap(), ttl_policy); + + assert!(matches!( + broker + .manage_topic( + "create-ttl-orders-0001", + "ttl-orders", + TopicManagementAction::Create, + broker.registry_revision(), + None, + Some(TopicPolicy::default()), + ) + .await, + Err(BrokerError::OperationConflict) + )); + assert!(matches!( + broker + .manage_topic( + "configure-ttl-orders-0001", + "ttl-orders", + TopicManagementAction::Configure, + broker.registry_revision(), + None, + Some(TopicPolicy { + delivery_mode: DeliveryMode::Reliable, + message_ttl_seconds: Some(30), + }), + ) + .await, + Err(BrokerError::InvalidTopicPolicy(_)) + )); +} + #[tokio::test] async fn expired_pending_tombstone_can_finish_and_unblock_the_topic() { let root = tempdir().unwrap(); @@ -2487,8 +2559,13 @@ async fn expired_pending_tombstone_can_finish_and_unblock_the_topic() { .unwrap(); broker.create_topic("orders").await.unwrap(); let operation_id = "tombstone-orders-0001"; - let fingerprint = - serde_json::to_string(&("topic", "orders", TopicManagementAction::Tombstone)).unwrap(); + let fingerprint = serde_json::to_string(&( + "topic", + "orders", + TopicManagementAction::Tombstone, + None::, + )) + .unwrap(); broker .inner .management_ops @@ -2508,6 +2585,7 @@ async fn expired_pending_tombstone_can_finish_and_unblock_the_topic() { TopicManagementAction::Tombstone, 0, Some(now_ms().saturating_sub(1)), + None, ) .await .unwrap(); @@ -2726,3 +2804,244 @@ fn runtime_integrity_errors_isolate_the_broker() { Err(BrokerError::StorageUnavailable) )); } + +#[tokio::test] +async fn ttl_discard_removes_unrouted_backlog_across_restart() { + let root = tempdir().unwrap(); + let config = BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }; + let broker = Broker::open(config.clone()).unwrap(); + broker.create_topic("ttl-events").await.unwrap(); + broker + .configure_topic_policy( + "ttl-events", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }, + ) + .await + .unwrap(); + broker + .publish( + "ttl-events", + vec![b"expires-without-a-channel".to_vec()], + Duration::ZERO, + ) + .await + .unwrap(); + drop(broker); + + tokio::time::sleep(Duration::from_millis(1_100)).await; + let broker = Broker::open(config).unwrap(); + while !broker.expire_due_topics(128).await.unwrap().is_empty() {} + broker + .create_channel("ttl-events", "late-workers") + .await + .unwrap(); + + assert!(broker + .fetch_batch( + "ttl-events", + "late-workers", + 1, + usize::MAX, + Duration::ZERO, + None, + ) + .await + .unwrap() + .is_empty()); + let topic = &broker.stats().topics[0]; + assert_eq!(topic.message_count, 0); + assert_eq!(topic.ttl_discarded_messages, 1); +} + +#[tokio::test] +async fn ttl_policy_changes_apply_to_backlog_without_reviving_expired_messages() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_channel("policy", "workers").await.unwrap(); + broker + .publish("policy", vec![b"old".to_vec()], Duration::ZERO) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1_100)).await; + + broker + .configure_topic_policy( + "policy", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(100), + }, + ) + .await + .unwrap(); + assert_eq!(broker.stats().topics[0].message_count, 1); + + broker + .configure_topic_policy( + "policy", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }, + ) + .await + .unwrap(); + assert!(broker + .fetch_batch("policy", "workers", 1, usize::MAX, Duration::ZERO, None) + .await + .unwrap() + .is_empty()); + + broker + .publish("policy", vec![b"survivor".to_vec()], Duration::ZERO) + .await + .unwrap(); + broker + .configure_topic_policy( + "policy", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(100), + }, + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1_100)).await; + broker + .configure_topic_policy("policy", TopicPolicy::default()) + .await + .unwrap(); + + let message = broker + .next_message("policy", "workers", None) + .await + .unwrap() + .unwrap(); + assert_eq!(message.body.as_ref(), b"survivor"); + broker + .finish("policy", "workers", message.id) + .await + .unwrap(); + assert!(broker + .next_message("policy", "workers", None) + .await + .unwrap() + .is_none()); + assert_eq!(broker.stats().topics[0].ttl_discarded_messages, 1); +} + +#[tokio::test] +async fn ttl_discard_evicts_durable_ephemeral_deferred_requeued_and_in_flight_state() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_topic("ttl-state").await.unwrap(); + broker + .configure_topic_policy( + "ttl-state", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }, + ) + .await + .unwrap(); + for channel in ["alpha", "beta", "tail#ephemeral"] { + broker.create_channel("ttl-state", channel).await.unwrap(); + } + broker + .publish( + "ttl-state", + vec![b"in-flight".to_vec(), b"retry".to_vec()], + Duration::ZERO, + ) + .await + .unwrap(); + broker + .publish( + "ttl-state", + vec![b"deferred".to_vec()], + Duration::from_secs(60), + ) + .await + .unwrap(); + + let alpha = broker + .fetch_batch("ttl-state", "alpha", 1, usize::MAX, Duration::ZERO, None) + .await + .unwrap() + .remove(0); + let beta = broker + .fetch_batch("ttl-state", "beta", 1, usize::MAX, Duration::ZERO, None) + .await + .unwrap() + .remove(0); + broker + .requeue("ttl-state", "beta", beta.id, Duration::from_secs(60)) + .await + .unwrap(); + let ephemeral = broker + .fetch_batch( + "ttl-state", + "tail#ephemeral", + 1, + usize::MAX, + Duration::ZERO, + None, + ) + .await + .unwrap() + .remove(0); + assert!(broker.stats().topics[0] + .channels + .iter() + .any(|channel| channel.deferred_count > 0)); + + tokio::time::sleep(Duration::from_millis(1_100)).await; + while !broker.expire_due_topics(128).await.unwrap().is_empty() {} + + let topic = &broker.stats().topics[0]; + assert_eq!(topic.message_count, 0); + assert_eq!(topic.ttl_discarded_messages, 3); + assert!(topic.channels.iter().all(|channel| { + channel.depth == 0 && channel.in_flight_count == 0 && channel.deferred_count == 0 + })); + broker.create_channel("ttl-state", "gamma").await.unwrap(); + assert!(broker + .fetch_batch("ttl-state", "gamma", 1, usize::MAX, Duration::ZERO, None,) + .await + .unwrap() + .is_empty()); + assert!(matches!( + broker.finish("ttl-state", "alpha", alpha.id).await, + Err(BrokerError::MessageNotInFlight) + )); + assert!(matches!( + broker + .requeue("ttl-state", "beta", beta.id, Duration::ZERO) + .await, + Err(BrokerError::MessageNotInFlight) + )); + assert!(matches!( + broker.touch( + "ttl-state", + "tail#ephemeral", + ephemeral.id, + Some(Duration::from_secs(30)), + ), + Err(BrokerError::MessageNotInFlight) + )); +} diff --git a/crates/queue/src/channel.rs b/crates/queue/src/channel.rs index 6f95be4..bff5b90 100644 --- a/crates/queue/src/channel.rs +++ b/crates/queue/src/channel.rs @@ -56,6 +56,7 @@ pub(crate) struct ChannelCheckpoint { struct InFlight { id: u64, + timestamp_ns: i64, deadline: Instant, token: u64, } @@ -352,7 +353,18 @@ impl ChannelState { .max(self.attempts.keys().copied().max().unwrap_or(0)) } + #[cfg(test)] pub fn reserve(&mut self, position: u64, id: u64, timeout: Duration) -> (u64, u16) { + self.reserve_timestamped(position, id, 0, timeout) + } + + pub fn reserve_timestamped( + &mut self, + position: u64, + id: u64, + timestamp_ns: i64, + timeout: Duration, + ) -> (u64, u16) { let token = self.next_token; self.next_token = self.next_token.wrapping_add(1).max(1); let attempts = self.attempts.entry(position).or_insert(0); @@ -362,6 +374,7 @@ impl ChannelState { position, InFlight { id, + timestamp_ns, deadline, token, }, @@ -404,6 +417,15 @@ impl ChannelState { .then_some(position) } + pub fn in_flight_timestamp(&self, id: u64, token: Option) -> Option { + let position = self.in_flight_position(id)?; + self.in_flight.get(&position).and_then(|flight| { + token + .is_none_or(|token| flight.token == token) + .then_some(flight.timestamp_ns) + }) + } + pub fn is_unacknowledged(&self, position: u64) -> bool { position > self.ack_floor_position && !self.acknowledged.contains(&position) } diff --git a/crates/queue/src/lib.rs b/crates/queue/src/lib.rs index 9a1498e..a3b6b39 100644 --- a/crates/queue/src/lib.rs +++ b/crates/queue/src/lib.rs @@ -27,7 +27,8 @@ pub use management::{ }; pub use model::{ BrokerLatencyStats, BrokerStats, ChannelGroupCommitStats, ChannelStats, Delivery, - DeliveryBatch, DeliveryBudgetStats, PublishGroupCommitStats, QueueAggregateStats, TopicStats, + DeliveryBatch, DeliveryBudgetStats, DeliveryMode, PublishGroupCommitStats, QueueAggregateStats, + TopicPolicy, TopicStats, TtlDiscardReport, }; #[doc(hidden)] diff --git a/crates/queue/src/management.rs b/crates/queue/src/management.rs index 639fcdc..9ec4ada 100644 --- a/crates/queue/src/management.rs +++ b/crates/queue/src/management.rs @@ -10,6 +10,7 @@ const FENCE_FORMAT: u8 = 1; #[serde(rename_all = "snake_case")] pub enum TopicManagementAction { Create, + Configure, Pause, Unpause, Empty, diff --git a/crates/queue/src/metadata.rs b/crates/queue/src/metadata.rs index f5f65f6..6d21152 100644 --- a/crates/queue/src/metadata.rs +++ b/crates/queue/src/metadata.rs @@ -24,6 +24,14 @@ pub(crate) struct TopicManifest { pub next_position: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub unrouted_from_position: Option, + #[serde(default)] + pub delivery_mode: crate::model::DeliveryMode, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message_ttl_seconds: Option, + #[serde(default)] + pub expired_through_position: u64, + #[serde(default)] + pub ttl_discarded_messages: u64, } pub(crate) fn load_optional(path: &Path) -> io::Result> { @@ -103,3 +111,22 @@ fn temporary_path(path: &Path) -> PathBuf { name.push(".tmp"); path.with_file_name(name) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::DeliveryMode; + + #[test] + fn legacy_topic_manifest_defaults_to_reliable_delivery() { + let manifest = parse_topic_manifest( + br#"{"format":7,"name":"events","paused":false,"next_position":2}"#, + ) + .unwrap(); + + assert_eq!(manifest.delivery_mode, DeliveryMode::Reliable); + assert_eq!(manifest.message_ttl_seconds, None); + assert_eq!(manifest.expired_through_position, 0); + assert_eq!(manifest.ttl_discarded_messages, 0); + } +} diff --git a/crates/queue/src/model.rs b/crates/queue/src/model.rs index 707c107..5fafe79 100644 --- a/crates/queue/src/model.rs +++ b/crates/queue/src/model.rs @@ -3,6 +3,49 @@ use rustqueue_telemetry::HistogramSnapshot; use serde::{Deserialize, Serialize}; use std::sync::Arc; +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum DeliveryMode { + #[default] + Reliable, + TtlDiscard, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct TopicPolicy { + pub delivery_mode: DeliveryMode, + pub message_ttl_seconds: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct TtlDiscardReport { + pub topic: String, + pub through_position: u64, + pub messages: u64, +} + +impl TopicPolicy { + pub fn validate(self) -> Result { + match (self.delivery_mode, self.message_ttl_seconds) { + (DeliveryMode::Reliable, None) => Ok(self), + (DeliveryMode::TtlDiscard, Some(seconds)) if seconds > 0 => Ok(self), + (DeliveryMode::Reliable, Some(_)) => { + Err("RELIABLE topics cannot define message_ttl_seconds") + } + (DeliveryMode::TtlDiscard, _) => { + Err("TTL_DISCARD topics require message_ttl_seconds greater than zero") + } + } + } + + pub const fn ttl_seconds(self) -> Option { + match self.delivery_mode { + DeliveryMode::Reliable => None, + DeliveryMode::TtlDiscard => self.message_ttl_seconds, + } + } +} + #[derive(Clone, Debug)] pub struct Delivery { pub id: u64, @@ -65,6 +108,8 @@ pub struct BrokerStats { pub delivery_budget: DeliveryBudgetStats, #[serde(default)] pub aggregate: QueueAggregateStats, + #[serde(default)] + pub ttl_discarded_messages: u64, pub topics: Vec, } @@ -152,6 +197,12 @@ pub struct TopicStats { pub name: String, pub paused: bool, #[serde(default)] + pub delivery_mode: DeliveryMode, + #[serde(default)] + pub message_ttl_seconds: Option, + #[serde(default)] + pub ttl_discarded_messages: u64, + #[serde(default)] pub published_count: u64, pub message_count: u64, #[serde(default)] diff --git a/crates/queue/src/outbox.rs b/crates/queue/src/outbox.rs index e0c7d0f..c3f8fc4 100644 --- a/crates/queue/src/outbox.rs +++ b/crates/queue/src/outbox.rs @@ -141,36 +141,36 @@ pub(crate) fn load(path: &Path) -> Result { pub(crate) fn retained_sources(directory: &Path) -> Result, BrokerError> { paths(directory)? .into_iter() - .map(|path| { - let mut file = File::open(&path)?; - let file_len = file.metadata()?.len(); - if file_len > MAX_OUTBOX_BYTES { - return Err(BrokerError::InvalidRecord( - "DLQ outbox file exceeds the maximum record size".into(), - )); - } - let mut bytes = [0u8; HEADER_LEN]; - file.read_exact(&mut bytes)?; - let header = parse_header(&bytes)?; - validate_layout(header, file_len)?; - let names_len = header - .source_topic_len - .checked_add(header.source_channel_len) - .and_then(|len| len.checked_add(header.target_topic_len)) - .ok_or_else(|| { - BrokerError::InvalidRecord("DLQ outbox name length overflow".into()) - })?; - let mut names = vec![0u8; names_len]; - file.read_exact(&mut names)?; - let mut cursor = 0; - let source_topic = read_string(&names, &mut cursor, header.source_topic_len)?; - let source_channel = read_string(&names, &mut cursor, header.source_channel_len)?; - validate_path(&path, header.message_id, &source_topic, &source_channel)?; - Ok((source_topic, header.message_id)) - }) + .map(|path| retained_source(&path)) .collect() } +fn retained_source(path: &Path) -> Result<(String, u64), BrokerError> { + let mut file = File::open(path)?; + let file_len = file.metadata()?.len(); + if file_len > MAX_OUTBOX_BYTES { + return Err(BrokerError::InvalidRecord( + "DLQ outbox file exceeds the maximum record size".into(), + )); + } + let mut bytes = [0u8; HEADER_LEN]; + file.read_exact(&mut bytes)?; + let header = parse_header(&bytes)?; + validate_layout(header, file_len)?; + let names_len = header + .source_topic_len + .checked_add(header.source_channel_len) + .and_then(|len| len.checked_add(header.target_topic_len)) + .ok_or_else(|| BrokerError::InvalidRecord("DLQ outbox name length overflow".into()))?; + let mut names = vec![0u8; names_len]; + file.read_exact(&mut names)?; + let mut cursor = 0; + let source_topic = read_string(&names, &mut cursor, header.source_topic_len)?; + let source_channel = read_string(&names, &mut cursor, header.source_channel_len)?; + validate_path(path, header.message_id, &source_topic, &source_channel)?; + Ok((source_topic, header.message_id)) +} + pub(crate) fn cleanup_temporary(directory: &Path) -> Result<(), BrokerError> { if !directory.exists() { return Ok(()); @@ -282,6 +282,15 @@ pub(crate) fn remove(path: &Path) -> Result<(), BrokerError> { Ok(()) } +pub(crate) fn remove_source_topic(directory: &Path, topic: &str) -> Result<(), BrokerError> { + for path in paths(directory)? { + if retained_source(&path)?.0 == topic { + remove(&path)?; + } + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/queue/src/telemetry.rs b/crates/queue/src/telemetry.rs index 0344919..75bec63 100644 --- a/crates/queue/src/telemetry.rs +++ b/crates/queue/src/telemetry.rs @@ -1,9 +1,11 @@ use crate::model::BrokerLatencyStats; use rustqueue_telemetry::LatencyHistogram; +use std::sync::atomic::AtomicU64; use std::sync::Arc; #[derive(Default)] pub(crate) struct QueueMetrics { + pub ttl_discarded_messages: AtomicU64, pub fsync: Arc, pub group_commit_wait: Arc, pub publish_topic_lock_wait: Arc, diff --git a/crates/queue/src/topic.rs b/crates/queue/src/topic.rs index 03b3c69..0ed3270 100644 --- a/crates/queue/src/topic.rs +++ b/crates/queue/src/topic.rs @@ -13,6 +13,7 @@ use crate::batch::{self, EncodedBatch}; use crate::channel::{ChannelCommand, ChannelRuntime, ChannelState}; use crate::channel_store::{checkpoint_paths, ChannelStore}; use crate::metadata::{load_topic_manifest, store_atomic, TopicManifest}; +use crate::model::{DeliveryMode, TopicPolicy}; use crate::BrokerError; use index::{MessageIndex, MessageIndexCache, MetadataReservation}; use parking_lot::Mutex; @@ -40,6 +41,7 @@ pub(crate) struct TopicHandle { pub channel_commit_gate: Mutex<()>, pub state: Mutex, pub wake: tokio::sync::watch::Sender, + policy_changes: tokio::sync::watch::Sender, } pub(crate) struct Topic { @@ -76,7 +78,6 @@ impl TopicHandle { storage_feature_level, index_cache, )?; - let (wake, _) = tokio::sync::watch::channel(0); topic.recover_channels()?; if topic.channels.len() > MAX_CHANNELS_PER_TOPIC { return Err(BrokerError::InvalidRecord(format!( @@ -85,12 +86,17 @@ impl TopicHandle { MAX_CHANNELS_PER_TOPIC ))); } + topic.reconcile_expired_boundary()?; topic.reconcile_unrouted_boundary()?; + let policy = topic.policy(); + let (wake, _) = tokio::sync::watch::channel(0); + let (policy_changes, _) = tokio::sync::watch::channel(policy); Ok(Arc::new(Self { commit_gate: Mutex::new(()), channel_commit_gate: Mutex::new(()), state: Mutex::new(topic), wake, + policy_changes, })) } @@ -111,6 +117,10 @@ impl TopicHandle { deleted: false, next_position: 1, unrouted_from_position: Some(1), + delivery_mode: DeliveryMode::Reliable, + message_ttl_seconds: None, + expired_through_position: 0, + ttl_discarded_messages: 0, }; store_atomic(&directory.join("manifest"), &manifest)?; let log = SegmentLog::open_with_feature_level( @@ -120,6 +130,7 @@ impl TopicHandle { storage_feature_level, )?; let (wake, _) = tokio::sync::watch::channel(0); + let (policy_changes, _) = tokio::sync::watch::channel(TopicPolicy::default()); Ok(Arc::new(Self { commit_gate: Mutex::new(()), channel_commit_gate: Mutex::new(()), @@ -142,6 +153,7 @@ impl TopicHandle { published_count: 0, }), wake, + policy_changes, })) } @@ -149,6 +161,19 @@ impl TopicHandle { self.wake .send_modify(|value| *value = value.wrapping_add(1)); } + + pub fn set_policy(&self, topic: &mut Topic, policy: TopicPolicy) -> Result { + let changed = topic.set_policy(policy)?; + if changed { + self.policy_changes.send_replace(policy); + self.signal(); + } + Ok(changed) + } + + pub fn subscribe_policy(&self) -> tokio::sync::watch::Receiver { + self.policy_changes.subscribe() + } } impl Topic { @@ -167,6 +192,17 @@ impl Topic { "topic manifest is not an active v7 topic".into(), )); } + TopicPolicy { + delivery_mode: manifest.delivery_mode, + message_ttl_seconds: manifest.message_ttl_seconds, + } + .validate() + .map_err(|error| BrokerError::InvalidRecord(error.into()))?; + if manifest.expired_through_position >= manifest.next_position { + return Err(BrokerError::InvalidRecord( + "topic manifest has an invalid TTL discard boundary".into(), + )); + } if manifest .unrouted_from_position .is_some_and(|position| position == 0 || position > manifest.next_position) @@ -501,12 +537,79 @@ impl Topic { .unrouted_from_position .unwrap_or(earliest) .max(earliest) + .max(self.manifest.expired_through_position.saturating_add(1)) .min(self.manifest.next_position), ) }; self.set_unrouted_from_position(position) } + fn reconcile_expired_boundary(&mut self) -> Result<(), BrokerError> { + let through_position = self.manifest.expired_through_position; + if through_position == 0 { + return Ok(()); + } + let channels: Vec<_> = self.channels.keys().cloned().collect(); + for channel in channels { + if self + .channels + .get(&channel) + .is_some_and(|runtime| runtime.state.ack_floor_position < through_position) + { + self.persist_channel(&channel, ChannelCommand::Evict { through_position })?; + } + } + self.messages.discard_scheduled_through(through_position); + Ok(()) + } + + pub fn policy(&self) -> TopicPolicy { + TopicPolicy { + delivery_mode: self.manifest.delivery_mode, + message_ttl_seconds: self.manifest.message_ttl_seconds, + } + } + + pub fn set_policy(&mut self, policy: TopicPolicy) -> Result { + let policy = policy + .validate() + .map_err(|error| BrokerError::InvalidTopicPolicy(error.into()))?; + if self.policy() == policy { + return Ok(false); + } + let mut manifest = self.manifest.clone(); + manifest.delivery_mode = policy.delivery_mode; + manifest.message_ttl_seconds = policy.message_ttl_seconds; + store_atomic(&self.manifest_path, &manifest)?; + self.manifest = manifest; + Ok(true) + } + + pub fn expiration_ns_for_timestamp(&self, timestamp_ns: i64) -> Option { + let ttl_ns = self + .policy() + .ttl_seconds()? + .saturating_mul(1_000_000_000) + .min(i64::MAX as u64) as i64; + Some(timestamp_ns.saturating_add(ttl_ns)) + } + + pub fn next_expiration_ns(&self) -> Result, BrokerError> { + let Some(ttl_seconds) = self.policy().ttl_seconds() else { + return Ok(None); + }; + let Some((_, timestamp_ns)) = self + .messages + .first_message_after(self.manifest.expired_through_position)? + else { + return Ok(None); + }; + let ttl_ns = ttl_seconds + .saturating_mul(1_000_000_000) + .min(i64::MAX as u64) as i64; + Ok(Some(timestamp_ns.saturating_add(ttl_ns))) + } + pub fn set_paused(&mut self, paused: bool) -> Result<(), BrokerError> { self.manifest.paused = paused; store_atomic(&self.manifest_path, &self.manifest)?; @@ -542,7 +645,8 @@ impl Topic { self.messages .retain_from_timestamp(cutoff, self.manifest.next_position) .saturating_sub(1) - }; + } + .max(self.manifest.expired_through_position); let mut state = ChannelState::new(name.into(), barrier, ephemeral, self.max_ack_gap); state.set_absent_ranges(Arc::clone(&self.position_gaps)); let store = if ephemeral { diff --git a/crates/queue/src/topic/delivery.rs b/crates/queue/src/topic/delivery.rs index 11782a6..a2a02f4 100644 --- a/crates/queue/src/topic/delivery.rs +++ b/crates/queue/src/topic/delivery.rs @@ -60,7 +60,12 @@ impl Topic { channel.state.defer_candidate(position); break; } - let (token, attempts) = channel.state.reserve(position, message.id, timeout); + let (token, attempts) = channel.state.reserve_timestamped( + position, + message.id, + message.timestamp_ns, + timeout, + ); bytes = bytes.saturating_add(next_bytes); reserved.push(ReservedDelivery { position, @@ -118,6 +123,13 @@ impl Topic { .get(channel) .ok_or(BrokerError::ChannelNotFound)? .state; + if state + .in_flight_timestamp(id, token) + .and_then(|timestamp| self.expiration_ns_for_timestamp(timestamp)) + .is_some_and(|expires_at| expires_at <= now_ns()) + { + return Err(BrokerError::MessageNotInFlight); + } match token { Some(token) => state.in_flight_position_with_token(id, token), None => state.in_flight_position(id), @@ -155,6 +167,14 @@ impl Topic { .channels .get(channel) .ok_or(BrokerError::ChannelNotFound)?; + if runtime + .state + .in_flight_timestamp(id, token) + .and_then(|timestamp| self.expiration_ns_for_timestamp(timestamp)) + .is_some_and(|expires_at| expires_at <= now_ns()) + { + return Err(BrokerError::MessageNotInFlight); + } let position = match token { Some(token) => runtime.state.in_flight_position_with_token(id, token), None => runtime.state.in_flight_position(id), @@ -183,10 +203,21 @@ impl Topic { token: Option, timeout: Duration, ) -> Result<(), BrokerError> { + let expired = self + .channels + .get(channel) + .ok_or(BrokerError::ChannelNotFound)? + .state + .in_flight_timestamp(id, token) + .and_then(|timestamp| self.expiration_ns_for_timestamp(timestamp)) + .is_some_and(|expires_at| expires_at <= now_ns()); + if expired { + return Err(BrokerError::MessageNotInFlight); + } let channel = self .channels .get_mut(channel) - .ok_or(BrokerError::ChannelNotFound)?; + .expect("validated channel remains present"); let position = match token { Some(token) => channel.state.in_flight_position_with_token(id, token), None => channel.state.in_flight_position(id), @@ -205,10 +236,23 @@ impl Topic { deliveries: &[(u64, u64)], timeout: Duration, ) -> Result<(), BrokerError> { + let expires_at = self + .channels + .get(channel) + .ok_or(BrokerError::ChannelNotFound)?; + if deliveries.iter().any(|(id, token)| { + expires_at + .state + .in_flight_timestamp(*id, Some(*token)) + .and_then(|timestamp| self.expiration_ns_for_timestamp(timestamp)) + .is_some_and(|expires_at| expires_at <= now_ns()) + }) { + return Err(BrokerError::MessageNotInFlight); + } let channel = self .channels .get_mut(channel) - .ok_or(BrokerError::ChannelNotFound)?; + .expect("validated channel remains present"); let positions: Vec<_> = deliveries .iter() .map(|(id, token)| { diff --git a/crates/queue/src/topic/index.rs b/crates/queue/src/topic/index.rs index a43c0e4..625fd60 100644 --- a/crates/queue/src/topic/index.rs +++ b/crates/queue/src/topic/index.rs @@ -207,6 +207,13 @@ impl MessageIndex { .collect() } + pub(crate) fn discard_scheduled_through(&mut self, through_position: u64) { + self.scheduled.retain(|_, positions| { + positions.retain(|position| *position > through_position); + !positions.is_empty() + }); + } + pub(crate) fn active_count(&self) -> usize { self.active.len() } @@ -279,6 +286,124 @@ impl MessageIndex { .or_else(|| self.active.front().map(|message| message.timestamp_ns)) } + pub(crate) fn first_message_after( + &self, + position: u64, + ) -> Result, BrokerError> { + let sealed_index = self + .sealed + .partition_point(|segment| segment.last_position <= position); + if let Some(segment) = self.sealed.get(sealed_index) { + let ordinal = position + .saturating_add(1) + .saturating_sub(segment.first_position) + .min(segment.count.saturating_sub(1)); + let message = self.load_ordinal(segment, ordinal)?; + return Ok(Some((message.position, message.timestamp_ns))); + } + Ok(self + .active + .iter() + .find(|message| message.position > position) + .map(|message| (message.position, message.timestamp_ns))) + } + + pub(crate) fn through_timestamp_after( + &self, + position: u64, + cutoff_ns: i64, + max_messages: u64, + ) -> Result, BrokerError> { + let mut through = None; + let mut messages = 0u64; + let sealed_index = self + .sealed + .partition_point(|segment| segment.last_position <= position); + for segment in self.sealed.iter().skip(sealed_index) { + let first_ordinal = position + .saturating_add(1) + .saturating_sub(segment.first_position) + .min(segment.count); + if first_ordinal >= segment.count || segment.first_timestamp_ns > cutoff_ns { + continue; + } + if segment.last_timestamp_ns <= cutoff_ns { + let expired = segment.count - first_ordinal; + let take = expired.min(max_messages.saturating_sub(messages)); + if take == 0 { + break; + } + through = if take == expired { + Some(segment.last_position) + } else { + Some( + self.load_ordinal(segment, first_ordinal + take - 1)? + .position, + ) + }; + messages = messages.saturating_add(take); + if take < expired || messages >= max_messages { + return Ok(through.map(|through| (through, messages))); + } + continue; + } + let mut low = first_ordinal; + let mut high = segment.count; + while low < high { + let ordinal = low + (high - low) / 2; + if self.load_ordinal(segment, ordinal)?.timestamp_ns <= cutoff_ns { + low = ordinal + 1; + } else { + high = ordinal; + } + } + if low > first_ordinal { + let take = (low - first_ordinal).min(max_messages.saturating_sub(messages)); + let last = self.load_ordinal(segment, first_ordinal + take - 1)?; + through = Some(last.position); + messages = messages.saturating_add(take); + } + return Ok(through.map(|through| (through, messages))); + } + let first_active = self + .active + .partition_point(|message| message.position <= position); + let end = self + .active + .partition_point(|message| message.timestamp_ns <= cutoff_ns); + let take = ((end - first_active) as u64).min(max_messages.saturating_sub(messages)); + if take > 0 { + through = self + .active + .get(first_active + take as usize - 1) + .map(|message| message.position); + messages = messages.saturating_add(take); + } + Ok(through.map(|through| (through, messages))) + } + + pub(crate) fn count_after_position(&self, position: u64) -> u64 { + let sealed = self + .sealed + .iter() + .map(|segment| { + if position < segment.first_position { + segment.count + } else if position < segment.last_position { + segment.last_position.saturating_sub(position) + } else { + 0 + } + }) + .sum::(); + sealed.saturating_add( + self.active + .iter() + .filter(|message| message.position > position) + .count() as u64, + ) + } + /// Returns a conservative position at or before the first message newer /// than the cutoff. At most one sealed segment is retained extra. pub(crate) fn retain_from_timestamp(&self, cutoff: i64, next_position: u64) -> u64 { diff --git a/crates/queue/src/topic/maintenance.rs b/crates/queue/src/topic/maintenance.rs index 0dfcc67..8cb0478 100644 --- a/crates/queue/src/topic/maintenance.rs +++ b/crates/queue/src/topic/maintenance.rs @@ -3,6 +3,13 @@ use crate::eviction::{self, ProtectiveEviction}; use crate::model::{QueueAggregateStats, TopicStats}; use rustqueue_storage::ScrubTarget; use std::collections::BTreeSet; +use std::sync::Arc; + +pub(crate) struct PreparedTtlDiscard { + pub through_position: u64, + pub messages: u64, + pub channels: Vec>, +} impl Topic { pub fn empty_channel(&mut self, channel: &str) -> Result<(), BrokerError> { @@ -51,8 +58,13 @@ impl Topic { TopicStats { name: self.name.clone(), paused: self.manifest.paused, + delivery_mode: self.manifest.delivery_mode, + message_ttl_seconds: self.manifest.message_ttl_seconds, + ttl_discarded_messages: self.manifest.ttl_discarded_messages, published_count: self.published_count, - message_count: self.messages.total_count(), + message_count: self + .messages + .count_after_position(self.manifest.expired_through_position), segment_count, segment_bytes, last_durable_position: self.durable_position, @@ -71,9 +83,10 @@ impl Topic { let scheduled = self.messages.deferred_positions(now_ms); let (segment_count, segment_bytes) = self.log.storage_usage(); aggregate.topic_count = aggregate.topic_count.saturating_add(1); - aggregate.message_count = aggregate - .message_count - .saturating_add(self.messages.total_count()); + aggregate.message_count = aggregate.message_count.saturating_add( + self.messages + .count_after_position(self.manifest.expired_through_position), + ); aggregate.segment_count = aggregate.segment_count.saturating_add(segment_count); aggregate.segment_bytes = aggregate.segment_bytes.saturating_add(segment_bytes); if let Some(pending) = self.pending_sync() { @@ -163,7 +176,8 @@ impl Topic { now_ns().saturating_sub(bootstrap_retention.as_nanos().min(i64::MAX as u128) as i64); let bootstrap_from = self .messages - .retain_from_timestamp(cutoff, self.manifest.next_position); + .retain_from_timestamp(cutoff, self.manifest.next_position) + .max(self.manifest.expired_through_position.saturating_add(1)); let channel_from = self .channels .values() @@ -217,6 +231,63 @@ impl Topic { Ok(removed) } + pub fn prepare_ttl_discard( + &mut self, + now_ns: i64, + max_messages: u64, + ) -> Result, BrokerError> { + let Some(ttl_seconds) = self.policy().ttl_seconds() else { + return Ok(None); + }; + let ttl_ns = ttl_seconds + .saturating_mul(1_000_000_000) + .min(i64::MAX as u64) as i64; + let cutoff_ns = now_ns.saturating_sub(ttl_ns); + let Some((through_position, messages)) = self.messages.through_timestamp_after( + self.manifest.expired_through_position, + cutoff_ns, + max_messages.max(1), + )? + else { + return Ok(None); + }; + let channels: Vec> = self.channels.keys().cloned().map(Arc::from).collect(); + for channel in &channels { + self.persist_channel_buffered(channel, ChannelCommand::Evict { through_position })?; + } + Ok(Some(PreparedTtlDiscard { + through_position, + messages, + channels, + })) + } + + pub fn commit_ttl_discard( + &mut self, + through_position: u64, + messages: u64, + ) -> Result<(), BrokerError> { + if through_position <= self.manifest.expired_through_position { + return Ok(()); + } + let mut manifest = self.manifest.clone(); + manifest.expired_through_position = through_position; + manifest.ttl_discarded_messages = manifest.ttl_discarded_messages.saturating_add(messages); + if !self.has_durable_channels() { + manifest.unrouted_from_position = Some( + manifest + .unrouted_from_position + .unwrap_or(1) + .max(through_position.saturating_add(1)) + .min(manifest.next_position), + ); + } + store_atomic(&self.manifest_path, &manifest)?; + self.manifest = manifest; + self.messages.discard_scheduled_through(through_position); + Ok(()) + } + pub fn scrub_targets(&self) -> Result, BrokerError> { Ok(self.log.scrub_targets(false)?) } diff --git a/crates/queue/tests/crash_boundaries.rs b/crates/queue/tests/crash_boundaries.rs index 13f6c46..023aa8d 100644 --- a/crates/queue/tests/crash_boundaries.rs +++ b/crates/queue/tests/crash_boundaries.rs @@ -1,6 +1,6 @@ #![cfg(all(unix, feature = "crash-injection"))] -use rustqueue_queue::{Broker, BrokerConfig}; +use rustqueue_queue::{Broker, BrokerConfig, DeliveryMode, TopicPolicy}; use std::collections::BTreeSet; use std::path::Path; use std::process::Command; @@ -63,6 +63,27 @@ async fn bodies(path: &Path) -> BTreeSet> { .collect() } +async fn ttl_fixture(path: &Path) { + let broker = Broker::open(config(path)).unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .configure_topic_policy( + "events", + TopicPolicy { + delivery_mode: DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }, + ) + .await + .unwrap(); + broker + .publish("events", vec![b"expires".to_vec()], Duration::ZERO) + .await + .unwrap(); + drop(broker); + tokio::time::sleep(Duration::from_millis(1_100)).await; +} + #[tokio::test] async fn sigkill_publish_boundaries_preserve_the_acknowledged_ledger() { for failpoint in [ @@ -181,3 +202,29 @@ async fn sigkill_gc_boundaries_reopen_without_corrupting_the_topic() { ); } } + +#[tokio::test] +async fn sigkill_ttl_boundaries_never_redeliver_or_double_count() { + for failpoint in [ + "ttl_discard_after_wal_append_before_fsync", + "ttl_discard_after_wal_fsync_before_manifest", + "ttl_discard_after_manifest_before_metric", + ] { + let root = tempfile::tempdir().unwrap(); + ttl_fixture(root.path()).await; + spawn_until_marker(root.path(), "ttl", failpoint); + + assert!(bodies(root.path()).await.is_empty()); + for _ in 0..2 { + let broker = Broker::open(config(root.path())).unwrap(); + let topic = broker + .stats() + .topics + .into_iter() + .find(|topic| topic.name == "events") + .unwrap(); + assert_eq!(topic.ttl_discarded_messages, 1); + drop(broker); + } + } +} diff --git a/crates/server/src/http.rs b/crates/server/src/http.rs index 73f8d0c..40bd6fe 100644 --- a/crates/server/src/http.rs +++ b/crates/server/src/http.rs @@ -297,6 +297,7 @@ impl From for ApiError { BrokerError::RevisionConflict { .. } => (StatusCode::CONFLICT, "E_REVISION_CONFLICT"), BrokerError::OperationConflict => (StatusCode::CONFLICT, "E_OPERATION_CONFLICT"), BrokerError::InvalidTombstone => (StatusCode::BAD_REQUEST, "E_BAD_TOMBSTONE"), + BrokerError::InvalidTopicPolicy(_) => (StatusCode::BAD_REQUEST, "E_BAD_TOPIC_POLICY"), BrokerError::InvalidChannel => (StatusCode::BAD_REQUEST, "E_BAD_CHANNEL"), BrokerError::MessageTooLarge | BrokerError::BatchTooLarge => { (StatusCode::BAD_REQUEST, "E_BAD_MESSAGE") @@ -373,9 +374,13 @@ mod tests { latency: Default::default(), delivery_budget: Default::default(), aggregate: Default::default(), + ttl_discarded_messages: 0, topics: vec![TopicStats { name: "events".into(), paused: false, + delivery_mode: Default::default(), + message_ttl_seconds: None, + ttl_discarded_messages: 0, published_count: 3, message_count: 3, segment_count: 1, diff --git a/crates/server/src/http/manage.rs b/crates/server/src/http/manage.rs index f909378..5cdaa80 100644 --- a/crates/server/src/http/manage.rs +++ b/crates/server/src/http/manage.rs @@ -2,8 +2,8 @@ use super::*; use crate::subscriptions::DeletePermit; use axum::extract::Path; use rustqueue_queue::{ - ChannelManagementAction, ChannelManagementCommand, ManagementFenceSnapshot, ManagementResult, - TopicManagementAction, + ChannelManagementAction, ChannelManagementCommand, DeliveryMode, ManagementFenceSnapshot, + ManagementResult, TopicManagementAction, TopicPolicy, }; #[derive(Debug, Deserialize)] @@ -12,6 +12,10 @@ pub(super) struct TopicManageRequest { topic: String, expected_revision: u64, tombstone_until_ms: Option, + #[serde(default, alias = "deliveryMode")] + delivery_mode: Option, + #[serde(default, alias = "messageTtlSeconds")] + message_ttl_seconds: Option, } #[derive(Debug, Deserialize)] @@ -31,6 +35,7 @@ pub(super) async fn manage_topic( ) -> Result, ApiError> { authorize(&headers, &state.tokens.console, "console")?; let action = parse_topic_action(&action)?; + let policy = topic_policy(&request, action)?; let result = state .broker .manage_topic( @@ -39,6 +44,7 @@ pub(super) async fn manage_topic( action, request.expected_revision, request.tombstone_until_ms, + policy, ) .await?; tracing::info!( @@ -253,6 +259,7 @@ pub(super) async fn sync_fences( fn parse_topic_action(value: &str) -> Result { match value { "create" => Ok(TopicManagementAction::Create), + "configure" => Ok(TopicManagementAction::Configure), "pause" => Ok(TopicManagementAction::Pause), "unpause" => Ok(TopicManagementAction::Unpause), "empty" => Ok(TopicManagementAction::Empty), @@ -265,6 +272,40 @@ fn parse_topic_action(value: &str) -> Result { } } +fn topic_policy( + request: &TopicManageRequest, + action: TopicManagementAction, +) -> Result, ApiError> { + let policy = request.delivery_mode.map(|delivery_mode| TopicPolicy { + delivery_mode, + message_ttl_seconds: request.message_ttl_seconds, + }); + match action { + TopicManagementAction::Create + if policy.is_none() && request.message_ttl_seconds.is_none() => + { + Ok(None) + } + TopicManagementAction::Create | TopicManagementAction::Configure => policy + .ok_or_else(|| { + ApiError::bad_request( + "E_BAD_TOPIC_POLICY", + "delivery_mode is required when configuring a topic policy", + ) + })? + .validate() + .map(Some) + .map_err(|detail| ApiError::bad_request("E_BAD_TOPIC_POLICY", detail)), + _ if policy.is_some() || request.message_ttl_seconds.is_some() => { + Err(ApiError::bad_request( + "E_BAD_TOPIC_POLICY", + "topic policy is only accepted for create or configure", + )) + } + _ => Ok(None), + } +} + fn parse_channel_action( value: &str, cleanup_enabled: bool, diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 043d563..a9a891b 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -86,6 +86,17 @@ async fn main() -> anyhow::Result<()> { storage_feature_level: config.storage.feature_level, require_management_fence_sync: config.security.console_management_enabled, })?); + loop { + let reports = broker + .expire_due_topics(usize::MAX) + .await + .context("reconcile expired Topic messages before readiness")?; + let complete = reports.is_empty(); + log_ttl_discards(reports); + if complete { + break; + } + } let config = Arc::new(config); let metrics = Arc::new(Metrics::default()); let accepting = Arc::new(AtomicBool::new(true)); @@ -151,6 +162,7 @@ async fn main() -> anyhow::Result<()> { initially_pressured, )); let mut storage_health_task = tokio::spawn(monitor_storage_health(Arc::clone(&broker))); + let mut ttl_expiration_task = tokio::spawn(run_ttl_expiration(Arc::clone(&broker))); let (completed_task, terminal_result) = tokio::select! { result = &mut tcp_task => ( @@ -177,6 +189,10 @@ async fn main() -> anyhow::Result<()> { Some(RuntimeTask::StorageHealth), unexpected_task_exit("storage health", result), ), + result = &mut ttl_expiration_task => ( + Some(RuntimeTask::TtlExpiration), + unexpected_task_exit("TTL expiration", result), + ), _ = shutdown_signal() => { info!("shutdown signal received"); (None, Ok(())) @@ -241,6 +257,12 @@ async fn main() -> anyhow::Result<()> { stop_background_task(&mut storage_health_task, "storage health").await, ); } + if completed_task != Some(RuntimeTask::TtlExpiration) { + record_cleanup_error( + &mut cleanup_error, + stop_background_task(&mut ttl_expiration_task, "TTL expiration").await, + ); + } let released = broker.release_all_in_flight(); record_cleanup_error( &mut cleanup_error, @@ -262,6 +284,7 @@ enum RuntimeTask { Scrub, DiskGuard, StorageHealth, + TtlExpiration, } fn unexpected_task_exit( @@ -341,6 +364,30 @@ async fn monitor_storage_health(broker: Arc) -> anyhow::Result<()> { } } +async fn run_ttl_expiration(broker: Arc) -> anyhow::Result<()> { + let mut tick = tokio::time::interval(std::time::Duration::from_millis(250)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tick.tick().await; + let reports = broker + .expire_due_topics(128) + .await + .context("expire due Topic messages")?; + log_ttl_discards(reports); + } +} + +fn log_ttl_discards(reports: Vec) { + for report in reports { + info!( + topic = report.topic, + through_position = report.through_position, + messages = report.messages, + "discarded Topic messages at their absolute TTL" + ); + } +} + async fn run_scrubber( broker: Arc, startup_delay: std::time::Duration, diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index a7cf51b..d11e48b 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -100,6 +100,8 @@ pub fn render_broker(stats: &BrokerStats, config: &MetricsConfig) -> String { rustqueue_topics {topic_count}\n\ # TYPE rustqueue_topic_messages_total gauge\n\ rustqueue_topic_messages_total {message_count}\n\ + # TYPE rustqueue_ttl_discarded_messages_total counter\n\ + rustqueue_ttl_discarded_messages_total {}\n\ # TYPE rustqueue_publish_unsynced_messages gauge\n\ rustqueue_publish_unsynced_messages {}\n\ # TYPE rustqueue_publish_unsynced_bytes gauge\n\ @@ -134,6 +136,7 @@ pub fn render_broker(stats: &BrokerStats, config: &MetricsConfig) -> String { stats.channel_group_commit.active_workers, stats.channel_group_commit.retired_workers, stats.channel_group_commit.rejected_workers, + stats.ttl_discarded_messages, stats.aggregate.unsynced_messages, stats.aggregate.unsynced_bytes, stats.delivery_budget.in_flight_bytes, @@ -216,7 +219,7 @@ pub fn render_broker(stats: &BrokerStats, config: &MetricsConfig) -> String { fn render_detailed_queue_metrics(output: &mut String, stats: &BrokerStats, config: &MetricsConfig) { let desired = usize::try_from(stats.aggregate.topic_count) .unwrap_or(usize::MAX) - .saturating_mul(5) + .saturating_mul(6) .saturating_add( usize::try_from(stats.aggregate.channel_count) .unwrap_or(usize::MAX) @@ -230,13 +233,14 @@ fn render_detailed_queue_metrics(output: &mut String, stats: &BrokerStats, confi # TYPE rustqueue_topic_publish_unsynced_messages gauge\n\ # TYPE rustqueue_topic_publish_unsynced_bytes gauge\n\ # TYPE rustqueue_topic_publish_sync_lag_seconds gauge\n\ + # TYPE rustqueue_topic_ttl_discarded_messages_total counter\n\ # TYPE rustqueue_channel_depth gauge\n\ # TYPE rustqueue_channel_in_flight gauge\n\ # TYPE rustqueue_channel_deferred gauge\n\ # TYPE rustqueue_channel_ack_gap gauge\n", ); for topic in &stats.topics { - if emitted.saturating_add(5) <= config.max_detailed_series { + if emitted.saturating_add(6) <= config.max_detailed_series { let topic_label = format!("topic=\"{}\"", escape_label(&topic.name)); output.push_str(&format!( "rustqueue_topic_messages{{{topic_label}}} {}\n\ @@ -250,7 +254,11 @@ fn render_detailed_queue_metrics(output: &mut String, stats: &BrokerStats, confi topic.unsynced_bytes, topic.sync_lag_ms as f64 / 1_000.0, )); - emitted += 5; + output.push_str(&format!( + "rustqueue_topic_ttl_discarded_messages_total{{{topic_label}}} {}\n", + topic.ttl_discarded_messages, + )); + emitted += 6; } for channel in &topic.channels { if emitted.saturating_add(4) > config.max_detailed_series { @@ -426,6 +434,9 @@ mod tests { topics: vec![TopicStats { name: "events".into(), paused: false, + delivery_mode: Default::default(), + message_ttl_seconds: None, + ttl_discarded_messages: 0, published_count: 7, message_count: 7, segment_count: 1, @@ -464,7 +475,7 @@ mod tests { assert!(output.contains("rustqueue_publish_sync_lag_seconds 0.007\n")); assert!(!output.contains("topic=\"events\"")); assert!(output.contains("rustqueue_detailed_queue_metric_series 0\n")); - assert!(output.contains("rustqueue_detailed_queue_metric_series_omitted 9\n")); + assert!(output.contains("rustqueue_detailed_queue_metric_series_omitted 10\n")); } #[test] @@ -473,13 +484,13 @@ mod tests { &broker_stats(), &MetricsConfig { detailed_queue_metrics: true, - max_detailed_series: 5, + max_detailed_series: 6, }, ); assert!(output.contains("rustqueue_topic_messages{topic=\"events\"} 7\n")); assert!(output.contains("rustqueue_topic_last_durable_position{topic=\"events\"} 5\n")); assert!(!output.contains("rustqueue_channel_depth{topic=")); - assert!(output.contains("rustqueue_detailed_queue_metric_series 5\n")); + assert!(output.contains("rustqueue_detailed_queue_metric_series 6\n")); assert!(output.contains("rustqueue_detailed_queue_metric_series_omitted 4\n")); } diff --git a/crates/server/src/tcp.rs b/crates/server/src/tcp.rs index 517d671..68c898c 100644 --- a/crates/server/src/tcp.rs +++ b/crates/server/src/tcp.rs @@ -120,6 +120,7 @@ struct RemoteDelivery { timestamp_ns: i64, attempts: u16, body: Bytes, + ttl_discard: bool, } #[derive(Clone, Copy, Debug)] diff --git a/crates/server/src/tcp/dead_letter.rs b/crates/server/src/tcp/dead_letter.rs index a4b932e..e6cc351 100644 --- a/crates/server/src/tcp/dead_letter.rs +++ b/crates/server/src/tcp/dead_letter.rs @@ -44,6 +44,9 @@ pub(super) async fn dead_letter_if_needed( } fn reason(config: &Config, topic: &str, delivery: &RemoteDelivery) -> Option { + if delivery.ttl_discard { + return None; + } if topic.ends_with(&config.queue.dead_letter_suffix) { return None; } @@ -92,6 +95,7 @@ mod tests { timestamp_ns, attempts, body: Bytes::from_static(b"body"), + ttl_discard: false, } } @@ -166,6 +170,7 @@ mod tests { timestamp_ns: second.timestamp_ns, attempts: second.attempts, body: bytes::Bytes::from_owner(second.body), + ttl_discard: false, }, ) .await diff --git a/crates/server/src/tcp/session.rs b/crates/server/src/tcp/session.rs index 6dfcc94..6be934c 100644 --- a/crates/server/src/tcp/session.rs +++ b/crates/server/src/tcp/session.rs @@ -76,10 +76,20 @@ pub(super) async fn run_session( output_buffer_tick.tick().await; let mut last_command = Instant::now(); let mut pending_fetch: Option> = None; + let mut subscription_policy_changes = None; let (channel_ops, mut channel_op_results, channel_ops_task) = start_channel_ops(broker.clone()); let session_result: anyhow::Result<()> = async { loop { + if subscription_policy_changes.is_none() { + if let Some(subscription) = state.subscription.as_ref() { + subscription_policy_changes = Some( + broker + .subscribe_topic_policy(&subscription.topic) + .map_err(|error| anyhow::anyhow!(error.to_string()))?, + ); + } + } let expired = expire_client_deadlines( &mut state.in_flight, &mut state.in_flight_deadlines, @@ -99,6 +109,12 @@ pub(super) async fn run_session( break; } if let Some(command) = pending.take() { + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ) + .await?; if !accepting.load(Ordering::Acquire) && publish_command(&command.command) { if config.limits.disconnect_on_retriable_publish_error { break; @@ -203,6 +219,11 @@ pub(super) async fn run_session( let Some(command) = command else { break; }; + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; let command = match command { Ok(command) => command, Err(CommandReadError::Io(_)) => break, @@ -274,6 +295,11 @@ pub(super) async fn run_session( if let Err((code, error)) = apply_channel_op_completion(completion, &mut state, metrics) { + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; write_broker_error(&mut writer, code, error).await?; } } @@ -307,7 +333,11 @@ pub(super) async fn run_session( .fetch_add(batch_bytes as u64, Ordering::Relaxed); if deliveries.is_empty() { metrics.fetch_empty.fetch_add(1, Ordering::Relaxed); - flush_timed(&mut writer, state.heartbeat).await?; + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; continue; } let write_timeout = delivery_write_timeout(state.heartbeat); @@ -337,13 +367,36 @@ pub(super) async fn run_session( ) .map_err(|error| anyhow::anyhow!(error.to_string()))?; let mut handed_off = Vec::with_capacity(deliveries.len()); - for delivery in deliveries { + for mut delivery in deliveries { if delivery_is_outstanding( &state.in_flight, delivery.id, ) { continue; } + let mut policy_changes = broker + .subscribe_topic_policy(&delivery_topic) + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + ensure_buffered_policy_is_current( + &writer, + subscription_policy_changes.as_mut(), + )?; + let expiration_ns = broker + .delivery_expiration_ns( + &delivery_topic, + delivery.timestamp_ns, + ) + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + delivery.ttl_discard = expiration_ns.is_some(); + let ttl_remaining = expiration_ns.map(remaining_until_ns); + if ttl_remaining.is_some_and(|remaining| remaining.is_zero()) { + broker + .expire_topic_if_due(&delivery_topic) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + delivery_guard.accept(delivery.id); + continue; + } if dead_letter_if_needed( config, broker, @@ -362,7 +415,7 @@ pub(super) async fn run_session( let token = delivery_guard .token(delivery.id) .ok_or_else(|| anyhow::anyhow!("delivery token is missing"))?; - let deadline = renew_delivery_lease( + let mut deadline = renew_delivery_lease( broker, &delivery_topic, &delivery_channel, @@ -371,6 +424,9 @@ pub(super) async fn run_session( state.message_timeout, ) .map_err(|error| anyhow::anyhow!(error.to_string()))?; + if let Some(remaining) = ttl_remaining { + deadline = cap_deadline_at_ttl(deadline, remaining); + } channel_ops .finish_sampled( delivery_topic.clone(), @@ -399,20 +455,28 @@ pub(super) async fn run_session( let token = delivery_guard .token(delivery.id) .ok_or_else(|| anyhow::anyhow!("delivery token is missing"))?; - tokio::time::timeout( + write_message_timed( + &mut writer, + &header, + &delivery.body, write_timeout, - writer.write_message_parts(&header, &delivery.body), + expiration_ns, + &mut policy_changes, ) - .await - .map_err(|_| anyhow::anyhow!("consumer delivery write timed out"))??; + .await?; let accepted_token = delivery_guard .accept_with_token(delivery.id) .ok_or_else(|| anyhow::anyhow!("delivery token is missing"))?; debug_assert_eq!(accepted_token, token); + let mut delivery_deadline = handoff_deadline; + if let Some(remaining) = expiration_ns.map(remaining_until_ns) { + delivery_deadline = + cap_deadline_at_ttl(delivery_deadline, remaining); + } state.record_delivery( delivery.id, InFlightDelivery { - deadline: handoff_deadline, + deadline: delivery_deadline, token, }, ); @@ -437,7 +501,10 @@ pub(super) async fn run_session( debug_assert_eq!(delivery.token, token); state.record_delivery( id, - InFlightDelivery { deadline, ..delivery }, + InFlightDelivery { + deadline: deadline.min(delivery.deadline), + ..delivery + }, ); } } @@ -448,6 +515,11 @@ pub(super) async fn run_session( if broker_storage_error(&error) { metrics.storage_errors.fetch_add(1, Ordering::Relaxed); } + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; write_error_timed( &mut writer, state.heartbeat, @@ -459,11 +531,22 @@ pub(super) async fn run_session( } } } + changed = wait_for_policy_change(&mut subscription_policy_changes), if subscription_policy_changes.is_some() => { + changed.map_err(|_| anyhow::anyhow!("subscribed Topic closed"))?; + if writer.has_pending() { + anyhow::bail!("Topic delivery policy changed with buffered consumer output"); + } + } _ = heartbeat_tick.tick(), if state.heartbeat.is_some() => { let heartbeat = state.heartbeat.unwrap(); if last_command.elapsed() >= heartbeat.saturating_mul(2) { break; } + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; tokio::time::timeout( connection_progress_timeout(state.heartbeat), write_frame(&mut writer, FrameType::Response, HEARTBEAT), @@ -472,7 +555,11 @@ pub(super) async fn run_session( .map_err(|_| anyhow::anyhow!("heartbeat write timed out"))??; } _ = output_buffer_tick.tick(), if state.output_buffer_timeout.is_some() && writer.has_pending() => { - flush_timed(&mut writer, state.heartbeat).await?; + flush_timed( + &mut writer, + state.heartbeat, + subscription_policy_changes.as_mut(), + ).await?; } _ = wait_for_in_flight_deadline(in_flight_deadline) => {} } @@ -513,6 +600,42 @@ pub(super) async fn run_session( session_result } +async fn wait_for_policy_change( + policy_changes: &mut Option>, +) -> Result<(), tokio::sync::watch::error::RecvError> { + policy_changes + .as_mut() + .expect("disabled policy change future is never polled") + .changed() + .await +} + +fn ensure_buffered_policy_is_current( + writer: &ClientWriter, + policy_changes: Option<&mut tokio::sync::watch::Receiver>, +) -> anyhow::Result<()> { + let Some(policy_changes) = policy_changes else { + return Ok(()); + }; + match policy_changes.has_changed() { + Ok(false) => Ok(()), + Ok(true) if writer.has_pending() => { + anyhow::bail!("Topic delivery policy changed with buffered consumer output") + } + Ok(true) => { + policy_changes.borrow_and_update(); + Ok(()) + } + Err(_) => anyhow::bail!("subscribed Topic closed"), + } +} + +fn cap_deadline_at_ttl(deadline: Instant, remaining: Duration) -> Instant { + Instant::now() + .checked_add(remaining) + .map_or(deadline, |ttl_deadline| deadline.min(ttl_deadline)) +} + async fn fetch_deliveries( broker: &Broker, request: FetchRequest, @@ -528,6 +651,8 @@ async fn fetch_deliveries( ) .await?; let (deliveries, delivery_guard) = batch.into_parts(); + let ttl_discard = broker.topic_policy(&request.topic)?.delivery_mode + == rustqueue_queue::DeliveryMode::TtlDiscard; let deliveries = deliveries .into_iter() .map(|delivery| RemoteDelivery { @@ -535,6 +660,7 @@ async fn fetch_deliveries( timestamp_ns: delivery.timestamp_ns, attempts: delivery.attempts, body: bytes::Bytes::from_owner(delivery.body), + ttl_discard, }) .collect(); Ok(FetchResponse { diff --git a/crates/server/src/tcp/time.rs b/crates/server/src/tcp/time.rs index 456821f..8859ad4 100644 --- a/crates/server/src/tcp/time.rs +++ b/crates/server/src/tcp/time.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + pub(super) fn now_ns() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -5,3 +7,8 @@ pub(super) fn now_ns() -> i64 { .as_nanos() .min(i64::MAX as u128) as i64 } + +pub(super) fn remaining_until_ns(deadline_ns: i64) -> Duration { + let remaining = deadline_ns.saturating_sub(now_ns()).max(0) as u64; + Duration::from_nanos(remaining) +} diff --git a/crates/server/src/tcp/writer.rs b/crates/server/src/tcp/writer.rs index fdb0bec..083832c 100644 --- a/crates/server/src/tcp/writer.rs +++ b/crates/server/src/tcp/writer.rs @@ -40,6 +40,39 @@ impl ClientWriter { } } +pub(super) async fn write_message_timed( + writer: &mut ClientWriter, + header: &[u8], + body: &[u8], + write_timeout: Duration, + expiration_ns: Option, + policy_changes: &mut tokio::sync::watch::Receiver, +) -> anyhow::Result<()> { + let effective_timeout = expiration_ns + .map(remaining_until_ns) + .map_or(write_timeout, |remaining| write_timeout.min(remaining)); + let write_frame = async { + writer.write_message_parts(header, body).await?; + if expiration_ns.is_some() { + writer.flush_pending().await?; + } + anyhow::Ok(()) + }; + tokio::select! { + biased; + changed = policy_changes.changed() => { + match changed { + Ok(()) => Err(anyhow::anyhow!("Topic delivery policy changed during consumer write")), + Err(_) => Err(anyhow::anyhow!("Topic closed during consumer write")), + } + } + result = tokio::time::timeout(effective_timeout, write_frame) => { + result + .map_err(|_| anyhow::anyhow!("consumer delivery write timed out"))? + } + } +} + pub(super) fn delivery_write_timeout(heartbeat: Option) -> Duration { heartbeat .unwrap_or(Duration::from_secs(30)) @@ -79,14 +112,31 @@ pub(super) async fn write_error_timed( pub(super) async fn flush_timed( writer: &mut ClientWriter, heartbeat: Option, + policy_changes: Option<&mut tokio::sync::watch::Receiver>, ) -> anyhow::Result<()> { - tokio::time::timeout( - connection_progress_timeout(heartbeat), - writer.flush_pending(), - ) - .await - .map_err(|_| anyhow::anyhow!("client output flush timed out"))??; - Ok(()) + if !writer.has_pending() { + return Ok(()); + } + let timeout = connection_progress_timeout(heartbeat); + if let Some(policy_changes) = policy_changes { + tokio::select! { + biased; + changed = policy_changes.changed() => { + match changed { + Ok(()) => Err(anyhow::anyhow!("Topic delivery policy changed with buffered consumer output")), + Err(_) => Err(anyhow::anyhow!("Topic closed with buffered consumer output")), + } + } + result = tokio::time::timeout(timeout, writer.flush_pending()) => { + result + .map_err(|_| anyhow::anyhow!("client output flush timed out"))? + } + } + } else { + tokio::time::timeout(timeout, writer.flush_pending()) + .await + .map_err(|_| anyhow::anyhow!("client output flush timed out"))? + } } impl AsyncWrite for ClientWriter { @@ -176,6 +226,95 @@ mod tests { assert_eq!(&received, b"headbody"); } + #[tokio::test] + async fn ttl_deadline_stops_a_partial_frame_and_requires_connection_close() { + let (mut peer, server) = tokio::io::duplex(32); + let io: BoxIo = Box::new(server); + let (_, write) = tokio::io::split(io); + let mut writer = ClientWriter::new(write, 1); + let body = vec![0xab; 1024 * 1024]; + let (_, mut policy_changes) = + tokio::sync::watch::channel(rustqueue_queue::TopicPolicy::default()); + let result = write_message_timed( + &mut writer, + b"header", + &body, + Duration::from_secs(1), + Some(now_ns().saturating_add(20_000_000)), + &mut policy_changes, + ) + .await; + assert!(result.is_err()); + + drop(writer); + let mut received = Vec::new(); + peer.read_to_end(&mut received).await.unwrap(); + assert!(received.len() < b"header".len() + body.len()); + } + + #[tokio::test] + async fn policy_change_stops_a_partial_frame_and_requires_connection_close() { + let (mut peer, server) = tokio::io::duplex(32); + let io: BoxIo = Box::new(server); + let (_, write) = tokio::io::split(io); + let mut writer = ClientWriter::new(write, 1); + let body = vec![0xcd; 1024 * 1024]; + let (policy, mut policy_changes) = + tokio::sync::watch::channel(rustqueue_queue::TopicPolicy::default()); + { + let write = write_message_timed( + &mut writer, + b"header", + &body, + Duration::from_secs(1), + None, + &mut policy_changes, + ); + tokio::pin!(write); + tokio::select! { + result = &mut write => panic!("partial frame completed before policy change: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(20)) => {} + } + policy.send_replace(rustqueue_queue::TopicPolicy { + delivery_mode: rustqueue_queue::DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }); + assert!(write.as_mut().await.is_err()); + } + drop(writer); + + let mut received = Vec::new(); + peer.read_to_end(&mut received).await.unwrap(); + assert!(!received.is_empty()); + assert!(received.len() < b"header".len() + body.len()); + } + + #[tokio::test] + async fn policy_change_discards_a_buffered_reliable_frame_before_flush() { + let (mut peer, server) = tokio::io::duplex(1024); + let io: BoxIo = Box::new(server); + let (_, write) = tokio::io::split(io); + let mut writer = ClientWriter::new(write, 4096); + let (policy, mut policy_changes) = + tokio::sync::watch::channel(rustqueue_queue::TopicPolicy::default()); + writer + .write_message_parts(b"header", b"body") + .await + .unwrap(); + policy.send_replace(rustqueue_queue::TopicPolicy { + delivery_mode: rustqueue_queue::DeliveryMode::TtlDiscard, + message_ttl_seconds: Some(1), + }); + + assert!(flush_timed(&mut writer, None, Some(&mut policy_changes)) + .await + .is_err()); + drop(writer); + let mut received = Vec::new(); + peer.read_to_end(&mut received).await.unwrap(); + assert!(received.is_empty()); + } + #[test] fn client_write_timeouts_are_bounded_when_heartbeats_are_disabled() { assert_eq!(delivery_write_timeout(None), Duration::from_secs(60)); diff --git a/deploy/helm/rustqueue/Chart.yaml b/deploy/helm/rustqueue/Chart.yaml index 1328f92..990e63f 100644 --- a/deploy/helm/rustqueue/Chart.yaml +++ b/deploy/helm/rustqueue/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rustqueue description: Kubernetes-native share-nothing NSQ-compatible message queue type: application -version: 0.8.4 -appVersion: "0.8.4" +version: 0.9.0 +appVersion: "0.9.0" kubeVersion: ">=1.28.0-0" keywords: - messaging diff --git a/deploy/helm/rustqueue/crds/rustqueue.io_rustqueuetopics.yaml b/deploy/helm/rustqueue/crds/rustqueue.io_rustqueuetopics.yaml index 05f4748..a47c880 100644 --- a/deploy/helm/rustqueue/crds/rustqueue.io_rustqueuetopics.yaml +++ b/deploy/helm/rustqueue/crds/rustqueue.io_rustqueuetopics.yaml @@ -29,6 +29,9 @@ spec: spec: type: object required: [queue, topic, phase] + x-kubernetes-validations: + - rule: "self.deliveryMode == 'RELIABLE' ? !has(self.messageTtlSeconds) : has(self.messageTtlSeconds) && self.messageTtlSeconds > 0" + message: "RELIABLE requires no TTL; TTL_DISCARD requires messageTtlSeconds greater than zero" properties: queue: {type: string, minLength: 1} topic: {type: string, minLength: 1} @@ -41,6 +44,11 @@ spec: enum: [PREPARING, ACTIVE, APPLYING, FAILED, TOMBSTONED] revision: {type: integer, format: int64, minimum: 0, default: 0} paused: {type: boolean, default: false} + deliveryMode: + type: string + enum: [RELIABLE, TTL_DISCARD] + default: RELIABLE + messageTtlSeconds: {type: integer, format: int64, minimum: 1, nullable: true} tombstoneUntilMs: {type: integer, format: int64, nullable: true} lastError: {type: string, nullable: true, maxLength: 1024} operation: @@ -51,7 +59,7 @@ spec: id: {type: string, minLength: 16, maxLength: 128} action: type: string - enum: [CREATE, PAUSE, UNPAUSE, EMPTY, DELETE] + enum: [CREATE, CONFIGURE, PAUSE, UNPAUSE, EMPTY, DELETE] completedOwners: type: array default: [] diff --git a/deploy/helm/rustqueue/values.yaml b/deploy/helm/rustqueue/values.yaml index c90514f..0dd8186 100644 --- a/deploy/helm/rustqueue/values.yaml +++ b/deploy/helm/rustqueue/values.yaml @@ -5,7 +5,7 @@ operator: replicas: 2 image: repository: rustqueue-operator - tag: "0.8.4" + tag: "0.9.0" pullPolicy: IfNotPresent log: rustqueue_operator=info resources: @@ -20,7 +20,7 @@ operator: queue: create: true name: rustqueue - image: rustqueue:0.8.4 + image: rustqueue:0.9.0 imagePullPolicy: IfNotPresent minBrokers: 1 maxBrokers: 500 diff --git a/docs/architecture/share-nothing-v7.md b/docs/architecture/share-nothing-v7.md index 8ede6da..6c12c5a 100644 --- a/docs/architecture/share-nothing-v7.md +++ b/docs/architecture/share-nothing-v7.md @@ -1,7 +1,7 @@ # RustQueue format v7 share-nothing architecture Status: accepted implementation contract -Target release: 0.8.4 +Target release: 0.9.0 Data format: v7, clean directories only ## 1. Goal @@ -97,6 +97,16 @@ This separates two guarantees: intentional destructive operations; protective eviction writes an audit record before advancing the unrouted boundary. +These guarantees describe the default `RELIABLE` mode. A Topic whose persisted +policy is `TTL_DISCARD` instead has a positive whole-second absolute lifetime +measured from its Broker-assigned publish timestamp. The same deadline applies +before routing, during DPUB/deferred and requeue delay, and after delivery while +the message is in flight. Expiry advances a persisted Topic prefix and evicts +that prefix from every durable and ephemeral Channel; late `FIN`, `REQ`, and +`TOUCH` therefore observe the normal not-in-flight error. Enabling or shortening +TTL applies to existing backlog by original publish time. Extending or disabling +TTL cannot restore an already expired prefix. + ### 2.4 Routing and scale - Producers connect to a node-local proxy DaemonSet. @@ -305,6 +315,12 @@ A complete segment may be deleted only when it is older than: - every active reader reference; - every pending DLQ outbox reference. +`TTL_DISCARD` first persists the Channel eviction WALs and Topic +`expiredThroughPosition`; ordinary GC later reclaims Segments once active +readers and other references release them. Scheduler work is deadline-ordered +and bounded per batch. Delivery reservation and socket handoff recheck the +absolute deadline, so scheduler delay cannot expose an expired message. + High disk watermark first rejects publishes with retryable `429`. Protective eviction remains opt-in and local: after the configured grace period it may delete the oldest complete segment, persist the resulting channel gaps, and @@ -333,6 +349,11 @@ finished until the DLQ message append is durable. Recovery retries incomplete outbox work. A crash may duplicate the DLQ entry but must not delete the source first. +DLQ termination applies only to `RELIABLE` Topics. `TTL_DISCARD` continues +normal retries until its absolute deadline, bypasses retention/max-attempt DLQ +rules, and removes any incomplete source outbox intent when the policy is +enabled. Expiry itself never publishes a DLQ message. + ## 5. Public behavior Retained NSQ commands: diff --git a/docs/operations/console.md b/docs/operations/console.md index 8bd400c..cea0efe 100644 --- a/docs/operations/console.md +++ b/docs/operations/console.md @@ -86,6 +86,12 @@ cookie, a CSRF token and a one-time 60-second action token bound to the resource UID, resource version, action and current owners. Durable Topics and Channels support create, pause, unpause, empty and delete. +Topic creation also selects `RELIABLE` (the default) or `TTL_DISCARD` with a +positive TTL in seconds. Existing Topics expose a **Configure TTL** action. +Enabling or shortening TTL previews the current stored/deferred/in-flight +impact and requires the exact Topic name because old backlog may expire +immediately. Extending or disabling TTL affects only content that has not +already expired. Policy changes are blocked while multiple owners are visible. Ephemeral Channels are observation-only. Empty and delete show an impact preview and require the exact resource name. Deletion first persists a tombstone, then synchronizes the fence to all brokers before removing data. diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index b7ad420..e3480e5 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -46,6 +46,31 @@ finished. Every reconcile checks the current Pod, revision, readiness, drain state, PVC request/capacity and compatibility report before taking the next step. +## Topic delivery policy + +`RustQueueTopic.spec` persists the desired delivery boundary: + +```yaml +spec: + deliveryMode: TTL_DISCARD + messageTtlSeconds: 3600 +``` + +`RELIABLE` requires `messageTtlSeconds` to be absent; `TTL_DISCARD` requires a +positive value. Existing and automatically created Topics remain `RELIABLE`. +Use the Console management flow for online changes so impact preview, +single-owner fencing, operation replay, and typed confirmation are enforced. +Do not edit the policy during a visible owner migration. + +Helm does not upgrade an already installed CRD. Before enabling this policy on +an upgraded cluster, apply the checked-in CRDs and wait for them to become +Established: + +```sh +kubectl apply -f deploy/helm/rustqueue/crds +kubectl wait --for=condition=Established crd/rustqueuetopics.rustqueue.io --timeout=60s +``` + ## Targeted Broker maintenance ```sh @@ -76,7 +101,7 @@ Canary approval is optional: ```sh helm upgrade rustqueue deploy/helm/rustqueue \ --namespace rustqueue \ - --set queue.image=registry.example/rustqueue:0.8.4 \ + --set queue.image=registry.example/rustqueue:0.9.0 \ --set queue.rollout.requireCanaryApproval=true rustqueuectl -n rustqueue rollout approve @@ -88,7 +113,7 @@ Useful controls: rustqueuectl -n rustqueue rollout pause rustqueuectl -n rustqueue rollout resume rustqueuectl -n rustqueue rollout retry -rustqueuectl -n rustqueue rollout rollback registry.example/rustqueue:0.8.4 +rustqueuectl -n rustqueue rollout rollback registry.example/rustqueue:0.9.0 rustqueuectl -n rustqueue rollout forward ``` diff --git a/docs/releases/v0.9.0.md b/docs/releases/v0.9.0.md new file mode 100644 index 0000000..ff9bade --- /dev/null +++ b/docs/releases/v0.9.0.md @@ -0,0 +1,100 @@ +# RustQueue 0.9.0 + +RustQueue 0.9.0 adds a Topic-scoped expiration policy for workloads that can +discard a message when no consumer has completed it within a fixed lifetime. +The existing reliable behavior remains the default, and the NSQ wire protocol +and publish acknowledgement contract are unchanged. + +## Topic TTL discard + +`RustQueueTopic.spec` and the Broker/Console observation models expose: + +```yaml +deliveryMode: RELIABLE | TTL_DISCARD +messageTtlSeconds: integer | null +``` + +- `RELIABLE` is the default and requires `messageTtlSeconds` to be absent. +- `TTL_DISCARD` requires a positive whole-second TTL. +- The TTL starts at the Broker-assigned write timestamp. It is absolute and is + not restarted by routing, deferral, requeue, redelivery or `TOUCH`. +- Expiration removes the message from the unrouted Topic backlog, durable and + ephemeral Channels, deferred/requeue state and in-flight state. It does not + publish a DLQ entry. +- `TTL_DISCARD` retries normally before the deadline and bypasses the + retention/max-attempt DLQ termination rules. The TTL is its only final + boundary. +- A delivery reservation and socket handoff recheck the deadline. If a slow or + blocked socket cannot finish a frame before expiration, the Broker closes + that consumer connection rather than leaving a partially parseable NSQ + stream. A frame that was fully written but not `FIN`ed still expires at the + same absolute deadline; later `FIN`, `REQ` and `TOUCH` receive the normal + not-in-flight error. +- Logical expiration is immediate at the deadline. Segment files are reclaimed + later by normal reader-safe GC, so physical disk usage may lag the logical + discard. + +The label-free `rustqueue_ttl_discarded_messages_total` counter is always +available. Optional detailed Topic metrics use the existing bounded series +budget. + +## Management and ownership fencing + +Topic creation may include the policy. Existing Topics can be changed with the +`CONFIGURE` operation through the Console management flow or the Broker's +authenticated `POST /v1/manage/topics/configure` endpoint. The request carries +the complete policy, including `delivery_mode` and `message_ttl_seconds`. + +Enabling TTL or shortening it can immediately discard old backlog according to +the original Broker write timestamp. Such changes require an impact preview +and exact Topic-name confirmation. Extending or disabling TTL affects only +messages that have not already expired and cannot revive discarded data. + +Policy changes are rejected while the Topic has multiple visible owners or is +mid-migration. The Console preview and apply challenge bind the operation to +the observed single owner, resource identity, revision and operation ID; +retries preserve the operation identity and completed-owner progress. + +The legacy NSQ-compatible `/topic/create` endpoint continues to create a +`RELIABLE` Topic. It does not accept a per-publish TTL. `PUB`, `MPUB` and +`DPUB` continue to acknowledge according to `queue.publish_ack_mode` and do +not wait for a Channel or consumer. + +## Compatibility and upgrade from 0.8.4 + +- Existing and automatically created Topics remain `RELIABLE` unless an + operator explicitly configures `TTL_DISCARD`. +- Older Topic manifests without the new policy fields load with the reliable + defaults. No manifest migration is required. +- The on-disk format remains v7. The v7 clean-directory boundary and its + existing reader/writer compatibility fences remain in force; no storage + migration is required from 0.8.4. +- NSQ V2 commands, lookup, TLS/mTLS, AUTH, compression, fan-out, ephemeral + Channels and publish acknowledgement modes retain their existing contract. + +For a 0.8.4 to 0.9.0 Kubernetes upgrade, apply the checked-in CRDs before +using the new Topic fields. Helm does not upgrade an already-installed CRD: + +```sh +kubectl apply -f deploy/helm/rustqueue/crds +kubectl wait --for=condition=Established \ + crd/rustqueuetopics.rustqueue.io --timeout=60s +helm upgrade rustqueue deploy/helm/rustqueue \ + --namespace rustqueue \ + --set queue.image=registry.example/rustqueue:0.9.0 +``` + +The CRD validation requires `RELIABLE` without a TTL and `TTL_DISCARD` with a +positive `messageTtlSeconds`. Use the Console preview/configure flow for +online policy changes; do not edit a Topic policy while ownership is visibly +migrating. Existing PVCs and v7 data are retained by the normal rollout. + +## Release assets + +- `rustqueue-0.9.0-linux-x86_64.tar.gz`: Linux x86_64 binaries, Console UI and + example configuration +- `rustqueue-0.9.0-linux-aarch64.tar.gz`: Linux ARM64 binaries, Console UI and + example configuration +- `rustqueue-0.9.0-source.tar.gz`: source archive for the tagged commit +- `rustqueue-0.9.0.tgz`: Helm Chart +- `SHA256SUMS-0.9.0`: SHA-256 checksums for all downloadable assets diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 472abb4..c2acc47 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2017,7 +2017,7 @@ dependencies = [ [[package]] name = "rustqueue-discovery" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2055,7 +2055,7 @@ dependencies = [ [[package]] name = "rustqueue-protocol" -version = "0.8.4" +version = "0.9.0" dependencies = [ "bytes", "serde", @@ -2065,7 +2065,7 @@ dependencies = [ [[package]] name = "rustqueue-proxy" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "axum", @@ -2085,7 +2085,7 @@ dependencies = [ [[package]] name = "rustqueue-queue" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "bytes", @@ -2105,7 +2105,7 @@ dependencies = [ [[package]] name = "rustqueue-storage" -version = "0.8.4" +version = "0.9.0" dependencies = [ "anyhow", "crc32c", @@ -2117,7 +2117,7 @@ dependencies = [ [[package]] name = "rustqueue-telemetry" -version = "0.8.4" +version = "0.9.0" dependencies = [ "serde", ] diff --git a/scripts/acceptance-multi-broker-k8s.sh b/scripts/acceptance-multi-broker-k8s.sh index 3ec9b60..6473fda 100755 --- a/scripts/acceptance-multi-broker-k8s.sh +++ b/scripts/acceptance-multi-broker-k8s.sh @@ -343,9 +343,9 @@ require jq } if [[ "$BUILD_IMAGES" == "1" ]]; then - BUILD_VERSION=0.8.4-e2e-a MAX_STORAGE_FEATURE_LEVEL=1 make image + BUILD_VERSION=0.9.0-e2e-a MAX_STORAGE_FEATURE_LEVEL=1 make image docker tag rustqueue:dev "$BROKER_IMAGE_A" - BUILD_VERSION=0.8.4-e2e-b MAX_STORAGE_FEATURE_LEVEL=2 make image-from-dist + BUILD_VERSION=0.9.0-e2e-b MAX_STORAGE_FEATURE_LEVEL=2 make image-from-dist docker tag rustqueue:dev "$BROKER_IMAGE_B" [[ "$(docker image inspect "$BROKER_IMAGE_A" -f '{{.Id}}')" != \ "$(docker image inspect "$BROKER_IMAGE_B" -f '{{.Id}}')" ]] || { diff --git a/site/index.html b/site/index.html index 0c37ef0..64efba8 100644 --- a/site/index.html +++ b/site/index.html @@ -407,7 +407,7 @@

One Helm command.

helm upgrade --install rustqueue deploy/helm/rustqueue \
   --namespace rustqueue --create-namespace \
-  --set queue.image=registry.example/rustqueue:0.8.4 \
+  --set queue.image=registry.example/rustqueue:0.9.0 \
   --set queue.storageClassName=ssd-rwo