diff --git a/.github/workflows/publish-crate.yml b/.github/workflows/publish-crate.yml index 5af11e09..741483cc 100644 --- a/.github/workflows/publish-crate.yml +++ b/.github/workflows/publish-crate.yml @@ -14,6 +14,7 @@ on: - 'windows-thread-ambient-sys-v*' - 'windows-threadpool-sys-v*' - 'windows-topology-sys-v*' + - 'windows-waitable-queues-v*' - 'wtf-string-v*' # Manual escape hatch for a tag whose commit cannot publish. `cargo publish # --locked` refuses when Cargo.lock disagrees with the manifests, and the @@ -38,6 +39,7 @@ on: - windows-thread-ambient-sys - windows-threadpool-sys - windows-topology-sys + - windows-waitable-queues - wtf-string permissions: @@ -107,7 +109,7 @@ jobs: - name: Wait for workspace-sibling dependencies on crates.io shell: bash run: | - workspace_crates="windows-file-enumeration-sys windows-file-watcher windows-file-watcher-example-test-harness windows-impersonation-token-sys windows-ioring-sys windows-namespace-request-sys windows-overlapped-io-sys windows-thread-ambient-sys windows-threadpool-sys windows-topology-sys wtf-string" + workspace_crates="windows-file-enumeration-sys windows-file-watcher windows-file-watcher-example-test-harness windows-impersonation-token-sys windows-ioring-sys windows-namespace-request-sys windows-overlapped-io-sys windows-thread-ambient-sys windows-threadpool-sys windows-topology-sys windows-waitable-queues wtf-string" metadata="$(cargo metadata --no-deps --format-version 1)" # `tr -d '\r'` is load-bearing on the Windows runner: jq.exe writes # CRLF, and `read` splits on LF alone, so without this the last field diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8368be57..05ca7294 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -9,5 +9,6 @@ "crates/windows-thread-ambient-sys": "0.2.0", "crates/windows-threadpool-sys": "0.1.3", "crates/windows-topology-sys": "0.1.0", + "crates/windows-waitable-queues": "0.0.1", "crates/wtf-string": "0.1.0" } diff --git a/Cargo.lock b/Cargo.lock index b9a7aac5..edb17299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -285,6 +291,14 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "windows-waitable-queues" +version = "0.0.1" +dependencies = [ + "portable-atomic", + "windows-sys", +] + [[package]] name = "wtf-string" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 753fa429..f255412d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/windows-thread-ambient-sys", "crates/windows-threadpool-sys", "crates/windows-topology-sys", + "crates/windows-waitable-queues", "crates/wtf-string", ] resolver = "2" diff --git a/crates/windows-waitable-queues/COMPLETED-CHECKLIST.md b/crates/windows-waitable-queues/COMPLETED-CHECKLIST.md new file mode 100644 index 00000000..39f207b0 --- /dev/null +++ b/crates/windows-waitable-queues/COMPLETED-CHECKLIST.md @@ -0,0 +1,78 @@ +# Completed checklists: windows-waitable-queues + +Append-only. Newest groups at the bottom. + +## Moved 2026-09-05 -- public API surface, closed against eight published queue crates + +# Checklist: public API surface + +Closes the gaps found by comparing this crate's public surface against the eight +most-depended-on Rust queue crates: `crossbeam-channel`, `crossbeam-queue`, +`flume`, `thingbuf`, `ringbuf`, `rtrb`, `concurrent-queue`, and +`std::sync::mpsc`. + +Design decisions are in [DESIGN-NOTES.md](DESIGN-NOTES.md). + +## What the comparison established + +Of seventeen capabilities present in three or more of those eight, this crate +already has nine, under the majority spelling in every case (`len`, `capacity`, +`is_empty`, `remaining`). The items below are the gaps that a reader coming from +any of those crates would notice. Deliberately **not** queued, because the +comparison showed them to be one- or two-crate features rather than +expectations: an explicit `close()` operation (1/8), `peek` (2/8, and no MPMC +crate offers it), bulk and slice transfers (2/8, both single-producer designs), +async (2/8), `force_push` (2/8), `sender_count` (1/8), and weak handles (1/8). + +**These are breaking changes, and that is why they are queued now.** The crate is +unpublished at `0.0.1`, so the cost of making them is zero and it will not be +zero again. + +## M1: the surface + +- [x] **API-1** -- Make `pop` distinguish an empty queue from a departed + producer, by returning `Result` with `Empty` and + `Disconnected` variants rather than `Option`. + + **The strongest signal is internal, not comparative.** `push` already + distinguishes `PushError::Full` from `PushError::Disconnected`, and `recv` + already returns `RecvError::Disconnected`; only `pop` collapses the two. The + crate has the vocabulary and one method declines to use it. + + Five of the eight crates distinguish these. The three that do not -- + `crossbeam-queue`, `rtrb`, `ringbuf` -- have **no handles and no + disconnection concept at all**, so they have nothing to distinguish. This + crate is channel-shaped: handles, `Drop`-based disconnection, and + `is_disconnected` on both sides. The shape implies the expectation. + + It also removes a protocol the caller is currently asked to remember. The + `Consumer` trait documents: *"Ask only after `pop` has returned `None`. + Draining to empty and then finding the producers gone is the only order that + cannot lose an item."* That is a `TryRecvError` written as prose, and prose + cannot be enforced. + +- [x] **API-2** -- Put `is_full` on the `Bounded` trait, so it is reachable + generically and from a consumer. + + Every concrete `Producer` answers `is_full` as an inherent method, but the + trait does not declare it, so generic code over `Bounded` cannot ask and no + `Consumer` offers it. The two surfaces disagree about which questions exist: + `remaining` is on the trait and therefore works on every consumer, yet is + spelled out inherently only on `reserving_mpsc`'s. + + Six of the eight crates offer `is_full`. Give it a default implementation in + terms of `remaining`, so no shape has to restate it, and add the inherent + method to each `Consumer` for the same reason the other accessors are + inherent: a caller should not need an import to ask. + +- [x] **API-3** -- Add `try_iter` and `IntoIterator` on the consumers, keeping + `drain` as the name that describes the semantics. + + `drain` is exactly `try_iter` -- take what is there, stop at the first empty + -- but `drain` is the one-crate spelling (flume) while `try_iter` is the + four-crate one, and `IntoIterator` on the receiving handle is four-crate as + well. Neither costs anything to add. + + It is also trait-only today, so `rx.drain()` does not compile without + `use windows_waitable_queues::Consumer`. Make the iterator reachable from the + concrete types. diff --git a/crates/windows-waitable-queues/COMPLETED-PLANS.md b/crates/windows-waitable-queues/COMPLETED-PLANS.md new file mode 100644 index 00000000..b087602a --- /dev/null +++ b/crates/windows-waitable-queues/COMPLETED-PLANS.md @@ -0,0 +1,5 @@ +# Completed plans: windows-waitable-queues + +| Path to CHECKLIST.md | Completion Date | Brief description | Design Notes | +|---|---|---|---| +| [COMPLETED-CHECKLIST.md](COMPLETED-CHECKLIST.md) | 2026-09-05 | Closed the public-surface gaps found by comparing this crate against the eight most-depended-on Rust queue crates: `pop` now distinguishes an empty queue from a departed producer, `is_full` is on the `Bounded` trait and both handles, and `try_iter`/`drain` are inherent. | [DESIGN-NOTES.md](DESIGN-NOTES.md) | diff --git a/crates/windows-waitable-queues/Cargo.toml b/crates/windows-waitable-queues/Cargo.toml new file mode 100644 index 00000000..b8de78e9 --- /dev/null +++ b/crates/windows-waitable-queues/Cargo.toml @@ -0,0 +1,81 @@ +# Copyright (c) 2026 Mike Grier + +[package] +name = "windows-waitable-queues" +# `0.0.x` on purpose: this crate has never been published, so this is a starting +# point rather than a record of a release. With `bump-minor-pre-major`, the six +# breaking commits behind it make the first published version `0.1.0` -- which is +# what a first release should look like. Leaving it at `0.1.0` would have made +# that first release `0.2.0`, skipping `0.1.0` entirely. Not `0.0.0`, which +# release-please special-cases into a jump to `1.0.0`. +version = "0.0.1" # x-release-please-version +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation = "https://docs.rs/windows-waitable-queues" +description = "Bounded producer/consumer queues whose readiness is a waitable Windows HANDLE, so a consumer can park on a queue and a kernel object in the same wait." +readme = "README.md" +keywords = ["windows", "queue", "spsc", "mpsc", "concurrency"] +categories = ["concurrency", "os::windows-apis", "data-structures"] + +# Deliberately publishable, unlike `windows-guard-alloc` next door: this is a +# general-purpose facility whose value is that it exists, and its first consumer +# (the I/O domain runtime) is not its only plausible one. Publishing is +# therefore an obligation accepted, not an oversight -- see DESIGN-NOTES.md D-8 +# for what it commits us to. +publish = true + +# The crate is Windows-only -- every public item is behind `cfg(windows)` and +# the implementation imports `std::os::windows::io` unconditionally -- so +# docs.rs must build it on a Windows target or the build fails outright. +[package.metadata.docs.rs] +default-target = "x86_64-pc-windows-msvc" +targets = ["x86_64-pc-windows-msvc"] + +[features] +# An experimental claim protocol, measured against the shipping one by +# `probe-queue-contention` so that SH-14.3 can be decided on evidence. Not +# covered by this crate's semver promise; it will either be merged into +# `reserving_mpsc` or deleted (SH-15.6). Non-default so nothing depends on it +# by accident. +experimental-permit-claim = [] + +# A 128-bit claim word for `reserving_mpsc`, adding the `Wide` layout. +# +# **The only thing in this crate that costs a third-party dependency, which is +# why it is a feature rather than always present.** Rust's standard library has +# no 128-bit atomic -- `core::sync::atomic` stops at 64 bits -- so a +# double-width compare-and-swap needs `portable-atomic`. Without this feature +# the crate depends on `windows-sys` alone and every layout uses `AtomicU64`. +# +# `default-features = false` on the dependency is load-bearing: with its +# defaults, `portable-atomic` silently substitutes a global lock where the +# native instruction is unavailable, which would put a mutex in the claim path +# while still compiling. With them off, `AtomicU128` does not exist on such a +# target and the build fails naming it. +# +# Most callers should not need this. `Perpetual` reaches roughly twenty years +# before its claim position recurs, on a plain `AtomicU64` at no measured cost, +# whereas the 128-bit exchange measured 2-3x slower on the claim itself. See +# `ClaimLayout` for the comparison. +dwcas = ["dep:portable-atomic"] + +[lib] +path = "src/lib.rs" + +[dependencies.portable-atomic] +version = "1.15.0" +default-features = false +optional = true + +[dependencies.windows-sys] +version = "0.61.2" +default-features = false +features = [ + "Win32_Foundation", + "Win32_Security", + "Win32_System_Threading", +] diff --git a/crates/windows-waitable-queues/DESIGN-NOTES.md b/crates/windows-waitable-queues/DESIGN-NOTES.md new file mode 100644 index 00000000..84052949 --- /dev/null +++ b/crates/windows-waitable-queues/DESIGN-NOTES.md @@ -0,0 +1,1418 @@ +# Design notes: windows-waitable-queues (Tier 1) + +This file records the decisions this crate's code is built against. D-1 to D-9 were taken during the +2026-08-30 design session and transcribed here so they steer the work rather than sitting in a session +record nothing is obliged to read; D-10 onwards were taken while building the shapes those decisions +called for, and record what the building settled or corrected. + +The naming decision -- plural, and no `-sys` suffix -- lives in the workspace +[DESIGN-NOTES.md](../../DESIGN-NOTES.md#the-waitable-queues-crate-is-named-plural-and-carries-no-sys-suffix) +rather than here, since it was taken before this directory existed. + +## Intent + +Bounded producer/consumer queues whose readiness is a waitable Windows `HANDLE`. + +The queues themselves are ordinary. What the crate exists for is the `HANDLE`: it lets a consumer park +on a queue **and** a kernel object in one wait, which is exactly what the existing Rust concurrent +queues cannot offer on Windows, and it is why depending on one of them was rejected rather than +preferred. + +## Decisions + +| ID | Decision | +|---|---| +| D-1 | **The crate exists because readiness must be a `HANDLE`, not because Rust lacks queues.** `crossbeam-channel` parks on a private primitive and its `Select` accepts only channel operations; `crossbeam-queue` never blocks. Neither can be seen by `WaitForMultipleObjects`, and neither can see an `IoRing` completion event. A consumer needing "a message **or** an I/O completion **or** shutdown" must otherwise poll one source while blocking on another. | +| D-2 | **Capabilities are sliced into narrow traits, not gathered into one.** The `std::io` shape -- `Read`, `Write`, `Seek`, `BufRead` -- rather than a single fat `WaitableQueue`. Forced by the shapes themselves: a poll-only queue cannot implement a trait containing `doorbell()`, and an unbounded one cannot implement `capacity()` meaningfully. | +| D-3 | **No trait ships until a second implementation exists to validate it.** The trait *shape* is fixed now so signatures stay compatible; the traits themselves land with the second shape. | +| D-4 | **Every shape is split into producer and consumer handles, and cardinality is carried by `Clone`.** Single-producer becomes a compile-time guarantee rather than a documented precondition. | +| D-5 | **The doorbell is level state owned by the queue: never unsignalled while the consumer has something to observe.** One-sided on purpose -- it may be signalled with nothing there, because the event stays set after the last take until the consumer's `arm()` clears it, and a late signal may arrive after a drain. A wake is a hint, never a proof; the guarantee is that a wake is never missing. The **reset** must not be separable from the observation that there is nothing to take; the **signal** may be. Manual-reset, and created lazily. Realized without a lock by [D-9](#d-9). | +| D-9 | **Without a lock, the reset is made inseparable from the observation by two things: ordering (clear, then re-check, and never wait if the re-check finds anything) and a `SeqCst` fence on each side.** `Consumer::arm` is the ordering step; the fences defeat the store-buffer hazard that ordering alone leaves open. The natural order -- check, then clear -- is asserted to hang by deliberate sabotage; the fences are beyond any test's reach and are M31.6's target. **Amended: this decision originally claimed the ordering alone sufficed.** | +| D-6 | **Overflow fails or reserves, and never overwrites.** For telemetry an overwritten entry is a lost sample; for an I/O submission it is a lost operation, and the two must not share a policy knob. | +| D-7 | **Shapes are plain modules, not Cargo features, until compile time justifies otherwise.** Two features are four configurations to test, against a benefit dead-code elimination already provides. | +| D-8 | **Published, and the obligation is accepted deliberately.** Unlike `windows-guard-alloc`, this is general-purpose and its first consumer is not its only plausible one. | +| D-10 | **The multi-producer shape is Vyukov's bounded array queue: a sequence number per slot, claimed by a compare-and-swap on the tail and published by a release store.** The sequence is what lets the consumer tell a *claimed* slot from a *written* one, which a plain fetch-and-add cannot. Lock-free rather than wait-free, bounded by construction, and no allocation after the constructor. | +| D-11 | **The capability traits shipped with this second shape, and the signatures `spsc` wrote down in advance held unchanged.** That is [D-3](#d-3)'s check actually being run rather than assumed. The load-bearing choice was `push(&self)`: `&mut self` would have been sound for one producer and would have made the trait unimplementable by this one. | +| D-12 | **A shape's *minimum* capacity belongs to the shape, not to the crate, and `slotwise_mpsc`'s is two.** One slot cannot encode three states when the lap stride is the capacity, so "published at `p`" and "free again at `p + capacity`" collide. Reported through `CapacityError` rather than worked around, because every available workaround puts a load back on the producer's hot path for every queue in order to serve a capacity of one. | +| D-13 | **The arming protocol is written once, in [`blocking.rs`](src/blocking.rs), and a shape binds to it by implementing a crate-private `Parked` trait.** The blocking receive loop *is* [D-9](#d-9), not glue around it; a second shape spelling it out again would be a second copy of a rule -- the exact mistake this crate has already paid for once. | +| D-14 | **`slotwise_mpsc`'s arming asks "would `pop` find something", not "is `len` zero".** The two disagree over a slot a producer has claimed but not published, and only the first answer lets the consumer park on it instead of spinning until that producer is rescheduled. | +| D-15 | **`Doorbell::clear` resets the event *before* clearing the flag that mirrors it, and the original order was a lost wakeup.** A producer signalling between the two lines set the flag and issued a real `SetEvent`; the `ResetEvent` that followed erased the signal and left the flag set, wedging the doorbell dark while it claimed to be lit. **Amends [D-9](#d-9)**, whose "there is no third case" holds only for a queue whose emptiness is one position comparison. | +| D-16 | **Its cost premise is falsified by [D-26](#d-26); the conclusion stands on capability instead -- see [D-29](#d-29).** Reservation is a capability a shape may lack, so the reserving multi-producer queue ships as a peer of `slotwise_mpsc` rather than replacing it. Honouring a reservation requires counting free slots, which requires the consumer's position -- a single shared line `slotwise_mpsc`'s push deliberately never reads. The original rationale added that this made reserving the *more expensive* shape and that both should ship rather than charge every caller for it; measurement reversed that, and the split is now justified by the capability alone. **Amends [D-6](#d-6)**, which assumed one queue would carry every policy. | +| D-17 | **The reservation count and the claim position live in one word, because a check-and-claim over both must be a single atomic operation.** Two atomics cannot be made correct with any amount of fencing: the pushing producer is load-then-store and the reserving one store-then-load, so the Dekker argument does not apply and both can miss each other. The 32/32 split is forced by the arithmetic, and caps this shape at 2^31 items. | +| D-18 | **Superseded by [D-37](#d-37), which adopts a 128-bit compare-and-swap for a separate wide shape.** Retained because its analysis of the *costs* is still correct and D-37 depends on it; what changed is that those costs are now paid by a **separate shape** rather than imposed on this one. **Originally: a 128-bit compare-and-swap is refused.** **Amended once before being superseded, because three of the four reasons originally given were wrong or incomplete, and the decisive one was missing.** It would *not* lift the cap "and nothing else": a 64-bit position also collapses SH-14.1's ABA recurrence, which was unknown when this was written. It is *not* outside the x86-64 baseline -- `rustc 1.98.0` emits `target_feature="cmpxchg16b"` for `x86_64-pc-windows-msvc`, so there is no floor to raise and no runtime detection to pay. What stands is the dependency (`AtomicU128` is still unstable, rust-lang/rust#99069) and, decisively, that **`i686-pc-windows-msvc` has no 128-bit atomic at all**: adopting this is not "widen the word" but "widen the word *and* drop 32-bit support". Revisit for a tagged pointer, or if 32-bit support is dropped for other reasons -- not before. | +| D-19 | **The coalesced loss latch is deliberately not generalised from the file watcher.** Coalescing there is sound because a desync is *idempotent* -- two mean the same as one, and the answer to both is a re-scan. A queue of arbitrary `T` has no such property, so what generalises is a loss *count*, which is M31.4's observability rather than a policy. | +| D-20 | **Undrained items are handed to a caller-supplied sink at teardown, and the sink is chosen at construction because `Drop` has nowhere to hand them back to.** Without one they are destroyed on whichever thread released the last handle -- which may be a pool callback that must not block, and closing a handle to a dead network path can block for a long time. The default is unchanged; what changes is that it is now a named choice. | +| D-21 | **A panicking disposal sink is caught and the teardown walk continues.** The sink is caller code inside a destructor: a panic escaping it abandons every item behind it -- the exact handles the mechanism exists to account for -- and during an unwind aborts the process. Catching declines to turn a caller's bug into a much larger one. | +| D-22 | **No `into_remaining`, because it would not close the hole and `drain` already covers what it would do.** A consumer can take everything available, but a producer may push afterwards, so an orderly drain covers only the orderly path. The last handle to drop is the only place that sees every survivor. | +| D-23 | **High-water tracking is opt-in at construction; refusals and doorbell rings are always on.** The difference is where each can be paid for: refusals sit on the failure path and rings on a path that already costs a syscall, but a peak has to observe *every* change -- and on `slotwise_mpsc` that means the producer reading the consumer's position, the shared line [D-16](#d-16) built a separate shape to avoid. Untracked reports `None`, not `0`. | +| D-24 | **Counting the doorbell's rings turns the skip optimisation into part of the observable contract, and that is the point rather than a side effect.** R9 asks for the count precisely so "disabling the skip must change the number" -- so the sabotage entry for removing the skip changed from a control expecting `survives` to a defect expecting `caught`. An optimisation nobody can measure is an assumption. | +| D-25 | **`Observable` deliberately does not restate depth.** [D-2](#d-2)'s sketch listed it, but `Bounded::len` already reports it from positions the queue keeps anyway. Naming it twice would give one number two spellings and two places to drift. What belongs on `Observable` is only what must be *accumulated*. | +| D-26 | **Measured: the tail claim contends badly, and `reserving_mpsc` is up to 4x FASTER than `slotwise_mpsc` under contention -- the opposite of what [D-16](#d-16) assumed.** Aggregate throughput *falls* as producers are added, for both shapes and far more than a bare contended atomic explains. D-16's premise, that reading the consumer's position makes the reserving shape the expensive one, is falsified everywhere except a single producer with a live consumer. | +| D-27 | **The gap is intrinsic to Vyukov's sequence protocol, not a fixable flaw in `slotwise_mpsc`'s retry loop.** Its producer must read a slot's sequence *before* claiming, and that slot marches through memory as the tail advances while other producers write it. Padding slots onto their own cache lines was tested and rejected: it recovers about a fifth at eight producers, for four times the memory, and leaves the shape still 2.8x slower. | +| D-28 | **Amended -- the blanket rejection is withdrawn; the verdict depends on thread placement, and the open question is an open question queued outside this crate.** Caching the peer's index was measured, and it engaged as designed. It cost ~1.8x on x64 with the threads across cores, and *won* 17x on ARM64 and 1.8x on x64 SMT siblings. Batch depth decides the sign, and batch depth is set by where the two threads are scheduled -- not by the architecture and not by our code. A prefetch-only "warming" control changed nothing on any host. | +| D-29 | **Both multi-producer shapes ship. The crate publishes what it measured and declines to choose for the caller.** [D-26](#d-26) falsified [D-16](#d-16)'s cost premise, which reopened merge-or-delete; the answer is neither. Vyukov's sequence protocol and the head-based one are independently researched designs, both in production use, and our own workload having settled which *we* want is not evidence about anyone else's. Deleting a shape because no visible consumer wants it is what PLATFORM INTEGRITY forbids. What the crate owes instead is the data and, through `probe-core-affinity`, the means to gather it on the caller's own hardware. | +| D-30 | **Both MPSC shapes are qualified by name; neither is `mpsc`.** A bare `mpsc` beside `reserving_mpsc` makes one canonical by implication, which contradicts this crate's own "no shape is the canonical one" and, after [D-29](#d-29), is simply false. `slotwise_mpsc` names its claim protocol -- it claims slot by slot, with no shared counter -- and avoids the reading `sequence_mpsc` invites, that it alone preserves FIFO order when both shapes do. Renamed before first publish, where it is free. | +| D-31 | **0.1.0 ships without machine-checked memory orderings, and says so in its own documentation.** Model-checking gates 1.0, not 0.1.0. It would close the *demonstrated* gap -- a weakened `Acquire` survives the whole suite -- but not the dangerous one: it cannot model `SetEvent`/`ResetEvent`, so it cannot cover the doorbell, and [D-15](#d-15)'s lost wakeup, the only ordering bug this crate has had, was found by sabotage instead. The risk it addresses is mostly regression risk, which is lowest before there are consumers. The disclosure, not the deferral, is the decision. | +| D-32 | **`Reserving::Reservation<'a>` gains a bound, before the crate publishes.** The associated type is currently unbounded, so a caller generic over the trait can claim a slot and drop it but never redeem it -- the trait cannot express the operation it exists for. Both implementors already have identical `send` and `is_disconnected` signatures, so the bound is additive; adding it after publication is a breaking change to every implementor. Done as SH-1.5: the [`Claim`](src/traits.rs) trait carries `send` and `is_disconnected`, and both reservation types implement it as forwarders. `Claim` must be in scope to call those methods on a claim whose concrete type the caller has not named, which is why it is re-exported at the crate root. | +| D-33 | **`PushError` is `#[non_exhaustive]`, and the one-directional doorbell is disclosed rather than fixed before 0.1.0.** The receive-side errors already carried the attribute and the send side lacked it by omission; adding it after publication is itself breaking, so it is taken now while the crate has no external consumers. Whether a producer can *wait* for room stays open as M32.3 -- it is additive, so it does not gate the release -- but the absence is stated in both the crate docs and the README, because `crossbeam-channel`'s `send` blocks and a reader arriving from it will assume this one does too. | +| D-34 | **Every bounded queue surveyed is ABA-safe for one of two reasons, and this crate's `reserving_mpsc` has neither.** Either the claim counter is a whole machine word, so recurrence is unreachable -- crossbeam, concurrent-queue, thingbuf, Vyukov, SCQ's `Head`/`Tail` -- or the authorizing compare-exchange is moved onto the cell, so the decision and the write are validated together (CRQ, SCQ). Ours packs the position into a 32-bit *subfield* and authorizes with an exchange that does not cover the separately-read `head`. Nikolaev (DISC 2019, section 3) states the width assumption the field relies on and states it for **CPU-word** width, which a subfield does not satisfy; DPDK's `rte_ring` is the same protocol as ours and its published justification covers modular arithmetic only. The generalisation -- ours, unstated in any source -- is that **the atomic operation authorizing the write must cover everything the decision depended on.** Survey in [DESIGN-SESSION-2026-09-02](design-sessions/DESIGN-SESSION-2026-09-02-claim-protocol-prior-art.md); the fix is the claim-protocol replacement recorded there. | +| D-35 | **Measured: the permit claim is 2.7x faster than `reserving_mpsc` at 16-32 producers, and 1.45x slower at one.** The safer claim is also the faster one everywhere contention exists, which was not the expected result -- it touches *two* shared lines where the shipping shape touches one plus a read, and [D-26](#d-26) had established that the shared line is what collapses. The mechanism is that both of its operations are unconditional read-modify-writes that never retry, where the shipping shape's compare-exchange retries once per lost race; the retries dominate long before the second line does. It is the only shape measured that gets *faster* per push as producers are added (42.8 ns at two to 19.5 at thirty-two) and the only one that stays within 1.5x of a bare contended `fetch_add`. **This decides the shape of the fix but not the fix**: the drained regime's refusal counts differ by orders of magnitude in a way this harness cannot attribute, which SH-15.5.1 exists to settle before SH-15.6 adopts anything. | +| D-36 | **Superseded by [D-41](#d-41): the hazard is now a layout choice, not a defect that must ship.** The reasoning below stands as the record of why it was right to disclose rather than delay while the only known fix was the claim-protocol replacement. **0.1.0 ships SH-14.1 disclosed rather than fixed, and the disclosure is a release blocker.** Following [D-31](#d-31)'s principle -- the disclosure, not the deferral, is the decision -- because the fix is a claim-protocol replacement ([D-35](#d-35)) whose adoption is still gated on an open question, and holding the release for it would trade a *documented* hazard for an undocumented rush. **The two gaps are not equally forgiving and the text says so**: an unverified ordering is a risk of a bug, this is a known one with a computed exposure, and its failure mode is silent -- no error, panic, or counter -- so a caller can neither detect nor mitigate it. That is precisely why it may not ship in silence. Stated in the crate docs, the README, and the shape's own module docs, each leading with **"on every target, not only 32-bit ones"**, because the natural spelling "32-bit position" invites the opposite reading and SH-6.1 already had to be corrected for exactly that. The shape-selection guidance in both documents was also amended: it previously said "start with `reserving_mpsc`" with no caveat, pointing callers at the hazardous shape by default. | +| D-37 | **Partly superseded by [D-41](#d-41): the wide word ships as a *layout* behind the non-default `dwcas` feature, not as a separate `reserving_mpsc_wide` shape, and the gate is the feature rather than the target.** What stands is the reasoning below about `portable-atomic`: `default-features = false` is load-bearing, because with defaults on it silently substitutes a global lock, and D-7's burden of proof is discharged rather than waived. What does not is the shape's name and the premise that the narrow word must keep SH-14.1 -- re-apportioning the narrow word removes the exposure for free, so the wide word is no longer the only way out. **The reserving claim word ships in two widths: the narrow one on every target, the wide one only where a 128-bit exchange is genuinely lock-free.** `reserving_mpsc` keeps its packed 64-bit word, keeps SH-14.1's hazard, keeps [D-36](#d-36)'s warnings, and is **never silently swapped** for the wide shape on targets that could host one -- a contract that changes with the target is what PLATFORM INTEGRITY rule 2 forbids, and a caller who read "2^32" must get 2^32. `reserving_mpsc_wide` is the same protocol with a `u128` word split 64/64: recurrence needs 2^64 pushes, and the capacity ceiling rises to 2^62. **The gate is one line of `Cargo.toml`: `default-features = false`.** Measured, not designed -- with the default feature set `portable-atomic` compiles on i686 and silently substitutes a global lock, but with defaults off `AtomicU128` **does not exist** there (`no AtomicU128 in the root`), nor on x86_64 built without `cmpxchg16b`. It exists exactly where a native lock-free exchange is guaranteed at compile time, so the `use` statement is the gate and it fails loudly. A `cfg(target_has_atomic = "128")` would be the *wrong* gate -- it is emitted even with `cmpxchg16b` disabled -- and a `const` assertion on `is_always_lock_free()`, though genuinely const-evaluable, is redundant where the type exists and unreachable where it does not. That is the standard SH-14.2 already set when it probed i686 to confirm `AtomicU64` was lock-free before widening `slotwise_mpsc`, recording that a hidden mutex "would have made this a bad trade". [D-7](#d-7)'s burden of proof for adding a Cargo feature is **discharged, not waived**: D-7 rejected feature-gating because the only benefit was compile time, and the cost here is a third-party dependency, which dead-code elimination does not remove from `Cargo.lock` or from an auditor's review. | +| D-38 | **One atomic, one discipline: an atomic that carries any acquire/release operation has acquire/release on *every* operation, and a relaxed load is never mixed in.** A relaxed operation is still **atomic** -- indivisible, untorn, and free of data-race UB -- but it is *unordered*: it behaves like a plain load or store with respect to placement, unanchored relative to the ordered operations on the same object and free to be moved by the optimizer or the processor. It is not pinned to its textual site, so reasoning about it in statement order is nonsense. **The two axes are independent, and conflating them is the mistake in both directions:** relaxed does not mean "no guarantees" (see [D-40](#d-40) -- the atomicity is often the whole point), and it does not mean "ordered but weakly". The failure mode is that it usually does what the source appears to say, until a change of code generator or a weaker processor makes it not; on x86-64 TSO a decorative `Acquire` and a `Relaxed` load emit near-identical code, so a test suite on this host cannot see the difference at all -- which is the same blindness [D-31](#d-31) measured. **When the two resolutions differ, promote the load.** The exception is a reference count ([D-39](#d-39)), and it is an exception for a stated reason rather than by convention. | +| D-39 | **The reference counts keep a relaxed increment against an `AcqRel` decrement, and that is the one sanctioned departure from [D-38](#d-38).** It is sanctioned because *no dependent memory is read on the strength of the relaxed increment*: a thread incrementing `producers` already holds a handle, so it needs no edge to learn the object exists, and the cache-coherence effect an acquire would buy is not required until the count reaches zero -- at which point the `AcqRel` decrement supplies it. That is the same argument `std::sync::Arc` makes, and it is a property of what the count is used for, not a general licence. A count whose value ever decided whether to *dereference* something would not qualify. | +| D-40 | **A relaxed atomic is chosen for its *atomicity*, and dropping to a plain field is never the way to "simplify" one.** [D-38](#d-38) says a relaxed operation is unordered; it is still indivisible, and that is frequently the entire reason the field is an atomic at all. Without it the implementation may synthesise a wide access out of narrower ones and observe a **torn** value, and concurrent access to a non-atomic field is a data race and therefore UB regardless. `reserving_mpsc`'s claim word is the worked example: it packs `reserved` and `position` into one `u64` and is uniformly relaxed ([D-38](#d-38)), yet a torn read would yield a pair that never existed as a state and would break the compare-and-swap protocol outright. On `i686-pc-windows-msvc` -- which [D-18](#d-18) deliberately keeps supported -- that load must be a `cmpxchg8b` or an 8-byte SSE load, *more* expensive than the two `mov`s a plain `u64` would get, and the compiler is obliged to pay it for exactly this reason. **Relaxed is a statement about ordering only; it is never a step toward removing the atomic.** | +| D-41 | **The claim word's apportionment is a caller's choice, and the recurrence behind SH-14.1 is a number the caller sets rather than one this crate imposes.** Supersedes [D-36](#d-36), whose premise was that the only fix was the [D-35](#d-35) claim-protocol replacement, gated on an open question -- so disclosing beat delaying. That premise was false, and measurement is what showed it: the 32/32 split followed from requiring the reservation half to hold the *entire capacity*, because every slot could be reserved at once. Capping outstanding reservations instead leaves the capacity bounded only by the ring, and the position is free to take 48 or 56 bits. `Balanced` (32/32), `Enduring` (16/48) and `Perpetual` (8/56) issue the **same** `lock cmpxchg` on the same `u64`, differing only in shift constants, and measured indistinguishable outside noise -- so the recurrence moves from about 37 seconds to about 20 years for no throughput and no dependency. The reservation half was the wrong half to spend bits on: it held 2^32 where the real bound is however many producers are mid-send. `Wide` (64/64 over a `u128`) is offered behind the non-default `dwcas` feature, because it is the one thing here that costs a third-party crate -- the standard library has no 128-bit atomic -- and it measured 2-3x slower on the claim; it buys a guarantee rather than a lifetime argument. **The default stays `Balanced`** so introducing the choice changed no existing caller's behaviour, and it is documented as *not* the recommended layout: leaving it because it is the status quo would preserve the hazard by inertia. | + +## D-2: capabilities are sliced, not gathered + +The first sketch of this crate had one `WaitableQueue` trait carrying push, pop, the doorbell, capacity, +and the loss latch. The engineer's observation that the shapes would be "sliced and diced by various +traits as we go along" is the correct instinct, and following it exposes that the fat trait is not merely +inelegant -- **it is unimplementable by the shapes that are planned.** A queue that is never waited on +has no doorbell to return; an unbounded queue has no capacity to report; a queue with no loss latch has +no losses to describe. + +So the contract is a set of narrow traits, each naming one capability, and a shape implements the subset +it genuinely has. The anticipated set, which is expected to grow: + +| Trait | Names | Held by | +|---|---|---| +| `Producer` | `push`, and the error a full or disconnected queue returns | producer handle | +| `Consumer` | `pop`, and drain-to-empty | consumer handle | +| `Waitable` | the readiness `HANDLE` | consumer handle | +| `Bounded` | `capacity`, `remaining` | either | +| `Reserving` | a slot claimed in advance for a message that must not be lost | producer handle | +| `LossReporting` | the coalesced loss latch | consumer handle | +| `Observable` | depth, high-water, doorbells actually rung | either | + +Two consequences worth stating, because they are what the slicing buys: + +- **A consumer can be generic over exactly what it needs.** The I/O domain runtime needs `Consumer` and + `Waitable` and nothing else; making it generic over a trait that also mentions reservation and loss + reporting would couple it to capabilities it never uses. +- **`Waitable` is not queue-specific and may not stay here.** "Hands out a `HANDLE` you can wait on" is a + property an event, a timer, or a completion port has too. If a second kind of thing wants to implement + it, the trait moves to a lower crate and this one depends on it. Recorded so that move is a planned + step rather than a surprise. + +## D-3: the traits ship with the second implementation, not the first + +Writing a trait against one implementation designs in a vacuum: every signature the single type happens +to have looks like a requirement, and nothing tests whether the abstraction is the right one. The +workspace already prefers duplicate-then-decide for exactly this reason -- keep the speculative path +separate until it is proven, then merge or delete. + +So the **shape** of the traits is fixed now, because it constrains the concrete types (D-4), while the +traits themselves are written when the second shape exists to be checked against them. The cheap +discipline that makes this work: write the intended signatures as a comment before writing the first +type, and confirm the type satisfies them. + +The failure this avoids is specific and unrecoverable-in-place. If the first shape ships +`pop(&mut self) -> Option` and the second ships `try_pop(&self) -> Result`, no trait unifies +them afterwards without a breaking change to one. + +## D-4: split handles, and cardinality carried by `Clone` + +The conventions for these shapes differ, and the difference is structural rather than cosmetic. An SPSC +queue is conventionally a split `Producer`/`Consumer` pair; a shared MPMC queue is conventionally one +`Arc` with `&self` on both ends. **No trait spans those two structures**, so a crate wanting a common +contract must choose one, and split handles are the choice that generalizes. + +It buys more than uniformity: + +| Shape | Producer | Consumer | +|---|---|---| +| SPSC | not `Clone` | not `Clone` | +| MPSC | `Clone` | not `Clone` | +| MPMC | `Clone` | `Clone` | + +Cardinality stops being a precondition in prose and becomes a fact the compiler enforces: a producer that +cannot be cloned cannot become a second producer. The alternative -- a shared `&self` queue documenting +"only one consumer" -- is precisely the rule-you-must-remember that +[`RingScope`](../windows-ioring-sys/DESIGN-NOTES.md#d-43) and `get(&mut self)` were introduced to +eliminate elsewhere in this workspace. + +The consumer handle also owns the doorbell, because the consumer is what waits; a producer merely rings. + +## D-5: the doorbell invariant, and which half must be under a lock + +The invariant is one sentence, and it is deliberately one-sided: **the event is never unsignalled +while the consumer has something to observe.** It is *level* state -- a function of the queue's +contents -- rather than a record of edges, which is why it is manual-reset. + +The converse does not hold, and stating it as "signalled exactly when" -- which an earlier wording +of this section and of [D-5](#d-5) both did -- promises more than the crate delivers, in a +paragraph immediately followed by the two bullets that contradict it. The event stays signalled +after the last item is taken until the consumer's own `arm()` clears it, and a late signal can +arrive after the consumer has already drained. So a wake is a **hint that there may be something**, +never a proof that there is; what the crate guarantees is that a wake is never *missing*. That is +why the consumer protocol is pop, `arm()`, re-check, and not "wait, then take". + +The asymmetry is the part that is easy to get wrong, and it was worked out by walking the interleavings: + +- **The signal may be given outside any lock.** A late `SetEvent` can at worst arrive after the consumer + already drained that item and parked, which produces a spurious wakeup: the consumer wakes, finds + nothing, parks again. Harmless, and consumers must tolerate it regardless. +- **The reset must not be separable from the observation that there is nothing to take.** Otherwise: + consumer drains to empty, producer pushes and signals, consumer resets -- clearing the signal for an + item that is still there -- and parks. That wakeup is lost and the item is stranded. + An earlier wording of this said the two must be *atomic*, which is how a lock achieves it but not the + only way, and taken literally it would have condemned the lock-free implementation this crate actually + ships. What is required is that no push can fall between them unnoticed; [D-9](#d-9) gets that from + ordering plus a re-check instead of from mutual exclusion. + +So a redundant signal is free and a stale reset is fatal, which is the whole reason the queue owns its +doorbell rather than accepting one. The same invariant, reached independently, is stated in +[windows-file-watcher's queue](../windows-file-watcher/src/queue.rs): signalling under the lock a +receiver holds while deciding there is nothing to take, "so a wakeup cannot be lost in the gap between +those two decisions, because there is no gap". + +**What is reused from that queue is the invariant, not the implementation.** It uses `Mutex` and +`Condvar`, which is right for change-notification cadence and wrong here, because a producer-side lock +serializes exactly what multi-producer exists to parallelize. + +**Created lazily**, so a consumer that only ever polls allocates no kernel object at all. Handed out as +a borrowed handle plus an owned duplicate, so a caller can choose whether to own it. + +**Skipping a redundant signal is an optimization, not a requirement.** Measured on ARM64: one +`SetEvent`/`ResetEvent` cycle is ~165 ns against a ~7 ns uncontended atomic, but one doorbell per drained +*batch* costs 20.6 ns per operation at a batch of eight and 5.2 at thirty-two -- so by a batch of about +twenty-three the doorbell already costs less per operation than the push it accompanies. The cheap skip +(the queue was already non-empty) needs no knowledge of the consumer and can be taken immediately; the +one requiring the consumer to publish whether it is parked is deferred until a measurement against real +work justifies its lost-wakeup risk. + +## D-9: the arming protocol, which is how a lock-free queue keeps D-5 + +[D-5](#d-5) says the reset must not be separable from the observation that there is nothing to take. A +lock-based queue gets that by doing both under the lock it already holds, which is what +[windows-file-watcher's queue](../windows-file-watcher/src/queue.rs) does. This crate's shapes are +lock-free by construction -- a producer-side lock serializes exactly what multi-producer exists to +parallelize -- so the property has to come from somewhere else. + +It comes from **ordering plus a re-check**, and the order is the reverse of the one that reads +naturally: + +1. Take everything available. +2. Clear the doorbell. +3. **Check emptiness again.** If anything is there, do not wait. +4. Wait. + +`Consumer::arm` is steps 2 and 3, and returns whether step 4 is safe. Step 3 is what carries the +guarantee: an item arriving before the clear is found by the check, and an item arriving after the clear +signals a doorbell that is no longer about to be reset. + +**"There is no third case" is what this decision originally said next, and it was wrong** -- see +[D-15](#d-15). It holds for `spsc`, where one producer and one position mean that *any* push before the +clear makes the check find something. It fails for `slotwise_mpsc`, where the check asks whether the *head* slot +is published: a producer publishing at a later position before the clear is the third case, invisible to +the check. The remedy is in `Doorbell::clear` rather than here, because what that case needs is not a +better check but a doorbell that is guaranteed able to ring again once the clear returns. + +**Check-then-clear is the lost wakeup**, and it is the easier code to write: a push landing between the +check and the clear both signals and has its signal erased, so the consumer sleeps on a queue that is +not empty and will never be signalled again. Not a stall -- a permanent hang. + +**Lazy creation is a third case of the same hazard.** A producer running while no event exists skips +signalling, because there is nothing to signal. So the doorbell must be created *before* the emptiness +check that decides to wait, which is why `arm` creates it rather than assuming a caller did. + +**The ordering above is necessary and, on its own, was not sufficient -- this decision originally said +it was.** A code review found the hole. Program order does not relate the producer's decision to skip +signalling to the consumer's emptiness check, because each side *stores* one location and then *loads* +another: the producer stores the queue position and loads the doorbell state, while the consumer stores +the doorbell state and loads the queue position. That is the store-buffer shape from Dekker's +algorithm, and release/acquire permits both loads to return stale values. When both do, the item is +queued, no signal was raised, and the consumer parks forever. + +The remedy is a `SeqCst` fence on each side -- before the loads in `Doorbell::signal`, after the stores +in `Doorbell::clear`. Every published eventcount carries the same fence in the same place for the same +reason, which is the clearest sign that this is a known shape rather than a local quirk. + +The original text had actually *identified* the sequential-consistency requirement and then dismissed +it, on the reasoning that the re-check closed the hole for free and the fence was only needed for a +different design (signalling at creation time). That reasoning was wrong, and the shape of the error is +worth keeping: the re-check closes the *program-order* version of the hazard, which is the one that is +easy to picture, and leaves the *visibility* version, which is not. Reasoning about interleavings in +terms of "what happens first" silently assumes the sequential consistency that is exactly what is +missing. + +Two temptations recorded as refused, both instances of +[PLATFORM INTEGRITY](../../.github/copilot-instructions.md) rule 2. The consumer's `ResetEvent` is a +syscall and is very probably a full barrier; and `stlr`/`ldar` on aarch64 are ordered more strongly +than the abstract model demands. Either would likely mask this defect on today's toolchain and today's +processors. Neither is a specified guarantee, and binding correctness to the incidental behaviour of a +code generator plus a particular processor -- rather than to the ordering primitives -- is the precise +trap that rule exists to name. + +**No test can catch this, and that is a property of the hazard.** Removing either fence leaves the +whole suite green, and no entry in [`sabotage.json`](sabotage.json) can express it, because the defect is a fact about +the memory model rather than an interleaving a scheduler can be coaxed into producing. It is the named +target of the planned `loom` verification. + +**This is asserted by sabotage, not by argument.** The suite reverses steps 2 and 3 deliberately and +requires the result to hang -- a real `WaitForSingleObject` that returns `WAIT_TIMEOUT` while an item +sits in the queue. The race is driven deterministically from one thread, because an interleaving that +must be hit to prove a point is not one to leave to the scheduler. Three further sabotages (push not +signalling, producer `Drop` not signalling, `clear` not resetting the mirror flag) are likewise caught +*as hangs*, which is the correct shape for this class of defect and the reason the sabotage harness +judges by exit code with a timeout rather than by reading output. + +**The signal side is cheapened, and a control proves that is all it is.** An `AtomicBool` mirrors the +event so a redundant `SetEvent` costs ~7 ns instead of ~81. Removing that optimization must leave the +suite green -- and does. Had it failed, the tests would have been asserting the implementation instead +of the contract. + +## D-6: overflow fails or reserves, and never overwrites + +Three policies, and the absence of a fourth: + +- **Fail fast.** A full queue returns the item to the caller in a typed error. That failure *is* the + backpressure, and it is why the shapes are bounded: an unbounded queue has no backpressure to offer, + only deferred memory growth. +- **Reserve.** A slot claimed in advance, so a message that must not be lost has somewhere to go. Taken + from [windows-file-watcher's queue](../windows-file-watcher/src/queue.rs), which needs it for exactly + the same reason: some messages are the ones a consumer cannot afford to miss. +- **Coalesced loss latch.** When a queue may lose, a drop latches a report the consumer is guaranteed to + observe, so loss is *counted* rather than silent. Also from the watcher. + +**Overwrite-oldest is deliberately not offered.** `crossbeam`'s `force_push` makes an `ArrayQueue` usable +as a ring buffer, which is right for telemetry, where an overwritten entry is a lost sample. Here an +entry is an I/O submission, and overwriting one is a lost *operation*. The two cases must not share a +knob, because a knob invites a consumer to choose the wrong one. + +## D-7: shapes are modules, not Cargo features + +An earlier position in the session was "one crate, feature-gated shapes". The first half stands -- one +crate, not a crate per family, so the shared vocabulary lives in one place. The second half does not +survive contact with the cost: two features are four configurations, and this workspace already runs a +`feature-matrix` CI job that would have to grow to cover them. The benefit -- not compiling a shape you +do not use -- is one dead-code elimination already provides for an unused type. + +Feature-gating remains available if compile time ever justifies it. It is not the default, and the burden +of proof is on adding a feature rather than on leaving one out. + +## D-8: published, and what that commits us to + +Publishing is an obligation rather than a status. It means the API is a contract that cannot be changed +casually, that a breaking change costs a major version, and that the crate must be documented for readers +who have never seen this workspace. + +It is accepted because this crate is general-purpose in a way `windows-guard-alloc` is not. That one is +`publish = false` precisely because its design trades memory for determinism and would be wrong for +anything but a test binary. These queues carry no such trap, the first consumer is not the only plausible +one, and a Windows Rust program that wants to wait on a queue and a kernel object together currently has +to write this itself. + +## D-10: the slot-wise MPSC shape is Vyukov's bounded array queue + +The obvious multi-producer array queue claims an index with a fetch-and-add, writes the slot, and lets the +consumer read it. It does not work, and the reason is worth stating because it is the whole justification +for the extra machinery: **the consumer cannot tell a slot that has been claimed from one that has been +written.** A producer preempted between the two leaves a hole, and a consumer reading through the hole +reads uninitialized memory. + +A sequence number per slot carries both facts at once. Slot `i` starts at `i`; a producer may claim +position `pos` only when the slot reads `pos`, and publishes by storing `pos + 1`; the consumer takes the +slot only when it reads exactly `pos + 1`, and frees it by storing `pos + capacity`, the position the next +lap will claim it at. A claimed-but-unwritten slot is therefore invisible to the consumer, and there is no +hole to read through. + +What this buys, and what it costs: + +- **Bounded by construction, so backpressure is free.** A full queue is a slot whose sequence has not come + round, which costs one load to discover. There is no separate count to maintain, no allocation to fail, + and no policy knob -- the refusal *is* the backpressure, which is [D-6](#d-6) in its cheapest form. +- **No allocation after the constructor**, which is what makes it usable on an I/O submission path. +- **Lock-free, not wait-free.** A producer that loses its compare-and-swap retries, with no bound on how + many times it may lose. What is guaranteed is that some producer always makes progress, and -- the + property that actually matters here -- that a producer suspended by the scheduler blocks no other + producer. It blocks only the consumer's view of the items queued behind it, and only until it resumes. +- **Order is claim order, not publication order.** If producer A claims position 5 and producer B claims + 6 and publishes first, the consumer must wait for A. This is not a defect to engineer around: it is what + makes the queue a FIFO at all. B's signal wakes a parked consumer that then finds nothing, which is a + spurious wakeup the protocol already tolerates, and A's own signal follows when it publishes. + +The head and the tail are padded onto separate cache lines. The padding is load-bearing and looks like +waste, which is why it is commented at both fields rather than at one: every successful push writes the +tail and every successful pop writes the head, so adjacent they would false-share, and each write would +invalidate the other side's copy of a value it only reads. That cost has no symptom other than being +slow, which is exactly the kind that survives a code review. + +## D-11: the traits shipped here, and the check D-3 demanded was actually run + +[D-3](#d-3) said no trait ships until a second implementation exists to validate it, and that the trait +*shape* would be fixed in advance so the concrete types could not diverge. `spsc` accordingly wrote its +intended signatures into its module documentation before its types existed. This milestone is where that +promissory note came due. + +**The signatures held unchanged.** `push`, `pop`, `is_disconnected`, `capacity`, `len`, `is_empty` are +what [`traits.rs`](src/traits.rs) says now and what that comment said then. The check is not rhetorical: +`slotwise_mpsc` is a lock-free array queue with a per-slot state machine and no structural resemblance to a +two-position ring, so a signature fitted to the first shape would have failed here rather than in a +consumer's code. + +**One choice turned out to be the load-bearing one, and it is worth naming.** `push(&self)` rather than +`push(&mut self)`. `&mut self` would have been perfectly sound for a single producer, is what several SPSC +crates use, and would have made this trait *unimplementable* by a shape whose whole point is several +threads pushing at once. It was chosen in advance on the argument that one spelling has to serve every +shape; this shape is the evidence that the argument was right. + +Two smaller decisions recorded so they are not re-litigated: + +- **The traits are also the names of the concrete handles.** `Producer` and `Consumer` are both a trait + and, in each shape's module, a type. That is deliberate -- the trait is named for the role, the handle + is named for the role, and the handle plays the role -- and `std` does the same with `fmt::Write` and + `io::Write`. A caller wanting only the methods imports them anonymously (`Consumer as _`). +- **`Reserving`, `LossReporting` and `Observable` from [D-2](#d-2)'s table are deliberately still absent.** + They belong to work that has not happened (M31.2, M31.4), and shipping an empty trait now would be the + design-in-a-vacuum D-3 forbids, one level up. + +## D-12: the minimum capacity belongs to the shape, and `slotwise_mpsc`'s is two + +`spsc` accepts a capacity of one. `slotwise_mpsc` cannot, and the reason is arithmetic rather than taste. Its slot +sequence distinguishes three states by counting -- `pos` is free, `pos + 1` is published, `pos + capacity` +is free again on the next lap -- and when `capacity == 1` the second and third are the *same number*. A +producer would read the sequence of the item it had just pushed, conclude the slot was free, and overwrite +an item the consumer had not read. + +**It is reported, not worked around.** The obvious workaround -- allocate two slots and refuse the second +-- reintroduces a load of the consumer's position on the producer's hot path, which is precisely the cost +the sequence protocol exists to avoid, and it would impose that cost on *every* queue in order to serve a +capacity of one. A caller that genuinely wants a one-item handoff wants `spsc`, which represents it +exactly. + +The consequence for the error type is small and was anticipated: `CapacityError` already carried a +`max_valid` on the argument that a bound "follows from how a shape represents its positions", and it now +carries a `min_valid` for the same reason. The suggestion methods respect it, so `bounded::(1)` on an +`slotwise_mpsc` reports `next_valid() == Some(2)` rather than a correction that would itself be refused. + +Each shape names its own minimum as a documented constant next to the code that needs it, rather than +passing a bare literal, so the number is never separated from the reason for it. + +## D-13: the arming protocol is stated once, and shapes bind to it + +The blocking receive loop is not glue around [D-9](#d-9) -- it *is* D-9, executed: drain, arm, check for +disconnection, and wait only if arming blessed it. Every step is load-bearing and the order is the whole +correctness argument. + +So it lives in [`blocking.rs`](src/blocking.rs), and a shape gains `recv` and `recv_timeout` by +implementing a crate-private `Parked` trait. A second shape spelling the loop out again would be a second +copy of a rule, free to drift, and -- the failure mode that actually bites -- free to *look* verified while +only the copy was tested. This crate has already paid for that once: the first lost-wakeup proof exercised +a hand-written duplicate of `Consumer::arm` and was structurally incapable of noticing the real `arm` +being reversed. The `ARM_RACE` hook is shared for the same reason. + +`Parked` is deliberately *not* one of the public capability traits. The public traits say what a caller may +ask of a queue; `Parked` says what the blocking loop needs from one, and the difference shows in `finish`, +whose contract is a precondition no external caller can check. + +## D-14: ``slotwise_mpsc`` arms on readiness, not on emptiness + +`Consumer::arm` must answer "is it safe to park?", and for `slotwise_mpsc` that is not the same question as "is the +queue empty". They disagree over a slot a producer has claimed but not yet published, and the disagreement +matters in both directions: + +- **`len` says non-empty**, because it counts the claim. Arming on that would refuse to bless the wait, and + the consumer would spin -- calling `pop`, getting `Empty`, re-arming, getting `false` -- until the + producer was rescheduled. Correct, and a burnt core. +- **Readiness says nothing is takeable**, so the consumer parks. That is safe precisely because the + producer's publishing release store is followed by a signal, so the wakeup is guaranteed to arrive. + +Arming therefore asks `Shared::has_ready_item`, which is the exact question `pop` answers: is the slot at +the head position published? `len` keeps its cheaper definition and its documented over-count, because it +is a metric rather than a control-flow input. + +This also places the `SeqCst` pairing from D-9 correctly for this shape: the producer stores the slot's +sequence and then loads the doorbell state, while the consumer stores the doorbell state and then loads +that same sequence. It is the same store-buffer shape, over the same two fences, with a different pair of +locations. + +## D-15: the clear order, and the assumption that hid a lost wakeup + +`Doorbell::clear` has two lines: reset the kernel event, and clear the `AtomicBool` that mirrors it so a +redundant `signal` can skip its syscall. **They originally ran flag-first, and that order is a permanent +hang.** A producer signalling between them finds a clear flag, sets it, and issues a real `SetEvent`; the +`ResetEvent` that follows erases that signal and leaves the flag set. The doorbell is then dark while +claiming to be lit, so every later `signal` skips, and a consumer parked on it never wakes. + +The flag is allowed to lie in exactly one direction -- claiming lit while the `SetEvent` has not landed +yet, which costs a skipped *redundant* signal. The order above produced the opposite lie, which costs the +one signal that mattered. + +**Why it survived review and a sabotage sweep.** The original argument was explicit and looks airtight: +the racing producer publishes *before* it signals, so the caller's re-check sees the item and does not +wait. It is sound -- for a queue whose re-check is guaranteed to see anything any producer published. +`spsc` is such a queue: one producer, one tail, and `is_empty` covers every push. So the argument was +tested against the only shape that could not falsify it, and it was written down as a general rule. + +`slotwise_mpsc` falsifies it. Its re-check asks whether the **head** slot is published ([D-14](#d-14)), so a +producer publishing at a later position is invisible to it. The consumer parks in exactly the wedged +state, the producer holding the head publishes, its `signal` is skipped, and the queue hangs with an item +sitting in it. + +**How it was found, which is the part worth keeping.** Not by review, and not by the test suite: the +suite passed 120 tests in 0.28 s, six runs in a row. It was found because the sabotage harness refuses to +sweep against a red baseline, and its *baseline* run -- the one that exists only to prove the suite is +green before any defect is injected -- hung once in +`slotwise_mpsc::tests::many_producers_deliver_every_item_exactly_once`. A single unreproducible hang is exactly +the finding it is tempting to dismiss as a slow machine, and the crate's own sabotage documentation +already says not to: "a flaky sabotage is a finding, not noise". The same applies to a flaky baseline. + +**The fix moves the guarantee from the caller to the type.** With the event reset first, the invariant is +a property of the doorbell rather than an obligation on whoever calls it: *once `clear` returns, the flag +is false, so the next `signal` cannot be skipped.* A producer signalling inside the window may still be +skipped, but it published before it signalled and therefore before the flag store, so the caller's +re-check observes whatever that publication made observable; and any producer that publishes after the +re-check finds the flag already false and rings for real. No caller has to reason about it, which is the +point -- the previous arrangement required every future shape to have a re-check strong enough to cover +the window, and no signature said so. + +**It is asserted deterministically, at the layer that owns it.** `race_hooks::CLEAR` fires inside the +real `clear`, between its two lines, and a test signals from there on one thread. The assertion is not +about the state immediately afterwards -- both orders leave the event dark -- but about what a consumer +depends on next: `signal` must still be able to ring. Reversed, the test fails every run; a sabotage +entry keeps it that way. A control with an empty window sits beside it, so the test cannot pass by +`clear` simply never leaving the doorbell ringable. + +**Two temptations refused.** Making `slotwise_mpsc` arm on `len` instead of readiness would also have masked this, +by restoring the property that any push makes the re-check find something -- but it would have left the +doorbell able to reach the inconsistent state, waiting for the next shape, and it would have cost the +consumer a spin whenever a claim was in flight. Adding a lock around the two lines would have fixed it +and thrown away the reason the flag exists. + +## D-16: reservation is a capability, so the reserving queue is a peer and not a replacement + +[D-6](#d-6) said overflow "fails or reserves, and never overwrites", and quietly assumed one queue would +carry both policies. Building the second one showed that assumption was wrong, and why. + +**The cost claim in this section is falsified; the structural claim is not.** Read what follows as an +account of *why the two shapes differ*, which remains correct, and not as an account of which is +cheaper, which [D-26](#d-26) reversed. [D-29](#d-29) records what the split rests on now. + +**Honouring a reservation costs the producer something on every push, including the pushes that never +reserve anything.** `slotwise_mpsc`'s producer never reads the consumer's position: it asks the slot's own +sequence number "are you free?", and those are spread across the slot array, so producers working at +different positions touch different cache lines. Avoiding a single shared position is not incidental to +Vyukov's design; it is most of the point of it. + +A reservation cannot be answered from that question. "Is this slot free" does not say **how many** slots +remain, and withholding one from the best-effort path requires exactly that count -- which requires the +consumer's position, on one line every thread in the system touches. + +So the choice was: pay that on every `slotwise_mpsc` push, or ship two shapes. Two shapes, for three reasons: + +- **The cost falls on the shape M31.5 exists to measure.** Degrading + `slotwise_mpsc`'s push before the contention benchmark runs would corrupt the measurement that decides whether + the deferred shapes are needed at all. +- **The crate is built for this.** It is named in the plural, [D-7](#d-7) makes shapes plain modules, and + [D-4](#d-4) already has shapes differing in what they can do. A third one is the pattern working, not + an exception to it. +- **It is [D-2](#d-2)'s argument reaching its sharpest case.** `slotwise_mpsc` does not implement `Reserving` + because it genuinely *cannot*, not because nobody got round to it -- which is exactly the situation + narrow traits were chosen for. A fat trait would have forced the cost on both shapes or excluded + reservation from the contract entirely. + +**The alternative shape considered and refused was a permit counter** -- one atomic that both producers +and the consumer read-modify-write, acquiring on push and releasing on pop. It is correct and simpler to +read, and it was rejected because it puts a second contended read-modify-write on the push path where the +packed word puts one shared *load*. Its only advantage is preserving the crate-wide capacity ceiling, and +[D-17](#d-17) explains why that ceiling is unreachable anyway. + +**The merge-or-delete decision is deferred to M31.5, deliberately and with a trigger.** If the benchmark +shows the shared-line read costs little at realistic contention, `slotwise_mpsc` and `reserving_mpsc` should merge +and the plain one should go. If it shows the read is expensive, both stay. What must not happen is the +duplicated path becoming permanent because nobody circled back, so the decision is recorded as an item on +M31.5 rather than as an intention here. + +## D-17: the reservation count and the claim position share one word + +**The obvious implementation is broken, and it is worth writing down why, because the brokenness is not +visible from reading either side on its own.** With the count in its own atomic: + +1. A pushing producer reads the count, sees room, and claims the position. +2. A reserving producer increments the count, reads the position, sees room, and grants. + +Each read before the other's write. The queue now owes a slot that does not exist, and the guarantee the +whole feature rests on is gone. + +**Sequentially consistent fences do not close this**, which is the part that surprises -- they *do* close +the superficially identical hazard in [D-9](#d-9). The Dekker argument needs store-then-load on both +sides. Here the pushing producer is **load**-then-store: it reads the count and then writes the position. +Writing the four operations into a single total order, `L_push < S_reserve < L_reserve < S_push` is +consistent with every side's program order, so both sides missing each other is permitted and no fence +forbids it. + +Two independent claimants on one resource must synchronise on **one location**. So the count and the +position become one location: a single `AtomicU64`, low 32 bits the position, high 32 the count. Every +operation that changes either changes both, with one compare-and-swap. + +Three consequences fall out, and all three are improvements: + +- **Redeeming is one exchange** that decrements the count as it advances the position, so + `occupied + reserved` -- the quantity the invariant is about -- is never momentarily wrong. +- **A racing `reserve` and `push` cannot both win.** The loser's exchange fails and it re-reads, which is + the ordinary lock-free retry rather than a special case. +- **The producer stops needing the slot sequence for the "free" direction**, because it now reads the + consumer's position anyway. So `reserving_mpsc`'s `pop` is one store shorter than `slotwise_mpsc`'s: nothing + writes a "free again" sequence. + +**The 32/32 split is forced, not chosen.** A position of `b` bits keeps a wrapping difference unambiguous +only up to `2^(b-1)`; the count can reach the capacity, so it needs `b` bits too; `b + b = 64` gives +`b = 32`. There is no cleverer division of the word, and the resulting ceiling is 2^31 items -- a ring +this shape allocates in full at construction, so at eight bytes an item it is already 17 GB. + +That ceiling is reported through `CapacityError`'s `max_valid`, which [D-12](#d-12) had already made a +property of the shape rather than of the crate. D-12 introduced that for the *minimum* and argued the +maximum worked the same way; this is that argument being cashed. + +**The invariants the packing depends on are `const` assertions, not tests**, because they are facts about +constants: a test can only report after the fact, on a build somebody chose to run. Worth recording that +the first version of those assertions was *tautological* -- it asserted that `BOUNDS_MAX` equalled its own +definition -- and widening the position to 40 bits sailed straight past it while silently narrowing the +count's field to 24 bits, which is the way the packing actually breaks. The assertions now name the +constraint that binds: the count's half must be wide enough to hold the whole capacity. + +## D-18: a 128-bit compare-and-swap is refused + +**Superseded by [D-37](#d-37).** A 128-bit exchange is now adopted, but for a **separate wide +shape** rather than for this one: `reserving_mpsc` keeps its packed 64-bit word on every target, and +`reserving_mpsc_wide` is a peer beside it. Read this decision for the cost analysis, which D-37 +depends on and does not repeat -- and note one correction it needs, below, that D-37's gate is built +around. + +**The i686 failure is not a build failure, which is worse.** The amendment below says i686 "has no +128-bit atomic at all", which reads as "it will not compile". It compiles: `portable-atomic`'s +default `fallback` feature silently substitutes a **global lock**, so the queue keeps working and +stops being lock-free, contending with any unrelated user of the fallback in the same process. And +`target_has_atomic="128"` is emitted even with `cmpxchg16b` disabled -- verified on 1.98 -- so a cfg +gate does not catch it either. That silent degradation, not a compile error, is the real reason this +shape may not simply widen its word. +D-37 turns it back into a build failure by the simplest available means: depending on +`portable-atomic` with **`default-features = false`**, which withholds the `fallback` feature, so +`AtomicU128` does not exist at all on a target that cannot do it natively. + +**Amended 2026-09-02. The refusal stands; almost none of its original reasoning does.** The first +version of this decision was written before SH-14.1 +existed, and it asserted target facts that were never checked against the toolchain. Both faults are +corrected below, and the correction is kept rather than silently rewritten because the *shape* of the +error is instructive: a decision can reach the right outcome and still leave every reason a reader +would rely on wrong. + +The natural question about [D-17](#d-17)'s packing is why not use `cmpxchg16b` (or `CASP` on aarch64) +and keep both halves full width. + +### What was claimed, and what is actually true + +**"It would lift the 2^31 cap and nothing else" -- wrong, and this is the substantive correction.** +A 64-bit position field would also collapse SH-14.1's ABA recurrence, which is a correctness hole and +not a capacity limit. The original decision denied the existence of what is now the option's main +benefit, purely because the hole had not yet been found. It remains true that a wider producer word +does nothing about *the cost that matters* -- free space is `capacity - (position - head) - reserved`, +`head` belongs to the consumer, and no width of producer-side exchange brings it into the producer's +word -- but "no performance benefit" was never the same claim as "no benefit". + +**"It is not in the x86-64 baseline" -- false on the pinned toolchain.** Checked rather than assumed: +`rustc 1.98.0 --print cfg --target x86_64-pc-windows-msvc` emits `target_feature="cmpxchg16b"` and +`target_has_atomic="128"`. There is no target-feature floor to raise and no runtime detection to pay +on the push path. The original text reasoned from the generic x86-64 baseline and never checked the +*Windows* target, which enables the feature by default. + +**"It is not even the same instruction on aarch64" -- true but not a cost.** `aarch64-pc-windows-msvc` +reports `target_has_atomic="128"` with no target feature required, because `ldxp`/`stxp` is ARMv8-A +baseline. That the instruction differs from x86-64's is what an atomics abstraction is for. + +**"There is no usable `AtomicU128`" -- true, verified.** Still unstable (rust-lang/rust#99069); a +test compile on 1.98.0 fails. Reaching a double-width exchange from stable means adding +`portable-atomic` to a workspace whose only third-party dependency is `windows-sys`, on a crate that +is [published](#d-8). **This is the one original reason that survives.** + +### The reason the decision actually rests on now + +**`i686-pc-windows-msvc` has no 128-bit atomic at all** -- `rustc --print cfg` reports +`target_has_atomic="64"` and no `"128"`. So this option is not "widen the claim word"; it is "widen +the claim word **and** drop 32-bit support", which collapses +SH-14.3's option 1 into its option 4. Narrowing the +platform is the engineer's decision under the repository's platform-integrity rule, not something a +correctness fix may take in passing. + +That is a stronger and simpler reason than the three it replaces, and it is the one to quote. + +### When to revisit + +For a **tagged pointer**, which is what the linked and sharded shapes parked in `M-inf.1` would need +-- or if 32-bit support is dropped for unrelated reasons, at which point the option becomes a live +candidate for SH-14.1 rather than a non-starter. Recorded so the question does not have to be +re-derived a third time. + +## D-19: the coalesced loss latch does not generalise + +[windows-file-watcher's queue](../windows-file-watcher/src/queue.rs) carries a third policy beside +fail-fast and reserve: a failed enqueue latches the affected `WatchId` in a set held *outside* the bounded +queue, where it coalesces, and is drained back in at the next successful enqueue. It is a good design and +this crate deliberately does not copy it. + +**Coalescing is sound there because a desync is idempotent.** Two lost notifications for one subscription +mean the same thing as one -- the client must re-scan -- so collapsing them loses nothing, and that is +what makes the latch lossless despite being bounded by the number of subscriptions rather than by the +number of losses. + +A queue of arbitrary `T` has no such property. There is no general way to collapse two lost `T`s into +one, and no general way to say what a client should do about them. What *does* generalise is the part +that does not depend on the payload: **a count of what was refused**, so loss is measured rather than +silent. That is observability, and it belongs to M31.4 rather than to the +overflow policy. + +So this crate's answer to a full queue is: refuse and hand the item back, or hold a reservation so the +refusal cannot happen to the messages that cannot survive it. A caller whose payload *is* idempotent can +build the watcher's latch on top of the typed refusal, which is the right layer for a decision that +depends on what the payload means. + +**Overwrite-oldest remains refused outright**, as [D-6](#d-6) said. `crossbeam`'s `force_push` makes an +`ArrayQueue` usable as a ring buffer, which is right for telemetry where an overwritten entry is a lost +sample. Here an entry may be an I/O submission, where it is a lost *operation*. The two must not share a +knob, because a knob invites a caller to pick the wrong one. + +## D-20: teardown hands undrained items back, and the decision is made at construction + +R8 asks that +descriptors in flight at teardown be **accounted, not dropped**, "because some own handles, and their +disposal must be allowed to block". The 2026-08-27 namespace session states the same hazard concretely: +an async open's completion carries an owned handle, and closing one to a dead network path is exactly the +blocking operation the whole facility exists to keep off a caller's thread. + +**The default answer to "who destroys the items nobody drained?" was bad in a way that is easy to miss.** +They were destroyed in place, inside the last `Arc` release -- so `T`'s destructor ran on whichever thread +happened to drop last. That thread is not knowable in advance and nobody chose it: it may be a thread-pool +callback that must not block, or a producer with no idea it was holding the last reference. Nothing told +the owner it had happened. + +**`Drop` cannot be made to hand them back.** It takes `&mut self`, returns nothing, and cannot fail; by +the time it runs every handle is gone, so there is nobody left to return anything *to*. Whatever the queue +is going to do with those items, it has to have been told beforehand. That is the whole reason [`Disposal`] +is supplied at construction rather than asked for at teardown -- not ergonomics, but the shape of the only +place that sees every survivor. + +So a queue built with a sink hands each survivor to it. The owner then decides where disposal happens: a +sink that moves items to a reaper thread keeps the blocking off the dropping thread entirely, while one +that disposes inline is perfectly fine when the dropping thread is allowed to block. Either way it is a +decision somebody made. + +**The default is unchanged and still destroys in place.** For items that own nothing -- which is most of +them -- that is exactly right, and a queue of `u32` should not have to think about any of this. What +changed is that the behaviour now has a name and an alternative. + +**The claim under test is about threads, not counts.** It would be easy to assert only that the sink +receives the items, which is the mechanism rather than the property. The suite instead records the +`ThreadId` a destructor runs on and asserts it is *not* the thread that released the last handle -- with a +control, without a sink, showing it *is*. That control matters: without it the first test would look +identical if destructors simply never ran anywhere observable. + +Each shape walks its own layout to find survivors, so the routing is asserted once per shape rather than +once for the crate. That is the lesson M31.2's sweep taught about the reservation guarantee, applied +before the sweep had to teach it again. + +## D-21: a panicking sink is caught, and the walk continues + +The sink is caller-supplied code running inside a destructor, which is the worst place for it to panic. +A panic escaping there does one of two bad things: during an unwind it aborts the process, and otherwise +it abandons every item not yet disposed -- precisely the handles the mechanism exists to account for. + +So the call is wrapped and the walk continues. This is deliberately **not** "swallowing an error": the +item has already been moved into the sink, so there is nothing left to report about it, and the item is +destroyed by the unwind rather than leaked. A sink that panics is a bug in the caller; catching only +declines to turn it into a much larger one. + +`AssertUnwindSafe` is the honest annotation rather than a way past the bound. The only state observable +after a panic is the caller's own closure, and the queue's invariants do not depend on the sink at all -- +teardown is already past the point where anything could observe them. + +## D-22: no `into_remaining`, because it would not close the hole + +The obvious API for shutdown is "consume the consumer, get everything that is left". It was considered +and refused, for two reasons that compound. + +**It does not close the hole.** A consumer can take everything *available*, but producers may still push +afterwards -- so it covers the orderly path and nothing else, and the disorderly path is the one that +strands handles. The last handle to drop remains the only place that sees every survivor, which is where +[D-20](#d-20) puts the mechanism. + +**And it adds nothing over what exists.** `Consumer::drain` already takes everything available; an +`into_remaining` would be that plus consuming the handle. Since the sink covers the case `drain` cannot, +the extra method would be surface without capability. + +The orderly shutdown therefore stays what it already was: drain to empty, observe +`Consumer::is_disconnected`, and take the final item with the receive loop's `finish` step. The sink is +for everything that does not go to plan. + +## D-23: high-water is opt-in; refusals and rings are not + +R9 asks for three numbers, and the interesting thing about them is that they do not cost the same. + +**A counter on a hot path is a shared line every thread writes** -- the same false-sharing cost the +positions are carefully padded apart to avoid. So each was placed where it is already paid for, and the +one that could not be placed that way became a switch: + +| Metric | Where it increments | Cost | +|---|---|---| +| Refusals | only when a push is refused | off the success path entirely | +| Doorbell rings | only when `SetEvent` is actually called | ~7 ns against a syscall measured at ~81 ns | +| Peak depth | must observe **every** change | see below -- and it varies by shape | + +Peak depth is the awkward one, and the awkwardness is not uniform: + +- **`spsc`** -- free. The producer already loads `head` to decide there is room and owns `tail`, so the + depth is a subtraction of two values in hand, and the counter's line is producer-owned. +- **`reserving_mpsc`** -- near-free, for the same reason: its producer reads `head` for the room check + that honours reservations. Only the counter's line is shared, and it is written rarely. +- **`slotwise_mpsc`** -- *not* free. Its producer never reads `head`; that is the whole property + [D-16](#d-16) built a separate shape to preserve, because `head` is the one line every thread touches. + Tracking makes it read that line on every push. + +Making it always-on would have imposed D-16's refused cost on every `slotwise_mpsc` user to serve a metric most of +them will never read -- and would have done it just before M31.5 measures +exactly that path. Omitting it from `slotwise_mpsc` would have narrowed the shape. So it is a switch, off by +default, and the cost lands only on queues that asked. Off, `slotwise_mpsc` pays one predictable branch on a field +written once at construction: the line is shared but read-only, which is the cheap kind. + +**Untracked reports `None`, not `0`.** They are different answers -- "nobody was counting" versus "it +never filled" -- and a caller sizing a queue from the second when the first was true would be reading a +number nobody recorded. + +**Two independent switches across three shapes is why `Options` is a builder.** As constructors that is +four functions per shape and twelve in the crate, and every future switch doubles it. The plain `bounded` +stays, because the default is the common case and should not have to say so. + +`record_depth` loads before it modifies. An unconditional `fetch_max` would be a read-modify-write on a +shared line for every push; a new maximum is rare after a queue warms up, so the common case becomes a +plain load of a rarely-written line and the read-modify-write is reached only when the value is actually +about to change. The load may be stale, and the `fetch_max` behind it is what keeps the result correct +regardless -- which is asserted by a concurrent test rather than argued. + +## D-24: counting the rings makes the skip part of the contract + +R9 asks for the ring count so that "disabling the skip must change the number". Following that literally +has a consequence worth naming, because it inverts something this crate had already written down. + +[`sabotage.json`](sabotage.json) carried an entry that removed the skip optimisation, expecting **`survives`**. It was a +*control*: skipping a redundant `SetEvent` changed no observable behaviour, so a suite that went red on +its removal would have been asserting the implementation instead of the contract -- and +[D-9](#d-9) records that the control earned its place by proving exactly that. + +Once the rings are counted, that stops being true. The count is observable, so the skip is observable, and +the same patch that had to survive now has to be **caught**. The entry changed sides in M31.4. + +**This is the requirement working, not a regression.** An optimisation nobody can measure is an +assumption, and R9's whole point is to stop this one being one. What it costs is that the skip is now part +of what the queue promises rather than a private cleverness -- so removing it later would be a behaviour +change, not a refactor. That is the right trade for a queue whose entire reason to exist is a wakeup +protocol, but it is a trade, and it should be made knowingly. + +The control it vacated is replaced rather than dropped: `slotwise_mpsc`'s guard around the `head` load is an +optimisation and not a correctness device, so removing *that* must still leave the suite green. A sweep +with no controls left is a sweep that has stopped asking whether its tests describe the contract. + +## D-25: `Observable` does not restate depth + +[D-2](#d-2)'s sketch of this trait read "depth, high-water, doorbells actually rung". Depth was dropped on +the way to shipping it. + +`Bounded::len` already reports depth, computed on demand from positions the queue keeps anyway. Naming it +again on `Observable` would give one number two spellings, two doc comments, and two places to drift -- +which is the restatement problem this workspace has already paid for, recorded in the root +[DESIGN-NOTES.md](../../DESIGN-NOTES.md). The trait carries only what must be **accumulated**: facts about +the past that the queue's present state cannot reconstruct. + +Both handles implement it, because both ends have a question. A producer wants to know how often it was +refused; a consumer wants to know how deep the backlog got and how often it was actually woken. + +## D-26: the measurement, and D-16's premise falsified + +Measured by `probe-queue-contention` in a **release** build on an AMD EPYC 7763, 8 cores / 16 logical +processors, Windows 11 Enterprise 10.0.26200, `x86_64`. Median of five repetitions after a discarded +warm-up; three independent invocations agreed to within noise. **Note the architecture**: every previous +measurement in this workspace was taken on the ARM64 development machine, so these numbers fill the x64 +gap rather than extending the ARM64 record, and the two are not interchangeable. + +Isolated regime -- producers only, capacity large enough that nothing is refused, so the curve is the +claim and nothing else: + +| producers | `slotwise_mpsc` ns/push | `reserving_mpsc` ns/push | contended `fetch_add` | +|---|---|---|---| +| 1 | 9.0 | 8.6 | 5.0 | +| 2 | 49.0 | 28.0 | 8.1 | +| 4 | 84.4 | 33.3 | 12.2 | +| 8 | 140.8 | 38.5 | 13.7 | +| 16 | 193.5 | 52.2 | 14.5 | +| 32 | 239.7 | 56.9 | 15.1 | + +**Two findings, and the second one was not the expected result.** + +**The tail claim contends, and severely.** Aggregate throughput *falls* as producers are added: `slotwise_mpsc` +from 111M to 4.2M pushes per second, `reserving_mpsc` from 116M to 17.6M. A bare contended `fetch_add` +falls only to a third and then plateaus, so most of both curves is the queue rather than what this +processor does to a fought-over line. + +**`reserving_mpsc` is up to 4x faster than `slotwise_mpsc` under contention**, which inverts [D-16](#d-16). That +decision shipped the two as peers on the reasoning that honouring a reservation costs the producer a read +of the consumer's position, making the reserving shape the expensive one. It is the cheaper one at every +producer count from two upward. The premise survives in exactly one place: a *single* producer against a +live consumer, where the drained regime measures 13.6 ns against 28.1 -- and at one producer the honest +answer is [`spsc`](crate::spsc) anyway. + +The drained regime otherwise shows the two within 16% of each other at two, four and eight producers, and +its sixteen- and thirty-two-producer rows are consumer-bound -- millions of refusals -- so they measure the +single consumer rather than the claim. + +## D-27: why, and why it is not a bug to fix + +The obvious response to D-26 is that `slotwise_mpsc` must have a defect. It does not, and the difference is worth +understanding because it is a property of the two *protocols* rather than of two implementations of one. + +Both do one compare-and-swap plus one load per attempt. The load is what differs: + +- **`slotwise_mpsc` reads `slots[tail & mask].sequence`** -- and must, because in Vyukov's protocol the slot's own + sequence is what says the slot is free. That address **marches through memory as the tail advances**, + and the slots it walks are being written by the very producers it is racing. +- **`reserving_mpsc` reads `head`** -- one fixed address, which stays hot in every core's cache and, in + the isolated regime, is never written at all. + +So the reserving shape's extra read is cheaper than the read it *replaces*, which is why the measurement +came out backwards from the prediction. + +**The false-sharing hypothesis was tested and rejected.** `Slot` is sixteen bytes, so four +consecutive positions share a cache line, and the obvious fix is to pad each slot onto its own. Measured: +at eight producers that moves `slotwise_mpsc` from 140.8 to 109.1 ns -- about a fifth -- for four times the +memory, and leaves it 2.8x slower than `reserving_mpsc`'s 38.5. False sharing between neighbouring slots +is a contributor, not the cause. The padding was reverted; the note on `Slot` that says slots deliberately +share lines is therefore correct, and now correct for a measured reason rather than an assumed one. + +The remedy that *would* close the gap is to stop reading the slot before claiming and decide freedom from +`head` instead -- which is precisely `reserving_mpsc`'s protocol. There is no third design here to +discover: the two shapes are not "one queue with and without reservations", they are two different claim +protocols, and this measurement is the comparison between them. + +**The merge-or-delete decision is therefore live and is the engineer's**, with the data above as its +basis. It is tracked as a checklist item rather than left here, because a decision recorded only in a +design note is not scheduled work. + + +## D-28: caching the peer's index was measured and rejected + +**Amended -- the rejection held on x64 only, and ARM64 reverses it by 17x. The blanket "no shape adopts +it" no longer follows from the evidence; the question is left open.** + +The engineer recalled a technique credited with taking queue throughput from millions to hundreds of +millions of operations per second: a load on the waiting side of the shared index. That memory is real +and it names a real optimisation -- **peer-index caching**, the standard trick in a high-performance +SPSC ring (Rigtorp). Each side keeps a plain, non-atomic copy of the *other* side's position. A +consumer whose cached `tail` says items are available drains them without touching the shared line at +all, and refreshes only when the cached copy says the ring is empty. One acquire load is amortised +over a whole batch, and the producer's release store stops invalidating a line the consumer reads +every iteration. + +None of the three shapes did this. `spsc::push` acquire-loads `head` on every push, `spsc::pop` +acquire-loads `tail` on every pop, and `reserving_mpsc::push` acquire-loads `head` on every push. + +**It was measured rather than adopted, and the measurement says do not adopt it.** +`probe-peer-index-cache` builds a minimal SPSC ring structurally identical to `spsc`'s and runs it +under three strategies -- baseline, peer-index caching, and a prefetch-only "warming" load kept as a +control -- while counting how many times each side actually reads the peer's position. Four release +runs on the x64 host agree: + +| strategy | ns/item | consumer reads | producer reads | +|---|---|---|---| +| baseline | 18.7 - 27.7 | ~2.03 M | ~2.03 M | +| peer-index caching | 36.6 - 39.0 | ~0.56 M | ~2.2 - 2.6 M | +| warming load only | 20.0 - 24.0 | ~2.03 M | ~2.04 M | + +The read counts are what make this conclusive, and they are the reason the probe counts them. **The +optimisation engaged**: consumer reads fell 3.6x. It engaged and still lost about 1.8x of throughput, +so this is not a failed implementation of the technique but a real result about our shape. + +The mechanism is visible in the same columns. Peer-index caching trades *freshness* for fewer reads. +That trade is free when a genuine backlog exists, because a stale index is still far behind the peer +and the batch it amortises over is deep. Here the batch is only about 3.6 items deep -- a spinning +consumer keeps the ring near empty -- so each side repeatedly idles on a stale bound it could have +refreshed, and the idling costs more than the reads saved. On the producer side the count goes *up*: +a cached index is consulted only when it says "no room", so a producer that is genuinely blocked +refreshes on every spin iteration and gains nothing whatsoever. + +The warming variant behaved exactly as a control should, which is what makes it worth having kept: it +removed no shared read (its counts match the baseline) and it moved no throughput. A discarded load +cannot help, because the authoritative load still happens and in a tight handoff loop the prefetch has +no time to land before it. **The engineer's "it is just for cache warming" reading is therefore not +the mechanism** -- the technique works by removing the load, not by warming the line for it. + +### The deeper-batching workload was found, and it is simply the other architecture + +The paragraph above says the trade "is free when a genuine backlog exists, because the batch it +amortises over is deep", and that here the batch is only ~3.6 items. **That mechanism is correct and it +is the reason the conclusion does not travel.** Re-running the identical release binary on the ARM64 +development host (Snapdragon X2 Elite, 12 cores, no SMT, no L3), median of three: + +| strategy | ns/item | consumer reads | producer reads | batch depth | +|---|---|---|---|---| +| baseline | 30.4 - 32.4 | ~2.1 M | ~2.1 M | ~1 | +| peer-index caching | **1.8** | ~9 - 19 K | ~3.4 - 3.6 K | **~150** | +| warming load only | 27.0 - 29.2 | ~2.0 M | ~2.1 M | ~1 | + +**17x faster, not 1.8x slower**, and the producer read count falls by ~580x rather than rising. Every +observable this decision rested on inverted. What did not change is the *explanation*: batch depth +decides the outcome, and batch depth is a property of how the two threads interleave -- core count, +whether siblings share a core, how the scheduler places them -- not of our code. x64 kept them +lock-step; ARM64 lets them decouple. + +**That last sentence was itself too coarse, and the x64 host disproved it.** See +"[the flip is placement, not architecture](#d-28-placement)" below: pinned to SMT siblings, the same +x64 machine that produced the rejection reverses it. The variable is placement, not instruction set. + +Two consequences, and the second is the uncomfortable one: + +- The blanket rule **"no shape adopts it"** does not follow from the evidence any more. It is now a + choice between hosts, an open question queued outside this crate, + which asks for a *policy* for a technique whose sign depends on the machine rather than for more + measurement. We have the measurement twice and it disagrees with itself. +- **The probe was printing this decision's conclusion as fixed prose.** It stated "the technique WORKED + and still lost", "roughly 3.6x", and "on the producer side the count goes UP" unconditionally, so on + ARM64 it contradicted its own table three lines above. Only the speedup ratio was computed. That is + fixed -- the interpretation is now derived from the run, including the batch depths, and it says + outright that the verdict has inverted by host. An instrument that reports its conclusion regardless + of what it measured is worse than no instrument, because it is believed. + +The two reasons below still stand, and neither is architecture-dependent: + +- **The shared read is a minority of the cost.** The model runs at 18.7-27.7 ns/item while the + shipping `spsc` runs at 58.6-62.8. This probe deliberately does not attribute that gap (the shipping + push also consults the reservation count, updates the depth metric and rings the doorbell), but it + does put a floor under the argument: whatever the shared read costs, removing it cannot be the large + win. +- **It would be a correctness hazard at the arming boundary.** `Consumer::arm` decides whether to + park, and that decision must be made against a fresh acquire load. A cached `tail` that says "empty" + when the producer has already published is a lost wakeup -- the same defect class as + [D-9](#d-9) and [D-15](#d-15), which this crate has now been bitten by twice. + +The technique is not wrong; it is right for a ring with a standing backlog. **This crate's ring is that +ring on one of our two architectures and is not on the other**, which is the whole finding. The +re-measurement conditions written down here were "if a later workload shows deep batching" -- what +actually surfaced it was not a later workload but a second machine, and that is the more useful trigger +to remember. The arming path must be exempted from any cache regardless of the result, for the reason +below. + +**Work is now scheduled by this decision**, where an earlier revision said none was. That sentence was +accurate when the answer was a flat rejection and is not accurate now: the open question is +that open question. It is recorded so the technique is +neither re-proposed without measurement nor adopted on the strength of whichever host someone happened +to benchmark on. + +### The flip is placement, not architecture -- and the sibling hypothesis was refuted backwards + +The ARM64 host asked the x64 host to test a specific prediction: **that SMT siblings sharing L1 would +stay in lockstep, giving shallow batches, and that this was the condition making caching lose.** ARM64 +has no SMT and physically cannot express that placement, so only the x64 host could answer it. + +`probe-core-affinity`, x64, medians of three runs (all three agreed to within 3%): + +| placement | base ns/item | cached ns/item | cached batch depth | verdict | +|---|---|---|---|---| +| SMT siblings (one core) | 10.7 | 5.9 - 6.0 | **116 - 163** | caching **WINS** 1.8x | +| same cache, same class | *not expressible* | | | | +| cross cache, same class | 19.3 - 21.8 | 38.7 - 42.1 | **1.7 - 1.8** | caching **LOSES** 2.0x | + +**The hypothesis is refuted, and refuted backwards.** Siblings do not stay in lockstep -- they produce +by far the *deepest* batches measured on this host, and caching wins there. The shallow batches are on +the *cross-core* row, which is where caching loses. Sharing a cache causes decoupling, not lockstep. + +This is the more important half of the finding: **the verdict flips inside a single machine.** The +earlier framing that "x64 keeps the threads lock-step and ARM64 lets them decouple" attributed to the +instruction set something that is a property of *placement*. The same x64 binary on the same x64 host +both wins and loses depending only on which two processors the threads land on. No decision keyed to +architecture can be correct, which is why the amended rule is placement-scoped. + +It also explains the original rejection without contradicting it. Unpinned threads land on separate +cores, which is exactly the losing row; re-running `probe-peer-index-cache` unpinned reproduces it +(baseline 18.4 - 23.8 ns, cached 35.0 - 38.8 ns). The first measurement was never wrong -- it was one +placement reported as though it were the machine. + +**Why the sign changes, unified across both hosts.** Caching wins when +`(cost of the shared read) x (reads saved)` exceeds the cost of idling on a stale bound. Both terms +move with placement: + +- *Siblings* share L1, so the handoff is cheap (10.7 vs 19.3 ns even with no caching). The two threads + interleave on one core's execution resources rather than running truly concurrently, so the producer + bursts ahead and the consumer drains deep batches. Deep batch, caching wins. +- *Across cores* the ring ping-pongs one item at a time (depth ~1.7) and, on this host, the read being + saved is cheap anyway -- crossing L2 while staying inside a shared L3, same package, same NUMA node, + same efficiency class. Little saved, staleness paid. Caching loses. +- *ARM64 across domains* saves a genuinely expensive read (215 ns baseline), so it wins 3.5x even at a + batch depth below 1. Cost per read, not just depth, is part of the trade. + +**The x64 host isolates the cache effect, which ARM64 could not.** ARM64's cache domains and core +classes are confounded -- crossing one crosses the other -- so it cannot separate "cache domain cost" +from "core speed cost". This host has a single L3 domain, a single efficiency class, and eight L2 +domains, so its `cross cache, same class` row varies *only* the cache domain, with class, package, +L3 and NUMA all held constant. That isolated crossing costs **1.8x - 2.0x** on the unoptimised +handoff. Conversely this host cannot express `same cache, same class` at all: its outermost +partitioning cache is L2, shared by exactly the two siblings of one core, so any two processors +sharing a cache domain are siblings. The two hosts are complementary rather than redundant, and +neither alone can produce the full table. + +The two hosts are in fact **disjoint** -- no placement is measured by both -- so every row rests on a +single machine and none cross-checks another. The per-placement coverage matrix, including the one row +(`same cache, cross class`) that neither host can express, is kept with the open question in +that open question rather than duplicated here. + +**A probe defect found while doing this, now fixed.** `probe-core-affinity` printed its placement +table from a hard-coded list of four variants that omitted `SameCoreSiblings`, while the +interpretation beneath it iterated over the placements actually measured. On an SMT host the table +therefore showed the sibling row as absent while the interpretation quoted a number for it -- the +single most important row on this machine, silently missing from the table that was supposed to +report it. This is the second time in this investigation that an instrument's *presentation* rather +than its measurement nearly produced a wrong conclusion (the first was the fixed-prose interpretation +noted above). The fix also makes the "near vs far" summary fall back to the sibling pair on hosts +where `same cache, same class` is not expressible, which would otherwise have printed nothing here. + + +## D-31: 0.1.0 ships without machine-checked orderings, and says so + +Model-checker verification of the memory orderings (M31.6) does **not** +gate `windows-waitable-queues` 0.1.0. It gates 1.0. The gap is disclosed in the crate documentation and +the README rather than left for an adopter to discover. + +The case for gating was strong and is worth stating before the case against. This is a lock-free +concurrency primitive whose whole value is correctness, and the suite's blindness to ordering defects is +**measured, not assumed**: weakening the producer's `Acquire` load of the consumer's position to +`Relaxed` left all twenty tests of the day green, while every logic defect injected beside it was +caught. Publishing with a known blind spot is a real decision. + +Three things decided it the other way. + +**A model checker would close the demonstrated gap but not the dangerous one.** It models atomics; it +cannot model `SetEvent` and `ResetEvent`. So it covers the queue shapes' positions and sequence numbers +-- which is where the weakened-acquire defect lives -- and it cannot cover the doorbell, whose entire +correctness argument ([D-9](#d-9), [D-15](#d-15)) is how an atomic mirror flag interleaves with those +two syscalls. Stubbing them verifies a model of `SetEvent` rather than `SetEvent`, which is the +"measures the model, not the thing" trap this workspace was already caught by once, in +[D-28](#d-28)'s probe. **The only ordering bug this crate has actually had was D-15's lost wakeup, it +was found by sabotage, and a model checker would not have found it.** Treating that work as "the +orderings are now verified" would therefore overstate it in exactly the direction that matters. + +**The risk it addresses is mostly regression risk, and that is lowest now.** The orderings are believed +correct and were argued at the time; the sabotage sweep *introduced* the weakening to prove the suite +was blind to it, rather than discovering one. Regression risk grows with contributors, changes and +consumers, all of which begin after publication. + +**Gating has a cost that is not paid by this crate.** It blocks 0.1.0, and through it the placement +tool and the measurements from other people's machines that the whole release sequence exists to +obtain -- see the placement-probe tooling. Every host available +here has one NUMA node; that is not fixable locally at any price. + +**The disclosure is what makes this a decision rather than a deferral.** The crate says what is +verified, says that stress testing here is known not to catch ordering defects, cites the measurement +that shows it, and says a model checker is planned before 1.0. An adopter then decides with the +information we have. A `0.x` version number carries the rest, and is meant literally. + +## D-29: both multi-producer shapes ship, and the caller is given the data instead of a verdict + +[D-26](#d-26) falsified [D-16](#d-16)'s premise -- reading the consumer's position was supposed to make +`reserving_mpsc` the expensive shape, and it is instead the faster one under contention, by up to 4x on +x64 and 6.4x on ARM64. That reopened a question D-16 had treated as settled: if the split does not buy +what it claimed, should the shapes merge, or should one be deleted? + +**Neither. Both ship, and the crate declines to choose between them on the caller's behalf.** + +The two are not one queue with a feature flag. `slotwise_mpsc` implements Vyukov's bounded array protocol, where +a producer asks a slot's own sequence number whether it is free; `reserving_mpsc` counts free slots +against the consumer's position, which is the only way a reservation can be answered at all. Both are +independently studied designs with production track records, chosen by different systems for different +reasons. **Our own workload having settled which we want is a fact about our workload**, and treating it +as a fact about queueing would be exactly the narrowing PLATFORM INTEGRITY forbids: the absence of a +visible consumer for a design is not evidence that none exists. + +What the crate owes a caller instead is honesty and equipment: + +- **The measurements, stated plainly**, including the regimes where each wins and the fact that the + answer inverted once already when a second architecture was tried. +- **The means to measure their own domain.** `probe-core-affinity` and the placement tool exist so a + caller can settle this on their own hardware and workload rather than inheriting ours. A queue + library that publishes one benchmark and calls it a recommendation is asserting a conclusion about + machines it has never seen. + +### What the split does *not* rest on + +Two justifications are available and both are refused, because a rationale that evaporates on +inspection is worse than none: + +- **Not capacity.** On a 64-bit target `slotwise_mpsc` reaches 2^62 slots and `reserving_mpsc` 2^31, and + that difference is unreachable: it counts slots allocated at construction, not items ever pushed, and + 2^31 slots is tens of gigabytes before the ring holds anything useful. See [D-17](#d-17) for why the + packing forces it. **On a 32-bit target the difference does not exist at all** -- the crate-wide + ceiling is 2^30 and `reserving_mpsc`'s packed 2^31 is clamped down to it, so both shapes stop in the + same place. Pinned by `the_shapes_ceilings_are_what_the_public_documentation_claims`, which is run + against `i686-pc-windows-msvc` as well as the host. +- **Not `slotwise_mpsc` being faster somewhere.** Its one measured advantage is a single producer with a live + consumer -- and at one producer the right shape is [`spsc`](#d-1), which is faster still and which + this crate also ships. A shape kept for a regime already better served elsewhere is kept on + sentiment. + +The split rests on **capability**: `reserving_mpsc` implements `Reserving` and `slotwise_mpsc` cannot, for the +structural reason D-16's surviving half explains. Everything else is profile, and profile is the +caller's to measure. + +### The obligation this creates + +Keeping both doubles the surface that every later decision must cover, and that is accepted knowingly +rather than discovered later. `M31.6`'s loom verification covers both shapes or neither is verified; +`M-inf.4`'s peer-index policy is decided for both or the crate ships two different answers to one +question. **A shape kept for others' benefit is still a shape this crate maintains**, and the moment +that maintenance is skipped for one of them, the argument above stops being true. + +## D-34: what the prior art actually protects, and why this crate is outside it + +SH-14.1 is not a bug in an unusual design. It is the +standard design, used below the width at which the standard correctness argument holds. That +distinction is the whole content of this decision, and it took a survey to establish -- the full +record, with citations and with the gaps flagged, is in +[DESIGN-SESSION-2026-09-02](design-sessions/DESIGN-SESSION-2026-09-02-claim-protocol-prior-art.md). + +### The protocol is mainstream + +`crossbeam-queue::ArrayQueue`, `concurrent-queue` and `thingbuf` were each read at their push path. +All three load a counter, load a second value, decide from the pair, then compare-exchange **only +the counter** and write. None re-validates after the exchange. That is our protocol. + +Searching all three for `ABA`, `wraparound`, `overflow` or any statement of a counter-width bound +returns nothing. The assumption is load-bearing in every one of them and written down in none. + +### What actually makes them safe + +Two mechanisms, and it is worth being blunt that neither is "the protocol is careful": + +- **By width.** The counter is a whole `usize`, so returning to a given bit pattern needs on the + order of 2^64 pushes. Nikolaev states this explicitly (DISC 2019, section 3, "ABA safety"): the + counters "will not wrap around until after the number of operations exceeds **the CPU word's + largest value**, a reasonable assumption made by other ABA-safe designs as well." +- **By structure.** CRQ and SCQ advance the shared counter with an unconditional fetch-and-add that + authorizes nothing, and put the authorizing compare-exchange on the *cell*, where the reuse + decision and the write live in one word. No observation survives the exchange unvalidated. + +`reserving_mpsc` has neither. Its position is a 32-bit half of a packed word, not a machine word, so +the width argument does not reach it -- and its exchange covers the claim word but not the `head` +its room decision was computed from. + +### The nearest published twin defends a different property + +DPDK's `rte_ring` is our protocol almost exactly: 32-bit indexes, room computed against a separately +loaded counterpart, compare-exchange to claim. Its *Programmer's Guide* (6.5.4) justifies it as "we +can do subtractions between 2 index values in a modulo-32bit base: that's why the overflow of the +indexes is not a problem." + +That defends **modular arithmetic of the difference**. It says nothing about a producer stalled +across a full recurrence, which is the hazard. Searching DPDK's ring library for "ABA" returns +nothing either. So the closest thing to a published defence of this design defends the wrong +property, and the gap is undocumented industry-wide rather than something we alone missed. + +### The generalisation + +Stated once, so it is not re-derived, and flagged as **ours**: no source phrases it this way, though +SCQ's cell compare-exchange and CRQ's double-width `CAS2` are both instances of it. + +> The atomic operation that authorizes the write must cover everything the decision depended on. +> Where it does not, correctness rests entirely on the counter being too wide to recur. + +This is the criterion any future claim protocol in this crate is judged against, and it is more +useful than the narrower "avoid ABA": it says *what to check*, and it explains why the same +structural window is harmless in crossbeam (word-width counter) and dangerous here (subfield). + +### What this decision does not decide + +Which fix to adopt. That is M15, which prototypes the central-permit claim and measures it rather +than arguing it -- necessary because [D-26](#d-26) already measured that the single shared line is +what collapses under contention, so a protocol that touches two shared lines instead of one is not +obviously cheaper. This decision records only the landscape and the criterion. + +## D-35: the permit claim measured, and the result that inverts the expectation + +Run by `probe-queue-contention` on the reference host (x86-64, 16 logical / 8 physical, SMT on), +release build, five repetitions per configuration with the median kept. The whole run was repeated +three times; the isolated numbers reproduced within noise except one outlier noted below. + +### Isolated regime -- producers only, nothing ever refused + +The cleanest measurement of the claim, because nothing else touches the queue. Nanoseconds per push: + +| producers | `slotwise_mpsc` | `reserving_mpsc` | `permit_mpsc` | contended `fetch_add` | +|---|---|---|---|---| +| 1 | 6.3 | 5.5 | 8.0 | 2.4 | +| 2 | 58.6 | 33.9 | 42.8 | 12.7 | +| 4 | 89.6 | 33.5 | 31.5 | 14.7 | +| 8 | 143.4 | 37.9 | 26.1 | 15.9 | +| 16 | 225.1 | 56.1 | 20.5 | 14.6 | +| 32 | 234.3 | 53.1 | 19.5 | 13.4 | + +`permit_mpsc` against `reserving_mpsc`, as a cost ratio: **1.45x, 1.26x, 0.94x, 0.69x, 0.37x, +0.37x**. The crossover is between two and four producers. + +### The result, and why it was not expected + +**The safer claim is also the faster one everywhere contention exists.** At sixteen and thirty-two +producers it is 2.7x cheaper per push than the shape it would replace, and 12x cheaper than +`slotwise_mpsc`. + +That is the opposite of what the design predicted. `permit_mpsc` touches **two** shared lines on the +push path -- the permit count and the ticket -- where `reserving_mpsc` touches one plus a read of +`head`, and [D-26](#d-26) had already established that the single shared line is what collapses +under contention. The expectation was therefore that adding a second one would cost. + +**The mechanism is retries, not lines.** Both of `permit_mpsc`'s operations are unconditional +read-modify-writes: `fetch_sub` on the permits and `fetch_add` on the ticket. Neither can fail, so +neither retries. `reserving_mpsc`'s claim is a `compare_exchange_weak` that retries once per lost +race, and at thirty-two producers almost every race is lost. The retry loop dominates the second +cache line long before the second cache line matters. + +Two corroborating observations, both from the same table: + +- **It is the only shape that gets *faster* per push as producers are added** -- 42.8 ns at two down + to 19.5 at thirty-two. Every other shape, and the bare atomic floor, degrades monotonically. A + claim that cannot fail has no retry storm to suffer, so added producers buy parallelism in the + slot writes without adding claim work. +- **It is the only shape that stays close to the floor.** At thirty-two producers it costs 19.5 ns + against a bare contended `fetch_add`'s 13.4 -- 1.46x, while doing two of them plus a slot write + plus a doorbell ring. `reserving_mpsc` is 4.0x the floor there and `slotwise_mpsc` 17x. + +### Where it loses, and why that is the honest reading + +**At one producer it is 1.45x slower** (8.0 ns against 5.5). Uncontended, `reserving_mpsc` pays one +compare-exchange that always succeeds plus a load of an uncontended line -- and a load is far +cheaper than a read-modify-write. `permit_mpsc` pays two read-modify-writes regardless. The permit +claim converts a shared *read* into a shared *read-modify-write*, which is the wrong trade when +there is no contention and the right one when there is. + +A single-producer queue is a real configuration, so this is a genuine cost and not a rounding error. +It is also exactly the regime in which [D-16](#d-16)'s surviving half already says `spsc` is the +right shape. + +### What this does NOT decide + +**The drained regime is not clean enough to read.** Its refusal counts differ between shapes by two +to five orders of magnitude and are unstable across runs -- `reserving_mpsc` recorded 0 refusals at +eight producers in one run and 2,363 in the next, and `permit_mpsc` recorded roughly 460,000 and +490,000. There are at least two candidate explanations, and this harness cannot separate them: the +permit shape is genuinely faster, so it attempts more pushes against a full queue; or its optimistic +overdraw refuses near-full more readily than the shipping shape's re-read does. Since the probe's own +caution is that a run with many refusals was waiting for the consumer rather than for the claim, +those rows price backpressure rather than admission. + +The drained ratios are recorded for completeness -- 2.19x, 0.85x, 0.94x, 0.75x, 0.92x, 0.66x -- but +the isolated regime is the one that answers the question asked, and the refusal question is queued as +`SH-15.5.1` rather than resolved here. + +**One outlier, recorded rather than dropped.** In the second of the three runs, `permit_mpsc` at +eight producers measured 56.7 ns against 26.1 and 26.8 in the other two, breaking an otherwise +monotone trend. Eight producers is exactly this host's physical core count, so scheduling variance +there is plausible; two of three runs agree closely and the trend either side of that point is +unambiguous. It is noted because a reader re-running this will likely see it too. + +### What it means for SH-14.1 + +The ABA hole and the contention cost turn out to be **the same load**. `reserving_mpsc`'s read of the +consumer's `head` is simultaneously the stale input that SH-14.1 exploits and the shared access D-26 +measured. Removing it for correctness removes it for performance as well, which is why this +measurement came out the way it did -- and why "closing the hole will cost throughput" was the wrong +thing to have worried about. + +## D-38: one atomic, one discipline + +An atomic that carries any acquire/release operation carries acquire/release on **every** operation. +A relaxed load is never mixed onto it. + +The reason is not that relaxed is "weaker and therefore riskier". It is that a relaxed operation has no +memory ordering **at all**, so it is not placed at any defined point relative to the ordered operations +on the same object, and the code generator and the processor are both free to move it. With respect to +placement it behaves like a plain load or store: it is not pinned to its site in the source. Reading +such a program in textual order -- statement one, then statement two, then statement three -- and +concluding what the relaxed operation observes is not a weak argument; it is not an argument, because +the premise that it happens there is false. + +**"Plain" above is about placement only, and the distinction is easy to lose.** A relaxed operation is +still fully atomic: indivisible, never torn, immune to the compiler inventing or duplicating accesses, +and coherent (all threads agree on a single modification order for that one location). What it gives up +is ordering with respect to *other* memory. The two axes are independent, and the confusion runs in both +directions -- "relaxed means no guarantees, so I may as well use a plain field" is as wrong as "relaxed +is ordered, just weakly", and it is the more dangerous of the two because the field it produces is a +data race. [D-40](#d-40) states the atomicity half, with this crate's claim word as the worked example. + +What makes this expensive rather than merely wrong is that it **usually does what the author expected**: +a simple load or a simple store at the obvious place. It survives review, it survives testing, and it +breaks later, when an optimizer version changes or the code runs on a processor with a weaker model. + +This crate is unusually exposed to that, for a reason already measured. On x86-64's TSO, a decorative +`Acquire` load and a `Relaxed` load compile to very nearly the same instruction, so **no test on this +host can distinguish them** -- which is precisely the blindness [D-31](#d-31) recorded when weakening a +real `Acquire` to `Relaxed` left all twenty tests of the day green. On AArch64 the same two loads are +`ldar` and `ldr`, and the difference is real. CI builds `aarch64-pc-windows-msvc`. + +### The resolutions, and which one to take + +For a mixed atomic there are two consistent repairs: **promote the loads to acquire**, or **demote the +stores to relaxed**. Both are valid, and choosing between them is a separate and more involved analysis +of what the atomic is actually for. + +**The standing answer here is to promote the load.** An acquire that turns out to have been unnecessary +is a performance claim someone can come back and make with a benchmark. A relaxed load that turns out to +have been load-bearing is a defect that appears only on hardware we do not own. The asymmetry is not +close, so it is settled in advance rather than re-argued per site. + +Demotion is correct only where the atomic has **no release operation anywhere** -- there being nothing to +pair with, an acquire load on it would read as a guarantee the type does not make. Three atomics are in +that position and are uniformly relaxed for that reason: `reserving_mpsc`'s claim word (every write is a +`Relaxed/Relaxed` compare-exchange), `permit_mpsc`'s `tail` and `head`, and `slotwise_mpsc`'s `tail` +(whose claim CAS is deliberately `Relaxed/Relaxed` and says so at the site). + +Those four remain atomics, and the atomicity is load-bearing even with no ordering attached to it -- +[D-40](#d-40). The claim word makes the point unmissable: it is a `u64` packing two `u32` halves, so a +torn read would produce a `(reserved, position)` pair that was never a state the queue was in. "Every +operation on it is relaxed" is therefore not a step toward making it a plain field; it is a statement +about ordering and nothing else. + +### What the audit found + +Every atomic in the crate was grouped by field and asked one question: does it have a release write, and +does it also have relaxed operations? Four atomics had **acquire loads with no release write anywhere** +-- the acquire pairing with nothing, synchronizing with nothing: + +| Atomic | Acquire loads | Release writes | +|---|---|---| +| `reserving_mpsc` claim word | 4 | 0 | +| `permit_mpsc::tail` | 1 | 0 | +| `permit_mpsc::head` | 1 | 0 | +| `slotwise_mpsc::tail` | 1 | 0 | + +Four more had a real release store with relaxed loads mixed in, and those loads were promoted: +`reserving_mpsc::head`, `slotwise_mpsc::head`, `spsc::head`, `spsc::tail`. + +One had a genuine load-bearing edge with a relaxed operation sitting in the middle of it: +`permit_mpsc`'s permit counter, where the `Release` increment frees a slot and the `Acquire` decrement +claims it, but the overdraw undo was `Relaxed`. That one is rescuable by the release-sequence rule, but +that rule was narrowed once already (C++20 dropped same-thread relaxed stores from it) and it is not +something a reader should have to reconstruct to trust a slot handoff. It is now `Release`, which costs +one `stlxr` over `stxr` on AArch64, on the contended slow path. + +Two categories were deliberately left alone. Reference counts are [D-39](#d-39). Reads through +`&mut self` in `Drop` are not atomic operations at all -- `get_mut` is a plain read, and there is no +second thread for an ordering to order against -- so `permit_mpsc`'s drop was converted to `get_mut`, +matching `reserving_mpsc`, which removes the question rather than answering it. + +### Why the audit is a script and not a reading + +The first two versions of the audit were both wrong, in opposite directions, and neither error was +visible without checking a result by hand. The first required the receiver on one line and so missed +every multi-line `self.shared\n.head\n.0\n.store(...)` chain, reporting three atomics as having no writes +at all. The second matched across newlines and started absorbing words out of comments, splitting one +atomic's operations into several phantom fields and inventing "decorative acquire" findings for atomics +whose release store it had filed elsewhere. + +Both produced confident, plausible, wrong tables. The lesson is not about regular expressions: an +ordering audit's output has to be checked against the source at a few points before it is believed, +because a wrong audit here is indistinguishable from a right one by inspection. diff --git a/crates/windows-waitable-queues/PLANS.md b/crates/windows-waitable-queues/PLANS.md new file mode 100644 index 00000000..85f260f4 --- /dev/null +++ b/crates/windows-waitable-queues/PLANS.md @@ -0,0 +1,9 @@ +# Plans: windows-waitable-queues + +Design decisions are in [DESIGN-NOTES.md](DESIGN-NOTES.md), and completed work +in [COMPLETED-PLANS.md](COMPLETED-PLANS.md). + +| Path to CHECKLIST.md | Status | Brief description | Design Notes | +|---|---|---|---| + +No checklist is open against this crate. diff --git a/crates/windows-waitable-queues/README.md b/crates/windows-waitable-queues/README.md new file mode 100644 index 00000000..0cfb8709 --- /dev/null +++ b/crates/windows-waitable-queues/README.md @@ -0,0 +1,459 @@ +# windows-waitable-queues + +Bounded producer/consumer queues whose readiness is a waitable Windows `HANDLE`. + +**Windows only.** Every public item is behind `cfg(windows)`; the crate builds to +an empty shell on other platforms. + +## Queues, and the moment a consumer has to wait + +A bounded queue -- a ring of slots with producers at one end and a consumer at +the other -- is how one thread hands work to another without the two sharing +mutable state. The producer pushes; the consumer pops; the ring is fixed in size, +so a full queue is backpressure rather than unbounded memory growth. + +The interesting moment is when the queue is **empty**. The consumer has nothing +to do and must decide how to wait for the next item. Spinning answers instantly +and burns a core doing it, so any queue meant for real work offers a blocking +receive instead, and needs something to sleep on until a producer wakes it. + +**What it sleeps on is the design decision this crate is about.** Every +general-purpose Rust queue picks an internal primitive of its own -- a condition +variable, a futex, a parking lot. That is exactly right when the queue is the +only thing the thread is waiting for. + +## On Windows, a thread is rarely waiting for only one thing + +The realistic wait is a disjunction: + +> a message arrived **or** my I/O completed **or** shutdown was signalled + +Windows is built for that. A `HANDLE` is the platform's universal waitable +currency: `WaitForSingleObject`, `WaitForMultipleObjects`, +`MsgWaitForMultipleObjects`, a thread-pool wait, and alertable waits all take +one, and an I/O completion, a process exit, a timer, and a cancellation event +are all handles. A thread can wait on any mixture of them in a single call. + +**A queue is the one thing in that list that is not a handle** -- because the +primitive it sleeps on is private to it. +[`crossbeam-channel`](https://docs.rs/crossbeam-channel) blocks in `recv` but +exposes no handle, and its `Select` composes only channel operations, with no +way to register a foreign OS object; +[`crossbeam-queue`](https://docs.rs/crossbeam-queue) does not block at all. +These are good queues; they simply cannot appear in the wait above. + +So the thread must poll one source while blocking on another -- burning a core, +or adding latency to whichever source lost. + +## What this crate contributes + +**The queue's readiness *is* a `HANDLE`.** That is the whole idea, and everything +else here follows from it: a queue that hands out a handle composes with +everything the platform can already wait on, so the disjunction above becomes one +call instead of a polling loop. + +Nothing is given up to get it. Every shape here can still be polled, or blocked +on directly through `recv`, without the caller ever touching a handle -- and the +kernel object is created lazily, so a consumer that only polls never allocates +one. + +## The shapes + +There is deliberately **no type named `Queue`**. What a queue must support -- +how many threads push, whether a slot can be claimed before the message exists +-- decides its *algorithm*, not merely its configuration, so these are separate +shapes rather than one type with switches. A caller names the shape it wants. + +| Shape | Producers | What it adds | Choose it when | +|---|---|---|---| +| `spsc` | one | nothing -- no compare-and-swap on either side | exactly one thread pushes | +| `slotwise_mpsc` | many | Vyukov's per-slot sequence protocol, so producers push without a lock | **the default** for many producers | +| `reserving_mpsc` | many | claiming a slot *before* the message exists | a message must not be lost to a full queue | +| `permit_mpsc` | many | an experimental claim protocol | never in production -- see below | + +Every shape has one consumer. `permit_mpsc` is behind the non-default +`experimental-permit-claim` feature and is outside the semver promise; it will +either be merged into `reserving_mpsc` or deleted. + +**Cardinality is enforced by the compiler, not by a sentence in a doc comment.** +Each shape splits into a producer handle and a consumer handle, and "single +producer" means the handle is not `Clone`. The handles are also not `Sync`, so +one that can be neither cloned nor shared is held by exactly one thread. + +| Shape | Producer | Consumer | Reserves | +|---|---|---|---| +| `spsc` | not `Clone` | not `Clone` | yes | +| `slotwise_mpsc` | `Clone` | not `Clone` | **no** | +| `reserving_mpsc` | `Clone` | not `Clone` | yes | +| `permit_mpsc` | `Clone` | not `Clone` | yes | + +The shapes also disagree about their smallest usable capacity, and the error +says so rather than the documentation: `spsc` accepts one slot, while the MPSC +shapes need two, because a per-slot sequence cannot distinguish "just published" +from "free again next lap" in a one-slot ring. + +The capability traits over the shapes -- `Producer`, `Consumer`, `Bounded`, +`Waitable`, `Reserving` -- each shipped with the second implementation that +validated them, rather than being designed against one. + +`reserving_mpsc` carries one further choice, and the next section is entirely +about it: it packs its claim position beside a reservation count in a single +word, and how those bits divide is the caller's to pick. + +The set is expected to grow. Shapes with many consumers, and shapes that signal +when space becomes available so a producer can wait for room, are both under +consideration for a future revision. + +The decisions all of this was built against are in +[DESIGN-NOTES.md](DESIGN-NOTES.md). + +## How long `reserving_mpsc` runs before its claim position recurs + +**`reserving_mpsc` can lose an item after 2^32 pushes under its default layout, +on every target -- not only 32-bit ones.** That layout gives the claim position +a 32-bit half of a packed word, so this reaches x86-64 and ARM64 exactly as it +reaches i686. Read that sentence before the paragraph below, because the phrase +"32-bit position" invites the opposite reading and this project has already had +to correct that misreading once. + +**This is a property of the default layout, not of the shape**, and that is a +change: it was previously a defect a caller had to live with. The claim word +packs an outstanding-reservation count beside the position, and how its bits are +divided is now a caller's choice. Reservations are bounded by how many producers +are mid-send -- hundreds at most -- so giving up a ceiling nobody reaches buys +positions: + +| Layout | Outstanding reservations | Pushes to recurrence | At sustained maximum rate | +|---|---|---|---| +| `Balanced` (default) | 2^32 | 2^32 | about 37 seconds | +| `Enduring` | 65,535 | 2^48 | about 28 days | +| `Perpetual` | 255 | 2^56 | about 20 years | +| `Wide` (needs `dwcas`) | 2^32 | 2^64 | unreachable | + +```rust +use windows_waitable_queues::reserving_mpsc::{self, Perpetual}; + +// The same queue, with a claim position that outlives the process. +let (tx, rx) = reserving_mpsc::bounded_as::(64)?; +# let _ = (tx, rx); +# Ok::<(), windows_waitable_queues::CapacityError>(()) +``` + +**A deeper position costs nothing measurable.** `Balanced`, `Enduring`, and +`Perpetual` all issue the same exchange on the same 64-bit word and differ only +in shift and mask constants; a probe comparing them found no difference outside +noise. `Wide` is the exception: it needs a 128-bit exchange, which measured 2-3x +slower on the claim, and it is the only thing in this crate that costs a +third-party dependency. + +The default remains `Balanced` so that no existing caller's behaviour changed +when the choice was introduced. It is not the recommended layout. + +**What happens.** A producer checks that there is room, is descheduled, and +resumes after other producers have driven the position field through a complete +wrap. Its claim then succeeds against a value that is numerically identical but +a whole generation later, and it writes into a slot whose emptiness was decided +long ago. If that slot now holds an item the consumer has not taken, the item is +overwritten. + +**The failure is silent.** No error, no panic, no counter moves. The consumer +receives a different item than the one that was sent, and nothing observable +says so -- which is why this is documented here rather than left to a caller to +discover, and why it cannot be mitigated after the fact. + +**The exposure, measured rather than estimated.** Under `Balanced`, 2^32 pushes +is 37 seconds to roughly four minutes of *sustained* pushing at this crate's own +measured rates -- about two minutes at two producers, which is the smallest +count that can trigger it at all. That is sustained throughput, not a total +accumulated over an uptime. Reaching the wrap is necessary but not sufficient: a +producer must also be stalled inside a window a few instructions wide. Rare, but +a preemption is enough, and "rare" over billions of pushes is not "never". + +The figures in the table above scale that same measurement by the position +width, so they are a floor on time rather than a forecast: a queue that must +drain cannot sustain the fastest rate measured, and a slower producer takes +proportionally longer to reach its wrap. + +**What to do about it.** + +- **Name a layout.** `Perpetual` puts the recurrence about twenty years out at + no measured cost, which takes it past any real deployment. This is the answer + for almost every caller who is exposed at all. +- **`slotwise_mpsc` does not have this hazard** under any layout. Its positions + are 64 bits on every target, so the equivalent wrap needs 2^64 claims. Prefer + it unless you need `Reserving`. +- **`spsc` never had it**, having no contended claim to race. +- **The default layout is sound below its wrap.** A queue that will not push 4.3 + billion items in one run, or that is not driven at sustained maximum rate by + two or more producers, is not exposed even on `Balanced`. + +This is disclosed on the same principle as the ordering gap below: an adopter +gets the information we have rather than an assurance we cannot support. The +difference between the two is worth stating plainly -- an unverified ordering is +a *risk* of a bug, while this is a known one with a computed exposure. What has +changed is that the exposure is now a number the caller sets rather than one the +crate imposes. + +## Cargo features + +Both are off by default, and the default build depends on `windows-sys` alone. + +**`dwcas`** adds the `Wide` claim layout, a 128-bit claim word for +`reserving_mpsc`. This is the only thing in the crate that costs a third-party +dependency: Rust's standard library has no 128-bit atomic -- `core::sync::atomic` +stops at 64 bits -- so the double-width compare-and-swap comes from +`portable-atomic`. Most callers do not need it; `Perpetual` reaches roughly +twenty years before its claim position recurs with no dependency and no measured +cost, while the 128-bit exchange measured 2-4x slower on the claim itself. Take +it when you want the recurrence gone as a guarantee rather than deferred by an +argument about deployment lifetimes. + +**`experimental-permit-claim`** adds `permit_mpsc`, a different claim protocol in +which the decision and the operation are one atomic rather than two. It is +**not** covered by this crate's semver promise: it will either be merged into +`reserving_mpsc` or deleted once it has been measured enough to decide. + +## How far the memory orderings are verified, and how far they are not + +Stated plainly, because a lock-free queue that is vague about this is asking to +be trusted rather than evaluated. + +**What is verified.** Every ordering was reasoned about when written, and the +reasoning is recorded in [DESIGN-NOTES.md](DESIGN-NOTES.md) beside the code it +justifies. The shapes are covered by an extensive unit suite and by a sabotage +suite that injects deliberate defects and requires each to be caught -- which is +how the one real ordering bug this crate has had was found: a lost wakeup where +the doorbell cleared its mirror flag before resetting the event. + +**What is not.** Stress testing cannot catch a *weakened memory ordering* here, +and that is measured rather than assumed: changing the producer's `Acquire` load +of the consumer's position to `Relaxed` left the entire suite green, while every +logic defect injected beside it was caught. A test can only observe the +interleavings the hardware and scheduler happen to produce, and neither x86-64 +nor ARM64 obliged. + +**So the orderings are not machine-checked.** Verification with a model checker +is planned before 1.0. Until then `0.x` is meant literally, and an adopter for +whom that matters has the same information we do rather than an assurance we +cannot support. + +One limit worth knowing even after that work lands: a model checker covers the +queue shapes' positions and sequence numbers, and **cannot** cover the doorbell, +whose correctness is the interleaving of an atomic flag with real `SetEvent` and +`ResetEvent` calls. Modelling those would verify a model of them rather than the +calls themselves. + +## Where these algorithms come from + +**None of the queue algorithms here are novel, and that is deliberate.** A +concurrent queue is a bad place to be original: the failure mode is a reordering +that shows up on one machine, under load, months later. Each shape implements a +published design, and what this crate adds is the waiting, not the queueing. + +- **`spsc`** is the classic single-producer single-consumer ring buffer, with the + two positions on separate cache lines so the ends stop invalidating each + other. The structure is old -- Lamport gave the concurrent reader/writer + treatment in 1983 -- and the padding is standard modern practice. +- **`slotwise_mpsc`** implements Dmitry Vyukov's bounded MPMC array queue, + specialised to one consumer. Each slot carries its own sequence number, so a + producer claims a position and asks *that slot* whether it is ready, which + keeps producers off any single shared line. It is among the most widely + reimplemented concurrent queues in existence. +- **`reserving_mpsc`** uses the other classic approach: count free slots against + the consumer's position, so space can be **claimed in advance**. Credit- and + ticket-based admission is long established in flow control, and counting is + the only way to answer "will there be room later?". + +Where this crate departs from a reference implementation it says so, and why, in +[DESIGN-NOTES.md](DESIGN-NOTES.md). The measured behaviour of both MPSC shapes is +below -- including one case where the published intuition turned out to be wrong +on our hardware. + +## Why not an existing queue crate + +Rust has excellent channel crates, and for most programs one of them is the right +answer. **They are not usable here for one structural reason: on Windows, +waiting is a kernel-object operation, and a queue whose readiness is not a +`HANDLE` cannot take part in one.** + +A thread that must wait for *an item arrived* **or** *an I/O completed* **or** +*this process exited* **or** *cancellation was requested* waits on all of them at +once, in a single `WaitForMultipleObjects`. Every participant has to be a kernel +object. A channel that signals readiness through a condition variable, a futex, +or a parked-thread list cannot be one of them -- however good its blocking +receive is, and however rich its `select`, because that select can only cover its +own channels. + +The alternatives are all worse in the same way: + +- **Poll on a timer.** Trades latency against wakeups, and the thread wakes to + discover nothing happened. +- **Dedicate a thread to blocking on the channel and signalling an event.** + Correct, and costs a thread plus a hop per item to convert a condition variable + back into the kernel object you needed in the first place. +- **Move everything to async.** A real answer if the program is already async; + not one for a thread whose other obligations are `HANDLE`s. + +So the queue owns a manual-reset event and keeps it **never unsignalled while +there is something to take**. That one-sided guarantee is the hard part and is +what this crate is actually for. + +It is one-sided deliberately. The event stays signalled after the last item is +taken until the consumer clears it with `arm()`, and a producer's signal may +land after the consumer has already drained -- so a wake means *there may be +something*, never *there is something*. What the crate guarantees is the +direction that matters: a wake is never missing. Follow the protocol the +blocking receivers use rather than treating the handle as a readiness +predicate. That protocol has **four** steps, and the fourth is the one that is +easy to leave out: + +1. take everything available -- `pop` until it reports `Empty`, and stop + outright if it reports `Disconnected` instead; +2. `arm()`, and if it returns `false`, start again -- something arrived; +3. **`pop` once more, and stop if it reports `Disconnected`.** `arm()` reports + only whether a later *push* can be missed, so on a queue with no producers + left it still returns `true` -- having just cleared the single doorbell ring + their drop left behind. Waiting on the strength of that `true` never wakes. + This step is not belt-and-braces either: a producer may push *and then* drop + between step 1 and here, and that item is delivered rather than discarded + because `pop` reports `Disconnected` only once the queue is genuinely empty; +4. only now, wait on the handle. + +Step 3 used to read "check `is_disconnected()`, and if the producers are gone, +take one last time" -- two calls whose order the caller had to get right, +because a `pop` returning `None` could not say which situation it was in. +`TryRecvError` collapses that into one question with the ordering built in. + +`recv` already does all four. The steps matter when driving the handle +yourself -- through a `ThreadpoolWait`, or a `WaitForMultipleObjects` across +several queues -- because then there is nothing to delegate to. + +## Choosing between `slotwise_mpsc` and `reserving_mpsc` + +They are **two different claim protocols**, not one queue with a switch. `slotwise_mpsc` +is Vyukov's bounded array queue: a producer asks a slot's own sequence number +whether it is free. `reserving_mpsc` counts free slots against the consumer's +position, which is the only way a reservation can be answered at all. Both are +well-studied designs in production use elsewhere, which is why this crate ships +both rather than picking one for you. + +**Start here:** + +- **Pushing more than ~4 billion items in one run, from two or more producers?** + Either use `slotwise_mpsc`, whose positions are 64 bits under every + configuration, or name a deeper layout on `reserving_mpsc` -- `Perpetual` + puts the recurrence about twenty years out at no measured cost. Under its + default layout `reserving_mpsc` can lose an item past that volume; see + [the section on recurrence](#how-long-reserving_mpsc-runs-before-its-claim-position-recurs) + above, which you should read before choosing. +- Need `reserve`? Only `reserving_mpsc` has it, and `slotwise_mpsc` structurally + cannot. That no longer forces a trade against the recurrence: choosing a + layout addresses it, so the capability can settle the choice on its own + merits. +- Otherwise, **start with `reserving_mpsc`.** It was the faster of the two at + every producer count we measured above one. +- Only one producer *and* one consumer? Use `spsc`, which beats both. + +**What we measured**, in ns per push, isolated regime, median of three runs. +Higher producer counts oversubscribe both hosts: + +| producers | `slotwise_mpsc` (x64) | `reserving` (x64) | `slotwise_mpsc` (ARM64) | `reserving` (ARM64) | +|---|---|---|---|---| +| 1 | 9.0 | 8.6 | 6.5 | 6.1 | +| 2 | 49.0 | 28.0 | 29.8 | 9.4 | +| 4 | 84.4 | 33.3 | 60.6 | 12.9 | +| 8 | 140.8 | 38.5 | 167.4 | 29.8 | +| 16 | 193.5 | 52.2 | 194.9 | 30.6 | +| 32 | 239.7 | 56.9 | 195.0 | 30.6 | + +x64 is an AMD EPYC 7763 slice (8 cores, 16 threads); ARM64 is a Snapdragon X2 +Elite (12 cores, no SMT). **Read these as two data points, not as a law.** This +comparison has already inverted once: it was designed on the assumption that +`slotwise_mpsc` would be the cheaper shape, and measurement said otherwise on both +machines. + +**Measure your own workload before treating any of this as settled.** Producer +count, how hard the consumer drains, and where the threads are scheduled all +move the answer -- thread placement alone moved an SPSC handoff by 5.6x on one +of these hosts. The `probe-core-affinity` tool in this repository exists so you +can run that measurement on your hardware instead of inheriting ours. + +Two things that look like reasons to choose and are not: + +- **Capacity.** On a 64-bit target `slotwise_mpsc` reaches 2^62 slots and + `reserving_mpsc` 2^31. On a 32-bit one the crate-wide ceiling is 2^30 and + **both** shapes land there -- `reserving_mpsc`'s packed 2^31 is clamped down + to it as well -- so the difference disappears and the comparison means + nothing. Either way it counts slots allocated up front, not items ever pushed: + a ring of 2^31 slots is tens of gigabytes before it holds anything useful. +- **`slotwise_mpsc` winning at one producer.** True in one regime, and at one producer + you want `spsc` anyway. + +## What it will not do + +- **It will not overwrite.** A full queue fails, or a reservation guarantees a + slot. Overwrite-oldest is right for telemetry, where a lost entry is a lost + sample; here an entry may be an I/O submission, where a lost entry is a lost + operation. +- **It will not let a producer wait for room.** The doorbell is one-directional: + a consumer can park until there is something to take, and there is no + equivalent for a producer waiting until there is somewhere to put. `push` + refuses immediately with `PushError::Full`, and `reserve` returns `None`; + neither blocks, and no handle is offered to wait on. + + **Said plainly because the obvious comparison misleads.** `crossbeam-channel`'s + `send` blocks on a full bounded channel, so a reader arriving from it will + expect the same here and get a refusal instead. A producer with nowhere to go + must decide what to do -- shed the item, retry on its own schedule, or grow a + buffer of its own -- rather than being parked by the queue. + + **A deliberate absence rather than an oversight, and under consideration for + a future revision.** It is not simply the doorbell mirrored: a blocking send + that parked on something `WaitForMultipleObjects` cannot see would reintroduce + the very composition problem that ruled out the existing channel crates. +- **It will not decide between two real queue designs on your behalf.** `slotwise_mpsc` + and `reserving_mpsc` are different claim protocols, both well studied and both + used in production and in research. `slotwise_mpsc` asks each slot's own sequence + number "are you free?"; `reserving_mpsc` counts free slots against the + consumer's position, which is what makes a reservation answerable at all -- + and why `slotwise_mpsc` does not implement the `Reserving` trait. It genuinely cannot, + which is the whole reason the traits are narrow. + **Which is faster is a property of your workload, not of the designs**, and we + publish what we measured rather than choosing for you -- see "Choosing between + them" below. +- **It will not allocate on push.** Bounded shapes allocate once, at + construction. +- **It will not create a kernel object you never use.** The doorbell is created + lazily, so a consumer that only polls allocates none. +- **It will not destroy your items on a thread you did not choose.** A queue + built with a `Disposal` sink hands whatever nobody drained back to you at + teardown, rather than running the destructors inside the last handle's drop. + That matters when an item owns a handle, because closing one can block -- + and the thread that happens to release last may be a pool callback that must + not. Without a sink the items are destroyed in place, which is the right + default for items that own nothing. +- **It will not round your capacity.** A capacity that a shape cannot represent + is refused, with the nearest valid neighbours on the error, rather than + silently turned into one the caller did not choose. + +## What it will tell you about itself + +Three numbers, through the `Observable` trait on either handle: + +- **`refused()`** -- pushes turned away for want of room. This is the loss count, + and it counts room only: a push refused because the consumer is gone is the end + of the stream, not backpressure. +- **`doorbell_rings()`** -- `SetEvent` calls, not signal attempts. The difference + between the two *is* the skip optimisation, which is what makes this the number + worth reporting. +- **`high_water()`** -- the deepest the queue got, or `None` if nobody asked for + it to be tracked. It is the one metric that cannot be made free, so it is + opt-in via `Options::tracking_high_water`; `None` rather than `0` so you cannot + mistake "nobody was counting" for "it never filled". + +Depth is not on that list because `len()` already reports it, from positions the +queue keeps anyway. + +## Licence + +Copyright (c) Mike Grier. diff --git a/crates/windows-waitable-queues/design-sessions/DESIGN-SESSION-2026-09-02-claim-protocol-prior-art.md b/crates/windows-waitable-queues/design-sessions/DESIGN-SESSION-2026-09-02-claim-protocol-prior-art.md new file mode 100644 index 00000000..ffa2f167 --- /dev/null +++ b/crates/windows-waitable-queues/design-sessions/DESIGN-SESSION-2026-09-02-claim-protocol-prior-art.md @@ -0,0 +1,265 @@ +# Design session 2026-09-02: what other bounded queues do about claim-protocol ABA + +Resulted in [D-34](../DESIGN-NOTES.md#d-34) and an amendment to +[D-18](../DESIGN-NOTES.md#d-18). + +Prompted by SH-14.1: `reserving_mpsc` can overwrite a live slot after 2^32 pushes, on every +target, because a producer's room decision is made against a separately-read `head` and the +compare-exchange that acts on it covers only the claim word. The question put to this session +was deliberately narrow -- *what does everyone else do* -- rather than *how should we fix it*, +because SH-14.3 had already enumerated four ways out and all four were unsatisfying. + +Sources were restricted to permissively licensed implementations (MIT, Apache-2.0, BSD, +public domain) and open-access papers, so that anything found could be cited and reasoned +from. **No code was copied.** What follows is a description of algorithms and a record of +citations; the mechanisms described are ideas, and any implementation this workspace adopts +is written here. + +## 1. The exact hazard, restated so the survey has something to match against + +1. Load the shared claim word `W`; extract `position`. +2. Decide "there is room" by comparing `position` against a separately-loaded `head`. +3. `compare_exchange(W, W with position + 1)` to claim the slot. +4. On success, write `slot[position % capacity]` and publish. + +A producer stalled between (2) and (3) resumes after other producers have driven the position +field through a complete wrap. The word recurs bit-for-bit, the exchange succeeds, and the +write proceeds on a room decision that is now generations stale. + +The essential point, and the thing to match implementations against: **the exchange protects +the counter; nothing protects the earlier observation.** + +## 2. Rust implementations read directly + +All three were read at their push path. All three are the same algorithm. + +### crossbeam-queue `ArrayQueue` (MIT OR Apache-2.0) + +`crossbeam-rs/crossbeam:crossbeam-queue/src/array_queue.rs`. The file's own header credits +Vyukov's page. `tail` is one `usize` packing `{lap, index}`; each slot carries a `stamp: +AtomicUsize`. The push path: + +- load `tail`; split into `index` and `lap` +- load `slot.stamp` +- **require `tail == stamp`** +- `compare_exchange_weak(tail, new_tail)` +- on success, write the value, then `slot.stamp.store(tail + 1, Release)` + +There is no re-check of `stamp` after the exchange. The predicate depends on `stamp`, which is +read separately and is not covered by the exchange -- structurally the same window as ours. +What makes it safe in practice is width: `tail` is a whole `usize`, so recurrence needs on the +order of 2^64 pushes on a 64-bit target. + +**On a 32-bit target `tail` is 32 bits, and the exposure is the same as ours.** No comment in +the file discusses this bound. + +### concurrent-queue (Apache-2.0 OR MIT) + +`smol-rs/concurrent-queue:src/bounded.rs`. The same algorithm, derived from crossbeam. Worth +noting for a different reason: it steals a `mark_bit` from the top of the tail word for the +"closed" flag, which narrows the recurrence space by a bit. A precedent for taking bits out of +a position word, and evidence that doing so is not treated as dangerous. + +### thingbuf (MIT) + +`hawkw/thingbuf:src/lib.rs`. `tail` is one `usize` packing `{gen, closed, idx}`; each slot has +a `state`. The push path requires `state == tail`, then compare-exchanges `tail`. Again no +post-exchange re-validation. + +### What none of them do + +Searched all three for `overflow`, `wrap around`, `wraparound`, `ABA`, `2^64`, `64-bit`: +**zero matches.** The assumption that the counter cannot recur is load-bearing in all three +and documented in none. + +## 3. The literature + +### Vyukov's bounded MPMC queue + +`1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue` (read via Internet Archive +snapshot 2024-01-12; the live domain is intermittently unreachable). Per-cell `sequence_` +initialised to the cell index; enqueue computes `dif = seq - pos` and compare-exchanges +`enqueue_pos_` when `dif == 0`. + +One genuine structural difference from ours worth recording: **Vyukov's enqueue never reads the +consumer's position at all.** The decision is made against the destination cell's sequence, +keyed to the same `pos` the exchange validates. That removes one stale input relative to our +design -- though the cell sequence is still read separately and still not covered by the +exchange. + +Vyukov does not discuss ABA, wraparound, or counter width anywhere on the page (verified +negative). He also explicitly disclaims lock-freedom for this queue. + +### Morrison and Afek, CRQ / LCRQ + +*Fast Concurrent Queues for x86 Processors*, PPoPP '13, pp. 103-112, DOI +`10.1145/2442516.2442527`. + +**The paper's own text could not be extracted** (the PDF returned raw compressed streams). +Everything below is from the MIT-licensed reference implementation +`chaoran/fast-wait-free-queue:lcrq.c` and from peer-reviewed secondary description in +Nikolaev's papers. Anyone quoting CRQ's prose must read the PDF first. + +From the source: a cell is `{uint64_t val; uint64_t idx;}` updated by a genuine double-width +`CAS2`. `idx` is a full 64-bit monotonically increasing position with bit 63 stolen as an +`unsafe` flag. Enqueue does `t = FAA(&rq->tail, 1)` -- **no compare-exchange on the tail at +all** -- and then validates the *cell*. + +Cycle recurrence is not a concern for CRQ because the epoch stored in the cell is the +full-width absolute position, compared with `<=`; there is no narrow modular subfield to recur. + +**"Closing" is a livelock escape, not an ABA device.** `close_crq()` sets bit 63 of the tail +when an enqueuer cannot find a usable cell; the producer then allocates a fresh ring. +Corroborated by Nikolaev DISC'19 section 5: CRQ "is not standalone due to its inherent +susceptibility to livelocks ... a slow path is taken, where the current CRQ instance is +'closed'." + +### Nikolaev, SCQ -- the citation that matters most + +*A Scalable, Portable, and Memory-Efficient Lock-Free FIFO Queue*, DISC 2019, LIPIcs vol. 146, +pp. 28:1-28:16, DOI `10.4230/LIPIcs.DISC.2019.28`. **Open access, CC-BY.** Extended version +arXiv:1908.04511. Reference implementation `rusnikola/lfqueue` (dual BSD-2-Clause / MIT). + +Section 3, under the heading "ABA safety", verbatim: + +> "The ABA problem is prevented by comparing cycles. As both `Head` and `Tail` are incremented +> sequentially, regardless of queue size, they will not wrap around until after the number of +> operations exceeds the CPU word's largest value, a reasonable assumption made by other +> ABA-safe designs as well." + +This appears to be the only explicit, citable statement in this literature of the assumption +everyone relies on. Note exactly what it licenses: **CPU-word width**. A 32-bit subfield of a +64-bit word does not satisfy it. That single sentence is the strongest available statement +that SH-14.1 is a real defect rather than a theoretical curiosity -- we are doing the standard +thing below the width at which the standard argument holds. + +The structural mechanism, Fig. 6 line 15: + +``` +if ( Cycle(Ent) < Cycle(T) and Index(Ent) = () and (IsSafe(Ent) or Load(&Head) <= T) ) + New = { Cycle(T), 1, index }; + if ( !CAS(&Entries[j], Ent, New) ) goto retry +``` + +Two things to take from it: the **compare-exchange is on the cell, not the counter** (the +counter is advanced by an unconditional fetch-and-add that authorizes nothing), and the reuse +decision plus the write are validated **together**, because both live in the word the exchange +covers. No observation survives across the exchange unvalidated. + +Cycle width is `word width - log2(slots) - 1` (the `-1` is the `IsSafe` bit), derived from +`lfring_cas1.h`. With their benchmark's 2^16 slots on 64-bit that is a 47-bit cycle. + +**The `threshold` is not an ABA device.** It is `2n - 1` (infinite array) or `3n - 1` (SCQ) and +its stated purpose is livelock-freedom and empty detection: "Livelocks occur when dequeuers +incessantly invalidate slots that enqueuers are about to use." A web summary claimed it was a +2^32 anti-aliasing constant; that is false. Recorded because the misreading is plausible. + +### Nikolaev and Ravindran, wCQ + +*wCQ: A Fast Wait-Free Queue with Bounded Memory Usage*, SPAA '22, DOI +`10.1145/3490148.3538572`; preprint arXiv:2201.02179. + +The passage that matters to this crate is not about ABA at all. On the family our shapes belong +to, section 1: + +> "such queues require a thread to reserve a ring buffer slot prior to writing new data. These +> approaches ... are technically blocking since one stalled (e.g., preempted) thread in the +> middle of an operation can adversely affect other threads." + +and it names DPDK's ring as a "straight-forward implementation ... erroneously dubbed as +'lock-free'". `reserving_mpsc` is squarely in that family. This is the citation behind +SH-inf.1's note that the crate should not repeat the error by implication. + +Also: "wCQ requires double-width CAS, which is nowadays widespread (i.e., x86 and +ARM/AArch64)", with a separate LL/SC construction for architectures lacking it. + +## 4. DPDK `rte_ring` -- the closest published twin of our exact protocol + +*DPDK Programmer's Guide*, section 6.5.4 "Modulo 32-bit Indexes"; code +`DPDK/dpdk:lib/ring/rte_ring_c11_pvt.h` (BSD-3-Clause). 32-bit indexes; room computed against a +separately loaded counterpart index; compare-exchange to claim. That is our protocol, in +production, at very large scale. + +Its published justification, verbatim: + +> "we can do subtractions between 2 index values in a modulo-32bit base: that's why the +> overflow of the indexes is not a problem." + +**That argument covers modular arithmetic of the difference and nothing else.** It does not +address a producer stalled across a full 2^32 recurrence. A search of `DPDK/dpdk path:lib/ring` +for "ABA" returns zero hits. + +So the closest thing to a published defence of our design defends a different property than +the one SH-14.1 attacks. This is the single most useful citation from the session: it shows the +shape is mainstream, shows the standard justification for it is insufficient for our hazard, +and shows nobody has written the gap down. + +Incidentally: DPDK's RTS and HTS modes pair head and tail into a single 64-bit compare-exchange. +That is a double-width fix in effect, but it is motivated by lock-waiter preemption, not ABA. + +## 5. Double-width compare-and-swap on this workspace's targets + +Checked against the pinned toolchain rather than against documentation, because the +documentation and D-18 disagreed. `rustc 1.98.0 --print cfg --target ...`: + +| target | `cmpxchg16b` feature | `target_has_atomic="128"` | +|---|---|---| +| `x86_64-pc-windows-msvc` | **set by default** | yes | +| `aarch64-pc-windows-msvc` | n/a (`ldxp`/`stxp` is ARMv8-A baseline) | yes | +| `i686-pc-windows-msvc` | n/a | **no** | + +And `core::sync::atomic::AtomicU128` was test-compiled: **still unstable** +(rust-lang/rust#99069), so reaching a 128-bit exchange from stable means a dependency such as +`taiki-e/portable-atomic` (Apache-2.0 OR MIT), whose own table records `cmpxchg16b` as "enabled +by default on Apple, Windows (except Windows 7, since Rust 1.78)". + +Consequences for [D-18](../DESIGN-NOTES.md#d-18), which refuses the 128-bit exchange: + +- "It is not in the x86-64 baseline ... does not enable the target feature by default" is + **false** on 1.98 for our target. No floor to raise, no runtime detection to pay. +- "There is no usable `AtomicU128`" is **true and verified**. The dependency cost stands. +- The fact D-18 never had, and the decisive one: **`i686` has no 128-bit atomic at all**, so a + 128-bit claim word is not "widen the word" but "widen the word *and* drop 32-bit support" -- + which collapses SH-14.3's option 1 into its option 4, an engineer's decision under the + platform-integrity rule. +- And the premise: D-18 says the exchange "would lift the 2^31 cap and nothing else", written + before SH-14.1 existed. It would also collapse the recurrence. + +## 6. What the survey concluded + +**Every design surveyed is safe for exactly one of two reasons**, and it is worth being blunt +that neither is "the protocol is careful": + +- **By width** -- Vyukov, crossbeam, concurrent-queue, thingbuf, SCQ's `Head`/`Tail`. The + counter is a whole machine word, so recurrence is unreachable. This is what Nikolaev states + explicitly and what the others rely on silently. +- **By structure** -- CRQ and SCQ. The counter is a fetch-and-add authorizing nothing, and the + authorizing compare-exchange is on the cell, where the decision and the write are validated + together. + +Ours is safe for neither reason. The position is a 32-bit subfield, not a machine word, and the +authorizing exchange does not cover `head`. + +The principle, stated once so it need not be re-derived -- **this is our inference, and no +source phrases it this way**, though SCQ's Fig. 6 and CRQ's `CAS2` are both instances: + +> The atomic operation that authorizes the write must cover everything the decision depended +> on. Where it does not, correctness rests entirely on the counter being too wide to recur. + +That is what makes the central-permit shape (M15 arm A) worth prototyping: admission becomes a +single atomic on one counter, so the predicate is a function solely of the word being modified, +and the position degrades to a ticket with no predicate at all. + +## 7. Gaps -- do not cite these without checking + +- **The LCRQ paper's own text.** Not extracted; all CRQ claims here are from MIT-licensed + reference source plus Nikolaev's secondary description. +- **Michael, *ABA Prevention Using Single-Word Instructions*, IBM RC23089 (2004).** Existence + well attested, full text not retrieved. It is the usual citation for tag-width reasoning. +- **Herlihy and Shavit, *The Art of Multiprocessor Programming*.** Not consulted. Whether it + treats bounded-counter ABA is unknown; section 10.6 and the `AtomicStampedReference` material + are the places to look. +- **wCQ section 5 (Correctness).** Only sections 1-3 were read; if wCQ restates a counter-width + assumption formally it would be there. +- **No published counter-argument was found** demonstrating a 64-bit monotonic counter being + wrapped in practice, and no paper states a stall-duration-versus-wrap-rate inequality. diff --git a/crates/windows-waitable-queues/sabotage.json b/crates/windows-waitable-queues/sabotage.json new file mode 100644 index 00000000..53fbdd8b --- /dev/null +++ b/crates/windows-waitable-queues/sabotage.json @@ -0,0 +1,575 @@ +{ + "package": "windows-waitable-queues", + "description": "Sabotages for the SPSC ring, the two MPSC shapes, the doorbell they share, and the blocking receive loop they share. Run with tools/run-sabotage.ps1; see tools/README-sabotage.md for the format and for why the results are read the way they are.", + "notCoveredHere": "The two SeqCst fences in doorbell.rs are deliberately ABSENT from this manifest, and their absence is not an oversight. Removing either leaves every test green, because the defect they prevent is a store-buffer reordering that no amount of stress testing reliably produces -- it is a fact about the memory model, not about any interleaving a scheduler will hand you. Adding them here with expect:'caught' would fail the sweep; adding them with expect:'survives' would assert they are harmless, which is false and far worse. They are verifiable only under a model checker, and are the named target of checklist item M31.6.", + "sabotages": [ + { + "name": "push does not signal the doorbell", + "file": "src/spsc.rs", + "expect": "caught", + "why": "A producer that never rings the bell leaves a parked consumer asleep on a queue with items in it. Caught as a hang, which is the correct shape for this defect.", + "find": [ + " self.doorbell.signal();", + " }", + "}", + "", + "impl Drop for Shared {" + ], + "replace": [ + " }", + "}", + "", + "impl Drop for Shared {" + ] + }, + { + "name": "producer drop does not signal", + "file": "src/spsc.rs", + "expect": "caught", + "why": "Disconnection is a wakeup, and the only one no other party can deliver. Without it a blocked consumer waits forever for an item that can no longer be sent: the queue stays correct and the program still hangs. NOTE the shape of this patch -- it deletes the live call. An earlier version inserted unreachable code beside it, which changed the file without changing behaviour, and the resulting pass was misread as a hole in the tests.", + "find": [ + " self.shared.doorbell.signal();", + " }", + "}", + "", + "/// A slot claimed in advance" + ], + "replace": [ + " }", + "}", + "", + "/// A slot claimed in advance" + ] + }, + { + "name": "arm checks emptiness before clearing", + "file": "src/spsc.rs", + "expect": "caught", + "why": "The lost wakeup itself, and the whole reason Consumer::arm exists. A push landing between the check and the clear both signals and has its signal erased, so the consumer sleeps on a queue that is not empty and will never be signalled again. This is the order that reads more naturally, which is exactly why it has to be proven wrong rather than assumed to be. MEASURED: this entry was caught in only one run of three while the sole deterministic test exercised a hand-written COPY of the reversed order rather than the real `arm` -- detection depended on two threads interleaving inside a window tens of nanoseconds wide. The `ARM_RACE_HOOK` in spsc.rs now drives the real `arm` through that window on one thread, and this is caught every run.", + "find": [ + " self.shared.doorbell.clear();", + " #[cfg(test)]", + " crate::race_hooks::ARM.run();", + " Ok(self.is_empty())" + ], + "replace": [ + " let empty = self.is_empty();", + " self.shared.doorbell.clear();", + " #[cfg(test)]", + " crate::race_hooks::ARM.run();", + " Ok(empty)" + ] + }, + { + "name": "arm does not create the doorbell before checking", + "file": "src/spsc.rs", + "expect": "caught", + "why": "Lazy creation is the same hazard a third time: a producer running while no event exists skips signalling, so the emptiness check has to come after the event exists to catch what that skip left behind.", + "find": [ + " self.shared.doorbell.handle()?;", + " self.shared.doorbell.clear();" + ], + "replace": [ + " self.shared.doorbell.clear();" + ] + }, + { + "name": "the final drain returns nothing", + "file": "src/spsc.rs", + "expect": "caught", + "why": "A producer may push and then drop in the window between a receive's first pop and its disconnection check. Reporting the disconnection without one last take silently discards an item that was successfully sent. This guard was originally unreachable from any test, and this sabotage is what found that out.", + "find": [ + " fn finish(&self) -> Option {", + " self.pop()", + " }" + ], + "replace": [ + " fn finish(&self) -> Option {", + " None", + " }" + ] + }, + { + "name": "clear resets the event but not the mirror flag", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "The mirror flag lets a redundant signal skip its syscall. If clear leaves it set, the next push believes the doorbell is already lit and skips the one signal that actually mattered.", + "find": [ + " self.signalled.store(false, Ordering::Release);" + ], + "replace": [ + "" + ] + }, + { + "name": "clear does not reset the event", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "Distinct from the mirror-flag sabotage below it: this leaves the flag correct and the kernel object wrong, so a consumer's wait returns immediately, forever. Added after a code review found that the two tests named for this property both asserted it vacuously -- they pushed before the event existed, so the doorbell was never lit and 'arming clears it' held trivially. Both now light the doorbell first, and this entry is what keeps them honest.", + "find": [ + " unsafe {", + " ResetEvent(event.as_raw_handle());", + " }" + ], + "replace": [ + " let _ = event;" + ] + }, + { + "name": "previous_valid is not clamped to the shape's bound", + "file": "src/error.rs", + "expect": "caught", + "why": "Rounding a request down to the nearest power of two gives 2^63 for anything at or above it, which exceeds the largest representable capacity. The suggestion exists so a caller can correct the call, and one that is itself refused is worse than none.", + "find": [ + " let clamped = rounded.min(self.largest_power_of_two_within_bound());", + " (clamped >= self.min_valid).then_some(clamped)" + ], + "replace": [ + " Some(rounded)" + ] + }, + { + "name": "next_valid is not clamped to the shape's bound", + "file": "src/error.rs", + "expect": "caught", + "why": "A request that is merely not a power of two can still sit between the largest valid power of two and the bound, and rounding it up overshoots. Found by the test written for previous_valid, which the review had not flagged -- the reviewer checked next_valid only on the TooLarge path.", + "find": [ + " (rounded >= self.min_valid && rounded <= self.max_valid).then_some(rounded)" + ], + "replace": [ + " Some(rounded)" + ] + }, + { + "name": "recv_timeout computes its deadline with the panicking add", + "file": "src/blocking.rs", + "expect": "caught", + "why": "`Instant + Duration` panics when the sum is not representable, and Duration::MAX is an ordinary way to spell 'effectively forever'. The rest of the function is careful about exactly this class of problem, which is what made the panicking operator easy to miss. Moved here from src/spsc.rs when the blocking receive loop was extracted so both shapes bind to one copy of the arming protocol; there is now exactly one site, and both shapes' suites indict it.", + "find": [ + " let Some(deadline) = Instant::now().checked_add(timeout) else {" + ], + "replace": [ + " let Some(deadline) = Some(Instant::now() + timeout) else {" + ] + }, + { + "name": "the event is auto-reset instead of manual-reset", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "An auto-reset event is an edge, not a level, and it does not count signals. A wait that reports one of several signalled handles would consume the queue's only notification while ignoring it. This exact mistake hung the crate's own doorbell probe for four hundred seconds.", + "find": [ + "CreateEventW(ptr::null(), TRUE, FALSE, ptr::null())" + ], + "replace": [ + "CreateEventW(ptr::null(), FALSE, FALSE, ptr::null())" + ] + }, + { + "name": "the event is created already signalled", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "A doorbell must not claim readiness before anything is pushed, or the first wait returns immediately and the consumer spins.", + "find": [ + "CreateEventW(ptr::null(), TRUE, FALSE, ptr::null())" + ], + "replace": [ + "CreateEventW(ptr::null(), TRUE, TRUE, ptr::null())" + ] + }, + { + "name": "signal always syscalls, skipping the flag optimisation", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "**This entry changed sides in M31.4, and the reason is the point of the milestone.** It was a CONTROL expecting 'survives': skipping a redundant SetEvent is an optimisation, so removing it changed no observable behaviour and the suite had to stay green. R9 asks for a count of doorbells actually rung precisely so that the skip rule becomes MEASURABLE rather than assumed -- and once the ring count is observable, disabling the skip is observable too, which is what 'disabling the skip must change the number' means. So the same patch that had to survive now has to be caught, by the tests asserting one ring for four pushes. If this is ever reported as survived again, the ring count has stopped counting syscalls.", + "find": [ + " if self.signalled.swap(true, Ordering::AcqRel) {" + ], + "replace": [ + " self.signalled.store(true, Ordering::Release);", + " if false {" + ] + }, + { + "name": "clear clears the mirror flag before resetting the event", + "file": "src/doorbell.rs", + "expect": "caught", + "why": "The historical order, and a lost wakeup. A producer signalling between the two lines finds a clear flag, sets it, and issues a real SetEvent; the ResetEvent that follows erases that signal and leaves the flag set, so the doorbell is dark while claiming to be lit and every later signal skips its syscall. It survived review and a whole sabotage sweep because the argument for it -- 'the caller's re-check sees the racing producer's item' -- is true for spsc and false for slotwise_mpsc, whose re-check asks only whether the HEAD slot is published. Found by a sabotage BASELINE hanging once in a run that was otherwise green six times over. The race_hooks::CLEAR hook drives the real clear through its own window on one thread, so this is caught every run rather than occasionally.", + "find": [ + " unsafe {", + " ResetEvent(event.as_raw_handle());", + " }", + " #[cfg(test)]", + " crate::race_hooks::CLEAR.run();", + " self.signalled.store(false, Ordering::Release);" + ], + "replace": [ + " self.signalled.store(false, Ordering::Release);", + " #[cfg(test)]", + " crate::race_hooks::CLEAR.run();", + " unsafe {", + " ResetEvent(event.as_raw_handle());", + " }" + ] + }, + { + "name": "slotwise_mpsc push does not signal the doorbell", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "A producer that never rings the bell leaves a parked consumer asleep on a queue with items in it. Caught as a hang, which is the correct shape for this defect.", + "find": [ + " self.shared.doorbell.signal();", + " Ok(())" + ], + "replace": [ + " Ok(())" + ] + }, + { + "name": "slotwise_mpsc: the last producer's drop does not signal", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "Disconnection is a wakeup, and the only one no other party can deliver. Without it a blocked consumer waits forever for an item that can no longer be sent. NOTE the shape of this patch -- it deletes the live call rather than inserting unreachable code beside it, because a sabotage that does not sabotage retires a question that was never asked.", + "find": [ + " self.shared.doorbell.signal();", + " }", + "}", + "", + "/// The reading half" + ], + "replace": [ + " }", + "}", + "", + "/// The reading half" + ] + }, + { + "name": "CONTROL: slotwise_mpsc signals on every producer's departure, not only the last", + "file": "src/slotwise_mpsc.rs", + "expect": "survives", + "why": "A control, not a defect. Ringing when a non-final producer leaves is a SPURIOUS wakeup: the consumer wakes, finds nothing, sees producers still alive, and parks again. The contract says a wakeup may be spurious, so the suite MUST stay green. If this is ever reported as caught, a test has started asserting that no extra wakeups occur -- which is asserting the implementation -- and that test is the thing to fix. Note this is NOT the same as the entry above: that one deletes the last producer's signal, which is a lost wakeup and a hang.", + "find": [ + " if self.shared.producers.fetch_sub(1, Ordering::AcqRel) != 1 {", + " return;", + " }" + ], + "replace": [ + " let _ = self.shared.producers.fetch_sub(1, Ordering::AcqRel);" + ] + }, + { + "name": "slotwise_mpsc arm checks readiness before clearing", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "The lost wakeup itself, in the second shape. A push landing between the check and the clear both signals and has its signal erased, so the consumer sleeps on a queue that is not empty and will never be signalled again. Driven deterministically through the REAL arm by the shared ARM_RACE hook rather than through a copy of it, which is what makes this caught every run rather than one in three.", + "find": [ + " self.shared.doorbell.clear();", + " #[cfg(test)]", + " crate::race_hooks::ARM.run();", + " // Deliberately not `is_empty`. The question is whether `pop` would find", + " // something, and a slot that a producer has claimed but not published", + " // is not something `pop` can find -- see `Shared::has_ready_item`.", + " Ok(!self.shared.has_ready_item())" + ], + "replace": [ + " let ready = self.shared.has_ready_item();", + " self.shared.doorbell.clear();", + " #[cfg(test)]", + " crate::race_hooks::ARM.run();", + " Ok(!ready)" + ] + }, + { + "name": "slotwise_mpsc arm does not create the doorbell before checking", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "Lazy creation is the same hazard a third time: a producer running while no event exists skips signalling, so the readiness check has to come after the event exists to catch what that skip left behind.", + "find": [ + " self.shared.doorbell.handle()?;", + " self.shared.doorbell.clear();" + ], + "replace": [ + " self.shared.doorbell.clear();" + ] + }, + { + "name": "slotwise_mpsc frees a slot one short of the next lap", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "The sequence protocol's whole arithmetic in one line. A slot freed at `pos + capacity - 1` is never equal to the position that next claims it, so every producer reads a negative difference and reports Full for ever: the queue works for exactly one lap and then wedges. An off-by-one here is invisible to any test that never wraps, which is why the wrap tests run a thousand rounds through four slots.", + "find": [ + " position.wrapping_add(self.shared.capacity as Position)," + ], + "replace": [ + " position.wrapping_add(self.shared.capacity as Position - 1)," + ] + }, + { + "name": "slotwise_mpsc cloning a producer does not count it", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "The count is what makes multi-producer disconnection work, and a clone that does not register makes the FIRST departure look like the last. The consumer then ends the stream while producers are still pushing into it.", + "find": [ + " self.shared.producers.fetch_add(1, Ordering::Relaxed);" + ], + "replace": [ + "" + ] + }, + { + "name": "slotwise_mpsc accepts a capacity of one", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "With one slot, 'published at position p' and 'free again at position p + capacity' are the SAME number, so a producer reads the sequence of the item it just pushed, concludes the slot is free, and overwrites an item the consumer has not read. spsc accepts one, which is exactly why the minimum belongs to the shape rather than to the crate -- and why it is asserted rather than assumed.", + "find": [ + "const BOUNDS: Bounds = Bounds {", + " min: 2,", + " max: MAX_ADMISSIBLE_CAPACITY,", + "};" + ], + "replace": [ + "const BOUNDS: Bounds = Bounds {", + " min: 1,", + " max: MAX_ADMISSIBLE_CAPACITY,", + "};" + ] + }, + { + "name": "reserving_mpsc: a best-effort push may take a reserved slot", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "The reservation guarantee itself. If the best-effort path ignores the outstanding count, the slot a reservation was promised gets taken by an ordinary push and the redemption overwrites a live item. Note this sabotage keeps the room check but drops the reservations from it, which is exactly the mistake an optimiser-minded reader would make: the count looks like bookkeeping until you ask who is holding the slot it accounts for.", + "find": [ + " occupied < capacity - reserved" + ], + "replace": [ + " let _ = reserved;", + " occupied < capacity" + ] + }, + { + "name": "reserving_mpsc: reserve does not check for room", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "A reservation handed out over a full queue is a promise that cannot be kept, which is worse than a refusal: the caller has already been told it may proceed. Reserve is the CHEAP place to fail -- no work has started -- and removing the check moves that failure to the one place the design exists to keep it away from.", + "find": [ + " if !self.shared.has_room_beyond_reservations(position, reserved) {", + " // Provisional for the reason `push`'s matching check is: a", + " // stale `word` and a freshly-read `head` need not describe the", + " // same instant, and once `head` passes a stale `position` the", + " // subtraction wraps and an empty queue refuses a reservation.", + " let current = self.shared.claim.0.load(Ordering::Relaxed);", + " if current != word {", + " word = current;", + " continue;", + " }", + " return None;", + " }" + ], + "replace": [] + }, + { + "name": "reserving_mpsc: redeeming does not release the reservation", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "Redeeming must decrement the count as it advances the position, in ONE exchange. Leaving the count up permanently withdraws a slot from the best-effort path on every send, so the queue's usable capacity bleeds away to zero over time. Caught quickly because a capacity-2 queue stops accepting anything after its first reserved send.", + "find": [ + " claim_word(reserved - 1, position.wrapping_add(1))," + ], + "replace": [ + " claim_word(reserved, position.wrapping_add(1))," + ] + }, + { + "name": "reserving_mpsc: dropping a reservation does not return the slot", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "An abandoned reservation must give its capacity back. Without this the queue leaks a slot per dropped reservation and eventually refuses everything, which is the same bleed as the entry above reached by the other path.", + "find": [ + " claim_word(reserved - 1, position_of(word))," + ], + "replace": [ + " claim_word(reserved, position_of(word))," + ] + }, + { + "name": "reserving_mpsc: an outstanding reservation does not hold the stream open", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "A reservation is a promise of a message still to come, so it must count as a producer. Without this the consumer is told the stream ended while a reservation is outstanding, and then handed an item afterwards -- losing exactly the message the reservation existed to protect. This is the defect the feature is FOR, so a test suite that missed it would be asserting the mechanism and not the purpose.", + "find": [ + " self.shared.producers.fetch_add(1, Ordering::Relaxed);" + ], + "replace": [ + "" + ] + }, + { + "name": "reserving_mpsc: send leaks the shared state via mem::forget", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "The real defect this shape shipped with for one test run. `mem::forget` suppresses the double-release correctly but also leaks the Arc the reservation holds, so the shared state is never dropped and every item still in the ring leaks with it. Caught by the drop-counting test rather than by review, which is why that test counts drops instead of merely checking the queue still works.", + "find": [ + " let this = core::mem::ManuallyDrop::new(self);", + " // SAFETY: `this` is a `ManuallyDrop`, so its own destructor never runs", + " // and the field is not read again after this move.", + " let shared = unsafe { core::ptr::read(&this.shared) };", + " shared.release_producer();" + ], + "replace": [ + " let shared = Arc::clone(&self.shared);", + " core::mem::forget(self);", + " shared.release_producer();" + ] + }, + { + "name": "high-water records the latest depth rather than the peak", + "file": "src/metrics.rs", + "expect": "caught", + "why": "A high-water mark that follows the depth down is not a high-water mark; it is `len` with extra steps, and a caller sizing a queue from it would read whatever the depth happened to be when they looked. Caught deterministically rather than by a race, because the sabotage removes the comparison as well as the fetch_max.", + "find": [ + " if depth > high_water.load(Ordering::Relaxed) {", + " high_water.fetch_max(depth, Ordering::Relaxed);", + " }" + ], + "replace": [ + " high_water.store(depth, Ordering::Relaxed);" + ] + }, + { + "name": "high-water is tracked even when nobody asked", + "file": "src/metrics.rs", + "expect": "caught", + "why": "Tracking must be genuinely off by default, because it is the one metric that costs the push path something -- on slotwise_mpsc it makes the producer read the consumer's position, which is the shared line that shape exists to avoid. If the default silently tracked, every slotwise_mpsc user would be paying for an answer they never asked for, and the only visible symptom would be a number appearing where None belongs.", + "find": [ + " high_water: if track_high_water {" + ], + "replace": [ + " high_water: if true {" + ] + }, + { + "name": "CONTROL: slotwise_mpsc reads head unconditionally, skipping the tracking guard", + "file": "src/slotwise_mpsc.rs", + "expect": "survives", + "why": "A control, and the replacement for the one M31.4 converted into a defect. The guard around slotwise_mpsc's `head` load is an OPTIMISATION, not a correctness device -- `record_depth` already returns early when tracking is off, so reading head regardless changes no observable behaviour and the suite MUST stay green. What it changes is the cost, which is the whole reason the guard is there. If this is ever reported as caught, a test has started asserting the implementation rather than the contract.", + "find": [ + " if self.shared.metrics.tracks_high_water() {" + ], + "replace": [ + " if true {" + ] + }, + { + "name": "teardown ignores the disposal sink and destroys in place", + "file": "src/disposal.rs", + "expect": "caught", + "why": "The whole mechanism in one line. If the policy always takes the default path, every queue silently reverts to running arbitrary destructors on whichever thread released the last handle -- which is the hazard, and it is invisible from any test that only counts survivors rather than observing where they were destroyed.", + "find": [ + " let Some(disposal) = self.disposal.as_mut() else {" + ], + "replace": [ + " let Some(disposal) = Option::<&mut Disposal>::None else {" + ] + }, + { + "name": "a panicking sink is not contained", + "file": "src/disposal.rs", + "expect": "caught", + "why": "The sink is caller code running inside a destructor. Letting a panic escape abandons every item not yet disposed -- exactly the handles this mechanism exists to account for -- and during an unwind it aborts the process outright. Caught by the test that panics on the fourth of ten items and requires the other nine to be disposed anyway.", + "find": [ + " let _ = catch_unwind(AssertUnwindSafe(|| (disposal.sink)(item)));" + ], + "replace": [ + " (disposal.sink)(item);" + ] + }, + { + "name": "spsc teardown destroys survivors instead of handing them over", + "file": "src/spsc.rs", + "expect": "caught", + "why": "Each shape walks its own layout to find survivors, so the routing has to be asserted once per shape -- covering one says nothing about the others, which is the same lesson M31.2's sweep taught about the reservation guarantee.", + "find": [ + " let item = unsafe { (*self.slots[pos & mask].get()).assume_init_read() };", + " self.teardown.dispose(item);" + ], + "replace": [ + " unsafe { (*self.slots[pos & mask].get()).assume_init_drop() };" + ] + }, + { + "name": "slotwise_mpsc teardown destroys survivors instead of handing them over", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "As for spsc, and this walk differs: it consults each slot's sequence rather than assuming the whole resident range is published.", + "find": [ + " let item = unsafe { slot.value.get_mut().assume_init_read() };", + " self.teardown.dispose(item);" + ], + "replace": [ + " unsafe { slot.value.get_mut().assume_init_drop() };" + ] + }, + { + "name": "reserving_mpsc teardown destroys survivors instead of handing them over", + "file": "src/reserving_mpsc.rs", + "expect": "caught", + "why": "The shape where it matters most: a reservation is taken because its message must not be lost, so a redeemed message abandoned at teardown would be lost after all -- just later and more quietly.", + "find": [ + " let item = unsafe { slot.value.get_mut().assume_init_read() };", + " self.teardown.dispose(item);" + ], + "replace": [ + " unsafe { slot.value.get_mut().assume_init_drop() };" + ] + }, + { + "name": "spsc: a best-effort push may take a reserved slot", + "file": "src/spsc.rs", + "expect": "caught", + "why": "The same guarantee on the single-producer shape, where the mechanism is a plain counter rather than a packed word. Worth its own entry precisely because the two implementations share nothing: a test that only covered the slotwise_mpsc path would leave this one unguarded.", + "find": [ + " if tail.wrapping_sub(head) + reserved >= self.shared.capacity {", + " // Report disconnection in preference to fullness" + ], + "replace": [ + " if tail.wrapping_sub(head) >= self.shared.capacity {", + " // Report disconnection in preference to fullness" + ] + }, + { + "name": "spsc: dropping a reservation does not return the slot", + "file": "src/spsc.rs", + "expect": "caught", + "why": "As for reserving_mpsc: an abandoned reservation must give its capacity back, or the queue bleeds a slot per drop until it refuses everything.", + "find": [ + " self.producer", + " .shared", + " .reserved", + " .store(reserved - 1, Ordering::Relaxed);" + ], + "replace": [ + " let _ = reserved;" + ] + }, + { + "name": "slotwise_mpsc reports Full for a full queue whose consumer is gone", + "file": "src/slotwise_mpsc.rs", + "expect": "caught", + "why": "Full invites a retry and Disconnected does not, and a full queue with no consumer will never drain -- so reporting Full here is telling the caller to spin forever. The preference has to be stated at the fullness branch specifically, because that branch returns before the general disconnection check below it is ever reached.", + "find": [ + " if !self.shared.consumer_live.load(Ordering::Acquire) {", + " // Not counted as a refusal: this is the end of the stream,", + " // not backpressure.", + " return Err(PushError::Disconnected(item));", + " }", + " self.shared.metrics.record_refusal();", + " return Err(PushError::Full(item));" + ], + "replace": [ + " self.shared.metrics.record_refusal();", + " return Err(PushError::Full(item));" + ] + } + ] +} diff --git a/crates/windows-waitable-queues/src/blocking.rs b/crates/windows-waitable-queues/src/blocking.rs new file mode 100644 index 00000000..82816873 --- /dev/null +++ b/crates/windows-waitable-queues/src/blocking.rs @@ -0,0 +1,207 @@ +// Copyright (c) Mike Grier. + +//! The blocking receive loop, written once for every shape that has one. +//! +//! # Why this is not simply copied into each shape +//! +//! The loop below is not glue -- it *is* the arming protocol, the contract +//! recorded as [D-9](../DESIGN-NOTES.md#d-9): drain, arm, and wait only if +//! arming blessed it, with the disconnection check placed between the arming +//! and the wait so a producer that vanished cannot leave a consumer parked. +//! Every step is load-bearing and the order is the whole correctness argument. +//! +//! A second shape spelling that sequence out again would be a second copy of a +//! rule, free to drift from the first and, worse, free to *look* verified while +//! only the copy was tested. This crate has already paid for that mistake once, +//! in a lost-wakeup test that exercised a hand-written duplicate of +//! `Consumer::arm` rather than the real one and so could not have noticed the +//! real one being reversed. So the protocol is stated here, and a shape binds +//! to it by implementing [`Parked`]. +//! +//! # Why [`Parked`] is not one of the public capability traits +//! +//! The public traits describe what a caller may *ask of* a queue. [`Parked`] +//! describes what this module needs *from* a queue in order to park on it, and +//! the difference shows in [`Parked::finish`], which no caller should ever +//! reach for: it is meaningful only after disconnection has already been +//! observed, and the public [`Consumer`](crate::Consumer) surface deliberately +//! does not offer a method whose contract is a precondition nobody can check +//! from outside. + +use std::io; +use std::os::windows::io::{AsRawHandle, BorrowedHandle}; +use std::time::{Duration, Instant}; + +use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; +use windows_sys::Win32::System::Threading::{INFINITE, WaitForSingleObject}; + +use crate::error::{RecvError, RecvTimeoutError}; + +#[cfg(test)] +mod tests; + +/// What a shape must offer for [`recv`] and [`recv_timeout`] to park on it. +pub(crate) trait Parked { + /// The item type the shape carries. + type Item; + + /// Takes the oldest item, or `None` if there is none right now. + fn pop(&self) -> Option; + + /// The last take before the end of the stream is reported. + /// + /// Separate from [`Parked::pop`] so a shape can name the step and a test + /// can call it directly. It guards a real and narrow race: a producer may + /// push *and then* drop in the window between this loop's first `pop` and + /// its disconnection check, and reporting the disconnection without one + /// last take would silently discard an item that was successfully sent. + fn finish(&self) -> Option; + + /// Clears the doorbell and reports whether waiting on it is safe. + /// + /// # Errors + /// + /// Whatever creating the doorbell reports. + fn arm(&self) -> io::Result; + + /// Whether every producer is gone. + fn is_disconnected(&self) -> bool; + + /// The doorbell to park on. + /// + /// # Errors + /// + /// Whatever creating the doorbell reports. + fn doorbell(&self) -> io::Result>; +} + +/// Takes the oldest item, blocking until one arrives. +/// +/// # Errors +/// +/// [`RecvError::Disconnected`] once every producer is gone *and* the queue is +/// drained -- items pushed before the last producer dropped are still +/// delivered. [`RecvError::Io`] if the doorbell cannot be created or waited on. +pub(crate) fn recv(consumer: &C) -> Result { + loop { + if let Some(item) = consumer.pop() { + return Ok(item); + } + if !consumer.arm()? { + continue; + } + if consumer.is_disconnected() { + return consumer.finish().ok_or(RecvError::Disconnected); + } + wait(consumer.doorbell()?, INFINITE)?; + } +} + +/// Takes the oldest item, blocking until one arrives or the deadline passes. +/// +/// The timeout bounds the whole call, not each individual wait: a consumer +/// woken spuriously does not get a fresh budget. +/// +/// # Errors +/// +/// [`RecvTimeoutError::Timeout`] if the deadline passes with the queue still +/// empty, which is not a malfunction. Otherwise as [`recv`]. +pub(crate) fn recv_timeout( + consumer: &C, + timeout: Duration, +) -> Result { + // `Instant + Duration` panics when the sum is not representable, and + // `Duration::MAX` is a perfectly ordinary way to spell "effectively + // forever". A library that panics on that is worse than one that blocks, so + // an unrepresentable deadline degrades to the untimed wait it was asking + // for rather than aborting the caller. + let Some(deadline) = Instant::now().checked_add(timeout) else { + return recv(consumer).map_err(|error| match error { + RecvError::Disconnected => RecvTimeoutError::Disconnected, + RecvError::Io(io) => RecvTimeoutError::Io(io), + }); + }; + loop { + if let Some(item) = consumer.pop() { + return Ok(item); + } + if !consumer.arm()? { + continue; + } + if consumer.is_disconnected() { + return consumer.finish().ok_or(RecvTimeoutError::Disconnected); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(RecvTimeoutError::Timeout); + } + wait(consumer.doorbell()?, wait_millis(remaining))?; + } +} + +/// The longest finite wait `WaitForSingleObject` accepts, in milliseconds. +/// +/// **Derived from `INFINITE`, not written as a number, because it is exactly +/// one less than it.** `INFINITE` is `u32::MAX`, so a clamp to `u32::MAX` does +/// not mean "wait a very long time" -- it means *wait forever*, and the loop +/// that was supposed to re-check the deadline never regains control to do so. +const MAX_FINITE_WAIT_MILLIS: u32 = INFINITE - 1; + +/// The shortest wait worth asking for, in milliseconds. +/// +/// Zero is a poll, not a wait, and the loop that calls this treats a return as +/// "check again" -- so a zero would busy-poll the doorbell rather than sleep on +/// it. +const MIN_WAIT_MILLIS: u32 = 1; + +/// How long to block for, given the time left on the caller's deadline. +/// +/// Saturating rather than wrapping: a duration longer than a `u32` of +/// milliseconds is roughly 49 days, and clamping it to that is a longer wait +/// than any caller meant, where truncating it would be a far shorter one. The +/// loop re-arms and waits again, so clamping costs an extra turn and nothing +/// else. +/// +/// **The clamp is to one below `INFINITE`.** An earlier version clamped to +/// `u32::MAX`, which is the same bit pattern as `INFINITE`: a `recv_timeout` +/// longer than about 49.7 days waited forever instead of timing out, silently +/// converting a bounded call into an unbounded one. The comment above was +/// already there and was right about everything except the one value it chose. +/// +/// **The `min` is not redundant with the `unwrap_or`**, and a boundary test is +/// what showed it. Changing only the fallback leaves the hole open from the +/// other side: a duration of exactly `u32::MAX` milliseconds *converts* +/// successfully, so the fallback never fires and `INFINITE` is returned by the +/// conversion itself. The two guards cover different inputs -- one the +/// durations too large to represent, the other the one that is representable +/// and still means forever. +/// **And never zero, which is the other end of the same argument.** The caller +/// has already returned `Timeout` if nothing remains, so every duration +/// reaching here is non-zero -- but anything under a millisecond truncates to +/// `0`, and a zero wait returns at once. The loop would then re-arm and re-wait +/// without sleeping, which is not merely a spin: arming clears the doorbell, +/// so it is a `ResetEvent` syscall per turn for the last fraction of the +/// budget. +/// +/// Waiting a whole millisecond can overshoot the deadline, and that is the +/// right trade for a blocking call. The timer granularity is coarser than a +/// millisecond anyway, so a caller needing sub-millisecond precision cannot get +/// it from a blocking wait at any price -- what they would get instead is a +/// burning core. +fn wait_millis(remaining: Duration) -> u32 { + u32::try_from(remaining.as_millis()) + .unwrap_or(MAX_FINITE_WAIT_MILLIS) + .clamp(MIN_WAIT_MILLIS, MAX_FINITE_WAIT_MILLIS) +} + +/// Block on a doorbell handle, translating the Win32 result. +fn wait(handle: BorrowedHandle<'_>, millis: u32) -> io::Result<()> { + // SAFETY: a live event handle borrowed for the duration of the call. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), millis) }; + match result { + // A timeout is not an error here: the caller's loop re-checks its own + // deadline and decides what a timeout means. + WAIT_OBJECT_0 | WAIT_TIMEOUT => Ok(()), + _ => Err(io::Error::last_os_error()), + } +} diff --git a/crates/windows-waitable-queues/src/blocking/tests.rs b/crates/windows-waitable-queues/src/blocking/tests.rs new file mode 100644 index 00000000..c1a5673c --- /dev/null +++ b/crates/windows-waitable-queues/src/blocking/tests.rs @@ -0,0 +1,330 @@ +// Copyright (c) Mike Grier. + +//! Tests for the blocking loop's timeout arithmetic. +//! +//! The loop itself is exercised through every shape's `recv_timeout`; what is +//! tested here is the one value that decides whether a bounded call stays +//! bounded, because the failing case takes 49 days to observe from the outside +//! and so can never be a test of the loop. + +use std::time::Duration; + +use windows_sys::Win32::System::Threading::INFINITE; + +use super::{MAX_FINITE_WAIT_MILLIS, MIN_WAIT_MILLIS, wait_millis}; + +#[test] +fn an_ordinary_duration_is_passed_through_in_milliseconds() { + assert_eq!(wait_millis(Duration::from_millis(1)), 1); + assert_eq!(wait_millis(Duration::from_millis(250)), 250); + assert_eq!(wait_millis(Duration::from_secs(1)), 1_000); + assert_eq!(wait_millis(Duration::from_secs(60)), 60_000); +} + +#[test] +fn a_duration_that_does_not_fit_is_clamped_rather_than_truncated() { + // Truncating would be the opposite failure: a caller who asked to wait a + // long time would be told "timed out" almost immediately. + let fifty_days = Duration::from_secs(50 * 24 * 60 * 60); + + assert_eq!(wait_millis(fifty_days), MAX_FINITE_WAIT_MILLIS); +} + +#[test] +fn the_clamp_is_never_the_value_that_means_wait_forever() { + // **The bug this file exists for.** `INFINITE` is `u32::MAX`, so clamping + // an oversized duration to `u32::MAX` does not mean "wait a very long + // time"; it means wait forever, and the loop that was supposed to re-check + // the deadline never runs again. A bounded call silently becomes unbounded. + // + // Every duration too large to fit lands on the clamp, so it is the clamp + // that has to be checked, not any particular duration. + assert_ne!( + MAX_FINITE_WAIT_MILLIS, INFINITE, + "the clamp is the value that means wait forever" + ); + + for excessive in [ + Duration::from_millis(u64::from(u32::MAX) + 1), + Duration::from_secs(50 * 24 * 60 * 60), + Duration::from_secs(u64::from(u32::MAX)), + Duration::MAX, + ] { + assert_ne!( + wait_millis(excessive), + INFINITE, + "a {excessive:?} timeout would have waited forever" + ); + } +} + +#[test] +fn the_largest_duration_that_still_fits_is_not_clamped() { + // The boundary, from the side that must not move. + let exact = Duration::from_millis(u64::from(MAX_FINITE_WAIT_MILLIS)); + + assert_eq!(wait_millis(exact), MAX_FINITE_WAIT_MILLIS); + assert_eq!( + wait_millis(exact + Duration::from_millis(1)), + MAX_FINITE_WAIT_MILLIS, + "one millisecond past the boundary must clamp, not wrap to zero" + ); +} + +#[test] +fn a_sub_millisecond_remainder_still_sleeps() { + // The busy-wait. Anything under a millisecond truncates to zero, and a zero + // wait returns immediately -- so the loop would re-arm and re-wait without + // sleeping for the last fraction of the budget. Arming clears the doorbell, + // which is a `ResetEvent` syscall, so the spin is a syscall storm rather + // than merely a hot loop. + for tiny in [ + Duration::from_nanos(1), + Duration::from_micros(1), + Duration::from_micros(999), + ] { + assert_eq!( + wait_millis(tiny), + MIN_WAIT_MILLIS, + "a {tiny:?} remainder would have polled instead of waiting" + ); + } +} + +#[test] +fn no_duration_ever_produces_a_zero_wait() { + // The property behind the case above, stated over the boundary values + // rather than over three samples. Zero is a poll; the caller has already + // returned `Timeout` when nothing remains, so a poll here is never what was + // wanted. + for remaining in [ + Duration::ZERO, + Duration::from_nanos(1), + Duration::from_micros(500), + Duration::from_millis(1), + Duration::from_secs(1), + Duration::MAX, + ] { + assert_ne!( + wait_millis(remaining), + 0, + "a {remaining:?} remainder produced a poll rather than a wait" + ); + } +} + +// The `Parked` protocol itself, across every shape that implements it. +// +// # Why this is here and not in each shape's suite +// +// `Parked` is what `recv` and `recv_timeout` are written against, and its four +// methods are a contract each shape restates. A mutation run found the whole +// group unguarded: `finish` could return `None` and `arm` could return +// `Ok(true)` on all three shapes with the suite green. +// +// Neither is cosmetic. `finish` is the last take before the end of a stream is +// reported, so a `None` silently discards an item that was successfully sent. +// `arm` reports whether parking is safe, so an unconditional `true` blesses a +// wait over a queue that already has an item -- which is a lost wakeup, the one +// ordering bug this crate has actually had (D-15). +// +// The methods are exercised **through the trait**, because that is the surface +// the loop uses. Calling the inherent method instead is what left these alive: +// it shadows the trait one, so a broken forwarder is never reached. + +use super::Parked; +use crate::{reserving_mpsc, slotwise_mpsc, spsc}; + +/// `finish` must hand back an item that arrived before disconnection was seen. +/// +/// Called directly rather than by scheduling the race it guards. That is the +/// stated reason it exists as a named step: the window between a receive's +/// first `pop` and its disconnection check cannot be hit reliably from a test, +/// so the step is reachable on its own instead. +fn finish_returns_the_owed_item(consumer: &C) +where + C: Parked, +{ + assert_eq!( + Parked::finish(consumer), + Some(1), + "an item pushed before the producer went is still owed to the consumer" + ); + assert_eq!( + Parked::finish(consumer), + None, + "and once taken it is gone, so the stream really has ended" + ); +} + +/// `arm` must refuse to bless a wait while an item is sitting there. +fn arm_refuses_to_park_over_an_item(consumer: &C, has_item: bool) +where + C: Parked, +{ + let safe = Parked::arm(consumer).expect("arming must succeed"); + if has_item { + assert!( + !safe, + "parking over a queued item is a wait nothing will wake" + ); + } else { + assert!(safe, "an empty queue is safe to park on"); + } +} + +#[test] +fn every_shape_hands_back_the_last_item_through_parked_finish() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + // Push, then drop the producer: the item is owed even though the stream has + // ended, which is exactly the state `finish` exists to resolve. + spsc_tx.push(1).expect("there is room"); + slot_tx.push(1).expect("there is room"); + res_tx.push(1).expect("there is room"); + drop(spsc_tx); + drop(slot_tx); + drop(res_tx); + + finish_returns_the_owed_item(&spsc_rx); + finish_returns_the_owed_item(&slot_rx); + finish_returns_the_owed_item(&res_rx); +} + +#[test] +fn every_shape_refuses_to_park_over_an_item_through_parked_arm() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + // Empty first, so the `true` answer is shown to be a real reading rather + // than the only answer this method ever gives. + arm_refuses_to_park_over_an_item(&spsc_rx, false); + arm_refuses_to_park_over_an_item(&slot_rx, false); + arm_refuses_to_park_over_an_item(&res_rx, false); + + spsc_tx.push(1).expect("there is room"); + slot_tx.push(1).expect("there is room"); + res_tx.push(1).expect("there is room"); + + arm_refuses_to_park_over_an_item(&spsc_rx, true); + arm_refuses_to_park_over_an_item(&slot_rx, true); + arm_refuses_to_park_over_an_item(&res_rx, true); +} + +#[test] +fn every_shape_reports_disconnection_and_pops_through_parked() { + // The other two methods of the same contract, so the trait is covered as a + // whole rather than only where mutants happened to survive. + fn pop_and_disconnection>( + consumer: &C, + expect_item: Option, + ) -> bool { + assert_eq!(Parked::pop(consumer), expect_item); + Parked::is_disconnected(consumer) + } + + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + spsc_tx.push(5).expect("there is room"); + slot_tx.push(5).expect("there is room"); + res_tx.push(5).expect("there is room"); + + assert!(!pop_and_disconnection(&spsc_rx, Some(5))); + assert!(!pop_and_disconnection(&slot_rx, Some(5))); + assert!(!pop_and_disconnection(&res_rx, Some(5))); + + drop(spsc_tx); + drop(slot_tx); + drop(res_tx); + + assert!(pop_and_disconnection(&spsc_rx, None)); + assert!(pop_and_disconnection(&slot_rx, None)); + assert!(pop_and_disconnection(&res_rx, None)); +} + +// That the wait actually waits. +// +// # Why a fake shape and a count, rather than a real queue and a clock +// +// `wait` is the one step in the loop whose removal changes no answer. A `wait` +// that returned immediately still delivers every item, still reports every +// disconnection, and still honours every deadline -- because the loop re-checks +// all three itself. What it stops doing is *sleeping*: the loop becomes a spin +// that re-arms the doorbell, which is a `ResetEvent` syscall per turn, for the +// whole of the caller's timeout. A mutation run found exactly this, with the +// suite green. +// +// Measuring CPU time would be the direct reading and the wrong instrument: the +// answer would then depend on how loaded the machine is, and a spin on an +// oversubscribed box can look like a sleep. Counting the loop's turns is the +// same evidence without the dependency -- a real wait comes round about twice +// however busy the host is, and a spin comes round thousands of times. + +/// A shape that is permanently empty and permanently connected. +/// +/// It never has an item and never disconnects, so the receive loop can only +/// leave by its deadline -- which makes the turn count a reading of the wait +/// and nothing else. +struct NeverReady { + /// A real event, never signalled, so the wait is a real kernel wait. + doorbell: crate::doorbell::Doorbell, + /// How many times the loop came round. + turns: std::sync::atomic::AtomicUsize, +} + +impl Parked for NeverReady { + type Item = u32; + + fn pop(&self) -> Option { + self.turns + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + None + } + + fn finish(&self) -> Option { + None + } + + fn arm(&self) -> std::io::Result { + Ok(true) + } + + fn is_disconnected(&self) -> bool { + false + } + + fn doorbell(&self) -> std::io::Result> { + self.doorbell.handle() + } +} + +#[test] +fn a_timed_receive_sleeps_out_its_budget_instead_of_spinning_through_it() { + let consumer = NeverReady { + doorbell: crate::doorbell::Doorbell::new(), + turns: std::sync::atomic::AtomicUsize::new(0), + }; + + let timeout = Duration::from_millis(150); + let outcome = super::recv_timeout(&consumer, timeout); + assert!( + matches!(outcome, Err(crate::RecvTimeoutError::Timeout)), + "nothing was ever pushed, so the only way out is the deadline" + ); + + // Two turns is the honest count -- pop, arm, wait the whole budget, then + // pop, arm, and find nothing left to wait for. The ceiling is loose enough + // that a wait returning a little early cannot fail it, and tight enough + // that a wait returning *immediately* cannot pass it: at a hundred and + // fifty milliseconds of spinning, the count runs to five figures. + let turns = consumer.turns.load(std::sync::atomic::Ordering::Relaxed); + assert!( + turns <= 16, + "the loop came round {turns} times in {timeout:?}, which is a spin rather than a wait" + ); +} diff --git a/crates/windows-waitable-queues/src/capacity.rs b/crates/windows-waitable-queues/src/capacity.rs new file mode 100644 index 00000000..ae72c33a --- /dev/null +++ b/crates/windows-waitable-queues/src/capacity.rs @@ -0,0 +1,159 @@ +// Copyright (c) Mike Grier. + +//! The capacity rule, stated once for every bounded shape. +//! +//! It lives here rather than inside a shape's module because every bounded +//! shape enforces the same rule for the same reason, and a second copy of a +//! rule is free to drift from the first. A test that wants to check a suggested +//! capacity asks [`validate_capacity`] rather than re-encoding the conditions, +//! which is the difference between checking the rule and checking a paraphrase +//! of it. +//! +//! # The bounds belong to the shape, not to the crate +//! +//! [`CapacityError`] carries both bounds rather than assuming crate-wide +//! constants, because both follow from how a shape represents its positions -- +//! and the shipped shapes disagree about both. +//! +//! - **The minimum.** `spsc` accepts a capacity of one; `slotwise_mpsc` cannot, because +//! its slot state machine encodes "published" as one past the claim position +//! and "free again" as one lap past it, and with a single slot those are the +//! same number. +//! - **The maximum.** Most shapes stop at [`WRAPPING_MAX_CAPACITY`], where a +//! wrapping difference between two positions stops being unambiguous. +//! `reserving_mpsc` stops far lower, because it packs its reservation count +//! into the same word as its position so that the two can be claimed +//! together. +//! +//! So a shape supplies its own [`Bounds`] and this module applies them, which +//! is the arrangement the error type was already shaped for. + +use crate::error::CapacityError; + +/// The largest capacity that keeps a wrapping position difference unambiguous. +/// +/// Positions are monotonic and wrap with the integer, so a shape needs the +/// difference between two of them to be readable as a signed quantity: +/// +/// - `spsc` computes the number of items held as `tail.wrapping_sub(head)`, +/// which is the true difference only while that difference cannot exceed half +/// the range. +/// - `slotwise_mpsc` compares a slot's sequence number against a position by +/// interpreting `sequence.wrapping_sub(position)` as an [`isize`], which is +/// the same requirement written a different way. +/// +/// A shape whose positions are narrower than a [`usize`] has a correspondingly +/// smaller bound, and says so in its own [`Bounds`]; this is the widest any +/// shape may be. +pub(crate) const WRAPPING_MAX_CAPACITY: usize = usize::MAX / 2; + +/// The largest capacity a [`usize`]-positioned shape actually accepts. +/// +/// # Why this exists rather than [`WRAPPING_MAX_CAPACITY`] being used directly +/// +/// The wrapping bound is `usize::MAX / 2`, which is `2^(BITS-1) - 1` -- odd, +/// and therefore not a power of two, and therefore **not a capacity any shape +/// in this crate accepts**. A shape whose `Bounds::max` was the wrapping bound +/// was reporting a ceiling it would itself refuse. +/// +/// That is not a cosmetic difference, because the number is handed to callers: +/// [`CapacityError::max_valid`](crate::CapacityError::max_valid) documents it +/// as "the largest capacity the shape that rejected this request will accept", +/// and a caller correcting a refusal by using it would have been refused again. +/// +/// `reserving_mpsc` had already noticed the same trap from the other side -- +/// its bounds clamp against this value rather than the wrapping one, with a +/// `const` assertion saying why -- so this is that reasoning applied to the two +/// shapes that had not adopted it, and hoisted to one definition rather than +/// two. +pub(crate) const MAX_ADMISSIBLE_CAPACITY: usize = 1_usize << (usize::BITS - 2); + +// Facts about constants, checked by the compiler rather than by a test. +const _: () = { + assert!( + MAX_ADMISSIBLE_CAPACITY.is_power_of_two(), + "a ceiling offered to a caller as a correction must itself be a capacity \ + this crate accepts" + ); + assert!( + MAX_ADMISSIBLE_CAPACITY <= WRAPPING_MAX_CAPACITY, + "the admissible ceiling must stay inside the range where a wrapping \ + position difference is still unambiguous" + ); + assert!( + MAX_ADMISSIBLE_CAPACITY.leading_zeros() == 1, + "it must be the *largest* such power of two, not merely one of them -- \ + stated as a bit position because the arithmetic identity would be \ + tautological, and because a mutation run showed `usize::BITS - 2` \ + surviving replacement by `usize::BITS / 2` on a 64-bit host, where the \ + value is not the one selected" + ); +}; + +#[cfg(test)] +mod tests; + +/// What one shape will accept as a capacity. +/// +/// A named pair rather than two loose arguments, so neither a call site nor a +/// test can silently transpose them, and so a shape's answer to "how small" and +/// "how large" is written in one place with the reasoning beside it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Bounds { + /// The smallest capacity this shape can represent. + pub(crate) min: usize, + /// The largest capacity this shape can represent. + pub(crate) max: usize, +} + +/// Whether a bounded shape will accept a capacity, and why not if it will not. +/// +/// A power of two is required so a position can be reduced to a slot index with +/// a mask rather than a division, and the requested number is the exact number +/// of items the queue holds -- not a hint, and not rounded. See +/// [`CapacityError`] for why a rejection is preferred to silently rounding. +/// +/// Separated from each shape's constructor so the rule can be *asked* rather +/// than restated. A test that wants to check a suggested capacity is acceptable +/// would otherwise have to either re-encode these conditions -- a second copy +/// of a rule, free to drift from this one -- or call the constructor, which for +/// a capacity near the bound means trying to allocate half the address space. +pub(crate) fn validate_capacity(capacity: usize, bounds: Bounds) -> Result<(), CapacityError> { + debug_assert!( + bounds.min.is_power_of_two(), + "a shape's minimum is suggested to callers verbatim, so it must itself be valid" + ); + // The maximum needs the same guard as the minimum, and its absence is what + // let two shapes report `usize::MAX / 2` -- an odd number this function + // rejects -- as the ceiling a caller should retry with. The asymmetry was + // the defect: both ends of the pair are handed to callers through + // `CapacityError`, so both have to be capacities this function accepts. + debug_assert!( + bounds.max.is_power_of_two(), + "a shape's maximum is suggested to callers verbatim as the value to retry with, so it \ + must itself be valid" + ); + debug_assert!( + bounds.max <= WRAPPING_MAX_CAPACITY, + "no shape may exceed the width at which a wrapping position difference is unambiguous" + ); + debug_assert!( + bounds.min <= bounds.max, + "a shape that accepts nothing at all would reject every capacity with a suggestion it \ + would also reject" + ); + + if capacity == 0 { + return Err(CapacityError::zero(bounds)); + } + if !capacity.is_power_of_two() { + return Err(CapacityError::not_power_of_two(capacity, bounds)); + } + if capacity < bounds.min { + return Err(CapacityError::too_small(capacity, bounds)); + } + if capacity > bounds.max { + return Err(CapacityError::too_large(capacity, bounds)); + } + Ok(()) +} diff --git a/crates/windows-waitable-queues/src/capacity/tests.rs b/crates/windows-waitable-queues/src/capacity/tests.rs new file mode 100644 index 00000000..6c7816d8 --- /dev/null +++ b/crates/windows-waitable-queues/src/capacity/tests.rs @@ -0,0 +1,233 @@ +// Copyright (c) Mike Grier. + +//! Tests for the capacity bounds themselves. +//! +//! These exist because the ceiling was **documented wrongly in four places at +//! once**: the crate docs, the README and two design-note sections all said the +//! widest shape reaches `2^63` slots. It does not, and `error.rs` said so +//! correctly in the same crate the whole time -- "the nearest power of two +//! below `usize::MAX` is 2^63, which exceeds the largest representable +//! capacity". Prose cannot notice that it disagrees with prose; a test can. +//! +//! No queue is constructed here. Validation is a pure function of the request +//! and the bounds, so the ceiling can be checked without asking an allocator +//! for exabytes. + +use super::{Bounds, MAX_ADMISSIBLE_CAPACITY, WRAPPING_MAX_CAPACITY, validate_capacity}; + +/// The bounds of the widest shape in this crate. +/// +/// **`MAX_ADMISSIBLE_CAPACITY`, not `WRAPPING_MAX_CAPACITY`, and that is a +/// correction.** This fixture used to carry the wrapping bound, which is +/// `usize::MAX / 2` -- odd, and so not a capacity `validate_capacity` accepts. +/// The tests below already knew that (see +/// `the_wrapping_ceiling_is_one_below_a_power_of_two`, which asserts exactly +/// it), so the fixture was encoding a `Bounds` no real shape should ever have +/// held -- and two shapes did hold it, reporting through +/// `CapacityError::max_valid` a ceiling they would themselves refuse. +const WIDEST: Bounds = Bounds { + min: 1, + max: MAX_ADMISSIBLE_CAPACITY, +}; + +/// The largest power-of-two capacity the wrapping bound admits, as a shift. +/// +/// **Derived from `usize::BITS`, not written as 62.** The bound is +/// `usize::MAX / 2`, which is `2^(BITS-1) - 1`, so the largest power of two +/// under it is `2^(BITS-2)`. Hard-coding the 64-bit answer made these tests +/// unbuildable on a 32-bit target -- `1_usize << 63` does not fit in a 32-bit +/// `usize` -- a strange way for a test *about* `usize` bounds to fail. +const LARGEST_ACCEPTED_SHIFT: u32 = usize::BITS - 2; + +/// One past it: the smallest power of two the bound refuses. +const SMALLEST_REFUSED_SHIFT: u32 = usize::BITS - 1; + +#[test] +fn the_wrapping_ceiling_is_one_below_a_power_of_two() { + // The fact every other assertion here rests on, stated so a reader does not + // have to do the arithmetic: `usize::MAX / 2` is odd, so it is not itself a + // capacity any shape accepts. + assert_eq!( + WRAPPING_MAX_CAPACITY, + (1_usize << SMALLEST_REFUSED_SHIFT) - 1 + ); + assert!(!WRAPPING_MAX_CAPACITY.is_power_of_two()); +} + +#[test] +fn the_largest_accepted_capacity_is_two_below_the_word_size() { + // On 64-bit that is 2^62 accepted and 2^63 refused, which is what four + // documents used to claim was the other way round. Expressed as shifts so + // the same assertion holds on a narrower word. + validate_capacity(1_usize << LARGEST_ACCEPTED_SHIFT, WIDEST) + .expect("the largest power of two under the bound is within it"); + + validate_capacity(1_usize << SMALLEST_REFUSED_SHIFT, WIDEST) + .expect_err("one power of two past the bound must be refused"); +} + +#[test] +fn every_power_of_two_up_to_the_ceiling_is_accepted() { + // A property rather than the two boundary samples above, so a bound that + // moved for some other reason cannot pass by coincidence. + for shift in 0..=LARGEST_ACCEPTED_SHIFT { + let capacity = 1_usize << shift; + assert!( + validate_capacity(capacity, WIDEST).is_ok(), + "2^{shift} should be accepted" + ); + } + for shift in SMALLEST_REFUSED_SHIFT..usize::BITS { + let capacity = 1_usize << shift; + assert!( + validate_capacity(capacity, WIDEST).is_err(), + "2^{shift} should be refused" + ); + } +} + +#[test] +fn a_capacity_that_is_not_a_power_of_two_is_refused_whatever_its_size() { + // Guards the other half of the rule, so a fix to the ceiling cannot be made + // by loosening the shape of what is accepted. + let largest = 1_usize << LARGEST_ACCEPTED_SHIFT; + for capacity in [3_usize, 6, 100, largest - 1, largest + 1] { + assert!( + validate_capacity(capacity, WIDEST).is_err(), + "{capacity} is not a power of two and must be refused" + ); + } +} + +#[test] +fn a_capacity_exactly_at_the_ceiling_is_accepted() { + // **A small explicit ceiling, kept after `WIDEST` was corrected.** This was + // originally the only test here that could reach the equality boundary at + // all: `WIDEST` carried `usize::MAX / 2`, which is not a power of two, so + // the power-of-two rule refused every capacity near it and the `>` in + // `validate_capacity` was never asked about equality -- widening it to `>=` + // changed nothing observable, and a mutation run found the comparison + // unguarded. + // + // `WIDEST` now carries a ceiling that *is* a legal capacity, so it reaches + // the boundary too. This stays because a bound of 8 states the property + // without depending on the word size, and because the two tests fail for + // different reasons if the rule breaks. + let bounds = Bounds { min: 2, max: 8 }; + + validate_capacity(8, bounds).expect("the ceiling itself must be accepted"); + validate_capacity(16, bounds).expect_err("one power of two above it must not be"); + + // The same at the other end, so the floor is not off by one either. + validate_capacity(2, bounds).expect("the floor itself must be accepted"); + validate_capacity(1, bounds).expect_err("below the floor must not be"); +} + +#[test] +fn the_ceiling_a_refusal_reports_is_itself_a_capacity_that_would_be_accepted() { + // The contract `CapacityError::max_valid` states -- "the largest capacity + // the shape that rejected this request will accept" -- was false for two + // shapes, which reported `usize::MAX / 2`. A caller correcting a refusal by + // using it would have been refused again, with the same suggestion. + // + // Asserted as a round trip rather than against a literal, so it holds for + // whatever bound each shape declares: take the ceiling out of a real + // refusal and feed it straight back. + let too_large = validate_capacity(MAX_ADMISSIBLE_CAPACITY * 2, WIDEST) + .expect_err("one past the ceiling must be refused"); + + validate_capacity(too_large.max_valid(), WIDEST).expect( + "the ceiling a refusal suggests must be one the same bounds accept, or the \ + suggestion sends a caller straight back into the error they just had", + ); + assert_eq!(too_large.max_valid(), MAX_ADMISSIBLE_CAPACITY); + + // The same for the other end, which was already correct -- included so the + // pair is stated together and a later edit cannot break one while the other + // still passes. + let too_small = validate_capacity(0, Bounds { min: 4, max: 64 }) + .expect_err("zero is refused whatever the bounds"); + validate_capacity(too_small.min_valid(), Bounds { min: 4, max: 64 }) + .expect("the floor a refusal suggests must also be acceptable"); +} + +#[test] +fn the_admissible_ceiling_is_the_largest_power_of_two_the_wrapping_bound_allows() { + // Only the relationships that are *not* already compile-time facts. That + // the ceiling is a power of two and sits inside the wrapping bound is + // asserted in `capacity.rs`'s `const` block, which is the stronger place -- + // it fails the build rather than a run somebody chose to make -- and clippy + // rightly rejects restating them here as constant-valued assertions. + // + // What is left is the tie between the ceiling and the shifts these tests + // reason in, so a bound moved for some other reason cannot pass by + // coincidence. + assert_eq!(MAX_ADMISSIBLE_CAPACITY, 1_usize << LARGEST_ACCEPTED_SHIFT); + assert_eq!( + WRAPPING_MAX_CAPACITY, + (1_usize << SMALLEST_REFUSED_SHIFT) - 1, + "the next power of two up is one past the wrapping bound, which is what \ + makes the admissible ceiling the largest one that fits" + ); +} + +#[test] +fn the_shapes_ceilings_are_what_the_public_documentation_claims() { + // The crate docs and the README compare the shapes by capacity, and that + // comparison is target-dependent -- which they did not say until a review + // round pointed it out. Asserted here so the claim is checked on whatever + // target the suite runs on rather than believed from a 64-bit reading. + // + // Written while correcting exactly that: a first draft of the corrected + // prose said `reserving_mpsc` keeps its 2^31 packed ceiling on a 32-bit + // target. It does not -- the clamp applies to it too -- and this assertion + // is what caught it. + // Asked through the public surface rather than by reaching for each + // shape's private `BOUNDS`: what the documentation describes is what a + // caller can observe, and a caller observes the ceiling by being refused. + // A capacity of 3 is refused by every shape for a reason that has nothing + // to do with the ceiling, so the error it carries reports the real one. + let spsc_ceiling = crate::spsc::bounded::(3) + .expect_err("3 is not a power of two") + .max_valid(); + let slotwise_ceiling = crate::slotwise_mpsc::bounded::(3) + .expect_err("3 is not a power of two") + .max_valid(); + + assert_eq!( + spsc_ceiling, MAX_ADMISSIBLE_CAPACITY, + "spsc's positions are full-width, so it goes as wide as any shape may" + ); + assert_eq!( + slotwise_ceiling, MAX_ADMISSIBLE_CAPACITY, + "slotwise_mpsc is bounded by allocation rather than by its own positions" + ); + + // `reserving_mpsc`'s default layout gives the position 32 bits, so its own + // ceiling is 2^31 -- but it is *also* clamped, and on a 32-bit target the + // clamp is the binding constraint. `BOUNDS_MAX` is the default layout's + // ceiling by definition; a deeper layout has its own, reachable through + // `ClaimLayout`, and is not what this constant reports. + let packed = 1_usize << 31; + let expected = if packed <= MAX_ADMISSIBLE_CAPACITY { + packed + } else { + MAX_ADMISSIBLE_CAPACITY + }; + assert_eq!(crate::reserving_mpsc::BOUNDS_MAX, expected); + + #[cfg(target_pointer_width = "64")] + { + assert_eq!(MAX_ADMISSIBLE_CAPACITY, 1_usize << 62); + assert_eq!(crate::reserving_mpsc::BOUNDS_MAX, 1_usize << 31); + } + #[cfg(target_pointer_width = "32")] + { + assert_eq!(MAX_ADMISSIBLE_CAPACITY, 1_usize << 30); + assert_eq!( + crate::reserving_mpsc::BOUNDS_MAX, + 1_usize << 30, + "the clamp binds here, so both shapes land on the same ceiling" + ); + } +} diff --git a/crates/windows-waitable-queues/src/disposal.rs b/crates/windows-waitable-queues/src/disposal.rs new file mode 100644 index 00000000..b7762f0c --- /dev/null +++ b/crates/windows-waitable-queues/src/disposal.rs @@ -0,0 +1,180 @@ +// Copyright (c) Mike Grier. + +//! What becomes of items still in the queue when it is torn down. +//! +//! # The hazard, which is not hypothetical +//! +//! A queue's items can own resources, and a descriptor for a completed async +//! open owns a **handle**. Closing a handle is not always cheap: closing one to +//! a dead network path can block for a long time, and keeping exactly that +//! operation off a caller's thread is the sort of thing the queue exists to +//! serve in the first place. +//! +//! So the question "who destroys the items nobody drained?" has a bad default +//! answer. Without this module, they are destroyed **in place, on whichever +//! thread happened to release the last handle** -- which may be a thread-pool +//! callback that must not block, or a producer that has no idea it is holding +//! the last reference. Nobody chose that thread, and nothing tells the owner it +//! happened. +//! +//! # Why `Drop` cannot simply hand them back +//! +//! The obvious fix -- return the remainder from teardown -- is not available. +//! [`Drop::drop`] takes `&mut self`, returns nothing, and cannot fail. By the +//! time it runs, every handle is already gone, so there is nobody left to +//! return anything *to*. Anything the queue is going to do with those items, it +//! must have been told in advance. +//! +//! Draining first does not close the hole either. A consumer can take +//! everything available, but a producer may push again afterwards, so an +//! orderly drain covers the orderly path and nothing else. **The last handle to +//! drop is the only place that sees every remaining item**, and it is the one +//! place with no way to report. +//! +//! # So the decision is made at construction +//! +//! A queue built with [`Disposal`] hands each surviving item to that sink +//! instead of destroying it. The owner therefore decides where disposal +//! happens: a sink that moves items to a reaper thread keeps the blocking off +//! the dropping thread entirely, and one that disposes inline is fine when the +//! dropping thread is allowed to block. Either way it is a decision somebody +//! made rather than one that fell out of which `Arc` clone happened to die +//! last. +//! +//! **The default is unchanged and still destroys in place**, because for the +//! overwhelmingly common case -- items that own nothing -- that is exactly +//! right, and a queue of `u32` should not have to think about any of this. What +//! changes is that the behaviour is now written down as a choice with a name. + +use core::fmt; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +/// Where a queue's surviving items go when it is torn down. +/// +/// See the [module documentation](self) for why this has to be supplied up +/// front rather than asked for at teardown. +/// +/// # Examples +/// +/// Handing the remainder to a channel a reaper thread drains, so a destructor +/// that blocks does so somewhere that is allowed to: +/// +/// ``` +/// use std::sync::mpsc; +/// use windows_waitable_queues::{Disposal, Options, spsc}; +/// +/// let (undelivered, reaper) = mpsc::channel(); +/// let (tx, rx) = spsc::bounded_with::( +/// 4, +/// Options::new().disposal(Disposal::new(move |item| { +/// // Cheap and non-blocking: the reaper thread does the real work. +/// let _ = undelivered.send(item); +/// })), +/// )?; +/// +/// tx.push(1).expect("a fresh queue has room"); +/// tx.push(2).expect("a fresh queue has room"); +/// drop((tx, rx)); +/// +/// // Nothing was destroyed behind the owner's back. +/// assert_eq!(reaper.into_iter().collect::>(), vec![1, 2]); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub struct Disposal { + sink: Box, +} + +impl Disposal { + /// Builds a sink from a closure. + /// + /// The closure is called once per surviving item, on the thread that + /// released the queue's last handle. **It should be cheap**: if disposal + /// can block, the useful shape is to move the item somewhere a thread that + /// may block will find it, rather than to do the blocking work here. + /// + /// `Send` because the thread that tears the queue down is whichever one + /// happened to drop last, and is not knowable in advance. Not `Sync`, + /// because it is only ever called from a teardown that has exclusive + /// access. + pub fn new(sink: impl FnMut(T) + Send + 'static) -> Self { + Self { + sink: Box::new(sink), + } + } +} + +impl fmt::Debug for Disposal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Disposal(..)") + } +} + +/// A shape's teardown policy: the sink if it was given one, otherwise the +/// default. +/// +/// Held by every shape's shared state and touched only from `Drop`, so it costs +/// the hot paths nothing but the space. +pub(crate) struct Teardown { + disposal: Option>, +} + +impl Teardown { + /// Hand each surviving item to `disposal`, or destroy it where it lies if + /// there is none. + pub(crate) const fn new(disposal: Option>) -> Self { + Self { disposal } + } + + /// Dispose of one surviving item. + /// + /// # A panicking disposal does not strand the items behind it + /// + /// This applies to the item's own `Drop` as much as to a sink, and an + /// earlier version guarded only the sink. Both are caller-supplied code + /// running inside a destructor -- `T` belongs to the caller too -- so a + /// panicking `T::drop` escaped this manual walk over the surviving slots, + /// abandoning every item behind it and risking the second-panic abort. The + /// default path had exactly the failure the sink path was written to + /// prevent, and the reasoning below never distinguished them. + /// + /// The disposal is caller-supplied code running inside a destructor, which is + /// the worst place for it to panic: a panic escaping here during an unwind + /// aborts the process, and one escaping otherwise abandons every item not + /// yet disposed -- precisely the handles this whole mechanism exists to + /// account for. + /// + /// So a panic is caught and the walk continues. That is deliberately *not* + /// "swallowing an error": the item has already been handed over or + /// destroyed, so there is nothing left to report about it, and the + /// alternative is to lose the rest of the queue as well. A sink or a `Drop` + /// that panics is a bug in the caller; this only declines to make it a much + /// larger one. + pub(crate) fn dispose(&mut self, item: T) { + let Some(disposal) = self.disposal.as_mut() else { + // The default. Written as an explicit drop rather than left to fall + // out of the binding going out of scope, because "destroy it here" + // is a decision this type exists to name -- and caught for the same + // reason the sink is: `T::drop` is the caller's code too. + let _ = catch_unwind(AssertUnwindSafe(move || drop(item))); + return; + }; + + // `AssertUnwindSafe` is the honest annotation rather than a way past + // the bound: the only state that could be observed after a panic is the + // caller's own closure, and the queue's own invariants do not depend on + // the sink at all -- teardown is already past the point where anything + // could observe them. + let _ = catch_unwind(AssertUnwindSafe(|| (disposal.sink)(item))); + } +} + +impl fmt::Debug for Teardown { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Teardown") + .field("hands_off", &self.disposal.is_some()) + .finish() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/disposal/tests.rs b/crates/windows-waitable-queues/src/disposal/tests.rs new file mode 100644 index 00000000..d0d991d3 --- /dev/null +++ b/crates/windows-waitable-queues/src/disposal/tests.rs @@ -0,0 +1,224 @@ +// Copyright (c) Mike Grier. + +//! Tests for the teardown policy in isolation, with no queue attached. +//! +//! The policy's behaviour *through* a queue is asserted in each shape's own +//! suite, because each walks its own layout to find the survivors and covering +//! one would say nothing about the others. What is tested here is the part +//! they share: that the default destroys, that a sink receives, and that a +//! panicking sink does not strand the items behind it. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::{Disposal, Teardown}; + +/// Counts its own drops, so a test can tell "handed to the sink" from +/// "destroyed where it lay" -- which is the entire distinction this module +/// exists to draw. +#[derive(Debug)] +struct DropCounter(Arc); + +impl Drop for DropCounter { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn the_default_policy_destroys_the_item() { + let drops = Arc::new(AtomicUsize::new(0)); + let mut teardown = Teardown::new(None); + + teardown.dispose(DropCounter(Arc::clone(&drops))); + assert_eq!( + drops.load(Ordering::Relaxed), + 1, + "with no sink there is nowhere else for it to go, so it is destroyed here" + ); +} + +#[test] +fn a_sink_receives_the_item_instead_of_it_being_destroyed() { + let drops = Arc::new(AtomicUsize::new(0)); + let collected = Arc::new(AtomicUsize::new(0)); + + let seen = Arc::clone(&collected); + let mut teardown = Teardown::new(Some(Disposal::new(move |item: DropCounter| { + seen.fetch_add(1, Ordering::Relaxed); + // Deliberately kept alive past the sink call, which is the whole point: + // the owner decides when -- and on which thread -- the destructor runs. + std::mem::forget(item); + }))); + + teardown.dispose(DropCounter(Arc::clone(&drops))); + + assert_eq!(collected.load(Ordering::Relaxed), 1, "the sink saw it"); + assert_eq!( + drops.load(Ordering::Relaxed), + 0, + "and teardown did not destroy it behind the owner's back" + ); +} + +#[test] +fn every_item_reaches_the_sink_in_order() { + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = Arc::clone(&order); + let mut teardown = Teardown::new(Some(Disposal::new(move |item: u32| { + seen.lock().expect("no test holds this poisoned").push(item); + }))); + + for value in 0..10 { + teardown.dispose(value); + } + + assert_eq!( + *order.lock().expect("no test holds this poisoned"), + (0..10).collect::>(), + "a sink is handed the survivors one at a time, in the order teardown walks them" + ); +} + +#[test] +fn a_panicking_sink_does_not_strand_the_items_behind_it() { + // The property that matters, and the reason the call is wrapped. A sink + // that panics on one item is a caller bug; losing every *later* item to it + // would turn that bug into the exact leak this mechanism exists to + // prevent, and inside an unwind it would abort the process outright. + let disposed = Arc::new(AtomicUsize::new(0)); + let seen = Arc::clone(&disposed); + + let mut teardown = Teardown::new(Some(Disposal::new(move |item: u32| { + seen.fetch_add(1, Ordering::Relaxed); + assert_ne!(item, 3, "deliberate panic from a caller-supplied sink"); + }))); + + for value in 0..10 { + teardown.dispose(value); + } + + assert_eq!( + disposed.load(Ordering::Relaxed), + 10, + "the walk must continue past a panicking sink, or one bad item loses the rest" + ); +} + +#[test] +fn a_panicking_sink_still_consumes_the_item_it_panicked_on() { + // The item was moved into the sink before it panicked, so it is destroyed + // by the unwind rather than leaked. Asserted so that "the panic is caught" + // is not mistaken for "the item is still somewhere". + let drops = Arc::new(AtomicUsize::new(0)); + let mut teardown = Teardown::new(Some(Disposal::new(|_item: DropCounter| { + panic!("deliberate panic from a caller-supplied sink"); + }))); + + teardown.dispose(DropCounter(Arc::clone(&drops))); + assert_eq!( + drops.load(Ordering::Relaxed), + 1, + "the item was already the sink's, so unwinding destroys it" + ); +} + +#[test] +fn a_sink_may_be_stateful_across_items() { + // `FnMut` rather than `Fn`, because the useful sinks accumulate: pushing + // into a channel, counting, or batching for a reaper. + let mut total = 0_u32; + let sum = Arc::new(AtomicUsize::new(0)); + let report = Arc::clone(&sum); + + let mut teardown = Teardown::new(Some(Disposal::new(move |item: u32| { + total += item; + report.store(total as usize, Ordering::Relaxed); + }))); + + for value in 1..=4 { + teardown.dispose(value); + } + assert_eq!(sum.load(Ordering::Relaxed), 10); +} + +#[test] +fn the_debug_form_says_which_policy_is_in_force() { + // Teardown is invisible until something goes wrong, so the one place it can + // be observed should say which of the two it is. + let plain: Teardown = Teardown::new(None); + assert!(format!("{plain:?}").contains("hands_off: false")); + + let handing: Teardown = Teardown::new(Some(Disposal::new(|_| {}))); + assert!(format!("{handing:?}").contains("hands_off: true")); +} + +/// An item whose own destructor panics on one chosen value. +/// +/// `T` is the caller's type, so its `Drop` is caller-supplied code exactly as a +/// sink is -- which is the whole point of the test below. +struct PanicsOnDrop { + value: u32, + dropped: Arc, +} + +impl Drop for PanicsOnDrop { + fn drop(&mut self) { + self.dropped.fetch_add(1, Ordering::Relaxed); + assert_ne!( + self.value, 3, + "deliberate panic from a caller-supplied destructor" + ); + } +} + +#[test] +fn a_panicking_item_destructor_does_not_strand_the_items_behind_it() { + // **The default policy had the defect the sink policy was written to + // avoid.** With no sink the item was dropped directly, so a panicking + // `T::drop` escaped this manual walk over the surviving slots -- abandoning + // every item behind it, and inside an unwind aborting the process on the + // second panic. The reasoning for catching the sink never distinguished the + // two, and neither does the code now. + let dropped = Arc::new(AtomicUsize::new(0)); + + let mut teardown = Teardown::new(None); + for value in 0..10 { + teardown.dispose(PanicsOnDrop { + value, + dropped: Arc::clone(&dropped), + }); + } + + assert_eq!( + dropped.load(Ordering::Relaxed), + 10, + "the walk must continue past a panicking destructor, or one bad item loses the rest" + ); +} + +#[test] +fn a_panicking_destructor_still_destroys_the_item_it_panicked_on() { + // Catching the panic must not turn into retaining the item: it was moved + // into the closure, so the unwind destroys it. Asserted separately so that + // "the panic is caught" cannot be mistaken for "the item survives". + let dropped = Arc::new(AtomicUsize::new(0)); + + let mut teardown = Teardown::new(None); + teardown.dispose(PanicsOnDrop { + value: 3, + dropped: Arc::clone(&dropped), + }); + + assert_eq!(dropped.load(Ordering::Relaxed), 1); +} + +#[test] +fn the_debug_rendering_names_the_type() { + // `Disposal` holds a boxed closure, so its rendering is deliberately opaque + // -- but opaque is not the same as empty. A `Debug` returning `Ok(default)` + // writes nothing at all and passes any test that only checks it does not + // panic, which is what a mutation run found here. + let rendered = format!("{:?}", Disposal::new(|_: u32| {})); + assert!(rendered.contains("Disposal"), "got {rendered}"); +} diff --git a/crates/windows-waitable-queues/src/doorbell.rs b/crates/windows-waitable-queues/src/doorbell.rs new file mode 100644 index 00000000..d11db9db --- /dev/null +++ b/crates/windows-waitable-queues/src/doorbell.rs @@ -0,0 +1,414 @@ +// Copyright (c) Mike Grier. + +//! A queue's readiness, expressed as a waitable Windows `HANDLE`. +//! +//! This is the part of the crate its name refers to. A queue shape owns a +//! [`Doorbell`] and keeps it in agreement with its own emptiness; a client that +//! wants to park on the queue *and* on an I/O completion *and* on a shutdown +//! event in one wait borrows the handle and hands it to +//! `WaitForMultipleObjects` alongside the others. +//! +//! # Manual-reset, and level-triggered +//! +//! The event is manual-reset, so it means "there is something to take" rather +//! than "something arrived". That is the difference between a state and an +//! edge, and only the state composes: `WaitForMultipleObjects` may report any +//! one of several signalled handles, so a waiter routinely learns about one +//! ready source while ignoring another. An auto-reset event consumed by that +//! wait would lose the second source's only edge. A level survives being +//! ignored, and will still be there on the next pass. +//! +//! The crate's own probe made this concrete before the design was fixed: an +//! auto-reset event does not count signals, so two pushes and one wait leave a +//! consumer blocked forever on an item that is sitting in the queue. +//! +//! # Created lazily, so polling is free +//! +//! A consumer that only ever calls `pop` in a loop of its own never needs a +//! kernel object, and should not be charged for one. The event is therefore +//! created on the first request for the handle and not before, following the +//! precedent already set by `windows-file-watcher`'s notification queue. +//! +//! The cost of that laziness is a race worth stating plainly: a producer that +//! runs while no event exists yet skips signalling, because there is nothing to +//! signal. If a consumer could create the doorbell and then immediately wait on +//! it, an item pushed during that window would never wake anyone. Closing that +//! hole takes **two** things, and an earlier version of this note claimed the +//! first was enough: +//! +//! 1. The doorbell must exist *before* the emptiness check that decides to +//! wait, which is what the arming protocol below arranges. +//! 2. The producer's decision to skip signalling and the consumer's emptiness +//! check must be sequentially consistent with respect to each other. Program +//! order alone does not give this. See "The store-buffer hazard" below. +//! +//! # The arming protocol, which is the whole correctness argument +//! +//! [`Doorbell`] cannot enforce this itself, because it cannot see the queue. A +//! shape that owns one must observe this order, and no other: +//! +//! 1. Take everything available. +//! 2. [`Doorbell::clear`]. +//! 3. **Check emptiness again.** If anything is there, do not wait -- go to 1. +//! 4. Wait on the handle. +//! +//! The re-check at step 3 is not an optimisation, and removing it is not a +//! missed wakeup once in a while -- it is a permanent hang. A producer that +//! pushes between steps 1 and 2 may signal before the clear at step 2 erases +//! it, leaving an item in the queue and the doorbell unsignalled. Nothing later +//! will signal again, because nothing later will arrive. +//! +//! Reversing steps 2 and 3 -- checking emptiness and then clearing -- fails the +//! same way and is the easier mistake to make, because it reads more naturally. +//! `spsc`'s test suite asserts this by reversing them deliberately and +//! requiring the result to hang. +//! +//! A lock-based queue gets this for free by clearing under the lock it already +//! holds while deciding there is nothing to take, which is what the file +//! watcher does. A lock-free queue has no such lock, so the ordering above is +//! the substitute, and it has to be written down because the compiler will not +//! ask about it. +//! +//! # The store-buffer hazard, which ordering alone does not fix +//! +//! The arming protocol says the consumer clears and then re-checks. The +//! producer pushes and then checks whether to signal. Written out as memory +//! operations, each side stores one location and then loads another: +//! +//! | Producer (`push` then [`Doorbell::signal`]) | Consumer ([`Doorbell::clear`] then re-check) | +//! |---|---| +//! | store the queue position (release) | store `signalled` / reset the event | +//! | load `event` and `signalled` | load the queue position (acquire) | +//! +//! This is the store-buffer shape -- the same one Dekker's algorithm runs +//! into -- and release/acquire does **not** forbid both loads from returning +//! stale values. When both do, the item is in the queue, the producer decided +//! no signal was needed, and the consumer decided it was safe to wait. That is +//! a permanent hang, not a stall. +//! +//! The remedy is sequential consistency on both sides, and it is not optional: +//! a `SeqCst` fence sits before the loads in [`Doorbell::signal`] and after the +//! stores in [`Doorbell::clear`]. Every published eventcount carries the same +//! fence in the same place for the same reason. +//! +//! Two temptations to record as refused. The consumer's `ResetEvent` is a +//! syscall and is very probably a full barrier, and `stlr`/`ldar` on aarch64 +//! happen to be ordered more strongly than the abstract model requires -- so on +//! today's compiler and today's processors this may well never misbehave. +//! Neither is a specified guarantee, and binding correctness to the incidental +//! behaviour of a code generator and a particular processor instead of to the +//! ordering primitives is exactly the trap this workspace has paid for before. +//! +//! **This hazard is invisible to the test suite**, which is a property of the +//! hazard and not a gap to be closed by trying harder: no amount of stress +//! testing reliably produces the interleaving, and none of the sabotages in +//! `sabotage.json` can express it. Removing either fence leaves every test +//! green. It is verifiable only under a model checker, which is what makes it +//! the named target of the `loom` work in checklist item M31.6. +//! +//! # Why a redundant signal is skipped, but a redundant clear is not +//! +//! The two directions are not symmetric, and the asymmetry is the reason this +//! type keeps a flag at all. +//! +//! A **late signal** is a spurious wakeup: a waiter wakes, finds nothing, and +//! waits again. A **stale clear** is a lost wakeup: a waiter sleeps on a +//! non-empty queue forever. Cheapening the signal side is therefore safe, and +//! cheapening the clear side is not. +//! +//! So `signal` keeps an [`AtomicBool`] mirroring the event and returns without +//! a syscall when the event is already signalled. On this crate's reference +//! machine `SetEvent` on an already-signalled event measured 81.2 ns against +//! 7.2 ns for an uncontended atomic, so a backlogged producer that would +//! otherwise pay a syscall per push pays roughly a tenth of one. +//! +//! # The flag must never outlive the signal it mirrors +//! +//! The flag is allowed to disagree with the event briefly, and that is sound in +//! exactly one direction: it may claim **signalled while the `SetEvent` has not +//! landed yet**, which costs a skipped redundant signal, never a skipped +//! necessary one. The opposite disagreement -- the flag claiming signalled over +//! an event that is *dark* -- is fatal, because every later [`Doorbell::signal`] +//! then skips its syscall and the doorbell can never be lit again. +//! +//! **[`Doorbell::clear`] therefore resets the event first and clears the flag +//! second, and that order is load-bearing.** Written the other way round -- flag +//! first, `ResetEvent` second, which is how this shipped originally -- a +//! producer signalling between the two lines finds a clear flag, sets it, and +//! issues a real `SetEvent`; the `ResetEvent` that follows then erases that +//! signal while leaving the flag set. The doorbell is wedged dark with the flag +//! claiming otherwise, and the next producer to publish skips the one signal +//! that mattered. +//! +//! The original argument for the other order was that the caller's re-check +//! covers it: a producer racing the clear publishes *before* it signals, so the +//! re-check sees the item and the caller does not wait. **That argument is +//! sound only when the re-check is guaranteed to see anything that producer +//! published**, and it silently assumed a queue whose emptiness is a single +//! position comparison. `slotwise_mpsc` broke the assumption -- its re-check asks whether +//! the *head* slot is published, so a producer publishing at a later position +//! is invisible to it, and the consumer parks in exactly the wedged state above. +//! The failure was a rare permanent hang, reproduced once in a sabotage +//! baseline and then not again in six runs. +//! +//! With the reset first, the invariant is a property of this type rather than a +//! property of its callers: **once `clear` returns, the flag is false, so the +//! next `signal` cannot be skipped.** A producer signalling inside the window +//! may still be skipped, but it published before it signalled and therefore +//! before the flag store, so the caller's re-check -- which follows -- observes +//! whatever that publication made observable, and any producer that publishes +//! *after* the re-check finds the flag already false and rings for real. + +use std::io; +use std::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle, FromRawHandle, OwnedHandle}; +use std::ptr; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering, fence}; + +use windows_sys::Win32::Foundation::{DUPLICATE_SAME_ACCESS, DuplicateHandle, FALSE, TRUE}; +use windows_sys::Win32::System::Threading::{ + CreateEventW, GetCurrentProcess, ResetEvent, SetEvent, +}; + +/// A lazily created manual-reset event that reports whether a queue has +/// anything to take. +/// +/// See the [module documentation](self) for the arming protocol every owner +/// must follow; this type cannot enforce it, because it cannot see the queue. +pub(crate) struct Doorbell { + /// The event, absent until somebody asks for the handle. + event: OnceLock, + /// Mirrors the event's state so a redundant [`Doorbell::signal`] can skip + /// its syscall. Only [`Doorbell::signal`] and [`Doorbell::clear`] write it. + signalled: AtomicBool, + /// How many times a real `SetEvent` has been issued. + /// + /// See [`Doorbell::rings`] for why this counts syscalls rather than calls, + /// and the [module documentation](self) for why the skipped signals are not + /// counted alongside it. + rings: AtomicU64, +} + +impl Doorbell { + /// A doorbell that owns no kernel object yet. + pub(crate) const fn new() -> Self { + Self { + event: OnceLock::new(), + signalled: AtomicBool::new(false), + rings: AtomicU64::new(0), + } + } + + /// Borrow the event, creating it on the first call. + /// + /// The borrow is deliberate: the event belongs to the queue and must not be + /// closed by a caller. Use [`Doorbell::owned`] where ownership is required, + /// such as arming a `ThreadpoolWait`. + /// + /// The event is created unsignalled regardless of what the queue holds, + /// because this type cannot see the queue. The owner is responsible for + /// bringing it into agreement, which the arming protocol does for free: the + /// re-check after [`Doorbell::clear`] runs after creation, so an item that + /// arrived before the doorbell existed is found rather than waited on. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub(crate) fn handle(&self) -> io::Result> { + if let Some(event) = self.event.get() { + return Ok(event.as_handle()); + } + // A racing caller may win the `set`, in which case ours is dropped and + // closed and theirs is used. Both are unsignalled, so the loser's + // disappearance costs nothing; only one event can ever be published. + let created = create_event()?; + let _ = self.event.set(created); + Ok(self + .event + .get() + .expect("the doorbell was just published") + .as_handle()) + } + + /// A duplicate of [`Doorbell::handle`] that the caller owns. + /// + /// The duplicate refers to the same event, so signalling reaches both, and + /// the caller may close its copy whenever it likes. This is the form a + /// `ThreadpoolWait` needs, since arming one takes ownership of its target. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` or `DuplicateHandle`. + pub(crate) fn owned(&self) -> io::Result { + duplicate(self.handle()?) + } + + /// Report that the queue has something to take. + /// + /// Does nothing when no handle has ever been requested, and nothing when + /// the event is already signalled. Both skips are safe; see the [module + /// documentation](self) for why the signal side may be cheapened and the + /// clear side may not. + /// + /// A failure of `SetEvent` is not reported. There is no useful reaction on + /// a producer's hot path, and the only documented failures are invalid + /// handles, which cannot occur for an event this type owns for its whole + /// lifetime. + pub(crate) fn signal(&self) { + // This fence is load-bearing and is not an abundance of caution. + // + // The caller has just published an item with a release store, and is + // about to LOAD state that decides whether to signal. The consumer does + // the mirror image: it stores that same state, then loads the queue's + // position. Store-then-load on each side, over two different locations, + // is the store-buffer (Dekker) shape, and release/acquire does not + // forbid both loads from seeing stale values. If both do, the item is + // queued, no signal is raised, and the consumer parks forever. + // + // Sequential consistency is the documented remedy, and every published + // eventcount carries the same fence in the same place for the same + // reason. Both skip paths below are loads, so the fence has to precede + // them rather than sit between them. + // + // Deliberately not relying on the fact that a particular compiler and a + // particular processor happen not to reorder this today, nor on the + // consumer's `ResetEvent` syscall incidentally acting as a barrier: the + // memory model permits the reordering, so the ordering must come from a + // specified primitive. + fence(Ordering::SeqCst); + + let Some(event) = self.event.get() else { + // Nobody can be waiting on a handle that does not exist yet, and + // the fence above guarantees that a consumer which publishes one + // after this load will see the item this push just added. + return; + }; + if self.signalled.swap(true, Ordering::AcqRel) { + // Already signalled, and a manual-reset event does not count, so + // setting it again would change nothing. + return; + } + // Counted here and nowhere else, which is what makes it free. This + // branch already costs a `SetEvent` -- measured at ~81 ns on this + // crate's reference machine against ~7 ns for an uncontended atomic -- + // so the increment is under a tenth of a cost that was already being + // paid, and it happens only on the rare path. + // + // **The skipped signals are deliberately not counted.** That would put + // a second read-modify-write on precisely the path the skip exists to + // cheapen, which is the one place in this type where an atomic is the + // whole cost rather than a rounding error on a syscall. + self.rings.fetch_add(1, Ordering::Relaxed); + + // SAFETY: a live manual-reset event owned by this type for as long as + // it exists; `SetEvent` has no other precondition. + unsafe { + SetEvent(event.as_raw_handle()); + } + } + + /// How many times this doorbell has actually rung. + /// + /// Counts `SetEvent` calls, not [`Doorbell::signal`] calls. The difference + /// between the two *is* the skip optimisation, which is why this number is + /// the one worth reporting: it makes the skip rule measurable rather than + /// assumed, and turning the skip off has to move it. + pub(crate) fn rings(&self) -> u64 { + self.rings.load(Ordering::Relaxed) + } + + /// Report that the queue appears to have nothing to take. + /// + /// **The caller must re-check emptiness after this returns**, and must not + /// wait if the re-check finds anything. See the [module + /// documentation](self); this is the step whose omission is a permanent + /// hang rather than an occasional stall. + pub(crate) fn clear(&self) { + let Some(event) = self.event.get() else { + return; + }; + + // **The event is reset first and the flag second, and swapping these + // two lines is a permanent hang.** See "the flag must never outlive the + // signal it mirrors" in the module documentation; the short form is + // that a producer signalling between them must never be able to leave + // the flag claiming "lit" over an event this call is about to darken. + // + // SAFETY: as in `signal`. + unsafe { + ResetEvent(event.as_raw_handle()); + } + #[cfg(test)] + crate::race_hooks::CLEAR.run(); + self.signalled.store(false, Ordering::Release); + + // The other half of the pair described in `signal`. The caller's + // re-check is a LOAD of the queue's state, and it follows this store of + // `signalled`; without a sequentially consistent fence on both sides, + // that load and the producer's load of `signalled` may both observe + // stale values, which is the lost wakeup. `ResetEvent` above is very + // probably a barrier in its own right, but that is an incidental + // property of an implementation rather than a documented guarantee, so + // it is not what this relies on. + fence(Ordering::SeqCst); + } + + /// Whether the event has been created, for tests and for asserting that + /// laziness actually holds. + #[cfg(test)] + pub(crate) fn is_armed(&self) -> bool { + self.event.get().is_some() + } +} + +impl std::fmt::Debug for Doorbell { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Doorbell") + .field("created", &self.event.get().is_some()) + // Acquire, matching every other operation on `signalled`, which + // carries an `AcqRel` swap and a `Release` store. Nothing here + // depends on the edge, but a lone relaxed load on such an atomic is + // a plain load with no defined position relative to them, and this + // is a `Debug` formatter -- there is no cost worth the exception. + .field("signalled", &self.signalled.load(Ordering::Acquire)) + .finish() + } +} + +/// Create an unnamed, unsignalled, manual-reset event. +fn create_event() -> io::Result { + // SAFETY: creates an unnamed event with default security attributes; both + // pointer arguments are null by design. + let raw = unsafe { CreateEventW(ptr::null(), TRUE, FALSE, ptr::null()) }; + if raw.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call returned a fresh, exclusively owned event handle. + Ok(unsafe { OwnedHandle::from_raw_handle(raw) }) +} + +/// Duplicate a handle into this process, so the caller owns its own copy. +fn duplicate(handle: BorrowedHandle<'_>) -> io::Result { + let mut duplicated = ptr::null_mut(); + // SAFETY: duplicates a live handle within this process with the same + // access; `duplicated` is a valid out-pointer for the call's duration. + let ok = unsafe { + DuplicateHandle( + GetCurrentProcess(), + handle.as_raw_handle(), + GetCurrentProcess(), + &raw mut duplicated, + 0, + FALSE, + DUPLICATE_SAME_ACCESS, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call succeeded, so `duplicated` is a fresh owned handle. + Ok(unsafe { OwnedHandle::from_raw_handle(duplicated) }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/doorbell/tests.rs b/crates/windows-waitable-queues/src/doorbell/tests.rs new file mode 100644 index 00000000..044a5102 --- /dev/null +++ b/crates/windows-waitable-queues/src/doorbell/tests.rs @@ -0,0 +1,425 @@ +// Copyright (c) Mike Grier. + +//! Tests for the doorbell in isolation, with no queue attached. +//! +//! These assert the properties the arming protocol is built on -- laziness, +//! level semantics, and that a redundant signal is skipped without losing a +//! necessary one. The protocol *itself* cannot be tested here, because it is a +//! statement about a queue this type cannot see; that is `spsc`'s job. + +use std::os::windows::io::AsRawHandle; +use std::sync::Arc; + +use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; +use windows_sys::Win32::System::Threading::WaitForSingleObject; + +use super::Doorbell; +use crate::race_hooks; + +/// Whether the doorbell is signalled right now, by asking the kernel rather +/// than by reading the mirror flag. +/// +/// A test that consulted the flag would be testing the flag against itself. The +/// zero timeout makes this a state query rather than a wait, and it does not +/// consume the signal because the event is manual-reset. +fn is_signalled(doorbell: &Doorbell) -> bool { + let handle = doorbell.handle().expect("the doorbell must be creatable"); + // SAFETY: a live event handle borrowed for the call; a zero timeout returns + // immediately and has no other precondition. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 0) }; + assert!( + result == WAIT_OBJECT_0 || result == WAIT_TIMEOUT, + "the wait must resolve to signalled or not, got {result:#x}" + ); + result == WAIT_OBJECT_0 +} + +#[test] +fn creates_no_kernel_object_until_asked() { + let doorbell = Doorbell::new(); + assert!( + !doorbell.is_armed(), + "a fresh doorbell must own no event, so a polling consumer pays nothing" + ); +} + +#[test] +fn signalling_an_unarmed_doorbell_creates_nothing() { + let doorbell = Doorbell::new(); + doorbell.signal(); + doorbell.signal(); + assert!( + !doorbell.is_armed(), + "a producer must not conjure a kernel object nobody asked for" + ); +} + +#[test] +fn clearing_an_unarmed_doorbell_creates_nothing() { + let doorbell = Doorbell::new(); + doorbell.clear(); + assert!(!doorbell.is_armed(), "clearing must not create the event"); +} + +#[test] +fn asking_for_the_handle_creates_the_event() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + assert!( + doorbell.is_armed(), + "the handle request must create the event" + ); +} + +#[test] +fn the_handle_is_stable_across_calls() { + let doorbell = Doorbell::new(); + let first = doorbell + .handle() + .expect("creation must succeed") + .as_raw_handle(); + let second = doorbell + .handle() + .expect("creation must succeed") + .as_raw_handle(); + assert_eq!( + first, second, + "the event is created once, so every borrow must name the same object" + ); +} + +#[test] +fn a_new_doorbell_is_unsignalled() { + let doorbell = Doorbell::new(); + assert!( + !is_signalled(&doorbell), + "a doorbell must not claim readiness before anything is pushed" + ); +} + +#[test] +fn signal_makes_it_signalled() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + doorbell.signal(); + assert!(is_signalled(&doorbell), "signalling must be observable"); +} + +#[test] +fn the_signal_is_a_level_and_survives_being_observed() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + doorbell.signal(); + + // Three observations, because the failure this guards against is an + // auto-reset event, where the first wait consumes the signal and the second + // blocks. That exact mistake hung the crate's own doorbell probe for four + // hundred seconds before the design was fixed. + for observation in 1..=3 { + assert!( + is_signalled(&doorbell), + "observation {observation} must still see the level" + ); + } +} + +#[test] +fn clear_makes_it_unsignalled() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + doorbell.signal(); + doorbell.clear(); + assert!(!is_signalled(&doorbell), "clearing must reset the level"); +} + +#[test] +fn a_signal_after_a_clear_is_delivered() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + + // The sequence that the skip-redundant-signals flag could plausibly break: + // if `clear` failed to reset the flag, this second signal would be skipped + // and the doorbell would stay dark with an item waiting. + doorbell.signal(); + doorbell.clear(); + doorbell.signal(); + + assert!( + is_signalled(&doorbell), + "a signal after a clear is the one signal that must never be skipped" + ); +} + +#[test] +fn many_clear_signal_cycles_stay_in_agreement() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + + for cycle in 0..64 { + doorbell.signal(); + assert!(is_signalled(&doorbell), "cycle {cycle} must signal"); + doorbell.clear(); + assert!(!is_signalled(&doorbell), "cycle {cycle} must clear"); + } +} + +#[test] +fn repeated_signals_remain_signalled() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + + // The redundant ones take the skip path. The observable state must not + // depend on how many were issued. + for _ in 0..16 { + doorbell.signal(); + } + assert!( + is_signalled(&doorbell), + "redundant signals must not clear it" + ); + + doorbell.clear(); + assert!( + !is_signalled(&doorbell), + "one clear must undo any number of signals, because the event is a level" + ); +} + +#[test] +fn repeated_clears_remain_clear() { + let doorbell = Doorbell::new(); + let _handle = doorbell.handle().expect("creation must succeed"); + doorbell.signal(); + for _ in 0..16 { + doorbell.clear(); + } + assert!( + !is_signalled(&doorbell), + "redundant clears must not signal it" + ); +} + +#[test] +fn a_signal_issued_before_the_handle_existed_is_not_delivered() { + let doorbell = Doorbell::new(); + + // This is the lazy-creation hole, asserted rather than hoped about: the + // producer ran while there was no event, so its signal went nowhere. The + // arming protocol's re-check is what makes this survivable, and that is + // tested against a real queue in `spsc`. + doorbell.signal(); + + assert!( + !is_signalled(&doorbell), + "a doorbell created after the fact cannot know what it missed, which is \ + precisely why the owner must re-check emptiness before waiting" + ); +} + +#[test] +fn the_owned_duplicate_names_the_same_event() { + let doorbell = Doorbell::new(); + let owned = doorbell.owned().expect("duplication must succeed"); + + // A distinct handle value, but the same underlying object: signalling + // through the queue's copy must be visible through the caller's. + doorbell.signal(); + + // SAFETY: `owned` is a live event handle; a zero timeout returns at once. + let result = unsafe { WaitForSingleObject(owned.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_OBJECT_0, + "the duplicate must observe the original's signal" + ); + + doorbell.clear(); + // SAFETY: as above. + let result = unsafe { WaitForSingleObject(owned.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_TIMEOUT, + "the duplicate must observe the original's clear" + ); +} + +#[test] +fn dropping_the_owned_duplicate_leaves_the_doorbell_usable() { + let doorbell = Doorbell::new(); + let owned = doorbell.owned().expect("duplication must succeed"); + drop(owned); + + // The caller closing its own copy must not close the queue's. If it did, + // this signal would be a use-after-close rather than a no-op. + doorbell.signal(); + assert!( + is_signalled(&doorbell), + "the queue's event must outlive any duplicate handed out" + ); +} + +#[test] +fn a_duplicate_taken_before_a_signal_still_sees_it() { + let doorbell = Doorbell::new(); + let owned = doorbell.owned().expect("duplication must succeed"); + doorbell.signal(); + + // SAFETY: `owned` is a live event handle; a zero timeout returns at once. + let result = unsafe { WaitForSingleObject(owned.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_OBJECT_0, + "duplication order must not affect what the duplicate observes" + ); +} + +#[test] +fn several_duplicates_all_observe_the_same_state() { + let doorbell = Doorbell::new(); + let handles: Vec<_> = (0..4) + .map(|_| doorbell.owned().expect("duplication must succeed")) + .collect(); + doorbell.signal(); + + for (index, handle) in handles.iter().enumerate() { + // SAFETY: each is a live event handle; a zero timeout returns at once. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_OBJECT_0, + "duplicate {index} must see the signal" + ); + } +} + +#[test] +fn a_waiting_thread_is_released_by_a_signal() { + use std::sync::Arc; + use std::thread; + + // The property the whole crate exists for, end to end through the kernel: + // a thread parked in a real blocking wait is released by `signal`. Every + // other test here uses a zero timeout, which never actually blocks. + let doorbell = Arc::new(Doorbell::new()); + let waiter = doorbell.clone(); + let handle = waiter.owned().expect("duplication must succeed"); + + let joiner = thread::spawn(move || { + // Five seconds is not a timing assertion; it is a bound so a broken + // doorbell fails the suite instead of hanging it forever. + // SAFETY: a live event handle owned by this thread for the call. + unsafe { WaitForSingleObject(handle.as_raw_handle(), 5_000) } + }); + + doorbell.signal(); + + let result = joiner.join().expect("the waiting thread must not panic"); + assert_eq!( + result, WAIT_OBJECT_0, + "a blocked waiter must be released by a signal, not by the timeout" + ); +} + +// --------------------------------------------------------------------------- +// The flag must never outlive the signal it mirrors. +// +// `clear` resets the event and *then* clears the flag. Written the other way +// round -- which is how this shipped originally -- a producer signalling +// between the two lines finds a clear flag, sets it, and issues a real +// `SetEvent`; the `ResetEvent` that follows erases that signal and leaves the +// flag set. The doorbell is then wedged dark while claiming to be lit, and +// every later `signal` skips its syscall. +// +// The window is two instructions wide, so the race is driven through the real +// `clear` by a hook rather than raced for on two threads: an interleaving that +// must be hit to prove a point is not one to leave to the scheduler. +// --------------------------------------------------------------------------- + +#[test] +fn a_signal_racing_a_clear_leaves_the_next_one_able_to_ring() { + // Shared rather than borrowed because the hook must be `'static`. One + // thread throughout -- the `Arc` is a lifetime device, not concurrency. + let doorbell = Arc::new(Doorbell::new()); + doorbell.handle().expect("the doorbell must be creatable"); + + // Start from the state that makes the wrong order fatal: already lit, so a + // producer racing the clear can find the flag either way depending on the + // order of the two lines. + doorbell.signal(); + assert!(is_signalled(&doorbell), "the setup must actually light it"); + + let racing = Arc::clone(&doorbell); + race_hooks::CLEAR.with(move || racing.signal(), || doorbell.clear()); + + // Nothing is asserted about the event's state right here, and the omission + // is deliberate. Whether the racing signal survived the clear depends on + // whether it was skipped, which depends on the flag optimisation -- and a + // signal that races a clear is entitled to leave the event lit, because + // that is a spurious wakeup and consumers tolerate those by contract. An + // earlier version of this test did assert it, and the sabotage sweep's + // control -- "signal always syscalls, skipping the flag optimisation" -- + // caught it, which is exactly what that control exists to do: it reported + // this test as asserting the implementation instead of the contract. + + // The assertion that matters, and the one the wrong order fails: whatever + // happened during the window, `clear` must leave the doorbell able to ring + // again. A queue's consumer parks immediately after this returns, and its + // wakeup is the next producer's `signal`. + doorbell.signal(); + assert!( + is_signalled(&doorbell), + "a signal racing a clear must not wedge the doorbell dark; the flag \ + would be claiming 'already lit' over an event nothing will ever set" + ); +} + +#[test] +fn a_clear_with_nothing_racing_it_still_re_arms() { + // The control for the test above: it must not pass merely because `clear` + // never leaves the doorbell ringable, so the same sequence is checked with + // an empty window. + let doorbell = Arc::new(Doorbell::new()); + doorbell.handle().expect("the doorbell must be creatable"); + doorbell.signal(); + + race_hooks::CLEAR.with(|| {}, || doorbell.clear()); + assert!(!is_signalled(&doorbell), "nothing raced it, so it is dark"); + + doorbell.signal(); + assert!(is_signalled(&doorbell), "and the next signal rings"); +} + +#[test] +fn the_debug_rendering_shows_whether_the_event_exists_and_its_state() { + // Both fields, and both values of the one that moves. This rendering is how + // a reader tells "the doorbell was never created" from "it was created and + // is unsignalled" -- the laziness being visible, per D-5 -- so an empty + // rendering loses exactly the distinction it exists to show. + let doorbell = Doorbell::new(); + let before = format!("{doorbell:?}"); + assert!(before.contains("Doorbell"), "got {before}"); + assert!( + before.contains("false"), + "an untouched doorbell is neither created nor signalled: {before}" + ); + + // Signalling one nobody has asked a handle for is a no-op by design -- the + // laziness D-5 describes -- so the rendering must still read false. This is + // the distinction the rendering exists to show, and it is why the test + // cannot simply signal and look for "true". + doorbell.signal(); + let unheard = format!("{doorbell:?}"); + assert!( + !unheard.contains("true"), + "a doorbell nobody is listening to was not created or signalled: {unheard}" + ); + + // Asking for the handle is what creates it; only then does a signal land. + doorbell.handle().expect("the doorbell must be creatable"); + doorbell.signal(); + let after = format!("{doorbell:?}"); + assert!( + after.contains("created: true"), + "the event now exists: {after}" + ); + assert!( + after.contains("signalled: true"), + "and has been rung: {after}" + ); +} diff --git a/crates/windows-waitable-queues/src/error.rs b/crates/windows-waitable-queues/src/error.rs new file mode 100644 index 00000000..9e4b8091 --- /dev/null +++ b/crates/windows-waitable-queues/src/error.rs @@ -0,0 +1,457 @@ +// Copyright (c) Mike Grier. + +//! Errors shared by every queue shape. +//! +//! They live at the crate root rather than inside a shape's module because the +//! shapes must agree on them: a trait cannot unify `push` across shapes if each +//! returns a differently-named error meaning the same thing. + +use core::fmt; +use std::io; + +use crate::capacity::Bounds; + +/// Why a capacity was rejected at construction. +/// +/// Constructing a queue is the one place a caller can get this wrong, so it is +/// reported rather than rounded away. Silently rounding 100 up to 128 would +/// hand back a bound the caller cannot see they got, and a bound is exactly the +/// number a caller chose deliberately. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CapacityError { + requested: usize, + /// The smallest capacity the rejecting shape accepts. + /// + /// Carried for the same reason as [`Self::max_valid`], and it is not always + /// one: `slotwise_mpsc` cannot represent a capacity below two, because its slot + /// state machine reuses a sequence number one lap later and a one-slot ring + /// would make "published" and "free again" the same value. + min_valid: usize, + /// The largest capacity the rejecting shape accepts. + /// + /// Carried on the error rather than assumed to be a crate-wide constant: + /// the bound follows from how a shape represents its positions, and the + /// shapes differ. Most stop where a wrapping difference between positions + /// stops being unambiguous; `reserving_mpsc` stops far lower, because it + /// packs its reservation count into the same word as its position so the + /// two can be claimed together. A suggestion computed against the wrong + /// bound is worse than no suggestion, because a caller will act on it. + max_valid: usize, + kind: CapacityErrorKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CapacityErrorKind { + Zero, + NotPowerOfTwo, + TooSmall, + TooLarge, +} + +impl CapacityError { + fn new(requested: usize, bounds: Bounds, kind: CapacityErrorKind) -> Self { + Self { + requested, + min_valid: bounds.min, + max_valid: bounds.max, + kind, + } + } + + pub(crate) fn zero(bounds: Bounds) -> Self { + Self::new(0, bounds, CapacityErrorKind::Zero) + } + + pub(crate) fn not_power_of_two(requested: usize, bounds: Bounds) -> Self { + Self::new(requested, bounds, CapacityErrorKind::NotPowerOfTwo) + } + + pub(crate) fn too_small(requested: usize, bounds: Bounds) -> Self { + Self::new(requested, bounds, CapacityErrorKind::TooSmall) + } + + pub(crate) fn too_large(requested: usize, bounds: Bounds) -> Self { + Self::new(requested, bounds, CapacityErrorKind::TooLarge) + } + + /// The smallest capacity the shape that rejected this request will accept. + #[must_use] + pub fn min_valid(&self) -> usize { + self.min_valid + } + + /// The largest capacity the shape that rejected this request will accept. + #[must_use] + pub fn max_valid(&self) -> usize { + self.max_valid + } + + /// The capacity that was asked for. + #[must_use] + pub fn requested(&self) -> usize { + self.requested + } + + /// The largest valid capacity not greater than the request, if there is + /// one. + /// + /// Offered so a caller can correct the call without working out the + /// arithmetic: a rejected 100 reports 64 here and 128 from + /// [`Self::next_valid`]. + /// + /// Never returns a value the shape would itself reject. Rounding a request + /// down to the nearest power of two is not sufficient on its own: the + /// nearest power of two below `usize::MAX` is 2^63, which exceeds the + /// largest representable capacity, so the answer is clamped to + /// [`Self::max_valid`], and a result below [`Self::min_valid`] is reported + /// as no suggestion at all. A suggestion that is itself refused would be + /// worse than none, because a caller acts on it and gets a second error. + #[must_use] + pub fn previous_valid(&self) -> Option { + match self.kind { + // Nothing valid lies below either of these: a request that was + // already too small has only larger answers, and zero has none. + CapacityErrorKind::Zero | CapacityErrorKind::TooSmall => None, + CapacityErrorKind::NotPowerOfTwo | CapacityErrorKind::TooLarge => { + let rounded = 1_usize << (usize::BITS - 1 - self.requested.leading_zeros()); + let clamped = rounded.min(self.largest_power_of_two_within_bound()); + (clamped >= self.min_valid).then_some(clamped) + } + } + } + + /// The largest power of two that does not exceed [`Self::max_valid`]. + /// + /// The clamp target for [`Self::previous_valid`]: `max_valid` need not be a + /// power of two -- for a ring of monotonic wrapping positions it is + /// `usize::MAX / 2`, which is `2^63 - 1` -- so clamping to it + /// directly would hand back a capacity that fails the power-of-two test + /// instead of the size test. + fn largest_power_of_two_within_bound(&self) -> usize { + if self.max_valid == 0 { + return 0; + } + 1_usize << (usize::BITS - 1 - self.max_valid.leading_zeros()) + } + + /// The smallest valid capacity not less than the request, if there is one. + /// + /// `None` when rounding up would leave the shape's bound behind, which is + /// not only the case for a request that was already too large: one that is + /// merely *not a power of two* can still sit between the largest valid + /// power of two and the bound, and rounding it up then overshoots. There is + /// genuinely no valid capacity at or above such a request, so saying so is + /// the honest answer -- [`Self::previous_valid`] is the one that can still + /// help. + #[must_use] + pub fn next_valid(&self) -> Option { + let rounded = match self.kind { + // The shape's own minimum, not one: a shape whose slot state + // machine needs two slots would reject a suggestion of one, and a + // suggestion that is itself refused is worse than none. + CapacityErrorKind::Zero | CapacityErrorKind::TooSmall => Some(self.min_valid), + CapacityErrorKind::NotPowerOfTwo => self.requested.checked_next_power_of_two(), + CapacityErrorKind::TooLarge => None, + }?; + (rounded >= self.min_valid && rounded <= self.max_valid).then_some(rounded) + } +} + +impl fmt::Display for CapacityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + CapacityErrorKind::Zero => { + write!(f, "a queue capacity of zero can never accept an item") + } + CapacityErrorKind::NotPowerOfTwo => { + // Each case spelled out, because `{:?}` on an `Option` puts + // Rust's own vocabulary into a message a user reads: "the + // nearest valid capacities are Some(64) and None" is both + // cluttered when there are two and wrong when there is one. + // What a caller needs is a number they can pass instead. + let requested = self.requested; + match (self.previous_valid(), self.next_valid()) { + (Some(lower), Some(upper)) => write!( + f, + "capacity {requested} is not a power of two; \ + the nearest valid capacities are {lower} and {upper}" + ), + (Some(lower), None) => write!( + f, + "capacity {requested} is not a power of two; \ + the nearest valid capacity below it is {lower}, \ + and none above it is representable" + ), + (None, Some(upper)) => write!( + f, + "capacity {requested} is not a power of two; \ + the nearest valid capacity above it is {upper}, \ + and none below it is large enough" + ), + (None, None) => write!( + f, + "capacity {requested} is not a power of two, \ + and this queue shape can represent no valid capacity near it" + ), + } + } + CapacityErrorKind::TooSmall => write!( + f, + "capacity {} is below the smallest this queue shape can represent, which is {}", + self.requested, self.min_valid + ), + CapacityErrorKind::TooLarge => write!( + f, + "capacity {} is above the largest this queue shape can represent, which is {}", + self.requested, self.max_valid + ), + } + } +} + +impl core::error::Error for CapacityError {} + +/// Why a push did not happen, carrying the item back. +/// +/// The item is returned rather than dropped, because a queue that swallows what +/// it refuses gives a caller no way to retry, redirect, or account for it. +/// +/// # Why this is `#[non_exhaustive]` +/// +/// Match it with a wildcard arm. Both receive-side errors already carry this +/// attribute, and the send side lacked it only by omission -- which mattered +/// more than an inconsistency, because **adding it later is itself a breaking +/// change**: every caller's exhaustive `match` would need the wildcard it does +/// not have. Free before the first publish, and a major bump after it. +/// +/// The concrete reason to keep the room open is the expected future work on +/// letting a producer *wait* for capacity rather than only being refused it -- +/// see the crate documentation. If that lands, the send side may need to +/// report something this enum cannot express +/// today. The crate's own precedent suggests a separate error type instead -- +/// [`RecvError`] and [`RecvTimeoutError`] are distinct rather than one extended +/// enum -- so a new variant here may never be needed. Deciding that under time +/// pressure, with the choice already foreclosed, is the outcome this avoids. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum PushError { + /// The queue is at capacity. + /// + /// **This is the backpressure signal**, not a malfunction. A bounded queue + /// exists so that a producer outrunning its consumer is told so, rather + /// than allowed to consume memory until something worse happens. + Full(T), + + /// Every consumer is gone, so nothing will ever take this item. + /// + /// Distinguished from [`Self::Full`] because the responses differ: a full + /// queue may drain, and a disconnected one never will, so retrying the + /// first is sensible and retrying the second is a spin. + Disconnected(T), +} + +impl PushError { + /// Takes the item back out. + #[must_use] + pub fn into_inner(self) -> T { + match self { + Self::Full(item) | Self::Disconnected(item) => item, + } + } + + /// Whether a later attempt could plausibly succeed. + #[must_use] + pub fn is_retryable(&self) -> bool { + matches!(self, Self::Full(_)) + } +} + +impl fmt::Display for PushError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Full(_) => write!(f, "the queue is at capacity"), + Self::Disconnected(_) => write!(f, "every consumer is gone"), + } + } +} + +impl core::error::Error for PushError {} + +/// The only way delivering into a reserved slot can fail: nobody is left to +/// take it. +/// +/// **There is deliberately no `Full` here, and the absence is the contract.** A +/// reservation's whole purpose is that the room is already the holder's, so a +/// full queue cannot refuse it. Returning [`PushError`] instead would name a +/// case that cannot occur and oblige every caller to handle it, which is how a +/// guarantee decays back into a thing you hope is true. +/// +/// The item comes back for the same reason it does from a refused push: a queue +/// that swallows what it cannot deliver leaves the caller no way to account for +/// it. That matters more here than elsewhere -- an item important enough to +/// reserve a slot for is exactly the kind whose disposal must not be silent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Disconnected(pub T); + +impl Disconnected { + /// Takes the item back out. + #[must_use] + pub fn into_inner(self) -> T { + self.0 + } +} + +impl fmt::Display for Disconnected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("every consumer is gone") + } +} + +impl core::error::Error for Disconnected {} + +/// Why a non-blocking take found nothing. +/// +/// **The two variants demand opposite reactions**, which is the whole reason +/// this is not an `Option`: [`TryRecvError::Empty`] means try again, and +/// [`TryRecvError::Disconnected`] means stop. A caller that cannot tell them +/// apart either spins on a stream that has ended or abandons one that has not. +/// +/// This mirrors [`PushError`] on the sending side, where `Full` and +/// `Disconnected` are likewise distinguished for the same reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum TryRecvError { + /// Nothing is queued **right now**. + /// + /// A statement about this instant and not about the stream: a producer may + /// push immediately afterwards. + Empty, + /// Every producer is gone *and* the queue has been drained. + /// + /// Reported only in that order, never merely because the producers went + /// away: a producer may push and then drop, and those items are still owed + /// to the consumer. Getting that order wrong loses the tail of the stream, + /// which is why it is settled here rather than left to each caller. + Disconnected, +} + +impl fmt::Display for TryRecvError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => f.write_str("the queue is empty"), + Self::Disconnected => { + f.write_str("every producer is gone and the queue has been drained") + } + } + } +} + +impl core::error::Error for TryRecvError {} + +/// Why a blocking receive gave up. +/// +/// There is no `Empty` variant, because a blocking receive does not return on +/// an empty queue -- it waits. Emptiness is only ever terminal when the +/// producer is gone as well, and that is [`RecvError::Disconnected`]. +#[derive(Debug)] +#[non_exhaustive] +pub enum RecvError { + /// Every producer has been dropped and the queue has been drained. + /// + /// Reported only after the queue is genuinely empty, never merely because + /// the producer went away: a producer may push and then drop, and those + /// items are still owed to the consumer. + Disconnected, + /// A Windows call failed while creating or waiting on the doorbell. + /// + /// Kept distinct from [`RecvError::Disconnected`] because the two demand + /// opposite reactions: disconnection is the orderly end of a stream, while + /// this means the wait itself is broken and retrying will not help. + Io(io::Error), +} + +impl From for RecvError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl fmt::Display for RecvError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Disconnected => { + f.write_str("the queue is empty and every producer has been dropped") + } + Self::Io(error) => write!(f, "waiting on the queue's doorbell failed: {error}"), + } + } +} + +impl core::error::Error for RecvError { + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + match self { + Self::Disconnected => None, + Self::Io(error) => Some(error), + } + } +} + +/// Why a blocking receive with a deadline gave up. +/// +/// Distinct from [`RecvError`] rather than a variant of it, so a caller that +/// cannot time out is not obliged to handle a case that cannot happen. +#[derive(Debug)] +#[non_exhaustive] +pub enum RecvTimeoutError { + /// The deadline passed with the queue still empty. + /// + /// The queue is still live, and this is not a malfunction: a caller polling + /// with a short deadline will see it constantly and should simply ask + /// again. + Timeout, + /// Every producer has been dropped and the queue has been drained. + Disconnected, + /// A Windows call failed while creating or waiting on the doorbell. + Io(io::Error), +} + +impl RecvTimeoutError { + /// Whether asking again could succeed. + /// + /// True only for [`RecvTimeoutError::Timeout`]. Both other variants are + /// terminal -- no further item will ever arrive, so retrying is a spin. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::Timeout) + } +} + +impl From for RecvTimeoutError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl fmt::Display for RecvTimeoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Timeout => f.write_str("the queue was still empty when the deadline passed"), + Self::Disconnected => { + f.write_str("the queue is empty and every producer has been dropped") + } + Self::Io(error) => write!(f, "waiting on the queue's doorbell failed: {error}"), + } + } +} + +impl core::error::Error for RecvTimeoutError { + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + match self { + Self::Timeout | Self::Disconnected => None, + Self::Io(error) => Some(error), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/error/tests.rs b/crates/windows-waitable-queues/src/error/tests.rs new file mode 100644 index 00000000..d3286ed4 --- /dev/null +++ b/crates/windows-waitable-queues/src/error/tests.rs @@ -0,0 +1,137 @@ +// Copyright (c) Mike Grier. + +//! Tests for the error types. +//! +//! # Why these exist as their own file +//! +//! This module had no tests at all, and a mutation run said so: ten survivors, +//! covering every `Display`, both `source` implementations, and -- the ones +//! that matter -- both `is_retryable` predicates, which could be replaced by a +//! constant `true` or a constant `false` with the whole suite still green. +//! +//! `is_retryable` is not decoration. It is what a caller branches on to decide +//! between backing off and giving up, so a constant answer is either an +//! infinite retry against a dead queue or a dropped item that would have gone +//! through a moment later. That the shapes' own suites exercise the *happy* +//! direction is what let the constant survive: asserting only that a full +//! queue is retryable never asks what a disconnected one says. + +use std::io; + +use super::{CapacityError, Disconnected, PushError, RecvError, RecvTimeoutError, TryRecvError}; +use crate::capacity::Bounds; + +/// An `io::Error` distinguishable from any other, so a `source` that returns +/// the wrong one is not mistaken for a right one. +fn io_error() -> io::Error { + io::Error::new(io::ErrorKind::BrokenPipe, "the doorbell broke") +} + +#[test] +fn a_push_refused_for_room_is_retryable_and_one_refused_for_disconnection_is_not() { + // Both directions, because a predicate asserted in one direction only is + // satisfied by the constant that agrees with it. + assert!( + PushError::Full(1_u32).is_retryable(), + "a full queue may have room later" + ); + assert!( + !PushError::Disconnected(1_u32).is_retryable(), + "a queue with no consumers will never take this item" + ); +} + +#[test] +fn a_refused_push_hands_back_the_item_whichever_way_it_failed() { + // The item is the caller's, and losing it is the failure this type exists + // to prevent: `into_inner` is the only way back. + assert_eq!(PushError::Full(7_u32).into_inner(), 7); + assert_eq!(PushError::Disconnected(9_u32).into_inner(), 9); + assert_eq!(Disconnected(11_u32).into_inner(), 11); +} + +#[test] +fn only_a_timeout_is_retryable_among_the_timed_receive_failures() { + assert!( + RecvTimeoutError::Timeout.is_retryable(), + "nothing arrived in time, and something still might" + ); + assert!( + !RecvTimeoutError::Disconnected.is_retryable(), + "no further item will ever arrive" + ); + assert!( + !RecvTimeoutError::from(io_error()).is_retryable(), + "a failed wait is not a reason to spin on the same call" + ); +} + +#[test] +fn only_the_io_failures_carry_a_source() { + use core::error::Error as _; + + assert!( + RecvError::Disconnected.source().is_none(), + "an ended stream is not caused by anything else" + ); + assert!( + RecvError::from(io_error()).source().is_some(), + "a failed wait must expose the failure underneath it" + ); + + assert!(RecvTimeoutError::Timeout.source().is_none()); + assert!(RecvTimeoutError::Disconnected.source().is_none()); + assert!(RecvTimeoutError::from(io_error()).source().is_some()); + + // And the source is the *right* error, not merely some error. + let source = RecvError::from(io_error()) + .source() + .expect("just asserted") + .to_string(); + assert!(source.contains("the doorbell broke"), "got {source}"); +} + +#[test] +fn every_error_renders_something_that_names_its_cause() { + // A `Display` that writes nothing satisfies any test that only checks it + // does not panic, so each rendering is asked for a word only it would use. + let cases: Vec<(String, &str)> = vec![ + (PushError::Full(1_u32).to_string(), "capacity"), + (PushError::Disconnected(1_u32).to_string(), "consumer"), + (Disconnected(1_u32).to_string(), "consumer"), + (TryRecvError::Empty.to_string(), "empty"), + (TryRecvError::Disconnected.to_string(), "producer"), + (RecvError::Disconnected.to_string(), "producer"), + (RecvError::from(io_error()).to_string(), "doorbell"), + (RecvTimeoutError::Timeout.to_string(), "deadline"), + (RecvTimeoutError::Disconnected.to_string(), "producer"), + (RecvTimeoutError::from(io_error()).to_string(), "doorbell"), + ]; + + for (rendered, expected) in cases { + assert!( + rendered.to_lowercase().contains(expected), + "{rendered:?} does not mention {expected:?}" + ); + } +} + +#[test] +fn a_capacity_error_renders_the_numbers_a_caller_needs_to_correct_the_call() { + // The whole value of this error is the three numbers, so a rendering that + // omits them leaves the caller guessing at a legal capacity. + let bounds = Bounds { + min: 2, + max: 1 << 20, + }; + let too_large = CapacityError::too_large(usize::MAX, bounds); + let rendered = too_large.to_string(); + + assert!( + rendered.contains(&usize::MAX.to_string()), + "the rejected capacity must appear: {rendered}" + ); + assert_eq!(too_large.requested(), usize::MAX); + assert_eq!(too_large.min_valid(), bounds.min); + assert_eq!(too_large.max_valid(), bounds.max); +} diff --git a/crates/windows-waitable-queues/src/lib.rs b/crates/windows-waitable-queues/src/lib.rs new file mode 100644 index 00000000..1c881722 --- /dev/null +++ b/crates/windows-waitable-queues/src/lib.rs @@ -0,0 +1,414 @@ +// Copyright (c) Mike Grier. + +//! Bounded producer/consumer queues whose readiness is a waitable Windows +//! `HANDLE`. +//! +//! **Windows only.** Every public item is behind `cfg(windows)`; the crate +//! builds to an empty shell on other platforms. +//! +//! # Why this exists +//! +//! There are good concurrent queues for Rust already. What none of them offers +//! on Windows is the one property this crate is named for: **you cannot wait on +//! them alongside a kernel object.** +//! +//! `crossbeam-channel` blocks in `recv`, but parks on its own internal +//! primitive and exposes no `HANDLE`; its `Select` is built purely from channel +//! operations, with no way to register a foreign OS object. +//! `crossbeam-queue` does not block at all. So a thread that must wake on +//! "a message arrived **or** my I/O completed **or** shutdown was signalled" +//! cannot express that wait, and must poll one source while blocking on +//! another -- which either burns a core or adds latency. +//! +//! On Windows a `HANDLE` is the universal waitable currency: +//! `WaitForSingleObject`, `WaitForMultipleObjects`, `MsgWaitForMultipleObjects`, +//! a thread-pool wait, and alertable waits all take one. So a queue whose +//! readiness *is* a `HANDLE` composes with everything the platform can wait on, +//! and one that hides its readiness behind a private primitive composes with +//! nothing. +//! +//! # What is here +//! +//! A family of queue shapes rather than one queue, which is why the crate is +//! named in the plural. They differ in producer and consumer cardinality, in +//! how they store their items, and in what they do when full. No shape is the +//! canonical one, so there is deliberately no type named `Queue`: a consumer +//! names the shape it wants. +//! +//! Each shape is split into a **producer handle** and a **consumer handle**, +//! and cardinality is carried by whether those handles are [`Clone`]. A +//! single-producer queue hands out a producer that cannot be cloned, so +//! "single producer" is a fact the compiler enforces rather than a sentence in +//! a doc comment. +//! +//! What the shapes have in common is described by the [capability +//! traits](traits) -- [`Producer`], [`Consumer`], [`Bounded`], [`Waitable`], +//! [`Reserving`] -- each naming one thing a queue can do, so a caller can be +//! generic over exactly what it needs and nothing more. [`Claim`] is the one +//! that is not a queue capability: it describes the *reservation* a +//! [`Reserving`] queue hands out, and is what lets a generic caller redeem one +//! rather than only drop it. Bring it into scope to call `send` on a claim +//! whose concrete type you have not named. +//! +//! # Waiting is one-directional: a producer cannot park on a full queue +//! +//! [`Waitable`] is a *consumer's* capability. A consumer can park until there +//! is something to take; there is **no equivalent for a producer waiting until +//! there is somewhere to put**. [`Producer::push`] refuses immediately with +//! [`PushError::Full`], [`Reserving::reserve`] returns `None`, and neither +//! offers a handle to wait on. +//! +//! **Stated here because the obvious comparison misleads.** `crossbeam-channel` +//! blocks in `send` on a full bounded channel, so a reader arriving from it +//! will expect the same and get a refusal. A producer with nowhere to go has to +//! decide for itself -- shed the item, retry on its own schedule, or buffer -- +//! rather than being parked by the queue. The refusal *is* the backpressure +//! (D-6), and it is typed so the item comes back rather than being swallowed. +//! +//! The absence is deliberate and not permanent. Whether a producer can wait, +//! and **what it would wait on**, is open: a blocking send that parks on +//! something `WaitForMultipleObjects` cannot see would reintroduce the very +//! composition problem that ruled out the existing channel crates, which is the +//! reason this one exists. Two properties already shape the answer -- a bounded +//! queue can offer such a wait and an unbounded one never can, so it belongs in +//! its own capability trait rather than in [`Waitable`]; and while every shape +//! here has a single consumer, two of them have many *producers*, so a +//! "there is room" signal has N waiters and is not the doorbell mirrored. +//! +//! # How long `reserving_mpsc` runs before its claim position recurs +//! +//! **[`reserving_mpsc`] can lose an item after 2^32 pushes under its default +//! layout, on every target -- not only 32-bit ones.** That layout gives the +//! claim position a 32-bit half of a packed word, so this reaches x86-64 and +//! ARM64 exactly as it reaches i686. Read that sentence before the paragraph +//! below, because the phrase "32-bit position" invites the opposite reading and +//! this project has already had to correct that misreading once. +//! +//! **This is a property of the default layout, not of the shape**, and that is +//! a change: it was previously a defect a caller had to live with. The claim +//! word packs an outstanding-reservation count beside the position, and how its +//! bits are divided is now a caller's choice. Reservations are bounded by how +//! many producers are mid-send -- hundreds at most -- so giving up a ceiling +//! nobody reaches buys positions: +//! +//! | Layout | Outstanding reservations | Pushes to recurrence | At sustained maximum rate | +//! |---|---|---|---| +//! | `Balanced` (default) | 2^32 | 2^32 | about 37 seconds | +//! | `Enduring` | 65,535 | 2^48 | about 28 days | +//! | `Perpetual` | 255 | 2^56 | about 20 years | +//! | `Wide` (needs `dwcas`) | 2^32 | 2^64 | unreachable | +//! +//! ``` +//! use windows_waitable_queues::reserving_mpsc::{self, Perpetual}; +//! +//! // The same queue, with a claim position that outlives the process. +//! let (tx, rx) = reserving_mpsc::bounded_as::(64)?; +//! # let _ = (tx, rx); +//! # Ok::<(), windows_waitable_queues::CapacityError>(()) +//! ``` +//! +//! **A deeper position costs nothing measurable.** `Balanced`, `Enduring`, and +//! `Perpetual` all issue the same exchange on the same 64-bit word and differ +//! only in shift and mask constants; a probe comparing them found no difference +//! outside noise. `Wide` is the exception: it needs a 128-bit exchange, which +//! measured 2-3x slower on the claim, and it is the only thing in this crate +//! that costs a third-party dependency. Prefer `Perpetual` unless you want the +//! guarantee rather than the twenty years. +//! +//! The default remains `Balanced` so that no existing caller's behaviour +//! changed when the choice was introduced. It is not the recommended layout. +//! +//! **What happens.** A producer checks that there is room, is descheduled, and +//! resumes after other producers have driven the position field through a +//! complete wrap. Its claim then succeeds against a value that is numerically +//! identical but a whole generation later, and it writes into a slot whose +//! emptiness was decided long ago. If that slot now holds an item the consumer +//! has not taken, the item is overwritten. +//! +//! **The failure is silent.** No error, no panic, no counter moves. The +//! consumer receives a different item than the one that was sent, and nothing +//! observable says so -- which is why this is documented here rather than left +//! to a caller to discover, and why it cannot be mitigated after the fact. +//! +//! **The exposure, measured rather than estimated.** Under `Balanced`, 2^32 +//! pushes is 37 seconds to roughly four minutes of *sustained* pushing at this +//! crate's own measured rates -- about two minutes at two producers, which is +//! the smallest count that can trigger it at all. That is sustained throughput, +//! not a total accumulated over an uptime. Reaching the wrap is necessary but +//! not sufficient: a producer must also be stalled inside a window a few +//! instructions wide. Rare, but a preemption is enough, and "rare" over +//! billions of pushes is not "never". +//! +//! The figures in the table above scale that same measurement by the position +//! width, so they are a floor on time rather than a forecast: a queue that must +//! drain cannot sustain the fastest rate measured, and a slower producer takes +//! proportionally longer to reach its wrap. +//! +//! **What to do about it.** +//! +//! - **Name a layout.** `Perpetual` puts the recurrence about twenty years out +//! at no measured cost, which takes it past any real deployment. This is the +//! answer for almost every caller who is exposed at all. +//! - **[`slotwise_mpsc`] does not have this hazard** under any layout. Its +//! positions are 64 bits on every target, so the equivalent wrap needs 2^64 +//! claims. Prefer it unless you need [`Reserving`]. +//! - **[`spsc`] never had it**, having no contended claim to race. +//! - **The default layout is sound below its wrap.** A queue that will not push +//! 4.3 billion items in one run, or that is not driven at sustained maximum +//! rate by two or more producers, is not exposed even on `Balanced`. +//! +//! This is disclosed on the same principle as the ordering gap below: an +//! adopter gets the information we have rather than an assurance we cannot +//! support. The difference between the two is worth stating plainly -- an +//! unverified ordering is a *risk* of a bug, while this is a known one with a +//! computed exposure. What has changed is that the exposure is now a number the +//! caller sets rather than one the crate imposes. +//! +//! # How far the memory orderings are verified, and how far they are not +//! +//! Stated plainly because a lock-free queue that is vague about this is asking +//! to be trusted rather than evaluated. +//! +//! **What is verified.** Every ordering was reasoned about when written and the +//! reasoning is recorded in `DESIGN-NOTES.md` beside the code it justifies. The +//! shapes are covered by an extensive unit suite and by a sabotage suite that +//! injects deliberate defects and requires each to be caught -- which is how the +//! one real ordering bug this crate has had was found: a lost wakeup where the +//! doorbell cleared its mirror flag before resetting the event. +//! +//! **What is not.** Stress testing cannot catch a *weakened memory ordering* +//! here, and that is measured rather than assumed: changing the producer's +//! `Acquire` load of the consumer's position to `Relaxed` left the entire suite +//! green, while every logic defect injected beside it was caught. A test can +//! only observe the interleavings the hardware and scheduler happen to produce, +//! and neither x86-64 nor ARM64 obliged. +//! +//! **So the orderings are not machine-checked.** Verification with a model +//! checker is planned before 1.0. Until then the `0.x` version is meant +//! literally, and an adopter for whom that matters now has the same information +//! we have rather than an assurance we cannot support. +//! +//! One limit worth knowing even after that work lands: a model checker covers +//! the queue shapes' positions and sequence numbers, and **cannot** cover the +//! doorbell, whose correctness is the interleaving of an atomic flag with real +//! `SetEvent` and `ResetEvent` calls. Modelling those would verify a model of +//! them rather than the calls themselves. +//! +//! # Where these algorithms come from +//! +//! **None of the queue algorithms here are novel, and that is deliberate.** A +//! concurrent queue is a bad place to be original: the failure mode is a +//! reordering that appears on one machine, under load, months later. Each shape +//! implements a published design, and the value this crate adds is the waiting, +//! not the queueing. +//! +//! - [`spsc`] is the classic single-producer single-consumer ring buffer, with +//! the producer's and consumer's positions on separate cache lines so the two +//! ends stop invalidating each other's line. The structure is old -- Lamport +//! gave the concurrent-reader/writer treatment in 1983 -- and the padding is +//! standard modern practice. +//! - [`slotwise_mpsc`] implements Dmitry Vyukov's bounded MPMC array queue, +//! specialised to one consumer. Each slot carries its own sequence number, so +//! a producer claims a position and then asks *that slot* whether it is ready, +//! which keeps producers off any single shared line. It is among the most +//! widely reimplemented concurrent queues in existence. +//! - [`reserving_mpsc`] uses the other classic approach: a producer counts free +//! slots against the consumer's position, so space can be *claimed in advance*. +//! Credit- or ticket-based admission of this kind is long-established in flow +//! control, and it is the only way to answer "will there be room later?". +//! +//! Where this crate departs from a reference implementation it says so, and why, +//! in `DESIGN-NOTES.md`. The measured behaviour of both MPSC shapes is below, +//! including one case where the published intuition turned out to be wrong on +//! our hardware. +//! +//! # Why not an existing queue crate +//! +//! Rust has excellent channel crates, and for most programs one of them is the +//! right answer. **They are not usable here for one structural reason: on +//! Windows, waiting is a kernel-object operation, and a queue whose readiness is +//! not a `HANDLE` cannot take part in one.** +//! +//! A thread that must wait for "an item arrived **or** an I/O completed **or** +//! this process exited **or** cancellation was requested" waits on all of them +//! at once, in a single `WaitForMultipleObjects`. Every participant in that wait +//! has to be a kernel object. A channel that signals readiness through an +//! internal condition variable, a futex, or a parked-thread list cannot be one +//! of them, however good its own blocking receive is -- and however rich its own +//! select mechanism, because that mechanism can only select over its own +//! channels. +//! +//! The alternatives to a waitable queue are all worse in the same way: +//! +//! - **Poll the queue on a timer.** Trades latency against wakeups, and the +//! thread is awake to discover nothing happened. +//! - **Dedicate a thread to blocking on the channel, which signals an event.** +//! Correct, and costs a thread and a hop per item to convert a condition +//! variable back into the kernel object you needed from the start. +//! - **Move everything to async.** A real answer for a program that is already +//! async; not one for a thread whose other obligations are `HANDLE`s. +//! +//! So the queue owns a manual-reset event and keeps it consistent with the +//! queue's state -- which is the hard part, and what this crate is actually +//! for. The event is created lazily, so a consumer that only ever polls never +//! allocates a kernel object at all. +//! +//! # Choosing between `slotwise_mpsc` and `reserving_mpsc` +//! +//! They are **two different claim protocols, not one queue with a switch**. +//! [`slotwise_mpsc`] is Vyukov's bounded array queue, where a producer asks a slot's own +//! sequence number whether it is free. [`reserving_mpsc`] counts free slots +//! against the consumer's position, which is the only way a reservation can be +//! answered at all. Both are well-studied designs in production use elsewhere, +//! which is why this crate ships both instead of picking one for you. +//! +//! - **Pushing more than ~4 billion items in one run, from two or more +//! producers?** Use [`slotwise_mpsc`]. [`reserving_mpsc`] has a known +//! item-loss defect past that volume, on every target -- see the section +//! above, which you should read before choosing. +//! - Need [`Reserving`]? Only [`reserving_mpsc`] has it; [`slotwise_mpsc`] structurally +//! cannot. Weigh that against the defect above rather than treating the +//! capability as settling the choice. +//! - Otherwise **start with [`reserving_mpsc`]**: it was the faster of the two +//! at every producer count above one that we measured. +//! - One producer *and* one consumer? Use [`spsc`], which beats both. +//! +//! Measured ns per push, isolated regime, median of three. An AMD EPYC 7763 +//! slice (8 cores, 16 threads) and a Snapdragon X2 Elite (12 cores, no SMT): +//! +//! | producers | `slotwise_mpsc` x64 | `reserving` x64 | `slotwise_mpsc` ARM64 | `reserving` ARM64 | +//! |---|---|---|---|---| +//! | 1 | 9.0 | 8.6 | 6.5 | 6.1 | +//! | 2 | 49.0 | 28.0 | 29.8 | 9.4 | +//! | 4 | 84.4 | 33.3 | 60.6 | 12.9 | +//! | 8 | 140.8 | 38.5 | 167.4 | 29.8 | +//! | 16 | 193.5 | 52.2 | 194.9 | 30.6 | +//! | 32 | 239.7 | 56.9 | 195.0 | 30.6 | +//! +//! **Read these as two data points, not as a law**, and measure your own +//! workload before treating them as settled. This comparison has already +//! inverted once: the split was designed on the assumption that `slotwise_mpsc` would be +//! the cheaper shape, and measurement disagreed on both machines. Producer +//! count, how hard the consumer drains, and where the threads are scheduled all +//! move the answer -- placement alone moved an SPSC handoff by 5.6x on one of +//! these hosts. +//! +//! Two things that look like reasons to choose and are not. **Capacity**: on a +//! 64-bit target `slotwise_mpsc` reaches 2^62 slots and `reserving_mpsc` 2^31. +//! On a 32-bit one the crate-wide ceiling is 2^30 and **both** shapes land +//! there -- `reserving_mpsc`'s packed 2^31 is clamped down to it too -- so the +//! difference disappears entirely and the comparison means nothing at all. +//! Either way it counts slots allocated up front rather than items ever pushed, +//! and 2^31 slots is tens of gigabytes before the ring holds anything useful. +//! **`slotwise_mpsc` winning at one producer**: true in one regime, and at one +//! producer you want [`spsc`]. +//! +//! # Shutting down +//! +//! A consumer learns that every producer is gone from +//! `is_disconnected`, and a producer learns the consumer is gone from a typed +//! [`PushError::Disconnected`] that hands the item back. The orderly shutdown +//! is therefore: drain to empty, then check. +//! +//! For everything that does not go to plan there is [`disposal`]. A queue torn +//! down with items still in it must do *something* with them, and by default it +//! destroys them inside the last handle's drop -- on whichever thread happened +//! to release it. When an item owns a handle that is a hazard rather than a +//! detail, because closing a handle can block and the dropping thread may be a +//! pool callback that must not. Building the queue with a [`Disposal`] sink +//! hands those items back instead. +//! +//! # Status +//! +//! [`spsc`], [`slotwise_mpsc`] and [`reserving_mpsc`] are implemented, each with its +//! doorbell: any of them can +//! be polled with no kernel object at all, blocked on directly, or waited on +//! alongside other handles. Shapes with many consumers, and shapes that signal +//! when space becomes available so a producer can wait for room, are under +//! consideration for a future revision. The decisions this crate is built +//! against are recorded in `DESIGN-NOTES.md` beside this file. + +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_docs)] +#![warn(unsafe_op_in_unsafe_fn)] + +// Every item is gated, so the crate builds to an empty shell off Windows rather +// than failing: the implementation rests on `std::os::windows::io` and +// `windows-sys` throughout, and the whole premise -- readiness that *is* a +// waitable `HANDLE` -- has no meaning on another platform. This mirrors the +// sibling Windows-only crates here, and the crate documentation above states +// the same contract, so the two cannot drift apart. + +#[cfg(windows)] +mod blocking; +#[cfg(windows)] +mod capacity; +#[cfg(windows)] +pub mod disposal; +#[cfg(windows)] +mod doorbell; +#[cfg(windows)] +mod error; +#[cfg(windows)] +mod metrics; +#[cfg(windows)] +mod options; +/// **Experimental, and not covered by this crate's semver promise.** +/// +/// A duplicate of [`reserving_mpsc`] differing only in its claim protocol, +/// built to be measured against it so that the ABA hole recorded as `SH-14.1` +/// can be closed on evidence rather than on judgement. It will either be merged +/// into `reserving_mpsc` or deleted. +#[cfg(all(windows, feature = "experimental-permit-claim"))] +pub mod permit_mpsc; +#[cfg(all(windows, test))] +mod race_hooks; +#[cfg(windows)] +pub mod reserving_mpsc; +#[cfg(windows)] +pub mod slotwise_mpsc; +#[cfg(windows)] +pub mod spsc; +#[cfg(windows)] +pub mod traits; + +#[cfg(windows)] +pub use disposal::Disposal; +#[cfg(windows)] +pub use error::{ + CapacityError, Disconnected, PushError, RecvError, RecvTimeoutError, TryRecvError, +}; +#[cfg(windows)] +pub use options::Options; +#[cfg(windows)] +pub use traits::{Bounded, Claim, Consumer, Drain, Observable, Producer, Reserving, Waitable}; + +/// Pads and aligns a value onto its own cache line. +/// +/// The producer's position and the consumer's position are written by different +/// threads on every operation. Left adjacent they would share a cache line, and +/// each write would invalidate the other thread's copy of a value it only ever +/// reads -- false sharing, which converts an uncontended queue into a +/// contended one while every load and store remains individually correct. +/// +/// 128 rather than 64: that is the cache line on aarch64, and on x86-64 the +/// adjacent-line prefetcher pulls pairs of 64-byte lines, so 64 does not +/// reliably separate them. +#[cfg(windows)] +#[repr(align(128))] +struct CacheAligned(T); + +// The README states this crate's wait protocol, and a review round found that +// statement had drifted from what `blocking::recv` actually does -- it named +// three steps where the code has four, and a caller following it would have +// waited forever at the end of the stream. That particular drift is fixed and +// pinned by a test, but the general risk is not: prose nothing executes can +// only rot. +// +// The README carries no code today, so this compiles nothing. It is here so +// that the first example somebody adds is compiled rather than trusted, which +// is the cheapest moment to close the gap. `cfg(doctest)` means the item exists +// only while rustdoc collects tests, so an ordinary build pays nothing. +#[cfg(all(doctest, windows))] +#[doc = include_str!("../README.md")] +struct ReadmeDoctests; diff --git a/crates/windows-waitable-queues/src/metrics.rs b/crates/windows-waitable-queues/src/metrics.rs new file mode 100644 index 00000000..5da17877 --- /dev/null +++ b/crates/windows-waitable-queues/src/metrics.rs @@ -0,0 +1,149 @@ +// Copyright (c) Mike Grier. + +//! Counters a queue keeps about itself. +//! +//! # What is here, and what is deliberately not +//! +//! Three numbers, and each is here because it answers a question the queue's +//! own state cannot: +//! +//! - **Refusals**, so backpressure is *measured* rather than inferred from a +//! caller's error handling. +//! - **Doorbell rings**, so the skip rule is measurable rather than assumed. +//! Counted by [`Doorbell`](crate::doorbell::Doorbell) rather than by +//! [`Metrics`], for the reason given below; a reader after the ring count +//! will not find it on this type. +//! - **Peak depth**, so a bound can be chosen from evidence. +//! +//! **Depth itself is not here**, and its absence is a decision. `Bounded::len` +//! already reports it, computed on demand from positions the queue keeps +//! anyway, so restating it as a metric would give one number two names and two +//! places to drift. What belongs here is only what has to be *accumulated*. +//! +//! # Why two of the three are free and one is not +//! +//! A counter on a hot path is a shared line every thread writes, which is the +//! same false-sharing cost the queues are carefully padded to avoid. So each +//! counter is placed where it is already paid for: +//! +//! - **Refusals** increment only when a push is *refused*, which is off the +//! success path entirely. +//! - **Rings** increment only when the doorbell actually calls `SetEvent`, +//! which is a syscall measured at ~81 ns against ~7 ns for an uncontended +//! atomic. That increment happens inside the doorbell, so the counter lives +//! on [`Doorbell`](crate::doorbell::Doorbell) rather than on [`Metrics`]: +//! keeping it here would mean reaching across to a line this type does not +//! own. The skipped signals -- the hot ones -- are deliberately *not* +//! counted, because that increment would land on exactly the path the skip +//! exists to cheapen. +//! - **Peak depth** cannot be placed that way, because it must observe every +//! change. It is therefore **opt-in**, and off by default; see +//! [`Metrics::record_depth`] and +//! [D-23](../DESIGN-NOTES.md#d-23). + +use core::fmt; +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +/// The counters one queue keeps. +pub(crate) struct Metrics { + /// Pushes refused for want of room. + /// + /// Written only on the failure path, so it costs a successful push nothing. + refused: AtomicU64, + /// The deepest the queue has been observed to get, if it is being tracked. + /// + /// An **upper bound** on the true peak rather than the peak exactly, and + /// never above the queue's capacity. The reasoning, and why the cheap + /// sample is preferred to an exact count, is on + /// [`Observable::high_water`](crate::Observable::high_water); it is stated + /// there because that is where a caller reads it. + /// + /// `Option` rather than a sentinel because "not tracked" and "never got + /// past empty" are different answers, and a caller acting on a `0` that + /// meant the former would be reading a number nobody recorded. + high_water: Option, +} + +impl Metrics { + /// Counters with peak depth left untracked, which is the default. + pub(crate) const fn new(track_high_water: bool) -> Self { + Self { + refused: AtomicU64::new(0), + high_water: if track_high_water { + Some(AtomicUsize::new(0)) + } else { + None + }, + } + } + + /// Whether peak depth is being tracked. + /// + /// Read on the push path by shapes whose producer does not otherwise know + /// the depth, so that they only pay for the load that computes it when + /// somebody asked for the answer. The field is written once at construction + /// and never again, so the line is shared but read-only -- which is the + /// cheap kind. + pub(crate) fn tracks_high_water(&self) -> bool { + self.high_water.is_some() + } + + /// Record that the queue reached `depth`. + /// + /// # Why this loads before it modifies + /// + /// The obvious spelling is an unconditional [`AtomicUsize::fetch_max`], and + /// it would be a read-modify-write on a shared line for **every push** -- + /// the cost this crate pads its positions apart to avoid. + /// + /// A new maximum is rare: it happens while a queue is filling and then + /// almost never again. So the common case is turned into a plain load of a + /// line that is written rarely and read often, and the read-modify-write is + /// reached only when the value is actually about to change. The load can be + /// stale, and the `fetch_max` that follows is what makes the result correct + /// anyway -- a racing pair of producers may both see an old maximum, but + /// `fetch_max` keeps the larger of the two regardless of which lands first. + pub(crate) fn record_depth(&self, depth: usize) { + let Some(high_water) = self.high_water.as_ref() else { + return; + }; + // `>` rather than `>=`, and a mutation run will report the two as + // indistinguishable -- correctly. `fetch_max(depth)` when `depth` + // already equals the maximum stores the value it read, so the weaker + // test only buys an extra read-modify-write on the shared line in the + // one case it admits. That is the cost this guard exists to avoid, so + // the difference is real; it is just not a difference in any answer, + // and no test can be written for it. Left documented rather than + // chased. + if depth > high_water.load(Ordering::Relaxed) { + high_water.fetch_max(depth, Ordering::Relaxed); + } + } + + /// Record that a push was refused for want of room. + pub(crate) fn record_refusal(&self) { + self.refused.fetch_add(1, Ordering::Relaxed); + } + + /// How many pushes have been refused for want of room. + pub(crate) fn refused(&self) -> u64 { + self.refused.load(Ordering::Relaxed) + } + + /// The deepest the queue has been observed to get, if tracked. + pub(crate) fn high_water(&self) -> Option { + Some(self.high_water.as_ref()?.load(Ordering::Relaxed)) + } +} + +impl fmt::Debug for Metrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Metrics") + .field("refused", &self.refused()) + .field("high_water", &self.high_water()) + .finish() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/metrics/tests.rs b/crates/windows-waitable-queues/src/metrics/tests.rs new file mode 100644 index 00000000..b5711356 --- /dev/null +++ b/crates/windows-waitable-queues/src/metrics/tests.rs @@ -0,0 +1,112 @@ +// Copyright (c) Mike Grier. + +//! Tests for the counters in isolation, with no queue attached. +//! +//! Their behaviour *through* a queue is asserted in each shape's own suite, +//! because each records depth from a different place -- and on `slotwise_mpsc` records +//! it only when asked. What is tested here is the arithmetic they share. + +use super::Metrics; + +#[test] +fn refusals_start_at_zero_and_accumulate() { + let metrics = Metrics::new(false); + assert_eq!(metrics.refused(), 0); + + for expected in 1..=5 { + metrics.record_refusal(); + assert_eq!(metrics.refused(), expected); + } +} + +#[test] +fn an_untracked_high_water_reports_none_rather_than_zero() { + // The distinction the `Option` exists to draw. A caller sizing a queue from + // `Some(0)` would conclude it never filled; from `None` it learns that + // nobody was counting, which is a different fact and demands a different + // response. + let metrics = Metrics::new(false); + assert!(!metrics.tracks_high_water()); + assert_eq!(metrics.high_water(), None); + + // And recording into it is a no-op rather than an error, so the shapes can + // call it unconditionally where the depth is free. + metrics.record_depth(9); + assert_eq!(metrics.high_water(), None); +} + +#[test] +fn a_tracked_high_water_starts_at_some_zero() { + // Distinct from `None`: this queue *is* counting, and has seen nothing. + let metrics = Metrics::new(true); + assert!(metrics.tracks_high_water()); + assert_eq!(metrics.high_water(), Some(0)); +} + +#[test] +fn high_water_keeps_the_peak_rather_than_the_latest() { + let metrics = Metrics::new(true); + + metrics.record_depth(3); + assert_eq!(metrics.high_water(), Some(3)); + + metrics.record_depth(7); + assert_eq!(metrics.high_water(), Some(7)); + + // The point of a high-water mark: it does not fall when the queue drains. + metrics.record_depth(1); + assert_eq!( + metrics.high_water(), + Some(7), + "a peak that receded is still a peak that happened" + ); + + metrics.record_depth(7); + assert_eq!(metrics.high_water(), Some(7), "and equal is not greater"); +} + +#[test] +fn concurrent_recorders_do_not_lose_the_peak() { + // `record_depth` loads before it modifies, so two threads can both observe + // a stale maximum. The `fetch_max` that follows is what makes the result + // correct anyway, and this is the test that says so: without it, the + // load-then-modify shortcut would be a lost update rather than an + // optimisation. + use std::sync::Arc; + use std::thread; + + const THREADS: usize = 4; + const PER_THREAD: usize = 500; + + let metrics = Arc::new(Metrics::new(true)); + let threads: Vec<_> = (0..THREADS) + .map(|offset| { + let metrics = Arc::clone(&metrics); + thread::spawn(move || { + for depth in 0..PER_THREAD { + metrics.record_depth(depth + offset * PER_THREAD); + } + }) + }) + .collect(); + for thread in threads { + thread.join().expect("no recorder may panic"); + } + + assert_eq!( + metrics.high_water(), + Some(THREADS * PER_THREAD - 1), + "the largest value any thread recorded must survive every race" + ); +} + +#[test] +fn the_debug_form_reports_both_counters() { + let metrics = Metrics::new(true); + metrics.record_refusal(); + metrics.record_depth(4); + + let shown = format!("{metrics:?}"); + assert!(shown.contains("refused: 1"), "{shown}"); + assert!(shown.contains("Some(4)"), "{shown}"); +} diff --git a/crates/windows-waitable-queues/src/options.rs b/crates/windows-waitable-queues/src/options.rs new file mode 100644 index 00000000..eebcbaea --- /dev/null +++ b/crates/windows-waitable-queues/src/options.rs @@ -0,0 +1,113 @@ +// Copyright (c) Mike Grier. + +//! The switches a queue is built with. +//! +//! # Why a builder rather than more constructors +//! +//! There are two independent choices at construction -- what becomes of +//! undrained items, and whether peak depth is tracked -- across three shapes. +//! Spelled as constructors that is four functions per shape and twelve in the +//! crate, and every future switch doubles it again. Spelled as one value passed +//! to one `bounded_with`, a new switch is a new method and nothing else moves. +//! +//! The plain [`bounded`](crate::spsc::bounded) constructor stays, because the +//! default is the overwhelmingly common case and it should not have to say so. +//! +//! # Both switches are off by default, for different reasons +//! +//! **Disposal** is off because destroying an item that owns nothing, where it +//! lies, is exactly right -- a queue of `u32` should not have to think about +//! teardown at all. See [`Disposal`]. +//! +//! **High-water tracking** is off because it is the one metric that cannot be +//! made free. Refusals and doorbell rings sit on paths that were already paying +//! for themselves, but a peak has to observe every change, and on +//! [`slotwise_mpsc`](crate::slotwise_mpsc) observing the depth means the producer reading the +//! consumer's position -- the single shared line that shape's push is built to +//! avoid touching. So it is a switch, and the cost lands only on queues that +//! asked for the answer. + +use core::fmt; + +use crate::disposal::Disposal; + +/// What a queue is built with, beyond its capacity. +/// +/// # Examples +/// +/// ``` +/// use windows_waitable_queues::{Options, spsc}; +/// +/// let (tx, rx) = spsc::bounded_with::(4, Options::new().tracking_high_water())?; +/// +/// tx.push(1).expect("a fresh queue has room"); +/// tx.push(2).expect("a fresh queue has room"); +/// assert_eq!(rx.pop(), Ok(1)); +/// +/// // `len` is the depth right now; `high_water` is the peak it reached. +/// assert_eq!(rx.len(), 1); +/// assert_eq!(rx.high_water(), Some(2)); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub struct Options { + pub(crate) disposal: Option>, + pub(crate) track_high_water: bool, +} + +impl Options { + /// The defaults: undrained items destroyed in place, peak depth untracked. + #[must_use] + pub fn new() -> Self { + Self { + disposal: None, + track_high_water: false, + } + } + + /// Hand undrained items to `disposal` at teardown instead of destroying + /// them where they lie. + /// + /// See [`Disposal`] for why this has to be decided here rather than asked + /// for at teardown. + #[must_use] + pub fn disposal(mut self, disposal: Disposal) -> Self { + self.disposal = Some(disposal); + self + } + + /// Track the deepest the queue gets, readable from + /// [`Observable::high_water`](crate::Observable::high_water). + /// + /// **This is the one option that costs the push path something**, which is + /// why it is off by default. A peak has to observe every change, so on + /// `slotwise_mpsc` it makes the producer read the consumer's position -- the shared + /// line that shape's push exists to avoid. On `spsc` and `reserving_mpsc` + /// the producer already knows the depth, so it costs those two almost + /// nothing. + /// + /// Untracked, `high_water` reports `None` rather than `0`, so a caller + /// cannot mistake "nobody was counting" for "it never filled". + #[must_use] + pub fn tracking_high_water(mut self) -> Self { + self.track_high_water = true; + self + } +} + +impl Default for Options { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Options { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Options") + .field("disposal", &self.disposal.is_some()) + .field("track_high_water", &self.track_high_water) + .finish() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/options/tests.rs b/crates/windows-waitable-queues/src/options/tests.rs new file mode 100644 index 00000000..b569aec0 --- /dev/null +++ b/crates/windows-waitable-queues/src/options/tests.rs @@ -0,0 +1,38 @@ +// Copyright (c) Mike Grier. + +//! Tests for the shape-construction options. +//! +//! The builder itself is exercised wherever a shape is built with options; what +//! is tested here is the rendering, which a mutation run found unguarded: a +//! `Debug` returning `Ok(default)` writes nothing and passes any test that only +//! checks formatting does not panic. + +use super::Options; +use crate::Disposal; + +#[test] +fn the_debug_rendering_shows_both_options_and_tracks_them_changing() { + // Both fields and both states of each, so a rendering stuck at one constant + // cannot satisfy this. + let bare = format!("{:?}", Options::::new()); + assert!(bare.contains("Options"), "got {bare}"); + assert!( + bare.contains("false"), + "a fresh Options has no disposal and no tracking: {bare}" + ); + + let configured = format!( + "{:?}", + Options::::new() + .tracking_high_water() + .disposal(Disposal::new(|_: u32| {})) + ); + assert!( + configured.contains("true"), + "a configured Options must show it: {configured}" + ); + assert!( + !configured.contains("false"), + "both fields were set, so neither should still read false: {configured}" + ); +} diff --git a/crates/windows-waitable-queues/src/permit_mpsc.rs b/crates/windows-waitable-queues/src/permit_mpsc.rs new file mode 100644 index 00000000..d078ab04 --- /dev/null +++ b/crates/windows-waitable-queues/src/permit_mpsc.rs @@ -0,0 +1,627 @@ +// Copyright (c) Mike Grier. + +//! **Experimental.** A reserving MPSC whose admission is a permit, not a room check. +//! +//! Not a shipping shape. This exists to be measured against +//! [`reserving_mpsc`](crate::reserving_mpsc) so that SH-14.3 can be decided on +//! evidence, and it is gated behind the non-default `experimental-permit-claim` +//! feature so that nothing depends on it by accident. It is exempt from this +//! crate's semver promise and will either be merged into `reserving_mpsc` or +//! deleted; see `SH-15.6`. +//! +//! # The one thing that differs +//! +//! Everything here -- the ring, the slot sequence, the publication order, the +//! doorbell ring, the reservation semantics -- is `reserving_mpsc`'s. **Only the +//! claim protocol changes**, because that is the variable under test and +//! anything else that differed would confound the measurement. +//! +//! `reserving_mpsc` decides "there is room" by reading the consumer's `head`, +//! and then compare-exchanges a claim word that does not contain `head`. The +//! decision and the operation that acts on it are separate, which is +//! the recurrence hazard described in the crate documentation: a producer stalled +//! between them resumes after the position field has recurred, its exchange +//! succeeds against a numerically equal but generations-later value, and it +//! writes a slot whose freedom was decided long ago. +//! +//! How wide that field is, and so how many pushes recurrence takes, is a layout +//! choice there -- 32 bits under the default and up to 64 under +//! [`reserving_mpsc::Wide`](crate::reserving_mpsc). **That moves the recurrence +//! out of reach without removing the separation that causes it**, which is why +//! this shape remains interesting: it addresses the structure rather than the +//! interval. +//! +//! Here the decision *is* the operation. A producer takes a permit from a count +//! of unspoken-for slots with one atomic, and that single modification both +//! decides and claims. The predicate is a function solely of the word being +//! modified, so recurrence of any *other* value cannot invalidate it -- which is +//! the criterion [D-34](../DESIGN-NOTES.md#d-34) records. The position stops +//! carrying any decision at all and becomes a ticket handed out by `fetch_add`, +//! an operation with no predicate to be wrong about. It may wrap freely. +//! +//! # Why this does not contradict D-17 +//! +//! [D-17](../DESIGN-NOTES.md#d-17) packs the reservation count and the position +//! into one word, and argues that two atomics cannot be made correct with any +//! amount of fencing: a pusher reads the count then writes the position while a +//! reserver writes the count then reads the position, each missing the other, +//! and no fence forbids it. That argument is sound and it is not evaded here. +//! +//! It concludes that "two independent claimants on one resource must synchronise +//! on one location". This shape agrees and picks a *different* single location. +//! Both claimants perform the same modification on `permits`; neither reads a +//! value the other writes elsewhere and then acts on it. The hazard D-17 +//! describes needs a load-then-store on one side and a store-then-load on the +//! other, and there is no such pair here. +//! +//! # What this does not change +//! +//! **It is still technically blocking**, and no rearrangement of the claim can +//! make it otherwise while items live in the ring: a producer holding ticket `p` +//! that is preempted before publishing stalls a consumer that must deliver `p` +//! in order. In-order delivery, inline storage, and non-blocking progress are +//! over-constrained together. See `SH-inf.1`. + +use core::cell::{Cell, UnsafeCell}; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; + +use crate::CacheAligned; +use crate::capacity::{Bounds, MAX_ADMISSIBLE_CAPACITY, validate_capacity}; +use crate::doorbell::Doorbell; +use crate::error::{CapacityError, Disconnected, PushError, TryRecvError}; +use crate::metrics::Metrics; + +/// A ticket, and the slot sequence numbers compared against one. +/// +/// **64 bits on every target, deliberately, rather than `usize`**, for the same +/// reason [`slotwise_mpsc`](crate::slotwise_mpsc) made the same choice: a +/// 32-bit counter laps in minutes at this crate's measured rates, and a shape +/// whose soundness depends on the target's pointer width is not one this crate +/// ships twice over. +/// +/// It matters *less* here than there, and that difference is the point of the +/// shape. In `slotwise_mpsc` the position is compared against a slot sequence to +/// decide whether a slot is free, so a lap is a correctness hazard. Here the +/// ticket carries **no decision at all** -- admission was already settled by the +/// permit -- so its only job is to name a distinct slot, and it could wrap +/// harmlessly. 64 bits makes it a *dumb* `fetch_add` that nobody has to reason +/// about again: no wrap analysis, no ambiguity bound to re-derive, and a +/// difference against `head` that stays meaningful for any queue this process +/// could construct. +type Position = u64; + +/// What this shape accepts as a capacity. See [`BOUNDS_MAX`]. +const BOUNDS: Bounds = Bounds { + min: 2, + max: BOUNDS_MAX, +}; + +/// The largest capacity this shape accepts. +/// +/// The crate-wide ceiling, matching [`slotwise_mpsc`](crate::slotwise_mpsc) +/// rather than [`reserving_mpsc`](crate::reserving_mpsc). That shape's far lower +/// 2^31 is forced by its packing -- half a word for the position, half for the +/// reservation count -- and this one has no packed word to be constrained by. +/// +/// **This was 2^31 while the ticket was 32 bits**, so that a measurement against +/// `reserving_mpsc` covered the same range on both. Widening the ticket removed +/// the reason: the shapes are now measured at whatever capacity the harness +/// picks, which is well below either ceiling, and matching an artificial limit +/// would only misreport what this shape can do. +pub const BOUNDS_MAX: usize = MAX_ADMISSIBLE_CAPACITY; + +const _: () = { + assert!( + BOUNDS.max.is_power_of_two(), + "the maximum is offered to a caller as a capacity it could use, so it must itself be one \ + this shape would accept" + ); + assert!( + BOUNDS.min <= BOUNDS.max, + "a shape that accepts nothing would reject every capacity with a suggestion it would also \ + reject" + ); + // The permit count starts at the capacity and never exceeds it, so the + // capacity is what must fit in the signed count's *positive* range. + // + // **The transient overdraft does not constrain this**, which is worth + // stating because an earlier version of this assertion assumed it did and + // reserved half the range for it. The overdraft goes *negative* -- each + // concurrent claimant subtracts one before undoing -- so it consumes the + // range below zero, of which there is a full 2^63, against a bound of one + // per thread in flight. It cannot meet the positive bound from below. + assert!( + BOUNDS.max as u64 <= i64::MAX as u64, + "the permit count must be able to hold the whole capacity" + ); + // `len` reads the queue's depth as `tail - head` in wrapping arithmetic, and + // that difference is unambiguous only up to half the ticket's range. So the + // capacity must fit below that half, not merely below `Position::MAX`. + // + // **Stated against the half rather than the maximum deliberately.** An + // earlier version of this assertion compared `BOUNDS.max` to `Position::MAX`, + // which is *tautological* on every target -- a `usize` capacity cannot exceed + // a `u64` maximum -- and so asserted nothing at all. That is precisely the + // trap `reserving_mpsc`'s own const block records having fallen into once. + // This form fails if `Position` is ever narrowed to `u32`, which is the + // change it exists to catch. + assert!( + (BOUNDS.max as u128) <= 1_u128 << (Position::BITS - 1), + "the ticket must be wide enough that a wrapping depth is unambiguous at any capacity this \ + shape accepts" + ); +}; + +/// Creates an experimental permit-claiming MPSC queue. +/// +/// `capacity` must be a power of two between two and [`BOUNDS_MAX`]. +/// +/// # Errors +/// +/// [`CapacityError`] when the capacity is outside what this shape accepts. +pub fn bounded(capacity: usize) -> Result<(Producer, Consumer), CapacityError> { + validate_capacity(capacity, BOUNDS)?; + + let mut slots = Vec::with_capacity(capacity); + for index in 0..capacity { + slots.push(Slot { + // Anything that is not `position + 1` for the position this slot + // first serves. Matches `reserving_mpsc`'s initialisation exactly. + sequence: AtomicU64::new(index as Position), + value: UnsafeCell::new(MaybeUninit::uninit()), + }); + } + + let shared = Arc::new(Shared { + metrics: Metrics::new(false), + slots: slots.into_boxed_slice(), + mask: capacity - 1, + capacity, + head: CacheAligned(AtomicU64::new(0)), + tail: CacheAligned(AtomicU64::new(0)), + permits: CacheAligned(AtomicI64::new(capacity as i64)), + producers: AtomicUsize::new(1), + consumer_live: AtomicBool::new(true), + doorbell: Doorbell::new(), + }); + + Ok(( + Producer { + shared: Arc::clone(&shared), + not_sync: PhantomData, + }, + Consumer { + shared, + not_sync: PhantomData, + }, + )) +} + +/// One cell of the ring. +struct Slot { + /// `position + 1` once the producer holding `position` has finished writing. + sequence: AtomicU64, + value: UnsafeCell>, +} + +struct Shared { + metrics: Metrics, + slots: Box<[Slot]>, + mask: usize, + capacity: usize, + /// The consumer's position. + /// + /// **No producer reads this**, which is the structural change: in + /// `reserving_mpsc` every push loads it to count free slots, and that load + /// is both the cost D-26 measured and the stale input SH-14.1 exploits. + /// Here it is consumer-private, kept atomic only so `len` can sample it. + head: CacheAligned, + /// The ticket dispenser. Only ever `fetch_add`. + /// + /// Carries no decision, so it has no predicate that a recurrence could + /// invalidate. At [`Position`]'s width it is a *dumb* increment: it would be + /// sound if it wrapped, and it cannot wrap, so neither property has to be + /// argued at a call site again. + tail: CacheAligned, + /// Slots not currently spoken for, as a signed count. + /// + /// Signed because the claim is an optimistic decrement that may overshoot; + /// see [`Shared::take_permit`]. + permits: CacheAligned, + producers: AtomicUsize, + consumer_live: AtomicBool, + doorbell: Doorbell, +} + +// SAFETY: as `reserving_mpsc`'s -- a slot is written by exactly one producer, +// the one whose ticket named that position, and read by exactly one consumer +// after it observes the release store that publishes it. +unsafe impl Sync for Shared {} +// SAFETY: as above. +unsafe impl Send for Shared {} + +impl Shared { + /// Takes one permit, or reports that none was available. + /// + /// **This single modification both decides and claims**, which is the whole + /// point of the shape. A permit in hand is a guarantee that a slot exists + /// for its holder; nothing observed before the modification has to still be + /// true afterwards, because nothing observed before the modification was + /// used. + /// + /// Optimistic rather than a compare-exchange loop: the decrement is + /// unconditional and is undone when it turns out to have overdrawn. That + /// costs one atomic on the success path where a loop costs one *plus* a + /// retry per losing race, and contention is the regime under test. + /// + /// **Signed, and that is load-bearing.** An unsigned count would wrap to a + /// huge value on overdraw, and a concurrent producer reading it would + /// conclude there was room and proceed -- admitting more claimants than + /// there are slots, which is precisely the failure this shape exists to + /// prevent. Signed, an overdraw is visibly negative to everyone: each + /// overdrawing thread sees its own non-positive result and undoes. + /// + /// The count can go no lower than `-(concurrent claimants)`, since each + /// subtracts one before undoing. + fn take_permit(&self) -> bool { + // Acquire: pairs with the consumer's release in `release_permit`, so a + // slot freed there is safe to overwrite by the time this returns. RMWs + // on one location form a release sequence, so this synchronizes with + // every earlier release on it, not merely the latest. + if self.permits.0.fetch_sub(1, Ordering::Acquire) > 0 { + return true; + } + // Overdrawn. Put it back; a concurrent claimant that saw the negative + // value is doing the same. + // + // Release, matching `release_permit`, even though this thread published + // nothing and needs no edge of its own. Unlike `tail` and `head`, this + // counter carries a real edge, and a relaxed RMW here would sit in the + // middle of it: if this undo reads from a consumer's release and a + // third thread's acquire then reads from this undo, that thread's + // synchronization with the consumer rests entirely on the release + // sequence rule. That rule holds, but it was narrowed once already + // (C++20 dropped same-thread relaxed stores from it), and it is not a + // thing a reader should have to reconstruct to trust a slot handoff. + // Uniform acquire/release on this counter costs one `stlxr` over + // `stxr` on ARM64, on the contended slow path, and removes the + // argument entirely. + self.permits.0.fetch_add(1, Ordering::Release); + false + } + + /// Returns one permit, freeing the slot it stood for. + /// + /// Release: the consumer's read of the slot must not become visible after + /// this, or a producer could take the permit and overwrite an item the + /// consumer had not finished taking. This store is what frees the slot, + /// exactly as advancing `head` is in `reserving_mpsc`. + fn release_permit(&self) { + self.permits.0.fetch_add(1, Ordering::Release); + } + + /// Writes and publishes `item` at `position`. + /// + /// # Safety + /// + /// The caller must hold a permit and the ticket naming `position`, so that + /// no other producer can write this slot and the consumer has finished with + /// whatever it held a lap ago. + unsafe fn publish(&self, position: Position, item: T) { + let slot = &self.slots[position as usize & self.mask]; + // SAFETY: the caller's ticket makes this thread the only writer, and its + // permit means the consumer has finished with the previous occupant. + unsafe { + (*slot.value.get()).write(item); + } + + // Release, and this is the publication: it must come after the write. + slot.sequence + .store(position.wrapping_add(1), Ordering::Release); + + // After the publication, never before. Kept here rather than omitted + // because `reserving_mpsc` rings on every publish, and a push path + // missing it would measure faster for a reason that has nothing to do + // with the claim protocol. + self.doorbell.signal(); + } + + /// Relaxed on both, because neither `tail` nor `head` ever receives a + /// release write: `tail` only ever moves by `fetch_add(Relaxed)`, and the + /// consumer's `head` store is deliberately relaxed (see `pop`, where the + /// permit is what frees the slot). An acquire load here would therefore + /// pair with nothing and synchronize with nothing -- it would read as a + /// guarantee this queue does not make. The real edges are `sequence` + /// (release in `publish`, acquire in `pop`) and `permits` (release in + /// `release_permit`, acquire in `try_take_permit`); this is a snapshot and + /// rides on neither. + fn len(&self) -> usize { + let tail = self.tail.0.load(Ordering::Relaxed); + let head = self.head.0.load(Ordering::Relaxed); + (tail.wrapping_sub(head) as usize).min(self.capacity) + } +} + +impl Drop for Shared { + fn drop(&mut self) { + // Every handle is gone, so `&mut self` proves this is the only thread. + // `get_mut` reads each position directly rather than atomically, which + // is why no ordering appears here at all: there is no second thread for + // one to order against, so the question does not arise. That is why + // this is not a relaxed read on an otherwise acquire/release atomic -- + // it is not an atomic read. `reserving_mpsc`'s drop does the same. + // + // A slot whose sequence marks it published still holds an item nobody + // took. + let mask = self.mask; + let head = *self.head.0.get_mut(); + let tail = *self.tail.0.get_mut(); + let mut position = head; + while position != tail { + let slot = &mut self.slots[position as usize & mask]; + if *slot.sequence.get_mut() == position.wrapping_add(1) { + // SAFETY: the sequence says a producer finished writing this + // slot and no consumer took it. Every handle is gone, so this + // is the only reader, and each position is visited once. + unsafe { + slot.value.get_mut().assume_init_drop(); + } + } + position = position.wrapping_add(1); + } + } +} + +/// A handle that can push. Clone it for more producers. +pub struct Producer { + shared: Arc>, + not_sync: PhantomData>, +} + +// SAFETY: the shared state is `Sync` for `T: Send`; the handle adds nothing. +unsafe impl Send for Producer {} + +impl Clone for Producer { + fn clone(&self) -> Self { + self.shared.producers.fetch_add(1, Ordering::Relaxed); + Self { + shared: Arc::clone(&self.shared), + not_sync: PhantomData, + } + } +} + +impl Drop for Producer { + fn drop(&mut self) { + if self.shared.producers.fetch_sub(1, Ordering::AcqRel) == 1 { + self.shared.doorbell.signal(); + } + } +} + +impl Producer { + /// Pushes an item, or hands it back. + /// + /// # Errors + /// + /// [`PushError::Full`] when no unreserved room remains, and + /// [`PushError::Disconnected`] when the consumer is gone. + pub fn push(&self, item: T) -> Result<(), PushError> { + if !self.shared.consumer_live.load(Ordering::Acquire) { + return Err(PushError::Disconnected(item)); + } + if !self.shared.take_permit() { + // Report disconnection in preference to fullness, matching + // `reserving_mpsc`: a full queue whose consumer is gone will never + // drain, so telling the caller to retry would be telling it to spin + // forever. + if !self.shared.consumer_live.load(Ordering::Acquire) { + return Err(PushError::Disconnected(item)); + } + self.shared.metrics.record_refusal(); + return Err(PushError::Full(item)); + } + + // The permit is held from here until the consumer takes the item. The + // ticket carries no decision, so a relaxed fetch-add is enough: what + // orders the write is the permit's acquire above and the release store + // that publishes the slot. + let position = self.shared.tail.0.fetch_add(1, Ordering::Relaxed); + + // SAFETY: this thread holds a permit and the ticket naming `position`. + unsafe { + self.shared.publish(position, item); + } + Ok(()) + } + + /// Claims a slot now for a message sent later. + /// + /// Takes a permit and **no ticket**, matching `reserving_mpsc`: an + /// outstanding reservation reduces the room available to other producers + /// without occupying a position, so it cannot stall the consumer however + /// long it is held. + /// + /// # Errors + /// + /// [`PushError::Full`] when no room remains. + pub fn reserve(&self) -> Result, PushError<()>> { + if !self.shared.take_permit() { + self.shared.metrics.record_refusal(); + return Err(PushError::Full(())); + } + self.shared.producers.fetch_add(1, Ordering::Relaxed); + Ok(Reservation { + shared: Arc::clone(&self.shared), + spent: false, + }) + } + + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.metrics.refused() + } +} + +/// A slot claimed in advance. +pub struct Reservation { + shared: Arc>, + spent: bool, +} + +// SAFETY: as `Producer`'s. +unsafe impl Send for Reservation {} + +impl Reservation { + /// Delivers the message the reservation was taken for. + /// + /// Cannot fail for want of room: the permit taken at `reserve` is still + /// held, so a slot is guaranteed. + pub fn send(mut self, item: T) -> Result<(), Disconnected> { + // **Checked, and the item comes back.** This used to publish + // unconditionally and return `()`, so redeeming against a departed + // consumer put the item into a ring nobody would ever read; it was + // destroyed at teardown and the caller was never told. That is a silent + // loss of exactly the message a reservation exists to guarantee. + // + // `reserving_mpsc::Reservation::send` has always returned + // `Disconnected` here. This module's documentation claims only the + // *admission* protocol differs between the two, so the divergence was + // undisclosed as well as wrong. Raised in the PR #56 review. + if !self.shared.consumer_live.load(Ordering::Acquire) { + // `self` is dropped on the way out with `spent` still false, which + // releases the permit and the producer count -- the right outcome, + // because this message is never being delivered. + return Err(Disconnected(item)); + } + self.spent = true; + let position = self.shared.tail.0.fetch_add(1, Ordering::Relaxed); + // SAFETY: the permit taken at `reserve` is still held and this ticket + // names a position no other producer can hold. + unsafe { + self.shared.publish(position, item); + } + Ok(()) + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if !self.spent { + // Never redeemed: give the room back. + self.shared.release_permit(); + } + if self.shared.producers.fetch_sub(1, Ordering::AcqRel) == 1 { + self.shared.doorbell.signal(); + } + } +} + +/// The single consuming handle. +pub struct Consumer { + shared: Arc>, + not_sync: PhantomData>, +} + +// SAFETY: as `Producer`'s. +unsafe impl Send for Consumer {} + +impl Drop for Consumer { + fn drop(&mut self) { + self.shared.consumer_live.store(false, Ordering::Release); + } +} + +impl Consumer { + /// Takes the oldest item. + /// + /// # Errors + /// + /// [`TryRecvError::Empty`] when nothing is queued right now, and + /// [`TryRecvError::Disconnected`] when every producer is gone *and* the + /// queue has been drained. + pub fn pop(&self) -> Result { + match self.take() { + Some(item) => Ok(item), + // Only on the empty path, so a successful take never pays for it. + None if self.is_disconnected() => Err(TryRecvError::Disconnected), + None => Err(TryRecvError::Empty), + } + } + + /// Whether every producer is gone. + /// + /// **A queue can be disconnected and still hold items**, because a producer + /// may push and then drop. [`Self::pop`] answers the composite question in + /// the order that cannot lose the tail of the stream. + /// + /// Acquire, so the release in the last producer's `Drop` makes every + /// producer's preceding pushes visible to a consumer that observes zero. + #[must_use] + pub fn is_disconnected(&self) -> bool { + self.shared.producers.load(Ordering::Acquire) == 0 + } + + /// The take itself, without the disconnection question. + fn take(&self) -> Option { + // Relaxed: this thread is the only writer of `head`. + let position = self.shared.head.0.load(Ordering::Relaxed); + let slot = &self.shared.slots[position as usize & self.shared.mask]; + // Acquire: pairs with the producer's release store in `publish`. + if slot.sequence.load(Ordering::Acquire) != position.wrapping_add(1) { + return None; + } + + // SAFETY: the sequence says the producer holding this position finished + // writing, and the acquire above makes that write visible. This is the + // only consumer and the position is given up below, so the item is read + // exactly once. + let item = unsafe { (*slot.value.get()).assume_init_read() }; + + // Relaxed is enough for `head` here, unlike `reserving_mpsc`, because no + // producer reads it. The release that actually frees the slot is the + // permit below. + self.shared + .head + .0 + .store(position.wrapping_add(1), Ordering::Relaxed); + self.shared.release_permit(); + Some(item) + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// How many pushes have been refused for want of room. + /// + /// Readable from this side as well as the producer's, matching the shipping + /// shapes: a measurement harness drops its producers before reading the + /// count, so a producer-only accessor would be unreachable exactly when the + /// number is wanted. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.metrics.refused() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/permit_mpsc/tests.rs b/crates/windows-waitable-queues/src/permit_mpsc/tests.rs new file mode 100644 index 00000000..530d97cd --- /dev/null +++ b/crates/windows-waitable-queues/src/permit_mpsc/tests.rs @@ -0,0 +1,634 @@ +// Copyright (c) Mike Grier. + +//! Tests for the experimental permit-claiming MPSC. +//! +//! These are not merely "does it enqueue". The shape exists to make a +//! particular hazard impossible, so the tests that matter are the ones about +//! *admission*: that the queue never admits more claimants than it has slots, +//! that a reservation holds room back without occupying a position, and that an +//! overdrawn permit count is always restored. + +use crate::error::TryRecvError; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; + +use super::*; +use crate::error::PushError; + +/// A payload that reports its own destruction, so leaks and double-drops in +/// teardown are observable rather than assumed. +#[derive(Debug)] +struct Tracked(Arc); + +impl Drop for Tracked { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn a_capacity_below_the_minimum_is_refused() { + assert!(bounded::(1).is_err()); + assert!(bounded::(0).is_err()); +} + +#[test] +fn a_capacity_that_is_not_a_power_of_two_is_refused() { + assert!(bounded::(3).is_err()); + assert!(bounded::(100).is_err()); +} + +#[test] +fn a_capacity_above_the_maximum_is_refused() { + assert!(bounded::(BOUNDS_MAX.wrapping_mul(2)).is_err()); +} + +// Deliberately no test that `BOUNDS_MAX` is itself an acceptable capacity. That +// is a fact about constants, and the module's `const _: () = { ... }` block +// already asserts both halves of it -- a const assertion fails the build rather +// than a run somebody chose to make, so a test here would be the weaker +// statement of a property already guaranteed. + +#[test] +fn an_item_pushed_is_the_item_popped() { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + tx.push(7).expect("room"); + assert_eq!(rx.pop(), Ok(7)); +} + +#[test] +fn popping_an_empty_queue_reports_nothing() { + let (_tx, rx) = bounded::(4).expect("a valid capacity"); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn items_come_back_in_the_order_they_went_in() { + let (tx, rx) = bounded::(8).expect("a valid capacity"); + for value in 0..8 { + tx.push(value).expect("room"); + } + for value in 0..8 { + assert_eq!(rx.pop(), Ok(value)); + } + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn the_queue_holds_exactly_its_capacity_and_then_refuses() { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for value in 0..4 { + tx.push(value).expect("room"); + } + assert_eq!(rx.len(), 4); + match tx.push(99) { + Err(PushError::Full(item)) => assert_eq!(item, 99), + other => panic!("expected Full, got {:?}", other.is_ok()), + } +} + +#[test] +fn a_refusal_hands_the_item_back_and_is_counted() { + let (tx, _rx) = bounded::(2).expect("a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert_eq!(tx.refused(), 0); + assert!(tx.push(3).is_err()); + assert_eq!(tx.refused(), 1); +} + +#[test] +fn a_refusal_leaves_the_permit_count_intact() { + // The optimistic decrement overdraws and must undo. If it did not, a single + // refusal would permanently cost the queue a slot -- so the queue must + // still accept exactly `capacity` items after many refusals. + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for value in 0..4 { + tx.push(value).expect("room"); + } + for _ in 0..1_000 { + assert!(tx.push(99).is_err()); + } + for value in 0..4 { + assert_eq!(rx.pop(), Ok(value)); + } + // Every slot came back. + for value in 0..4 { + tx.push(value).expect("room after draining"); + } + assert_eq!(rx.len(), 4); +} + +#[test] +fn a_concurrent_refusal_storm_leaves_the_permit_count_intact() { + // The overdraft is bounded by the number of concurrent claimants, so the + // count can go transiently negative. What must not happen is that it fails + // to return to exactly the capacity. + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for value in 0..4 { + tx.push(value).expect("room"); + } + thread::scope(|scope| { + for _ in 0..8 { + let tx = tx.clone(); + scope.spawn(move || { + for _ in 0..2_000 { + assert!(tx.push(99).is_err()); + } + }); + } + }); + for value in 0..4 { + assert_eq!(rx.pop(), Ok(value)); + } + for value in 0..4 { + tx.push(value).expect("room after draining"); + } + assert!(tx.push(99).is_err(), "capacity must not have grown"); +} + +#[test] +fn the_ring_is_reused_across_many_laps() { + // Far more pushes than slots, so every slot serves many positions. This is + // ring wraparound, not position wraparound. + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for value in 0..10_000 { + tx.push(value).expect("room"); + assert_eq!(rx.pop(), Ok(value)); + } + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn a_reservation_holds_room_back_from_other_producers() { + let (tx, _rx) = bounded::(4).expect("a valid capacity"); + let _reservation = tx.reserve().expect("room"); + // Three slots remain for best-effort pushes. + for value in 0..3 { + tx.push(value).expect("room"); + } + assert!( + tx.push(99).is_err(), + "the reserved slot must not be available to a push" + ); +} + +#[test] +fn a_reservation_delivers_even_when_the_queue_is_otherwise_full() { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + let reservation = tx.reserve().expect("room"); + for value in 0..3 { + tx.push(value).expect("room"); + } + assert!(tx.push(99).is_err()); + // Reserved is guaranteed: this cannot fail. + reservation.send(42).expect("the consumer is still here"); + assert_eq!(rx.len(), 4); + for value in 0..3 { + assert_eq!(rx.pop(), Ok(value)); + } + assert_eq!(rx.pop(), Ok(42)); +} + +#[test] +fn a_reservation_dropped_unredeemed_gives_the_room_back() { + let (tx, _rx) = bounded::(4).expect("a valid capacity"); + { + let _reservation = tx.reserve().expect("room"); + for value in 0..3 { + tx.push(value).expect("room"); + } + assert!(tx.push(99).is_err()); + } + tx.push(99) + .expect("the dropped reservation released its slot"); +} + +#[test] +fn an_outstanding_reservation_does_not_block_the_consumer() { + // The semantic that distinguishes this from taking a ticket at reserve + // time: a reservation withholds capacity but occupies no position, so items + // pushed after it are delivered without waiting for it. + let (tx, rx) = bounded::(8).expect("a valid capacity"); + let reservation = tx.reserve().expect("room"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + reservation.send(3).expect("the consumer is still here"); + assert_eq!(rx.pop(), Ok(3)); +} + +#[test] +fn every_reservation_the_capacity_allows_can_be_taken_at_once() { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + let reservations: Vec<_> = (0..4).map(|_| tx.reserve().expect("room")).collect(); + assert!(tx.push(99).is_err(), "every slot is spoken for"); + for (value, reservation) in reservations.into_iter().enumerate() { + reservation + .send(value as u32) + .expect("the consumer is still here"); + } + for value in 0..4 { + assert_eq!(rx.pop(), Ok(value)); + } +} + +#[test] +fn a_push_to_a_departed_consumer_is_reported_as_disconnection() { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + drop(rx); + match tx.push(1) { + Err(PushError::Disconnected(item)) => assert_eq!(item, 1), + _ => panic!("expected Disconnected"), + } +} + +#[test] +fn a_full_queue_whose_consumer_is_gone_reports_disconnection_not_fullness() { + // Telling a caller to retry a queue that will never drain is telling it to + // spin forever, so disconnection wins over fullness. + let (tx, rx) = bounded::(2).expect("a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(rx); + match tx.push(3) { + Err(PushError::Disconnected(_)) => {} + _ => panic!("expected Disconnected on a full, consumerless queue"), + } +} + +#[test] +fn undrained_items_are_dropped_exactly_once_at_teardown() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for _ in 0..3 { + tx.push(Tracked(Arc::clone(&drops))).expect("room"); + } + // Take one, so teardown must drop exactly the two that remain. + drop(rx.pop().expect("an item")); + assert_eq!(drops.load(Ordering::Relaxed), 1); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 3, + "every item must be dropped exactly once" + ); +} + +#[test] +fn teardown_after_a_lap_drops_only_the_live_items() { + // Slots hold stale bit patterns from earlier laps; teardown must not drop + // those a second time. + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, rx) = bounded::(4).expect("a valid capacity"); + for _ in 0..12 { + tx.push(Tracked(Arc::clone(&drops))).expect("room"); + drop(rx.pop().expect("an item")); + } + assert_eq!(drops.load(Ordering::Relaxed), 12); + // Two live items remain at teardown. + tx.push(Tracked(Arc::clone(&drops))).expect("room"); + tx.push(Tracked(Arc::clone(&drops))).expect("room"); + } + assert_eq!(drops.load(Ordering::Relaxed), 14); +} + +#[test] +fn many_producers_deliver_every_item_exactly_once() { + const PRODUCERS: u32 = 8; + const EACH: u32 = 2_000; + + let (tx, rx) = bounded::(64).expect("a valid capacity"); + let mut seen = vec![0_u32; (PRODUCERS * EACH) as usize]; + + thread::scope(|scope| { + for producer in 0..PRODUCERS { + let tx = tx.clone(); + scope.spawn(move || { + for index in 0..EACH { + let value = producer * EACH + index; + // Bounded queue: retry rather than lose the item. + while tx.push(value).is_err() { + std::hint::spin_loop(); + } + } + }); + } + let mut taken = 0; + while taken < PRODUCERS * EACH { + if let Ok(value) = rx.pop() { + seen[value as usize] += 1; + taken += 1; + } else { + std::hint::spin_loop(); + } + } + }); + + assert!( + seen.iter().all(|&count| count == 1), + "every item must arrive exactly once" + ); +} + +#[test] +fn the_queue_never_admits_more_claimants_than_it_has_slots() { + // The property the shape exists for, stated as an observable: at no moment + // may the number of items held exceed the capacity. A permit system that + // over-admitted would show up here as a length beyond the bound. + const CAPACITY: usize = 16; + let (tx, rx) = bounded::(CAPACITY).expect("a valid capacity"); + + thread::scope(|scope| { + for _ in 0..8 { + let tx = tx.clone(); + scope.spawn(move || { + for value in 0..4_000 { + let _ = tx.push(value); + } + }); + } + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(400); + while std::time::Instant::now() < deadline { + assert!( + rx.len() <= CAPACITY, + "the queue reported holding more than its capacity" + ); + let _ = rx.pop(); + } + }); + while rx.pop().is_ok() {} +} + +#[test] +fn a_producer_and_a_reservation_contend_for_the_same_room_without_overdrawing() { + // `reserve` and `push` are two claimants on one resource. If they could + // both be admitted to the last slot, the queue would owe a slot that does + // not exist -- which is the hazard D-17's packing exists to prevent, and + // which this shape prevents with a single shared permit count instead. + for _ in 0..200 { + let (tx, rx) = bounded::(2).expect("a valid capacity"); + tx.push(0).expect("room"); + // One slot left, two claimants racing for it. + let reserver = tx.clone(); + let pusher = tx.clone(); + let (reserved, pushed) = thread::scope(|scope| { + let a = scope.spawn(move || reserver.reserve().ok()); + let b = scope.spawn(move || pusher.push(1).is_ok()); + (a.join().expect("no panic"), b.join().expect("no panic")) + }); + let claims = usize::from(reserved.is_some()) + usize::from(pushed); + assert!(claims <= 1, "both claimants took the same single slot"); + if let Some(reservation) = reserved { + reservation.send(2).expect("the consumer is still here"); + } + drop(rx); + } +} + +#[test] +fn redeeming_against_a_departed_consumer_hands_the_item_back() { + // The whole point of a reservation is that the message it stands for is + // not lost. This used to publish into a ring nobody would ever read: the + // item was destroyed at teardown and the caller was never told, which is a + // silent loss of exactly the message the reservation guaranteed. + // + // `reserving_mpsc::Reservation::send` has always answered `Disconnected` + // here, and this module claims only the *admission* protocol differs, so + // the divergence was undisclosed as well as wrong. Raised in the PR #56 + // review. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let reservation = tx.reserve().expect("an empty queue has room"); + drop(rx); + + let returned = reservation + .send(7) + .expect_err("a departed consumer must not swallow the item"); + assert_eq!(returned.0, 7, "the item itself must come back, not a copy"); +} + +#[test] +fn a_refused_redemption_gives_the_room_back() { + // The complement, so the refusal cannot be bought by leaking the slot the + // reservation was holding: the permit must return to the pool exactly as + // it does when a reservation is simply dropped. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + let reservation = tx.reserve().expect("an empty queue has room"); + drop(rx); + let _ = reservation.send(7); + + // Both slots are free again, so both may be reserved. + assert!(tx.reserve().is_ok(), "the refused slot must be reusable"); + assert!(tx.reserve().is_ok(), "and the queue's other slot with it"); +} + +// --------------------------------------------------------------------------- +// The consumer's own accessors, and the disconnection question `pop` answers. +// +// A mutation run found every one of these uncovered: `capacity`, `is_empty`, +// `refused` and `is_disconnected` could each be replaced by a constant with the +// suite still passing. This shape is experimental, but a caller who enables the +// feature gets the same surface as the shipping shapes and is entitled to the +// same evidence that it works. +// --------------------------------------------------------------------------- + +#[test] +fn the_consumer_reports_the_capacity_it_was_built_with() { + let (_tx, rx) = bounded::(8).expect("8 is a valid capacity"); + assert_eq!(rx.capacity(), 8); + + let (_tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert_eq!( + rx.capacity(), + 2, + "a second capacity, so a constant cannot satisfy both" + ); +} + +#[test] +fn the_consumer_sees_the_queue_fill_and_empty() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!(rx.is_empty(), "a fresh queue holds nothing"); + assert_eq!(rx.len(), 0); + + tx.push(1).expect("an empty queue has room"); + assert!(!rx.is_empty(), "and not once an item is pushed"); + assert_eq!(rx.len(), 1); + + assert_eq!(rx.pop(), Ok(1)); + assert!(rx.is_empty(), "and empty again once it is taken"); + assert_eq!(rx.len(), 0); +} + +#[test] +fn the_consumer_counts_refusals() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert_eq!(rx.refused(), 0, "nothing has been refused yet"); + + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one slot remains"); + assert!(tx.push(3).is_err(), "the third does not fit"); + + assert_eq!(rx.refused(), 1, "and the refusal is counted"); + assert!(tx.push(4).is_err()); + assert_eq!(rx.refused(), 2, "and counted again, rather than latched"); +} + +#[test] +fn the_consumer_learns_when_every_producer_is_gone() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + assert!(!rx.is_disconnected(), "two producers are alive"); + + drop(tx); + assert!( + !rx.is_disconnected(), + "and one still is -- disconnection is every producer, not any" + ); + + drop(second); + assert!(rx.is_disconnected(), "now none are"); +} + +#[test] +fn an_empty_queue_is_distinguishable_from_a_finished_one() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + + drop(tx); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn a_departed_producers_items_are_delivered_before_the_disconnection() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + drop(tx); + + assert_eq!(rx.pop(), Ok(1), "the item comes first"); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and only then the end of the stream" + ); +} + +#[test] +fn a_dropped_producer_that_was_not_the_last_leaves_the_stream_open() { + // The `Drop` impl decrements a count and only signals at zero. A mutation + // run found both the decrement and its `== 1` test uncovered, so this + // asserts the boundary from both sides rather than only that dropping + // everything eventually disconnects. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + let third = tx.clone(); + + drop(second); + drop(third); + assert!( + !rx.is_disconnected(), + "two of three gone is not the end of the stream" + ); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + + drop(tx); + assert!(rx.is_disconnected(), "the last one is"); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn a_dropped_reservation_releases_its_hold_on_the_stream() { + // A reservation counts as a producer, so dropping the last *handle* while a + // reservation is outstanding must not end the stream -- and dropping the + // reservation afterwards must. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + drop(tx); + assert!( + !rx.is_disconnected(), + "a reservation is a promise of a message still to come" + ); + + drop(slot); + assert!( + rx.is_disconnected(), + "and releasing it without sending ends the stream" + ); +} + +#[test] +fn only_the_last_producer_to_leave_rings_the_doorbell() { + // **The count and the signal are separate consequences of the same drop, + // and only the count is visible through the public surface.** This shape + // exposes no `doorbell`, `arm` or `recv`, so a test written against + // `is_disconnected` alone cannot see whether the signal happened -- a + // mutation run proved it, surviving `==` -> `!=` here while every such test + // passed. The ring is what a waiting consumer would depend on, so it is + // asserted directly through the shared state rather than left unobserved. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + // **The handle must be requested first, or the signal is a no-op.** The + // doorbell does nothing until a handle exists, and this shape exposes no + // way to ask for one -- so through its public surface the ring is + // unobservable, which is precisely why the mutation survived. Asking + // through the shared state makes the signalling testable now, and is the + // behaviour the shape needs to already be correct if it ever gains the + // waitable surface its siblings have. + let _handle = rx + .shared + .doorbell + .handle() + .expect("the event can be created"); + let before = rx.shared.doorbell.rings(); + + drop(second); + assert_eq!( + rx.shared.doorbell.rings(), + before, + "a producer leaving while another remains has ended nothing, so waking a consumer \ + would be a spurious wakeup" + ); + + drop(tx); + assert_eq!( + rx.shared.doorbell.rings(), + before + 1, + "the last one to leave ends the stream, and a consumer parked on the doorbell has to \ + be told or it waits forever" + ); +} + +#[test] +fn only_the_last_reservation_to_leave_rings_the_doorbell() { + // A reservation counts as a producer, so the same boundary applies to it, + // and it has its own `Drop` impl -- which a mutation run also found + // unobserved. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + // As above: no handle, no signal. + let _handle = rx + .shared + .doorbell + .handle() + .expect("the event can be created"); + let before = rx.shared.doorbell.rings(); + + drop(tx); + assert_eq!( + rx.shared.doorbell.rings(), + before, + "the handle went but the reservation still promises a message" + ); + + drop(slot); + assert_eq!( + rx.shared.doorbell.rings(), + before + 1, + "and releasing it unsent is what finally ends the stream" + ); +} diff --git a/crates/windows-waitable-queues/src/race_hooks.rs b/crates/windows-waitable-queues/src/race_hooks.rs new file mode 100644 index 00000000..4ad83635 --- /dev/null +++ b/crates/windows-waitable-queues/src/race_hooks.rs @@ -0,0 +1,112 @@ +// Copyright (c) Mike Grier. + +//! Test-only: hooks that drive a two-statement sequence through its own race +//! window, deterministically and on one thread. +//! +//! # Why these exist at all +//! +//! Three places in this crate consist of statements whose *order* -- or whose +//! freshness -- is the whole correctness argument, and whose wrong form is a +//! permanent hang or a wrong answer rather than an occasional stall: +//! `Consumer::arm`, [`Doorbell::clear`], and the reserving queue's claim loop. +//! Proving such an order is load-bearing means placing a racing operation +//! strictly between the statements, and that is not an interleaving a scheduler +//! can be asked for -- the window is tens of nanoseconds wide. +//! +//! # Why a hook rather than a hand-written copy of the code +//! +//! The first attempt at proving `arm` was a duplicate of it with the two +//! statements swapped, driven deterministically. It could only ever show that +//! *a* reversed order is wrong; it was structurally incapable of noticing the +//! **real** `arm` being reversed, which left that case covered only by +//! whatever interleavings the scheduler happened to produce. Measured: +//! sabotaging the real `arm` was then caught in one run out of three. +//! +//! A second copy of a rule is a check of the copy, not of the rule. So the real +//! code carries the hook, and a test drives the real code through the exact +//! window. +//! +//! # Why one facility for both +//! +//! Giving each site its own thread-local would reintroduce, one layer down, the +//! duplication this file exists to avoid. The hooks are thread-local, so two +//! suites running concurrently in one process cannot see each other's. +//! +//! [`Doorbell::clear`]: crate::doorbell::Doorbell::clear + +use core::cell::RefCell; +use std::thread::LocalKey; + +mod tests; + +type Slot = RefCell>>; + +thread_local! { + static ARM_HOOK: Slot = const { RefCell::new(None) }; + static CLEAR_HOOK: Slot = const { RefCell::new(None) }; + static CLAIM_HOOK: Slot = const { RefCell::new(None) }; +} + +/// One named race window. +pub(crate) struct Hook(&'static LocalKey); + +/// Fires inside `Consumer::arm`, between clearing the doorbell and checking +/// whether anything is takeable. +pub(crate) const ARM: Hook = Hook(&ARM_HOOK); + +/// Fires inside [`Doorbell::clear`](crate::doorbell::Doorbell::clear), between +/// resetting the event and clearing the flag that mirrors it. +pub(crate) const CLEAR: Hook = Hook(&CLEAR_HOOK); + +/// Fires inside the reserving queue's claim loop, between reading the claim +/// word and testing whether that claim leaves room. +/// +/// The window this opens is not an ordering one: the two readings the room test +/// combines -- a position from the claim word, and `head` -- are taken at +/// different instants, so a claim that goes stale here makes the test answer +/// about a state that never existed. Firing a racing producer and consumer in +/// this window is what makes that reachable on one thread. +pub(crate) const CLAIM: Hook = Hook(&CLAIM_HOOK); + +impl Hook { + /// Runs the installed hook, if any. Called from the code under test. + pub(crate) fn run(&self) { + self.0.with(|hook| { + // Taken out for the call rather than held borrowed across it, so a + // hook that re-enters this window cannot trip a `RefCell` + // re-entrancy panic. + let taken = hook.borrow_mut().take(); + if let Some(mut race) = taken { + race(); + *hook.borrow_mut() = Some(race); + } + }); + } + + /// Installs a hook for the duration of a closure. + /// + /// **The removal is a guard rather than a statement after `body`**, so an + /// unwind takes the hook with it. A test that installs a hook and then + /// fails an assertion is the ordinary case, not an exotic one, and a hook + /// surviving that would fire inside whatever ran next on this thread -- + /// turning one failure into an unrelated second one, in a facility the + /// crate's central correctness argument rests on. + pub(crate) fn with(&self, race: impl FnMut() + 'static, body: impl FnOnce() -> R) -> R { + self.0 + .with(|hook| *hook.borrow_mut() = Some(Box::new(race))); + let _installed = Installed(self.0); + body() + } +} + +/// Removes a hook when it goes out of scope, however that happens. +struct Installed(&'static LocalKey); + +impl Drop for Installed { + fn drop(&mut self) { + // `try_with`, not `with`: a hook can outlive its thread-local during + // thread teardown, and panicking there would replace whatever unwind is + // already in progress. + let _ = self.0.try_with(|hook| *hook.borrow_mut() = None); + } +} diff --git a/crates/windows-waitable-queues/src/race_hooks/tests.rs b/crates/windows-waitable-queues/src/race_hooks/tests.rs new file mode 100644 index 00000000..513c194e --- /dev/null +++ b/crates/windows-waitable-queues/src/race_hooks/tests.rs @@ -0,0 +1,89 @@ +// Copyright (c) Mike Grier. + +//! Tests for the hook facility itself. +//! +//! These test the *test infrastructure*, which is worth doing precisely because +//! everything else about the arming protocol is proved through it. A hook that +//! misbehaves does not fail loudly; it makes some other test fail for a reason +//! that has nothing to do with what that test is about. + +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::ARM; + +#[test] +fn a_hook_fires_while_installed_and_not_afterwards() { + // The static lives inside the test rather than at module scope: this + // workspace runs tests as threads in one process, so a module-scope counter + // could be moved by another test. + static FIRED: AtomicUsize = AtomicUsize::new(0); + + ARM.with( + || { + FIRED.fetch_add(1, Ordering::Relaxed); + }, + || { + ARM.run(); + ARM.run(); + }, + ); + assert_eq!(FIRED.load(Ordering::Relaxed), 2, "the hook did not fire"); + + ARM.run(); + assert_eq!( + FIRED.load(Ordering::Relaxed), + 2, + "the hook fired after it was removed" + ); +} + +#[test] +fn a_panic_inside_the_body_still_removes_the_hook() { + // The defect this guards. Removing the hook with a statement after `body` + // means an unwind skips it, and a test that installs a hook and then fails + // an assertion -- the ordinary way for a test to fail -- would leave it + // installed to fire inside whatever ran next on this thread. + static FIRED: AtomicUsize = AtomicUsize::new(0); + + let panicked = catch_unwind(AssertUnwindSafe(|| { + ARM.with( + || { + FIRED.fetch_add(1, Ordering::Relaxed); + }, + || panic!("the body fails, as a failing test does"), + ); + })); + assert!(panicked.is_err(), "the panic must reach the caller"); + + ARM.run(); + assert_eq!( + FIRED.load(Ordering::Relaxed), + 0, + "a hook survived an unwind and fired later" + ); +} + +#[test] +fn a_hook_that_re_enters_its_own_window_does_not_trip_the_refcell() { + // `run` takes the hook out for the call rather than holding the borrow + // across it. Without that, a hook whose body reaches the same window again + // would panic on a `RefCell` double borrow -- and the panic would look like + // a fault in the queue rather than in the harness. + static DEPTH: AtomicUsize = AtomicUsize::new(0); + + ARM.with( + || { + if DEPTH.fetch_add(1, Ordering::Relaxed) == 0 { + ARM.run(); + } + }, + || ARM.run(), + ); + + assert_eq!( + DEPTH.load(Ordering::Relaxed), + 1, + "re-entering the window should find the hook taken out, not re-run it" + ); +} diff --git a/crates/windows-waitable-queues/src/reserving_mpsc.rs b/crates/windows-waitable-queues/src/reserving_mpsc.rs new file mode 100644 index 00000000..e1cfa4df --- /dev/null +++ b/crates/windows-waitable-queues/src/reserving_mpsc.rs @@ -0,0 +1,2047 @@ +// Copyright (c) Mike Grier. + +//! The multi-producer, single-consumer bounded array queue **that can reserve**. +//! +//! Everything [`slotwise_mpsc`](crate::slotwise_mpsc) is, plus [`Producer::reserve`]: a slot +//! claimed in advance, so that a later delivery cannot be refused for want of +//! room. *Reserved is guaranteed, unreserved is best-effort.* +//! +//! # The claim position recurs, and how soon is a layout choice +//! +//! **Under the default layout this shape can lose an item after 2^32 pushes, on +//! every target and not only 32-bit ones** -- [`Balanced`] gives the claim +//! position a 32-bit half of the packed word below, so this reaches x86-64 and +//! ARM64 exactly as it reaches i686. +//! +//! A producer that has checked for room, been descheduled, and resumed after +//! other producers drove the position through a full wrap will claim +//! successfully against a numerically identical but generations-later value, +//! and write into a slot whose emptiness was decided long ago. **The failure is +//! silent**: the consumer receives a different item than was sent, and no error, +//! panic, or counter reports it. +//! +//! Under `Balanced`, 2^32 pushes is 37 seconds to about four minutes of +//! *sustained* pushing at this crate's measured rates, roughly two minutes at +//! two producers. The wrap alone is not enough -- a producer must also stall +//! inside a window a few instructions wide -- but a preemption suffices. +//! +//! **[`ClaimLayout`] is how far away that is.** [`Perpetual`] moves it to 2^56 +//! pushes, about twenty years at the same rate, for the cost of a reservation +//! ceiling of 255 and nothing measurable besides -- it is the same exchange on +//! the same word, differing only in shift constants. [`Enduring`] sits between +//! them, and the `dwcas` feature adds a 128-bit word that removes the +//! recurrence outright. +//! +//! ``` +//! use windows_waitable_queues::reserving_mpsc::{self, Perpetual}; +//! +//! let (tx, rx) = reserving_mpsc::bounded_as::(64)?; +//! # let _ = (tx, rx); +//! # Ok::<(), windows_waitable_queues::CapacityError>(()) +//! ``` +//! +//! The default stays `Balanced` so that introducing the choice changed no +//! existing caller's behaviour; it is not the recommended layout. +//! [`slotwise_mpsc`](crate::slotwise_mpsc) does not have this hazard under any +//! layout, its positions being 64 bits on every target; [`spsc`](crate::spsc) +//! never had it. The full statement is in the [crate documentation](crate). +//! +//! # Why this is a separate shape rather than a method on `slotwise_mpsc` +//! +//! Because the two ask different questions to claim a slot, and only this one's +//! question can answer a reservation. They are two claim protocols, not one +//! queue with a switch. +//! +//! Honouring a reservation costs the producer a read of the consumer's position +//! on **every** push, including the pushes that never reserve anything -- which +//! is what `slotwise_mpsc` avoids and why it cannot offer reservation at all. +//! +//! **That cost is not what makes either shape slower.** This one measured +//! *faster* than `slotwise_mpsc` under contention on both architectures tried, by up to +//! 6.4x, because the slot sequence `slotwise_mpsc` reads instead marches through memory +//! while other producers write it. See the crate documentation for the numbers +//! and for how to choose. +//! +//! `slotwise_mpsc`'s producer never reads the consumer's position. It asks a different +//! question -- "is the slot I am about to claim free?" -- and reads that from +//! the slot's own sequence number, which is spread across the slot array, so +//! producers working at different positions touch different cache lines. +//! Avoiding a single shared position is not incidental to that design; it is +//! most of the point of it. +//! +//! A reservation cannot be honoured from that question. "Is this slot free" does +//! not tell you **how many** slots remain, and holding one back for a reserver +//! requires exactly that count -- which requires the consumer's position, on one +//! line every thread in the system touches. +//! +//! So the two ship as peers ([D-16](../DESIGN-NOTES.md#d-16)): `slotwise_mpsc` for a +//! caller who wants the cheapest possible push and can treat a refusal as +//! backpressure, this shape for a caller with a message it must not lose. That +//! is the narrow-trait argument from [D-2](../DESIGN-NOTES.md#d-2) reaching +//! its sharpest case -- `slotwise_mpsc` does not implement +//! [`Reserving`](crate::Reserving) because it genuinely cannot, not because +//! nobody got round to it. +//! +//! # The claim word, which is why reservation is sound here +//! +//! The reservation count and the claim position live in **one** [`AtomicU64`]: +//! the low 32 bits are the position, the high 32 the number of outstanding +//! reservations. Every operation that changes either changes both together, with +//! one compare-and-swap. +//! +//! That is not tidiness, it is the correctness argument, and the obvious +//! alternative is broken in a way worth recording. With the count in its own +//! atomic: +//! +//! 1. A pushing producer reads the count, sees room, and claims the position. +//! 2. A reserving producer increments the count, reads the position, sees room, +//! and hands out the reservation. +//! +//! Each read before the other's write, and the queue now owes a slot that does +//! not exist. **Sequentially consistent fences do not close this**, unlike the +//! superficially similar hazard in the internal `Doorbell`: the +//! Dekker argument needs store-then-load on both sides, and the pushing producer +//! is load-then-store -- it *reads* the count and then *writes* the position. In +//! a total order over the four operations, both sides missing each other is +//! consistent, so no fence forbids it. Two independent claimants on one resource +//! must synchronise on one location, so the count and the position become one +//! location. +//! +//! With that, redeeming a reservation is a single compare-and-swap that +//! decrements the count and advances the position at once -- so the quantity the +//! invariant is about, `occupied + reserved`, is never momentarily wrong. +//! +//! # What the packing costs, and what it does not +//! +//! Splitting a 64-bit word 32/32 caps this shape at +//! a maximum of 2^31 items, and that split is forced rather than chosen: +//! a position of `b` bits keeps a wrapping difference unambiguous only up to +//! `2^(b-1)`, and the count needs `b` bits because it can reach the capacity, so +//! `b + b = 64` gives `b = 32`. There is no cleverer division of the word. +//! +//! **A 128-bit compare-and-swap is deliberately not used *here*** +//! ([D-37](../DESIGN-NOTES.md#d-37)). It would not remove the cost that +//! matters -- the consumer's position still has to be read -- and 2^31 slots is +//! a ring this shape allocates in full at construction. +//! +//! The operative reason is that widening *this* shape's word would change what +//! it offers depending on the target: `i686-pc-windows-msvc` has no lock-free +//! 128-bit exchange, so the same module would be lock-free on one target and +//! silently mutex-backed on another. A wider claim ships instead as its own +//! shape (`reserving_mpsc_wide`, not yet built -- see D-37), to exist only +//! where the exchange is genuinely lock-free. That keeps *this* module's +//! contract the same on every target, which is the property being protected +//! here: a caller who wants 2^62 slots and no wrap hazard will ask for it by +//! name rather than get it by accident of where they compiled. + +use core::cell::{Cell, UnsafeCell}; +use core::fmt; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::io; +use std::os::windows::io::{BorrowedHandle, OwnedHandle}; +use std::sync::Arc; +use std::time::Duration; + +use crate::CacheAligned; +use crate::blocking::{self, Parked}; +use crate::capacity::{Bounds, MAX_ADMISSIBLE_CAPACITY, WRAPPING_MAX_CAPACITY, validate_capacity}; +use crate::disposal::Teardown; +use crate::doorbell::Doorbell; +use crate::error::{ + CapacityError, Disconnected, PushError, RecvError, RecvTimeoutError, TryRecvError, +}; +use crate::metrics::Metrics; +use crate::options::Options; + +/// How the claim word's 64 bits are divided between the two things it packs. +/// +/// The word carries an outstanding-reservation count and a claim position, and +/// it must carry both because a single compare-and-swap has to update them +/// together (see the [module documentation](self)). Dividing 64 bits between +/// them is therefore a trade, and this trait is where a caller chooses which +/// side to spend them on. +/// +/// **The two things being traded are not equally valuable, and the shipping +/// default spends the bits on the less valuable one.** The reservation count +/// bounds how many messages may be held in flight at once -- in practice the +/// number of producers mid-send, so hundreds or thousands. The position decides +/// how many pushes occur before it recurs, and a recurrence is the `SH-14.1` +/// hazard: a producer descheduled across a full wrap can claim against a +/// numerically identical but generations-later value. +/// +/// | Layout | reserved / position | Outstanding reservations | Pushes to recurrence | +/// |---|---|---|---| +/// | [`Balanced`] | 32 / 32 | 2^32 | 2^32 | +/// | [`Enduring`] | 16 / 48 | 65,535 | 2^48 | +/// | [`Perpetual`] | 8 / 56 | 255 | 2^56 | +/// +/// At this crate's disclosed sustained rate of about 116 million pushes per +/// second, those recurrences are roughly **37 seconds**, **28 days**, and +/// **20 years** respectively. The rate is the one `reserving_mpsc`'s own hazard +/// note quotes; a queue that must drain cannot sustain the fastest rate +/// measured, so treat these as a floor on time rather than a forecast. +/// +/// **Choosing a deeper position costs nothing measurable.** All three issue the +/// same `lock cmpxchg` on the same `u64` and differ only in shift and mask +/// constants; a probe comparing them found no difference outside noise. The +/// trade is entirely against the reservation ceiling. +/// +/// This trait is sealed: the layouts are a fixed set because each one's +/// constants are checked against each other at compile time, and a caller +/// supplying its own could pick a division this shape cannot honour. +pub trait ClaimLayout: sealed::Sealed { + /// The integer the two halves are packed into. + /// + /// `u64` for every layout the crate offers by default. The `dwcas` feature + /// adds `Wide`, whose word is a `u128` -- and the arithmetic is done in + /// this type rather than uniformly in the wider one, so a `u64` layout + /// issues `u64` instructions exactly as it did before the type became a + /// parameter. + type Word: ClaimWord; + + /// How wide [`Self::Word`] is, in bits. + const WORD_BITS: u32; + + /// How many of the claim word's bits carry the position. + const POSITION_BITS: u32; + + /// Isolates the position half of the claim word. + /// + /// A position is carried in a `u64` whatever the word's width, since no + /// layout gives it more than 64 bits. At exactly 64 the shift that would + /// build this mask overflows, so the whole-width case is spelled out. + const POSITION_MASK: u64 = if Self::POSITION_BITS >= 64 { + u64::MAX + } else { + (1u64 << Self::POSITION_BITS) - 1 + }; + + /// The largest outstanding-reservation count the word's other half holds. + /// + /// **A ceiling on reservations, not on capacity.** An earlier form of this + /// shape required the count's half to be wide enough for the whole + /// capacity, because every slot could be reserved at once. That is what made + /// a large capacity consume the position's bits. Capping the reservations + /// instead leaves the capacity bounded only by the ring. + /// + /// **Capped at [`u32::MAX`] however wide the field is**, because the count + /// is reported to callers as a `u32`. A field wider than that would let the + /// queue hold a count it could not describe. + const MAX_RESERVED: u64 = { + let field = Self::WORD_BITS - Self::POSITION_BITS; + if field >= 32 { + u32::MAX as u64 + } else { + (1u64 << field) - 1 + } + }; + + /// The largest capacity this layout accepts. + /// + /// A wrapping position difference is unambiguous only up to half the + /// position space, and the crate-wide ceiling applies as well -- on a + /// 32-bit target it is the narrower of the two, and a shift by the position + /// width would overflow `usize` outright. + const BOUNDS_MAX: usize = { + let ring_bits = Self::POSITION_BITS - 1; + if ring_bits >= usize::BITS { + MAX_ADMISSIBLE_CAPACITY + } else { + let packed = 1_usize << ring_bits; + if packed <= MAX_ADMISSIBLE_CAPACITY { + packed + } else { + MAX_ADMISSIBLE_CAPACITY + } + } + }; + + /// The relationships this layout's constants depend on. + /// + /// **Forced at construction rather than left to be evaluated.** An + /// associated constant in a generic context is only evaluated where it is + /// used, so assertions written here and never mentioned would compile for + /// every layout including a broken one. The constructors name it, so + /// creating a queue is what checks it. + /// + /// Note what is deliberately *not* asserted: that `BOUNDS_MAX` is at most + /// `MAX_RESERVED`. That assertion is what previously tied the capacity to + /// the reservation field, and removing it is the point of the decoupling + /// above. + const VALID: () = { + assert!( + Self::POSITION_BITS >= 32, + "the reservation count is read out as a u32, so a count field wider than 32 bits -- \ + that is, a position narrower than 32 -- would be truncated on the way out" + ); + assert!( + Self::POSITION_BITS < Self::WORD_BITS, + "the count needs at least one bit, so the position cannot take the whole word" + ); + assert!( + Self::POSITION_BITS <= 64, + "a position is carried in a u64 between the packing and the ring, so a layout giving \ + it more bits than that would lose them on the way out" + ); + assert!( + Self::MAX_RESERVED <= u32::MAX as u64, + "the count is read back out through `reserved_of`'s cast to `u32`, so a field the \ + word could hold but the cast could not would make this constant's name a lie" + ); + assert!( + Self::MAX_RESERVED >= 1, + "a layout that permits no reservation at all would make `reserve` always fail, which \ + is the one capability this shape exists to provide" + ); + assert!( + Self::BOUNDS_MAX >= 2, + "two is the smallest capacity this shape accepts, so a layout offering less accepts \ + nothing" + ); + assert!( + Self::BOUNDS_MAX.is_power_of_two(), + "the maximum is offered to a caller as a capacity it could use, so it must itself be \ + one this shape would accept" + ); + assert!( + Self::BOUNDS_MAX <= WRAPPING_MAX_CAPACITY, + "a shape may be narrower than the crate-wide bound but never wider" + ); + }; +} + +mod sealed { + /// Prevents a caller outside this crate from adding a layout. + pub trait Sealed {} + /// Prevents a caller outside this crate from adding a word width. + pub trait SealedWord {} +} + +/// The integer a claim word is packed into, and the atomic that holds it. +/// +/// Exists so a layout can choose between a `u64` and a `u128` word without the +/// narrow layouts paying for the wide one: each is monomorphised to the +/// instructions its own width needs. Sealed for [`ClaimLayout`]'s reason, and +/// because an implementation that got the packing wrong would corrupt the two +/// halves into each other. +pub trait ClaimWord: sealed::SealedWord + Copy + PartialEq { + /// The atomic this word lives in. + type Atomic; + + /// A fresh atomic holding a zeroed word. + fn zeroed() -> Self::Atomic; + + /// Read the word. + fn load(cell: &Self::Atomic, order: Ordering) -> Self; + + /// Attempt to replace `current` with `new`. + fn compare_exchange_weak( + cell: &Self::Atomic, + current: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> Result; + + /// Read the word through a unique borrow, without synchronization. + fn read_mut(cell: &mut Self::Atomic) -> Self; + + /// Pack a reservation count and a position together. + fn pack(reserved: u32, position: u64, position_bits: u32, position_mask: u64) -> Self; + + /// Read the position back out. + fn position(self, position_mask: u64) -> u64; + + /// Read the reservation count back out. + fn reserved(self, position_bits: u32) -> u32; +} + +impl sealed::SealedWord for u64 {} +impl ClaimWord for u64 { + type Atomic = AtomicU64; + + // `AtomicU64::new(0)` and `AtomicU64::default()` are the same value, so a + // mutation run reports this as a survivor. It is an equivalent mutant: the + // explicit zero is kept because the protocol depends on the initial claim + // word being zero, which `default()` states only by coincidence. + #[inline] + fn zeroed() -> Self::Atomic { + AtomicU64::new(0) + } + + #[inline] + fn load(cell: &Self::Atomic, order: Ordering) -> Self { + cell.load(order) + } + + #[inline] + fn compare_exchange_weak( + cell: &Self::Atomic, + current: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> Result { + cell.compare_exchange_weak(current, new, success, failure) + } + + #[inline] + fn read_mut(cell: &mut Self::Atomic) -> Self { + *cell.get_mut() + } + + #[inline] + // **The `|` could equally be `^`, or `+`, and a mutation run reports as + // much.** The halves are disjoint by construction -- the shift clears every + // bit the position occupies -- so all three agree on every input and no test + // can tell them apart. `|` says "these are separate fields" where the others + // say "these are numbers". Recorded at both `pack` impls as well as on + // `claim_word`, because that is where the operation actually lives: a run + // reported it here after the word became a type parameter and the note + // stayed behind on the caller. + fn pack(reserved: u32, position: u64, position_bits: u32, position_mask: u64) -> Self { + ((reserved as u64) << position_bits) | (position & position_mask) + } + + #[inline] + fn position(self, position_mask: u64) -> u64 { + self & position_mask + } + + #[inline] + fn reserved(self, position_bits: u32) -> u32 { + (self >> position_bits) as u32 + } +} + +#[cfg(feature = "dwcas")] +impl sealed::SealedWord for u128 {} +#[cfg(feature = "dwcas")] +impl ClaimWord for u128 { + type Atomic = portable_atomic::AtomicU128; + + // Equivalent-mutant note as on the `u64` impl above. + #[inline] + fn zeroed() -> Self::Atomic { + portable_atomic::AtomicU128::new(0) + } + + #[inline] + fn load(cell: &Self::Atomic, order: Ordering) -> Self { + cell.load(order) + } + + #[inline] + fn compare_exchange_weak( + cell: &Self::Atomic, + current: Self, + new: Self, + success: Ordering, + failure: Ordering, + ) -> Result { + cell.compare_exchange_weak(current, new, success, failure) + } + + #[inline] + fn read_mut(cell: &mut Self::Atomic) -> Self { + *cell.get_mut() + } + + #[inline] + // Equivalent-mutant note as on the `u64` impl above. + fn pack(reserved: u32, position: u64, position_bits: u32, position_mask: u64) -> Self { + ((reserved as u128) << position_bits) | ((position & position_mask) as u128) + } + + #[inline] + fn position(self, position_mask: u64) -> u64 { + (self as u64) & position_mask + } + + #[inline] + fn reserved(self, position_bits: u32) -> u32 { + (self >> position_bits) as u32 + } +} + +/// The shipping division: 32 bits each. +/// +/// Holds 2^32 outstanding reservations and recurs after 2^32 pushes -- about +/// **37 seconds** of sustained maximum-rate pushing. This is the default +/// because it is what the shape shipped with, not because it is the best +/// choice: the reservation ceiling it buys is far beyond any real use, and it +/// is paid for with the whole of the `SH-14.1` exposure. Prefer [`Enduring`] or +/// [`Perpetual`] unless you genuinely hold more than 65,535 reservations at +/// once. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Balanced; +impl sealed::Sealed for Balanced {} +impl ClaimLayout for Balanced { + type Word = u64; + const WORD_BITS: u32 = 64; + const POSITION_BITS: u32 = 32; +} + +/// A deeper position: 16 bits of reservations, 48 of position. +/// +/// Holds 65,535 outstanding reservations and recurs after 2^48 pushes -- about +/// **28 days** of sustained maximum-rate pushing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Enduring; +impl sealed::Sealed for Enduring {} +impl ClaimLayout for Enduring { + type Word = u64; + const WORD_BITS: u32 = 64; + const POSITION_BITS: u32 = 48; +} + +/// The deepest position: 8 bits of reservations, 56 of position. +/// +/// Holds 255 outstanding reservations and recurs after 2^56 pushes -- about +/// **20 years** of sustained maximum-rate pushing, which puts the recurrence +/// beyond any real deployment rather than merely far away. +/// +/// 255 reservations is the whole of the trade, and it is a real limit rather +/// than a nominal one: [`Producer::reserve`] returns `None` once that many are +/// outstanding, however empty the queue is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Perpetual; +impl sealed::Sealed for Perpetual {} +impl ClaimLayout for Perpetual { + type Word = u64; + const WORD_BITS: u32 = 64; + const POSITION_BITS: u32 = 56; +} + +/// A 128-bit claim word: 64 bits of position, and the count in the other half. +/// +/// Requires the `dwcas` feature, which is what brings in the `portable-atomic` +/// dependency this crate otherwise does not have. The position needs 2^64 +/// pushes to recur, which no deployment reaches -- not "not for twenty years", +/// but not at all. +/// +/// **Read the cost before choosing it.** The 128-bit exchange measured 2-3x +/// slower than a `u64` one on the claim itself, and the penalty grows with +/// producer count; against a draining consumer the difference is much smaller. +/// [`Perpetual`] reaches about twenty years on a plain `AtomicU64` at no +/// measured cost, so this is worth taking only when a guarantee is wanted in +/// place of an argument about deployment lifetimes. +/// +/// The reservation ceiling is [`u32::MAX`] rather than the 64 bits the field +/// could hold, because the count is reported to callers as a `u32`. +#[cfg(feature = "dwcas")] +#[cfg_attr(docsrs, doc(cfg(feature = "dwcas")))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Wide; +#[cfg(feature = "dwcas")] +impl sealed::Sealed for Wide {} +#[cfg(feature = "dwcas")] +impl ClaimLayout for Wide { + type Word = u128; + const WORD_BITS: u32 = 128; + const POSITION_BITS: u32 = 64; +} + +/// The position after `position`, wrapping at the width the layout gives it. +/// +/// **Centralised because the width is no longer the type's.** A position is +/// carried in a `u64` but is only `L::POSITION_BITS` wide, so it wraps where the +/// packing says rather than where `u64` would. Spelling that as +/// `wrapping_add(1) & L::POSITION_MASK` at each of the dozen sites that need it +/// would be a dozen chances to omit the mask, and an omitted mask is not a +/// compile error -- it is a position that escapes its half of the claim word +/// and silently corrupts the reservation count beside it. +#[inline] +const fn advance(position: u64) -> u64 { + position.wrapping_add(1) & L::POSITION_MASK +} + +/// How far `position` leads `head`, in the modular arithmetic the position +/// width defines. +/// +/// Masked for [`advance`]'s reason. When the position was a `u32` the type +/// supplied this wrap for free; it no longer does. +#[inline] +const fn distance(position: u64, head: u64) -> u64 { + position.wrapping_sub(head) & L::POSITION_MASK +} + +/// The two handles a constructor hands back. +/// +/// Named because the layout parameter makes the pair long enough to obscure the +/// error type beside it, not because a caller is expected to write it: the +/// constructors return it and a caller destructures it immediately. +pub type Pair = (Producer, Consumer); + +// The layouts' relationship to one another, checked by the compiler rather than +// by a test. These are facts about constants, so a test could only report after +// the fact, on a build somebody chose to run -- and the trade they describe is +// the whole reason more than one layout exists. +const _: () = { + assert!( + ::MAX_RESERVED < ::MAX_RESERVED, + "a deeper position must cost reservations, or it would be free and there would be no \ + choice to offer" + ); + assert!( + ::MAX_RESERVED < ::MAX_RESERVED, + "the layouts must order consistently, or the table documenting them is wrong" + ); + assert!( + ::POSITION_MASK > ::POSITION_MASK, + "and the reservations given up must buy positions with them" + ); + assert!( + ::POSITION_MASK > ::POSITION_MASK, + "as above, across the whole ordering" + ); +}; + +/// The largest capacity the default layout accepts. +/// +/// Retained as a plain constant because it is public API and a caller may name +/// it. It is [`Balanced`]'s ceiling; other layouts have their own, reachable as +/// `::BOUNDS_MAX`. +pub const BOUNDS_MAX: usize = ::BOUNDS_MAX; + +/// The capacities a layout accepts. +/// +/// The minimum is two for the same reason [`slotwise_mpsc`](crate::slotwise_mpsc)'s is: with a +/// single slot, "published at `p`" and "free again on the next lap" would be the +/// same sequence number. The maximum is the layout's own, since the position +/// width decides how large a wrapping difference stays unambiguous. +const fn bounds() -> Bounds { + Bounds { + min: 2, + max: L::BOUNDS_MAX, + } +} + +/// Reads the position out of a claim word. +#[inline] +fn position_of(word: L::Word) -> u64 { + word.position(L::POSITION_MASK) +} + +/// Reads the outstanding-reservation count out of a claim word. +#[inline] +fn reserved_of(word: L::Word) -> u32 { + word.reserved(L::POSITION_BITS) +} + +/// Builds a claim word from its two halves. +/// +/// **Why the word is one `AtomicU64` and not two `AtomicU32`s**, given that every +/// operation on it is `Relaxed` (see D-38 in DESIGN-NOTES.md): relaxed is a +/// statement about *ordering*, and says nothing about atomicity. The two halves +/// are read and written as a unit, so the load must be indivisible -- a torn read +/// would return a `(reserved, position)` pair that was never a state this queue +/// was in, and the compare-and-swap protocol would be building on a value that +/// never existed. On `i686-pc-windows-msvc`, which D-18 keeps supported, that +/// costs a `cmpxchg8b` or an 8-byte SSE load rather than the two `mov`s a plain +/// `u64` would get. That cost is the point, not an overhead to optimize away. +/// +/// The `|` could equally be `^`, or `+`, and a mutation run will report as much. +/// The halves are disjoint by construction -- the shift clears every bit the +/// position occupies -- so all three agree on every input, and no test can tell +/// them apart. `|` is kept because it says "these are separate fields" where the +/// others say "these are numbers"; the equivalence is recorded here so it is not +/// investigated again. +#[inline] +fn claim_word(reserved: u32, position: u64) -> L::Word { + L::Word::pack(reserved, position, L::POSITION_BITS, L::POSITION_MASK) +} + +/// Creates a reserving multi-producer, single-consumer bounded array queue. +/// +/// One producer handle is returned; further producers are made by cloning it, +/// and the queue is disconnected when the last of them -- and the last +/// outstanding [`Reservation`] -- is gone. +/// +/// `capacity` must be a power of two between two and [`BOUNDS_MAX`], and is the +/// exact number of items the queue holds -- not a hint, and not rounded. +/// +/// # Errors +/// +/// Returns [`CapacityError`] if `capacity` is zero, is not a power of two, is +/// less than two, or exceeds [`BOUNDS_MAX`]. +/// +/// # Examples +/// +/// A slot taken before the work that will fill it, so the delivery cannot fail +/// for want of room: +/// +/// ``` +/// use windows_waitable_queues::reserving_mpsc; +/// +/// let (tx, rx) = reserving_mpsc::bounded::(2)?; +/// +/// // Claimed up front, while failing is still cheap. +/// let slot = tx.reserve().expect("a fresh queue has room"); +/// +/// // The rest of the queue fills. Best-effort pushes cannot take the +/// // reserved slot, so one of these is refused. +/// tx.push(1).expect("one slot remains unreserved"); +/// assert!(tx.push(2).is_err(), "the other belongs to the reservation"); +/// +/// // And the reservation is still honoured, on a queue that is otherwise full. +/// slot.send(99).expect("the room was already ours"); +/// +/// assert_eq!(rx.pop(), Ok(1)); +/// assert_eq!(rx.pop(), Ok(99)); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub fn bounded(capacity: usize) -> Result, CapacityError> { + build(capacity, Options::new()) +} + +/// Creates a queue whose claim word is divided as `L` says. +/// +/// [`bounded`] is this with `L` left at [`Balanced`], and the two are otherwise +/// identical. See [`ClaimLayout`] for what the division trades: a lower ceiling +/// on outstanding reservations against a longer run before the claim position +/// recurs. +/// +/// A separate entry point rather than a defaulted parameter on [`bounded`], +/// because Rust permits generic defaults on types but not on functions. The +/// types carry the default, so a caller who never names a layout never sees +/// one. +/// +/// ``` +/// use windows_waitable_queues::reserving_mpsc::{self, Perpetual}; +/// +/// // 255 outstanding reservations, and a claim position that recurs after +/// // 2^56 pushes rather than 2^32. +/// let (tx, rx) = reserving_mpsc::bounded_as::(4)?; +/// tx.push(1).expect("an empty queue has room"); +/// assert_eq!(rx.pop(), Ok(1)); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +/// +/// # Errors +/// +/// As [`bounded`], against `L`'s own capacity ceiling. +pub fn bounded_as(capacity: usize) -> Result, CapacityError> { + build(capacity, Options::new()) +} + +/// Creates a queue with both a layout and non-default behaviour. +/// +/// [`bounded_as`] with [`Options`], as [`bounded_with`] is to [`bounded`]. +/// +/// # Errors +/// +/// As [`bounded`], against `L`'s own capacity ceiling. +pub fn bounded_with_as( + capacity: usize, + options: Options, +) -> Result, CapacityError> { + build(capacity, options) +} + +/// Creates a queue with something other than the default behaviour. +/// +/// Identical to [`bounded`] except for what [`Options`] asks for. +/// +/// **This is the shape where disposal matters most.** A reservation exists +/// because its message must not be lost; a message redeemed into a queue that +/// is then torn down undrained would be lost after all, just later and more +/// quietly. Pairing a reservation with a disposal sink is what closes that. +/// +/// [`Options::tracking_high_water`] costs this shape almost nothing, unlike +/// [`slotwise_mpsc`](crate::slotwise_mpsc): the producer already reads the consumer's position +/// to decide whether there is room beyond the reservations, so the depth is a +/// subtraction of two numbers it is already holding. +/// +/// # Errors +/// +/// As [`bounded`]. +pub fn bounded_with(capacity: usize, options: Options) -> Result, CapacityError> { + build(capacity, options) +} +fn build( + capacity: usize, + options: Options, +) -> Result, CapacityError> { + validate_capacity(capacity, bounds::())?; + + let mut slots = Vec::with_capacity(capacity); + for index in 0..capacity { + slots.push(Slot { + // Anything that is not `position + 1` for the position this slot + // first serves, so the consumer sees it as unpublished. The + // position's own value is the natural choice and matches the state + // the slot returns to on every later lap. + sequence: AtomicU64::new(index as u64), + value: UnsafeCell::new(MaybeUninit::uninit()), + }); + } + + // Names `L::VALID` so the layout's own const assertions are evaluated. + // An associated constant in a generic context is only checked where it is + // used, so a layout whose constants contradict each other would otherwise + // compile untouched until something happened to mention them. + let () = L::VALID; + + let shared = Arc::new(Shared { + layout: PhantomData, + teardown: Teardown::new(options.disposal), + metrics: Metrics::new(options.track_high_water), + slots: slots.into_boxed_slice(), + mask: capacity - 1, + capacity, + head: CacheAligned(AtomicU64::new(0)), + claim: CacheAligned(::zeroed()), + producers: AtomicUsize::new(1), + consumer_live: AtomicBool::new(true), + doorbell: Doorbell::new(), + }); + + Ok(( + Producer { + shared: Arc::clone(&shared), + not_sync: PhantomData, + }, + Consumer { + shared, + not_sync: PhantomData, + }, + )) +} + +/// One cell of the ring: an item, and a sequence number saying whether it has +/// been published. +struct Slot { + /// `position + 1` once the producer that claimed `position` has finished + /// writing, and anything else before that. + /// + /// **This shape uses the sequence for one direction only.** In + /// [`slotwise_mpsc`](crate::slotwise_mpsc) it answers both "has this been published?" for the + /// consumer and "is this slot free?" for the producer. Here the producer + /// answers the second from the consumer's position instead -- it has to read + /// that position anyway, to count free slots for the reservations -- so + /// nothing ever stores a "free again" value and the consumer's `pop` is one + /// store shorter than `slotwise_mpsc`'s. + sequence: AtomicU64, + value: UnsafeCell>, +} + +struct Shared { + /// Ties the shared state to the layout its arithmetic is done in. + /// + /// Carries no data: the layout is entirely a set of compile-time constants, + /// so this exists only because a type parameter must appear in the type. + layout: PhantomData, + /// What becomes of undrained items at teardown. + /// + /// Read only by [`Shared::drop`], which holds `&mut self`, so it needs no + /// synchronization and costs the hot paths nothing but its space. + teardown: Teardown, + /// The counters this queue keeps about itself. See [`crate::metrics`]. + metrics: Metrics, + slots: Box<[Slot]>, + mask: usize, + capacity: usize, + /// Where the consumer will next read. Written only by the consumer. + /// + /// Padded onto its own cache line, and here the padding earns its place + /// twice over: unlike `slotwise_mpsc`, *every* producer reads this on *every* push, + /// so letting the claim word share the line would put the consumer's writes + /// directly in their path. + head: CacheAligned, + /// The outstanding-reservation count and the claim position, packed. + /// + /// One word because they must be claimed together; see the [module + /// documentation](self) for why two atomics cannot be made correct with any + /// amount of fencing. + claim: CacheAligned<::Atomic>, + /// How many producer handles and outstanding reservations are alive. + /// + /// **A reservation counts as a producer**, which is not bookkeeping + /// pedantry: a reservation is a promise of a message still to come, so a + /// consumer that saw the stream end while one was outstanding would be told + /// the queue was finished and then handed an item. That would lose exactly + /// the message the reservation existed to protect. + producers: AtomicUsize, + consumer_live: AtomicBool, + /// Readiness as a waitable `HANDLE`. Costs nothing until somebody asks for + /// the handle, so a polling consumer never allocates a kernel object. + doorbell: Doorbell, +} + +// SAFETY: a slot is written by exactly one producer -- the one whose +// compare-and-swap claimed that position -- and read by exactly one consumer, +// which reads it only after observing the release store of `position + 1` that +// publishes it. The write of the item therefore happens-before the read, and no +// two threads ever touch the same slot's contents at the same time. `T: Send` is +// required and sufficient because an item is moved between threads and never +// referenced from both. +// +// The `teardown` field is deliberately NOT covered by that argument, because it +// cannot be: it holds a boxed FnMut, which is Send but not Sync, so this +// impl is forcing Sync onto a field that does not have it. That is sound for +// a narrower reason -- the field is unreachable through a shared reference. It +// is private, no method reads it, and the only access is from Drop, which +// holds &mut self and runs when the last handle is already gone. So no two +// threads can reach it at all, concurrently or otherwise. +unsafe impl Sync for Shared {} +// SAFETY: as above; sending the shared state is sending the items it holds. +unsafe impl Send for Shared {} + +impl Shared { + /// The capacity as the width the positions are counted in. + /// + /// Lossless by construction: [`BOUNDS`] caps the capacity at 2^31. + fn capacity_u64(&self) -> u64 { + debug_assert!(self.capacity <= BOUNDS_MAX); + self.capacity as u64 + } + + /// Whether a *best-effort* claim may take the slot at `position`, given the + /// reservations currently outstanding. + /// + /// Written as a subtraction from the capacity rather than as + /// `occupied + reserved >= capacity`, because both terms can reach 2^31 and + /// their sum would overflow the width the positions are counted in. The + /// invariant guarantees `reserved <= capacity`, so this cannot underflow. + /// + /// **The answer is only meaningful for a claim word that is still current.** + /// `position` comes from a claim word and `head` is read here, so the two + /// need not describe the same instant: if other producers claim and publish + /// past a stale `position` and the consumer drains them, `head` overtakes it + /// and the subtraction wraps to near [`u32::MAX`] -- "full" computed from a + /// pair of readings that never coexisted. Callers therefore treat a `false` + /// as provisional and re-read the claim before reporting it (see + /// [`Producer::push`]). + fn has_room_beyond_reservations(&self, position: u64, reserved: u32) -> bool { + let capacity = self.capacity_u64(); + debug_assert!( + u64::from(reserved) <= capacity, + "reservations may never exceed the capacity they are claimed from" + ); + let occupied = distance::(position, self.head.0.load(Ordering::Acquire)); + occupied < capacity - u64::from(reserved) + } + + /// Items currently held, as a snapshot. + /// + /// Counts slots a producer has claimed but not yet finished writing, for the + /// reason `slotwise_mpsc`'s does: counting only published items would need a walk of + /// the ring, and this number is a metric rather than a control-flow input. + /// + /// **Clamped to the capacity**, for the reason given on `slotwise_mpsc`'s + /// twin: the claim word and `head` are two loads at two instants, so a + /// consumer draining past the sampled position makes the wrapping + /// subtraction produce a number near `u32::MAX`. A bounded queue must never + /// report holding more than it can. + fn len(&self) -> usize { + let position = position_of::(L::Word::load(&self.claim.0, Ordering::Relaxed)); + let head = self.head.0.load(Ordering::Acquire); + (distance::(position, head) as usize).min(self.capacity) + } + + /// How many further items a best-effort push could still place, as a + /// snapshot. + /// + /// **Not `capacity - len()`, which is what the [`Bounded`](crate::Bounded) + /// default computes and is wrong for this shape.** `len` deliberately + /// excludes outstanding reservations, so on an empty queue of four with one + /// slot reserved the default answers four while only three items fit -- and + /// a caller sizing a batch from it would be told there is room the + /// reservation is holding. + /// + /// The claim word is read **once**: the position and the reservation count + /// share it precisely so the two cannot be sampled at different instants, + /// and reading it twice would reintroduce the skew this shape packs them + /// together to avoid. `head` is still a second load, so the result is + /// clamped for the reason `len` is. + fn remaining(&self) -> usize { + let word = L::Word::load(&self.claim.0, Ordering::Relaxed); + let head = self.head.0.load(Ordering::Acquire); + let capacity = self.capacity_u64(); + let occupied = distance::(position_of::(word), head).min(capacity); + let spoken_for = occupied.saturating_add(u64::from(reserved_of::(word))); + capacity.saturating_sub(spoken_for) as usize + } + + /// Whether the consumer would find an item right now. + /// + /// Asks precisely what [`Consumer::pop`] asks -- is the slot at the head + /// position published? A claimed-but-unpublished slot answers `false`, which + /// is the right answer: the consumer may safely park on it, because the + /// producer's publishing store is followed by a signal. + fn has_ready_item(&self) -> bool { + // Acquire, matching every other load of `head`. This thread is `head`'s + // only writer, so coherence alone would make a relaxed load read its + // own latest value -- but `head` carries a release store (in `pop`), and + // a relaxed load on an atomic that also carries acquire/release + // operations is a plain load: unanchored, free to be moved by the + // optimizer or the processor, with no defined position relative to the + // ordered operations on the same object. Uniform acquire is what makes + // the load mean, at this point in the source, what it appears to mean. + let position = self.head.0.load(Ordering::Acquire); + let slot = &self.slots[position as usize & self.mask]; + slot.sequence.load(Ordering::Acquire) == advance::(position) + } + + /// Give up one unit of the producer count, signalling if it was the last. + /// + /// Shared by [`Producer`] and [`Reservation`] because they are the same + /// obligation: both represent a message that may still arrive, and the last + /// of either to leave is the one that ends the stream. + fn release_producer(&self) { + // `AcqRel` carries both halves. The release half publishes everything + // this producer pushed to whichever thread observes the count reaching + // zero, so a consumer that sees the disconnection can trust that + // draining to empty really has drained everything. The acquire half + // makes *this* thread -- when it is the one that drives the count to + // zero -- see the other producers' pushes, which is what makes the + // signal below meaningful. + if self.producers.fetch_sub(1, Ordering::AcqRel) != 1 { + return; + } + + // Disconnection is a wakeup like any other, and the only one nobody else + // can deliver. A consumer blocked on the doorbell would otherwise wait + // forever for an item that can no longer be sent. + // + // Only the *last* departure rings: an earlier one changes nothing a + // consumer could act on, and waking it to discover that would be a + // spurious wakeup per departing thread. + self.doorbell.signal(); + } + + /// Write an item into a claimed position and publish it. + /// + /// # Safety + /// + /// The caller must have claimed `position` by advancing the claim word, and + /// must not have published it already. A position is claimed by exactly one + /// producer, so this is the only writer of the slot. + unsafe fn publish(&self, position: u64, item: T) { + // Gated, matching `slotwise_mpsc`: untracked, this costs one predictable + // branch on a field written once at construction, and the shared `head` + // line is not touched at all. + // + // **Before the publication below, and that placement is load-bearing.** + // The subtraction is only non-negative while the consumer cannot have + // passed `position`, and what holds it back is precisely that `position` + // is not published yet. Taken afterwards, the consumer is free to drain + // past it, the subtraction wraps, and `fetch_max` keeps a vast number + // forever -- the defect `slotwise_mpsc`'s twin comment records measuring. + // + // **Clamped, which its twin does not need to be.** There, the producer's + // acquire load of the slot's sequence synchronizes-with the consumer + // freeing that slot, so the `head` read here cannot be older than + // `position - capacity + 1` and the depth is bounded by construction. + // This shape has a second entry point with no such edge: + // [`Reservation::send`] redeems without a room check, so the only `head` + // its thread is ordered against is the one *`reserve`* read -- which may + // be arbitrarily old by the time the reservation is redeemed. A stale + // read can only over-report, never under-report, so clamping to the + // capacity keeps the value an upper bound on a depth the queue really + // reached rather than an unbounded one. See [`Observable::high_water`] + // for what that bound is contracted to mean. + // + // [`Observable::high_water`]: crate::Observable::high_water + // **This load is unconditional because the slot write below needs it, + // not because the metric does.** It is the acquire half of the pair + // [`Consumer::pop`] describes: freeing a slot is `head.store(Release)`, + // and a producer may only write that slot after acquiring a `head` that + // has passed it. [`Producer::push`] gets that edge from the room check + // in [`Self::has_room_beyond_reservations`]; [`Reservation::send`] + // deliberately has no room check, so without this load its non-atomic + // write would race the consumer's non-atomic read of the previous + // occupant -- a data race, and undefined behaviour, however reliably a + // given target's codegen happens to order it today. + // + // Placing it here rather than in `send` covers every path with one + // load. + // + // **The load must be fresh *enough*, and a single acquire load does not + // guarantee that.** An earlier version of this comment argued that + // because the claim invariant makes `head >= position - capacity + 1` + // true at the exchange, and `head` never moves backwards, a later load + // "can only be fresher". That conflates what `head` *is* in modification + // order with what a load is *guaranteed to observe*: an acquire load may + // legally return any earlier value in the modification order, and + // synchronizes only with the release store whose value it actually + // reads. + // + // Nothing else forces freshness here. The claim exchange is `Relaxed`, + // so it carries no edge; and while `reserve` does read `head`, a + // `Reservation` is `Send`, so the thread that redeems one **need never + // have read `head` at all** -- leaving no coherence constraint to + // inherit. A reservation held across a full lap and redeemed elsewhere + // is exactly the case. Raised in PR #56 review. + // + // So the load is repeated until it observes a `head` that has actually + // passed this position's previous occupant. That is the store which + // frees the slot, so observing it (or any later one, by the same + // consumer and therefore sequenced after its read) is precisely the + // edge the write below needs. The loop terminates because the claim + // invariant makes the condition already true in modification order -- + // this waits to *see* it, not for it to *become* true. + let mut head = self.head.0.load(Ordering::Acquire); + while distance::(position, head) >= self.capacity_u64() { + std::hint::spin_loop(); + head = self.head.0.load(Ordering::Acquire); + } + if self.metrics.tracks_high_water() { + let depth = (distance::(position, head) + 1) as usize; + // **The clamp is unreachable from here, and is kept deliberately.** + // The wait above exits only once `position - head < capacity`, so + // `depth <= capacity` already holds and `min` never binds. It was + // load-bearing when this was a single unvalidated load: a stale + // `head` then made the depth an unbounded over-report, and + // `the_high_water_mark_never_exceeds_the_capacity` drove exactly + // that. Waiting for a fresh `head` removes the over-report at its + // source, so that test was replaced by + // `publish_waits_for_a_head_that_has_freed_the_slot`, which asserts + // the fix instead of the mitigation. + // + // Kept because it costs one register-to-register `min` on a path + // already doing an atomic load, and because it bounds the metric by + // the shape's own contract rather than by an argument a future + // change to the wait might invalidate silently. A mutation run will + // report it as a survivor; that is expected, and it is unreachable + // code rather than a missing test. + self.metrics.record_depth(depth.min(self.capacity)); + } + + let slot = &self.slots[position as usize & self.mask]; + // SAFETY: the caller's claim makes this thread the only writer, and the + // acquire load of `head` above -- repeated until it observed a value + // past this position's previous occupant -- synchronizes-with the + // `head.store` by which the consumer freed this slot a lap ago, so its + // read of the previous occupant happens-before this write. + // + // The claim alone is not enough. It establishes that the slot is + // *logically* free -- `occupied + reserved <= capacity` with + // `reserved >= 1` -- but a non-atomic write racing a non-atomic read + // needs a happens-before edge, not merely a logical guarantee that the + // read is over. The load above is that edge. + unsafe { + (*slot.value.get()).write(item); + } + + // Release, and this is the publication: it must come after the write, + // and this is what forbids the compiler and the processor from moving it + // earlier. Until it lands, the consumer sees the slot as + // claimed-but-empty and skips it. + slot.sequence + .store(advance::(position), Ordering::Release); + + // After the publication, never before: the doorbell says "there is + // something to take", and that must not become true before the item is + // actually takeable. A consumer woken early would find nothing, clear + // the doorbell, and go back to sleep on an item that is about to exist. + // + // A producer may signal while an *earlier* position is still + // unpublished, so the consumer wakes and finds nothing. That is a + // spurious wakeup, which the protocol tolerates by construction: the + // producer holding the earlier slot signals in its turn. + self.doorbell.signal(); + } +} + +impl Drop for Shared { + fn drop(&mut self) { + // Every handle is gone, so no synchronization is needed and the + // positions can be read directly. A slot between the two positions still + // holds an item nobody took, and dropping the queue must drop those + // rather than leak them. + // + // The sequence is consulted per slot rather than assuming every position + // in the range holds an item. A producer cannot be mid-push here -- it + // would have to hold a handle, and there are none -- so in practice + // every one does; the check states the invariant the read depends on + // instead of leaving it to that argument. + let mask = self.mask; + let head = *self.head.0.get_mut(); + let tail = position_of::(L::Word::read_mut(&mut self.claim.0)); + let mut position = head; + while position != tail { + let published = advance::(position); + let slot = &mut self.slots[position as usize & mask]; + if *slot.sequence.get_mut() == published { + // SAFETY: the slot's sequence says the producer finished writing + // it and the consumer never took it, so it holds an initialized + // item. It is read exactly once, because `position` advances + // every iteration and the slot is never read again. + let item = unsafe { slot.value.get_mut().assume_init_read() }; + self.teardown.dispose(item); + } + position = advance::(position); + } + } +} + +/// A writing half of a [`reserving_mpsc`](self) queue. +/// +/// [`Clone`], so producers multiply by cloning rather than by sharing: each +/// thread owns its own handle. Not [`Sync`], so a handle is used by one thread +/// at a time. +pub struct Producer { + shared: Arc>, + /// Removes [`Sync`] without removing [`Send`]. A [`Cell`] is exactly that + /// shape, and no value of it is ever created. + not_sync: PhantomData>, +} + +impl Producer { + /// Appends an item, best-effort. + /// + /// **Cannot take a reserved slot.** A queue with one free slot and one + /// outstanding reservation refuses this, which is the reservation doing its + /// job rather than a malfunction. + /// + /// # Errors + /// + /// [`PushError::Full`] when no unreserved room remains, which is the + /// backpressure signal, and [`PushError::Disconnected`] when the consumer is + /// gone. Either way the item comes back. + pub fn push(&self, item: T) -> Result<(), PushError> { + // Relaxed: this load only proposes a claim. The compare-and-swap below + // is what makes it, and fails if the proposal was stale, so a stale read + // costs a retry rather than correctness. + let mut word = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + let position = loop { + let position = position_of::(word); + let reserved = reserved_of::(word); + #[cfg(test)] + crate::race_hooks::CLAIM.run(); + + if !self.shared.has_room_beyond_reservations(position, reserved) { + // Provisional, not authoritative. `position` came from `word` + // and `head` was read inside the check, so a `word` that has + // since moved makes the two readings describe different + // instants -- and once `head` passes a stale `position` the + // subtraction wraps, so an *empty* queue reports full. Re-read + // the claim: if it moved, this answer was computed from a + // snapshot that never existed, so retry rather than refuse. + let current = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + if current != word { + word = current; + continue; + } + // Report disconnection in preference to fullness: a full queue + // whose consumer is gone will never drain, and telling the + // caller to retry would be telling it to spin forever. + if !self.shared.consumer_live.load(Ordering::Acquire) { + // Not counted as a refusal: this is the end of the stream, + // not backpressure. + return Err(PushError::Disconnected(item)); + } + self.shared.metrics.record_refusal(); + return Err(PushError::Full(item)); + } + if !self.shared.consumer_live.load(Ordering::Acquire) { + return Err(PushError::Disconnected(item)); + } + + // Relaxed on both sides is sufficient: this exchange orders nothing + // but the claim itself. The item's visibility comes from the release + // store that publishes the slot, and the freedom to write the slot + // comes from the acquire load of `head` inside the room check. + // + // The reservation count is carried through unchanged, which is what + // makes a racing `reserve` fail its own exchange and re-read rather + // than have its increment silently overwritten. + match L::Word::compare_exchange_weak( + &self.shared.claim.0, + word, + claim_word::(reserved, advance::(position)), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break position, + Err(actual) => word = actual, + } + }; + + // SAFETY: this thread's compare-and-swap claimed `position`, which no + // other producer can also have claimed, and it has not been published. + unsafe { + self.shared.publish(position, item); + } + Ok(()) + } + + /// Claims one slot for a message that must not be lost. + /// + /// See [`Reserving::reserve`](crate::Reserving::reserve) for what a + /// reservation is for. The short form: failing here is cheap, because no + /// work has been started yet, whereas failing at delivery means blocking or + /// losing the message. + /// + /// The queue stays connected while a reservation is outstanding, so a + /// consumer will not be told the stream ended and then handed the item. + #[must_use = "a reservation withholds capacity from every other producer until it is used or dropped"] + pub fn reserve(&self) -> Option> { + let mut word = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + loop { + let position = position_of::(word); + let reserved = reserved_of::(word); + #[cfg(test)] + crate::race_hooks::CLAIM.run(); + + if !self.shared.has_room_beyond_reservations(position, reserved) { + // Provisional for the reason `push`'s matching check is: a + // stale `word` and a freshly-read `head` need not describe the + // same instant, and once `head` passes a stale `position` the + // subtraction wraps and an empty queue refuses a reservation. + let current = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + if current != word { + word = current; + continue; + } + return None; + } + + // The count's own half of the word can overflow into the position's + // before the capacity is exhausted, once a layout gives it fewer + // bits than the capacity has slots. Refusing here is what decouples + // the two ceilings: the capacity is bounded by the ring, and the + // reservations by whatever the layout left room for. + // + // **Checked against the word this iteration read, not against a + // separate load.** The count and the position share the word + // precisely so a decision about one cannot be made against a stale + // reading of the other, and the exchange below re-validates the + // whole word -- so a racing `reserve` that got there first makes + // this one fail and re-read rather than exceed the ceiling. + if u64::from(reserved) >= L::MAX_RESERVED { + return None; + } + + // The position is carried through unchanged: a reservation claims + // capacity, not an order. Where the item lands is decided when the + // reservation is redeemed, so a slot held for a long time does not + // stall everything queued behind it. + match L::Word::compare_exchange_weak( + &self.shared.claim.0, + word, + claim_word::(reserved + 1, position), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + // Relaxed: this thread already holds a live producer handle, + // so the count cannot reach zero during this call and no + // other thread's decision depends on when the increment + // becomes visible. The pairing that matters is in + // `release_producer`. + self.shared.producers.fetch_add(1, Ordering::Relaxed); + return Some(Reservation { + shared: Arc::clone(&self.shared), + not_sync: PhantomData, + }); + } + Err(actual) => word = actual, + } + } + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Slots currently claimed by a reservation and not yet redeemed, as a + /// snapshot. + #[must_use] + pub fn outstanding_reservations(&self) -> usize { + reserved_of::(L::Word::load(&self.shared.claim.0, Ordering::Relaxed)) as usize + } + + /// Whether the next best-effort push would be refused, as a snapshot. + /// + /// True when the queue is full *or* every remaining slot is reserved, since + /// those are indistinguishable to a best-effort caller. Advisory only: + /// another producer may take the last slot between this call and the push. + #[must_use] + pub fn is_full(&self) -> bool { + self.remaining() == 0 + } + + /// How many further items a best-effort push could still place, as a + /// snapshot. + /// + /// **Reservations are subtracted**, unlike `capacity() - len()`: a reserved + /// slot is spoken for, so counting it as room would promise a push that + /// [`push`](Self::push) is guaranteed to refuse. Advisory only, like every + /// other gauge here. + #[must_use] + pub fn remaining(&self) -> usize { + self.shared.remaining() + } + + /// Whether the consumer has been dropped. + #[must_use] + pub fn is_disconnected(&self) -> bool { + !self.shared.consumer_live.load(Ordering::Acquire) + } +} + +impl Clone for Producer { + fn clone(&self) -> Self { + // Relaxed, for the reason given in `reserve`. + self.shared.producers.fetch_add(1, Ordering::Relaxed); + Self { + shared: Arc::clone(&self.shared), + not_sync: PhantomData, + } + } +} + +// Hand-written rather than derived: deriving would demand `T: Debug`, which +// would make a handle to a queue of non-`Debug` items un-printable for no +// reason. The item type is not the handle's business, so the handle reports the +// queue's state instead. +impl fmt::Debug for Producer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("reserving_mpsc::Producer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("reserved", &self.outstanding_reservations()) + .field("producers", &self.shared.producers.load(Ordering::Relaxed)) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Producer { + fn drop(&mut self) { + self.shared.release_producer(); + } +} + +/// A slot claimed in advance, which [`Reservation::send`] redeems. +/// +/// Owned rather than borrowed from the [`Producer`], and [`Send`], because that +/// is the shape the use case has: an operation reserves its completion slot when +/// it is submitted and redeems it from whichever thread the completion arrives +/// on. ([`spsc`](crate::spsc)'s reservation borrows instead, because there the +/// producer handle *is* the single-producer guarantee and letting a reservation +/// outlive it would create a second one.) +/// +/// Dropping it returns the slot to the queue. +#[must_use = "a reservation withholds capacity from every other producer until it is used or dropped"] +pub struct Reservation { + shared: Arc>, + /// See [`Producer::not_sync`]. A reservation may be *moved* between threads + /// but is used by one at a time, exactly like the handle that made it. + not_sync: PhantomData>, +} + +impl Reservation { + /// Delivers into the reserved slot. + /// + /// **This cannot fail for want of room**, which is the entire purpose: the + /// slot was withheld from every other producer from the moment the + /// reservation was taken. See [`Disconnected`] for why that is the only + /// error and why the type says so. + /// + /// # Errors + /// + /// [`Disconnected`] if the consumer is gone, carrying the item back so it + /// can be accounted for rather than silently dropped. + pub fn send(self, item: T) -> Result<(), Disconnected> { + if !self.shared.consumer_live.load(Ordering::Acquire) { + // Dropping `self` on the way out releases the slot and the producer + // count, which is what should happen: this message is never coming. + return Err(Disconnected(item)); + } + + // Redeem and claim in ONE exchange: the count falls by one as the + // position rises by one, so `occupied + reserved` -- the quantity the + // whole invariant is about -- is never momentarily wrong, and no + // concurrent producer can observe a state in which this slot looks + // available. + // + // There is no room check here, and its absence is the guarantee. The + // invariant `occupied + reserved <= capacity` with `reserved >= 1` means + // `occupied < capacity`, so the slot at this position is one the + // consumer has already finished with. + let mut word = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + let position = loop { + let position = position_of::(word); + let reserved = reserved_of::(word); + debug_assert!( + reserved >= 1, + "this reservation is outstanding, so the count cannot be zero" + ); + + match L::Word::compare_exchange_weak( + &self.shared.claim.0, + word, + claim_word::(reserved - 1, advance::(position)), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break position, + Err(actual) => word = actual, + } + }; + + // SAFETY: the exchange above claimed `position` for this thread alone, + // and the invariant argued for in the comment means the slot is free. + unsafe { + self.shared.publish(position, item); + } + + // The slot has been given up as part of the exchange above, so the + // `Drop` that would give it up again must not run. The producer count, + // however, still has to be released -- this reservation's promise is now + // fulfilled, and if it was the last outstanding one the stream ends + // here. + // + // **`mem::forget` would be wrong here, and was wrong here**: this type + // owns an `Arc`, and forgetting it leaks that strong reference, so the + // shared state is never dropped and every item still in the ring leaks + // with it. `ManuallyDrop` plus a move-out suppresses only *this type's* + // `Drop` while leaving the `Arc`'s own to run exactly once. + let this = core::mem::ManuallyDrop::new(self); + // SAFETY: `this` is a `ManuallyDrop`, so its own destructor never runs + // and the field is not read again after this move. + let shared = unsafe { core::ptr::read(&this.shared) }; + shared.release_producer(); + // `shared` falls out of scope here, releasing the reference this + // reservation held. + Ok(()) + } + + /// Whether the consumer has been dropped, so redeeming would fail. + #[must_use] + pub fn is_disconnected(&self) -> bool { + !self.shared.consumer_live.load(Ordering::Acquire) + } +} + +impl fmt::Debug for Reservation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("reserving_mpsc::Reservation") + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + // Give the slot back. Only the count moves: the position is untouched, + // because an unredeemed reservation never occupied a position. + let mut word = L::Word::load(&self.shared.claim.0, Ordering::Relaxed); + loop { + let reserved = reserved_of::(word); + debug_assert!( + reserved >= 1, + "this reservation is outstanding, so the count cannot be zero" + ); + match L::Word::compare_exchange_weak( + &self.shared.claim.0, + word, + claim_word::(reserved - 1, position_of::(word)), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => word = actual, + } + } + self.shared.release_producer(); + } +} + +/// The reading half of a [`reserving_mpsc`](self) queue. +/// +/// Neither [`Clone`] nor [`Sync`], which is what makes "single consumer" a fact +/// the compiler checks rather than a rule to remember. +pub struct Consumer { + shared: Arc>, + /// See [`Producer::not_sync`]. + not_sync: PhantomData>, +} + +impl Consumer { + /// Takes the oldest item. + /// + /// # Errors + /// + /// [`TryRecvError::Empty`] when nothing is queued right now, and + /// [`TryRecvError::Disconnected`] when every producer is gone *and* the + /// queue has been drained -- in that order, so the tail of a stream whose + /// producers have already departed is still delivered. See + /// [`Consumer::pop`](crate::Consumer::pop) for why that ordering is a + /// guarantee rather than an implementation detail. + pub fn pop(&self) -> Result { + match self.take() { + Some(item) => Ok(item), + // Only on the empty path, so a successful take never pays for this + // load. The queue must be observed empty *before* disconnection is + // reported, which is exactly what this ordering enforces. + None if self.is_disconnected() => Err(TryRecvError::Disconnected), + None => Err(TryRecvError::Empty), + } + } + + /// The take itself, without the disconnection question. + fn take(&self) -> Option { + // Acquire, matching every other load of `head`. Sole-writer coherence + // would suffice to read this thread's own latest value, but `head` also + // carries the release store below, and a relaxed load mixed onto such an + // atomic is a plain load the code generator may move. See + // `has_ready_item` for the full argument. + let position = self.shared.head.0.load(Ordering::Acquire); + let slot = &self.shared.slots[position as usize & self.shared.mask]; + // Acquire: pairs with the producer's release store, so an item it + // published is visible here. + if slot.sequence.load(Ordering::Acquire) != advance::(position) { + return None; + } + + // SAFETY: the sequence says the producer that claimed this position + // finished writing it, and the release/acquire pair above makes that + // write visible here. This is the only consumer, and the position is + // given up below, so the item is read exactly once. + let item = unsafe { (*slot.value.get()).assume_init_read() }; + + // Release, and this is what frees the slot: a producer reads `head` with + // an acquire load to count free slots, so this store must not become + // visible before the read above completes, or that producer could claim + // the position and overwrite an item this thread had not finished + // taking. + // + // Note that nothing stores a "free again" sequence here, unlike `slotwise_mpsc`. + // Advancing `head` *is* the release, because this shape's producers + // decide freedom from `head` rather than from the sequence. + self.shared + .head + .0 + .store(advance::(position), Ordering::Release); + Some(item) + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Slots currently claimed by a reservation and not yet redeemed, as a + /// snapshot. + /// + /// Offered on the consumer as well as the producer because it is the + /// difference between "nothing is coming" and "something was promised": + /// a drained queue with an outstanding reservation is not an idle one. + #[must_use] + pub fn outstanding_reservations(&self) -> usize { + reserved_of::(L::Word::load(&self.shared.claim.0, Ordering::Relaxed)) as usize + } + + /// How many further items a best-effort push could still place, as a + /// snapshot. + /// + /// The same number [`Producer::remaining`] reports, and offered here for + /// the same reason `outstanding_reservations` is: a consumer deciding + /// whether to keep draining wants the producers' view of the room left, and + /// that view subtracts reservations rather than treating them as free. + #[must_use] + pub fn remaining(&self) -> usize { + self.shared.remaining() + } + + /// Whether a further best-effort push would be refused for want of room. + /// + /// The consumer's view of the question the producer answers, so a caller + /// holding only this handle need not import [`Bounded`](crate::Bounded). + #[must_use] + pub fn is_full(&self) -> bool { + crate::Bounded::is_full(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// The inherent form of [`Consumer::drain`](crate::Consumer::drain), so it + /// works without importing the trait. + pub fn drain(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// An alias for [`Self::drain`] under the name most of the ecosystem uses. + pub fn try_iter(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Whether every producer and every outstanding reservation is gone. + /// + /// **A queue can be disconnected and still hold items**, because a producer + /// may push and then drop -- so this alone does not mean the stream is + /// finished, and acting on it while items remain would discard them. + /// [`Self::pop`] answers the composite question in the only order that + /// cannot lose the tail, and is what a drain loop should use. + #[must_use] + pub fn is_disconnected(&self) -> bool { + self.shared.producers.load(Ordering::Acquire) == 0 + } + + /// Borrows the queue's readiness as a waitable `HANDLE`. + /// + /// The event is created on the first call, so a consumer that only ever + /// polls with [`Self::pop`] is charged for no kernel object. + /// + /// # Waiting on it correctly + /// + /// **Do not simply wait and then drain.** Use [`Self::arm`] to decide + /// whether waiting is safe, or the wait can miss an item and block forever; + /// [`spsc::Consumer::doorbell`](crate::spsc::Consumer::doorbell) carries the + /// worked example, and the protocol is identical here. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn doorbell(&self) -> io::Result> { + self.shared.doorbell.handle() + } + + /// A duplicate of [`Self::doorbell`] that the caller owns. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` or `DuplicateHandle`. + pub fn doorbell_owned(&self) -> io::Result { + self.shared.doorbell.owned() + } + + /// Clears the doorbell and reports whether a later push could be missed. + /// + /// `true` means the queue had nothing takeable after the doorbell was + /// cleared, so any later push is guaranteed to signal. `false` means + /// something arrived in the meantime. + /// + /// **`true` is not by itself permission to wait indefinitely.** It answers + /// only whether a later *push* can be missed, and says nothing about the + /// end of the stream: with every producer gone it still returns `true`, + /// having just cleared the single ring their drop left behind. See + /// [`Waitable::arm`](crate::Waitable::arm) for the four-step protocol an + /// indefinite wait needs, and the example on [`Self::doorbell`] for it + /// written out. + /// + /// Clearing must come before the check, which is the reverse of the order + /// that reads naturally; see [D-9](../DESIGN-NOTES.md#d-9). + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn arm(&self) -> io::Result { + // Before the clear, and so before the check: a producer running while no + // event exists skips signalling, so the check has to come after the + // event exists to catch what that skip left behind. + self.shared.doorbell.handle()?; + self.shared.doorbell.clear(); + #[cfg(test)] + crate::race_hooks::ARM.run(); + // Deliberately not `is_empty`: the question is whether `pop` would find + // something, and a claimed-but-unpublished slot is not something `pop` + // can find. + Ok(!self.shared.has_ready_item()) + } + + /// The last take before reporting the end of the stream. + /// + /// Called only after [`Self::is_disconnected`] has returned `true`, which + /// makes the answer final rather than a snapshot. It guards a race that is + /// real and narrow: a producer may push *and then* drop in the window + /// between a receive's first `pop` and its disconnection check. + fn finish(&self) -> Option { + self.take() + } + + /// Takes the oldest item, blocking until one arrives. + /// + /// # Errors + /// + /// [`RecvError::Disconnected`] once every producer *and every outstanding + /// reservation* is gone and the queue is drained. [`RecvError::Io`] if the + /// doorbell cannot be created or waited on. + pub fn recv(&self) -> Result { + blocking::recv(self) + } + + /// Takes the oldest item, blocking until one arrives or the deadline passes. + /// + /// # Errors + /// + /// [`RecvTimeoutError::Timeout`] if the deadline passes with the queue still + /// empty, which is not a malfunction. Otherwise as [`Self::recv`]. + pub fn recv_timeout(&self, timeout: Duration) -> Result { + blocking::recv_timeout(self, timeout) + } +} + +impl Parked for Consumer { + type Item = T; + + fn pop(&self) -> Option { + Self::take(self) + } + + fn finish(&self) -> Option { + Self::finish(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } + + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } +} + +/// See [`Producer`]'s impl for why this is hand-written. +impl fmt::Debug for Consumer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("reserving_mpsc::Consumer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("reserved", &self.outstanding_reservations()) + .field("producers", &self.shared.producers.load(Ordering::Relaxed)) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Consumer { + fn drop(&mut self) { + self.shared.consumer_live.store(false, Ordering::Release); + } +} + +impl crate::Producer for Producer { + type Item = T; + + fn push(&self, item: T) -> Result<(), PushError> { + Self::push(self, item) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Claim for Reservation { + type Item = T; + + fn send(self, item: T) -> Result<(), Disconnected> { + Self::send(self, item) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Reserving for Producer { + type Item = T; + type Reservation<'a> + = Reservation + where + Self: 'a; + + fn reserve(&self) -> Option> { + Self::reserve(self) + } + + fn outstanding_reservations(&self) -> usize { + Self::outstanding_reservations(self) + } +} + +impl crate::Consumer for Consumer { + type Item = T; + + fn pop(&self) -> Result { + Self::pop(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Bounded for Producer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + // Overridden, because the default `capacity - len` counts a reserved slot + // as room: `len` excludes reservations by design, so an empty queue of four + // holding one reservation would answer four while only three items fit. + fn remaining(&self) -> usize { + self.shared.remaining() + } +} + +impl crate::Bounded for Consumer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + // The consumer's view has to agree with the producer's: both describe the + // same queue, and a caller generic over `Bounded` should not get a different + // answer depending on which handle it holds. + fn remaining(&self) -> usize { + Self::remaining(self) + } +} + +impl Shared { + /// The counters, as the [`Observable`](crate::Observable) trait reports + /// them. Written once so the two handles cannot drift apart. + fn refused(&self) -> u64 { + self.metrics.refused() + } + + fn doorbell_rings(&self) -> u64 { + self.doorbell.rings() + } + + fn high_water(&self) -> Option { + self.metrics.high_water() + } +} + +impl Producer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl Consumer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl crate::Observable for Producer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Observable for Consumer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Waitable for Consumer { + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } + + fn doorbell_owned(&self) -> io::Result { + Self::doorbell_owned(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/reserving_mpsc/tests.rs b/crates/windows-waitable-queues/src/reserving_mpsc/tests.rs new file mode 100644 index 00000000..7223392d --- /dev/null +++ b/crates/windows-waitable-queues/src/reserving_mpsc/tests.rs @@ -0,0 +1,1767 @@ +// Copyright (c) Mike Grier. + +//! Tests for the reserving MPSC bounded array queue. +//! +//! The shape's queueing behaviour is `slotwise_mpsc`'s and is covered there; what is +//! tested here is the part that is different -- the reservation, the packed +//! claim word, and the ways the two interact with everything else. +//! +//! The load-bearing property is stated once and asserted from several angles: +//! **a granted reservation is always redeemable.** A test that only checked +//! "reserve then send works on an idle queue" would assert nothing, because the +//! failure mode is a reservation granted while a racing producer takes the last +//! slot -- so the interesting cases all put the queue under pressure first. + +use super::{ + BOUNDS_MAX, Balanced, ClaimLayout, Consumer, Enduring, Perpetual, Producer, Reservation, + advance, bounded, bounded_as, bounded_with, claim_word, position_of, reserved_of, +}; +use crate::error::TryRecvError; + +/// The default layout's constants, named once so the packing tests read as +/// prose rather than as turbofish. +const POSITION_MASK: u64 = ::POSITION_MASK; +/// As [`POSITION_MASK`]. +const MAX_RESERVED: u64 = ::MAX_RESERVED; +use crate::race_hooks; +use crate::{Disposal, Options}; +// The trait is imported anonymously because this module also names the concrete +// `Consumer` type, and only its `drain` method is wanted here. That the two can +// coexist is the point made in `traits`: the trait is named for the role and the +// handle is named for the role, and a caller who wants only the methods says so. +use crate::{Bounded, PushError, RecvError, Reserving}; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +/// Counts its own drops, so a test can prove an item was destroyed rather than +/// leaked. `Arc` rather than a `static`, so tests that run +/// concurrently in one process cannot see each other's counts. +#[derive(Debug)] +struct DropCounter(Arc); + +impl Drop for DropCounter { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// Fills every slot a best-effort producer is allowed to take. +/// +/// Returns how many went in, which is the capacity less whatever is reserved. +fn fill(producer: &Producer, item: T) -> usize { + let mut pushed = 0; + while producer.push(item.clone()).is_ok() { + pushed += 1; + } + pushed +} + +// --------------------------------------------------------------------------- +// The packed claim word. +// +// Tested directly as well as through the queue: the packing is arithmetic, and +// arithmetic is worth checking at its edges rather than only through the six +// layers of queue that happen to use it. +// --------------------------------------------------------------------------- + +#[test] +fn the_claim_word_round_trips_both_halves() { + // Written against the split's own constants rather than against `u32`'s + // extremes, because the position is no longer 32 bits by definition: it is + // carried in a `u64` and bounded by `POSITION_MASK`. Spelling the edges as + // `u32::MAX` would have quietly stopped testing the edges the moment the + // apportionment changed, while still passing. + let max_reserved = MAX_RESERVED as u32; + for &reserved in &[0_u32, 1, 2, 1000, max_reserved - 1, max_reserved] { + for &position in &[0_u64, 1, 2, 1000, POSITION_MASK - 1, POSITION_MASK] { + let word = claim_word::(reserved, position); + assert_eq!( + (reserved_of::(word), position_of::(word)), + (reserved, position), + "packing must be lossless in both halves, including at their extremes" + ); + } + } +} + +#[test] +fn the_two_halves_do_not_bleed_into_each_other() { + // The mistake packing invites: a position that wraps must not carry into + // the reservation count, and a count must not appear as a position. + let word = claim_word::(0, POSITION_MASK); + assert_eq!( + reserved_of::(word), + 0, + "a maximal position leaves the count at zero" + ); + + let word = claim_word::(MAX_RESERVED as u32, 0); + assert_eq!( + position_of::(word), + 0, + "a maximal count leaves the position at zero" + ); + + // And an increment of the position at its maximum wraps within its own half + // rather than incrementing the count, which is what the queue relies on + // every time a position laps. + let wrapped = claim_word::(7, advance::(POSITION_MASK)); + assert_eq!( + ( + reserved_of::(wrapped), + position_of::(wrapped) + ), + (7, 0) + ); +} + +// The relationship between the split and the ceiling is deliberately NOT tested +// here. It is a fact about constants, so it lives as a `const` assertion beside +// `BOUNDS` in the parent module, where changing the split without changing the +// ceiling fails to compile. A test would have been the weaker instrument: it can +// only report after the fact, and only on a build somebody chose to run. + +#[test] +fn a_capacity_above_this_shapes_ceiling_is_refused_even_though_others_accept_it() { + // The bound is a property of the shape, which is exactly what D-12 argued + // and what this shape is the second instance of. `slotwise_mpsc` takes this capacity + // happily; the packing means this one cannot. + let error = bounded::(BOUNDS_MAX * 2).expect_err("beyond the packed position's range"); + assert_eq!(error.max_valid(), BOUNDS_MAX); + assert_eq!( + error.previous_valid(), + Some(BOUNDS_MAX), + "and the correction offered is this shape's own ceiling" + ); +} + +// --------------------------------------------------------------------------- +// The reservation guarantee. +// --------------------------------------------------------------------------- + +#[test] +fn a_reserved_slot_is_delivered_into_a_queue_that_is_otherwise_full() { + // The whole contract in one test: reserve, let the best-effort path fill + // everything it is allowed to, and redeem anyway. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!(!tx.is_full(), "an empty queue is not full"); + let slot = tx.reserve().expect("a fresh queue has room"); + assert!( + !tx.is_full(), + "nor is one holding a single reservation against four slots" + ); + + let pushed = fill(&tx, 1); + assert_eq!(pushed, 3, "the reservation withheld exactly one slot"); + // Both directions, and this shape's is the interesting one: `is_full` + // counts held reservations as occupied, so the queue is full at three + // items rather than four. An `is_full` stuck at either constant would + // report that wrongly, and only the positive case was ever asserted. + assert!(tx.is_full(), "and now nothing more may be pushed"); + + slot.send(99).expect("the room was already ours"); + + let drained: Vec = rx.drain().collect(); + assert_eq!( + drained, + vec![1, 1, 1, 99], + "the reserved item lands where it was redeemed, not where it was claimed" + ); +} + +#[test] +fn a_reservation_withholds_a_slot_from_the_best_effort_path() { + let (tx, _rx) = bounded::(8).expect("8 is a valid capacity"); + assert_eq!( + fill(&tx, 0), + 8, + "with nothing reserved, every slot is available" + ); + + let (tx, _rx) = bounded::(8).expect("8 is a valid capacity"); + let reservations: Vec<_> = (0..3).map(|_| tx.reserve().expect("room")).collect(); + assert_eq!(tx.outstanding_reservations(), 3); + assert_eq!( + fill(&tx, 0), + 5, + "three reserved leaves five for the best-effort path" + ); + drop(reservations); +} + +#[test] +fn dropping_a_reservation_returns_the_slot() { + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + assert_eq!(tx.outstanding_reservations(), 1); + + drop(slot); + assert_eq!(tx.outstanding_reservations(), 0); + assert_eq!( + fill(&tx, 0), + 4, + "a released reservation is capacity given back, not capacity lost" + ); +} + +#[test] +fn the_consumer_can_see_that_something_was_promised_even_with_nothing_queued() { + // The consumer's own `outstanding_reservations`, which is a *second* + // accessor rather than a view of the producer's -- and one no test reached, + // so a mutation run found it could return a constant. The distinction it + // exists to draw is in the name: a drained queue with a reservation + // outstanding is not an idle one, and a consumer deciding whether to park + // has to be able to tell the two apart. + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + + assert_eq!( + rx.outstanding_reservations(), + 0, + "a fresh queue has promised nothing" + ); + + // Two, not one: a count asserted only at one is satisfied by a method that + // always answers one. + let first = tx.reserve().expect("room"); + let second = tx.reserve().expect("room"); + assert_eq!(rx.outstanding_reservations(), 2); + assert_eq!( + rx.outstanding_reservations(), + tx.outstanding_reservations(), + "the two handles read the same claim word, so they cannot disagree" + ); + + // The state the method is for: nothing to pop, and yet not idle. + assert!(rx.is_empty(), "nothing has been sent"); + assert_eq!( + rx.outstanding_reservations(), + 2, + "an empty queue with two slots promised is waiting, not finished" + ); + + first.send(7).expect("the room was ours"); + assert_eq!( + rx.outstanding_reservations(), + 1, + "one redeemed, one still out" + ); + assert_eq!(rx.pop(), Ok(7)); + assert_eq!( + rx.outstanding_reservations(), + 1, + "and taking the item does not release the *other* promise" + ); + + drop(second); + assert_eq!( + rx.outstanding_reservations(), + 0, + "a dropped reservation is a promise withdrawn" + ); +} + +#[test] +fn a_redeemed_reservation_does_not_also_release_its_slot() { + // The double-release bug this shape's `send` avoids by consuming `self` and + // suppressing the drop. If both ran, the count would underflow and the + // queue would over-admit for ever afterwards. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + for round in 0..10 { + let slot = tx.reserve().expect("room"); + assert_eq!(tx.outstanding_reservations(), 1); + slot.send(round).expect("the room was ours"); + assert_eq!( + tx.outstanding_reservations(), + 0, + "redeeming releases the claim exactly once" + ); + assert_eq!(rx.pop(), Ok(round)); + } + assert_eq!( + fill(&tx, 0), + 4, + "and the capacity is intact after ten cycles" + ); +} + +#[test] +fn reserving_fails_when_every_slot_is_spoken_for() { + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _first = tx.reserve().expect("room"); + let _second = tx.reserve().expect("room"); + assert!( + tx.reserve().is_none(), + "reservations are drawn from the same capacity as everything else" + ); + + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + fill(&tx, 0); + assert!( + tx.reserve().is_none(), + "and a full queue has nothing left to promise" + ); +} + +#[test] +fn a_full_queue_refuses_a_best_effort_push_and_hands_the_item_back() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + match tx.push(3) { + Err(PushError::Full(returned)) => assert_eq!(returned, 3), + other => panic!("expected Full, got {other:?}"), + } + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(2)); +} + +#[test] +fn a_push_refused_for_a_reservation_is_still_reported_as_full() { + // A best-effort caller cannot tell "no slots" from "the only slot is + // reserved", and should not have to: both mean "no room for you", both are + // backpressure, and both clear when the queue drains. + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _slot = tx.reserve().expect("room"); + tx.push(1).expect("one slot is unreserved"); + + assert!( + matches!(tx.push(2), Err(PushError::Full(2))), + "the reserved slot is not available to the best-effort path" + ); + assert!(!tx.is_empty(), "yet the queue is demonstrably not empty"); +} + +// --------------------------------------------------------------------------- +// The guarantee under contention, which is the reason the claim word is packed. +// --------------------------------------------------------------------------- + +/// How many producer threads the concurrent tests use. +/// +/// Fixed rather than derived from the machine's core count, so a failure +/// reproduces on the machine that reported it. +const PRODUCERS: usize = 4; + +#[test] +fn every_granted_reservation_is_redeemable_under_contention() { + // **The test the packed claim word exists to pass.** With the count in its + // own atomic, a pushing producer and a reserving one can each read before + // the other's write, and the queue grants a slot that does not exist. That + // shows up here as a `send` finding no room -- which, because the invariant + // it violates is checked by a debug assertion in `send`, aborts the test + // rather than quietly corrupting the ring. + // + // A small capacity and many threads, because the race needs the queue to be + // near-full continuously for the two paths to collide at the boundary. + const ROUNDS: usize = 2_000; + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|producer| { + let handle = tx.clone(); + thread::spawn(move || { + let mut granted = 0_usize; + for round in 0..ROUNDS { + // Alternate the two paths so both are contending for the + // same last slot rather than taking turns. + if round % 2 == 0 { + let _ = handle.push(producer); + } else if let Some(slot) = handle.reserve() { + granted += 1; + slot.send(producer).expect("a granted slot is guaranteed"); + } + } + granted + }) + }) + .collect(); + drop(tx); + + // Drain continuously, so the queue keeps returning to the near-full + // boundary instead of simply staying full. + let mut received = 0_usize; + while let Ok(item) = rx.recv() { + assert!(item < PRODUCERS, "items must not be torn or invented"); + received += 1; + } + + let granted: usize = threads + .into_iter() + .map(|thread| thread.join().expect("no producer may panic")) + .sum(); + + assert!( + granted > 0, + "the run must actually have exercised reservations" + ); + assert!( + received >= granted, + "every reservation that was granted must have been delivered: \ + {granted} granted, only {received} items arrived in total" + ); +} + +#[test] +fn a_reservation_holds_capacity_against_every_other_producer() { + // Not just against the thread that took it. Reserve on one thread, fill + // from others, and redeem: the slot must have survived their contention. + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + let slot = tx.reserve().expect("room"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|_| { + let handle = tx.clone(); + thread::spawn(move || while handle.push(0).is_ok() {}) + }) + .collect(); + for thread in threads { + thread.join().expect("no producer may panic"); + } + + assert_eq!(tx.len(), 7, "seven taken, one withheld"); + slot.send(99).expect("the withheld slot is still ours"); + assert_eq!(rx.len(), 8); +} + +#[test] +fn a_reservation_can_be_redeemed_from_another_thread() { + // The shape of the real use case, and the reason this shape's reservation + // is owned rather than borrowed: claim the slot where the work is + // submitted, redeem it wherever the completion lands. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + fill(&tx, 1); + + thread::spawn(move || { + slot.send(99).expect("the room was claimed before the move"); + }) + .join() + .expect("the redeeming thread must not panic"); + + let drained: Vec = rx.drain().collect(); + assert_eq!(drained.last(), Some(&99)); +} + +#[test] +fn a_reservation_is_send_but_not_sync() { + fn assert_send() {} + assert_send::>(); + assert_send::>(); + assert_send::>(); + + // `!Sync` is asserted by the absence of any test that shares one across + // threads: the compiler refuses to write it. +} + +// --------------------------------------------------------------------------- +// Disconnection, which a reservation participates in. +// --------------------------------------------------------------------------- + +#[test] +fn an_outstanding_reservation_keeps_the_stream_open() { + // **A reservation is a promise of a message still to come.** If dropping + // the last producer ended the stream while one was outstanding, the + // consumer would be told the queue was finished and then handed an item -- + // losing exactly the message the reservation existed to protect. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(tx); + + assert!( + !rx.is_disconnected(), + "a promise outstanding is a producer outstanding" + ); + + slot.send(7).expect("the consumer is alive"); + assert!( + rx.is_disconnected(), + "and redeeming the last one does end the stream" + ); + assert_eq!(rx.pop(), Ok(7), "with the promised item still owed"); +} + +#[test] +fn dropping_an_outstanding_reservation_also_ends_the_stream() { + // The other half: a promise abandoned is still a promise resolved, so the + // consumer must not be left waiting on it for ever. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(tx); + assert!(!rx.is_disconnected()); + + drop(slot); + assert!( + rx.is_disconnected(), + "an abandoned promise resolves the stream" + ); +} + +#[test] +fn a_blocked_consumer_is_woken_by_the_last_reservation_being_redeemed() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(tx); + + let sender = thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + slot.send(7).expect("the consumer is alive"); + }); + + assert_eq!( + rx.recv().expect("the reservation is redeemed"), + 7, + "a parked consumer must be woken by a reserved delivery like any other" + ); + assert!(matches!(rx.recv(), Err(RecvError::Disconnected))); + sender.join().expect("the sender must not panic"); +} + +#[test] +fn a_blocked_consumer_is_woken_by_the_last_reservation_being_dropped() { + // Caught as a hang if the drop path forgets to release the producer count + // or to ring the doorbell. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(tx); + + let abandoner = thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + drop(slot); + }); + + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "abandoning the last promise must wake a parked consumer" + ); + abandoner + .join() + .expect("the abandoning thread must not panic"); +} + +#[test] +fn redeeming_into_a_departed_consumer_hands_the_item_back() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(rx); + + assert!(slot.is_disconnected()); + let error = slot.send(7).expect_err("nobody is left to take it"); + assert_eq!( + error.into_inner(), + 7, + "an item important enough to reserve for must not be dropped silently" + ); +} + +#[test] +fn an_abandoned_reservation_leaves_the_queue_usable() { + // A reservation that fails to be redeemed must not poison the capacity. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + for _ in 0..50 { + let slot = tx.reserve().expect("room"); + drop(slot); + } + assert_eq!(tx.outstanding_reservations(), 0); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert!(tx.push(3).is_err()); + assert_eq!(rx.pop(), Ok(1)); +} + +// --------------------------------------------------------------------------- +// Queue behaviour, kept honest against the shape it is a variant of. +// --------------------------------------------------------------------------- + +#[test] +fn items_come_out_in_the_order_they_went_in() { + let (tx, rx) = bounded::(8).expect("a power-of-two capacity"); + for value in 0..8 { + tx.push(value).expect("room for eight"); + } + let drained: Vec = rx.drain().collect(); + assert_eq!(drained, (0..8).collect::>()); +} + +#[test] +fn the_ring_wraps_many_times_without_losing_order() { + // The test that indicts the position arithmetic, and it matters more here + // than in `slotwise_mpsc`: this shape decides a slot is free from the consumer's + // position rather than from the slot's own sequence, so an error in the + // wrapping subtraction is a use-after-free rather than a wrong answer. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for round in 0..2000 { + tx.push(round).expect("the previous item was taken"); + assert_eq!(rx.pop(), Ok(round)); + } + assert!(rx.is_empty()); +} + +#[test] +fn a_partly_full_ring_wraps_correctly_with_a_reservation_held_throughout() { + // Keeps a reservation outstanding across hundreds of laps, so the count + // must survive every position wrap in the packed word. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + let slot = tx.reserve().expect("room"); + + for round in 0..500 { + tx.push(round).expect("three slots remain unreserved"); + assert_eq!(rx.pop(), Ok(round)); + assert_eq!(tx.outstanding_reservations(), 1, "round {round}"); + } + + slot.send(99).expect("still ours after five hundred laps"); + assert_eq!(rx.pop(), Ok(99)); +} + +#[test] +fn zero_sized_items_round_trip() { + let (tx, rx) = bounded::<()>(2).expect("a power-of-two capacity"); + let slot = tx.reserve().expect("room"); + tx.push(()).expect("room"); + assert!(matches!(tx.push(()), Err(PushError::Full(())))); + slot.send(()).expect("the room was ours"); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn dropping_the_queue_drops_the_items_it_still_holds() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(8).expect("a power-of-two capacity"); + let slot = tx.reserve().expect("room"); + for _ in 0..5 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + slot.send(DropCounter(Arc::clone(&drops))) + .expect("the room was ours"); + assert_eq!(drops.load(Ordering::Relaxed), 0, "nothing dropped yet"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 6, + "every undrained item must be dropped, not leaked -- including the reserved one" + ); +} + +#[test] +fn dropping_the_queue_after_a_wrap_drops_only_what_is_resident() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for _ in 0..6 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + rx.pop().expect("an item"); + } + assert_eq!(drops.load(Ordering::Relaxed), 6); + for _ in 0..3 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + } + assert_eq!( + drops.load(Ordering::Relaxed), + 9, + "the three still resident must also be dropped" + ); +} + +#[test] +fn many_producers_deliver_every_item_exactly_once() { + const PER_PRODUCER: usize = 500; + let (tx, rx) = bounded::<(usize, usize)>(16).expect("a valid capacity"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|producer| { + let handle = tx.clone(); + thread::spawn(move || { + for sequence in 0..PER_PRODUCER { + let mut item = (producer, sequence); + while let Err(PushError::Full(returned)) = handle.push(item) { + item = returned; + std::hint::spin_loop(); + } + } + }) + }) + .collect(); + drop(tx); + + let mut per_producer = [0_usize; PRODUCERS]; + while let Ok((producer, sequence)) = rx.recv() { + assert_eq!( + sequence, per_producer[producer], + "a producer's own items must arrive in that producer's order" + ); + per_producer[producer] += 1; + } + for thread in threads { + thread.join().expect("no producer may panic"); + } + assert!(per_producer.iter().all(|count| *count == PER_PRODUCER)); +} + +// --------------------------------------------------------------------------- +// The doorbell, which behaves as it does everywhere else. +// --------------------------------------------------------------------------- + +#[test] +fn polling_never_creates_a_kernel_object() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + slot.send(1).expect("the room was ours"); + while rx.pop().is_ok() {} + drop(tx); + while rx.pop().is_ok() {} + + assert!( + !rx.shared.doorbell.is_armed(), + "a poll-only consumer must allocate no kernel object, reservations included" + ); +} + +#[test] +fn a_reserved_delivery_lights_the_doorbell() { + // A reserved send is a delivery like any other, so it must ring. If it did + // not, a consumer parked on the doorbell would sleep through precisely the + // message that was important enough to reserve a slot for. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + let slot = tx.reserve().expect("room"); + assert!(rx.arm().expect("arming must succeed"), "nothing yet"); + + slot.send(1).expect("the room was ours"); + assert!( + !rx.arm().expect("arming must succeed"), + "a reserved delivery must be visible to the arming protocol" + ); +} + +#[test] +fn the_real_arm_finds_an_item_that_lands_inside_its_window() { + // The same deterministic indictment of the reversed order used by the other + // shapes, driven through this one's `arm`. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + let safe_to_wait = race_hooks::ARM.with( + move || { + tx.push(1).expect("there is room"); + }, + || rx.arm().expect("arming must succeed"), + ); + + assert!( + !safe_to_wait, + "an item landing between the clear and the check must be found, not waited past" + ); +} + +#[test] +fn the_real_arm_still_blesses_a_wait_when_its_window_stays_empty() { + let (_tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + let safe_to_wait = race_hooks::ARM.with(|| {}, || rx.arm().expect("arming must succeed")); + assert!(safe_to_wait, "nothing arrived, so waiting is right"); +} + +// --------------------------------------------------------------------------- +// The claim word going stale under the room test, which is the one window where +// "full" can be computed from two readings that never coexisted. +// --------------------------------------------------------------------------- + +/// Drains the queue after filling it, so `head` overtakes a claim position read +/// before any of it happened. +/// +/// Returned as a closure that fires **once**: the hook sits inside the claim +/// loop, so a closure that acted on every call would move the queue forward +/// again on each retry and the loop could never catch up with it. +fn advance_past(tx: Producer, rx: Rc>, items: u32) -> impl FnMut() { + let mut fired = false; + move || { + if fired { + return; + } + fired = true; + for i in 0..items { + tx.push(i).expect("the queue starts empty, so this fits"); + } + for _ in 0..items { + rx.pop().expect("what was just pushed is takeable"); + } + } +} + +#[test] +fn a_push_whose_claim_goes_stale_retries_instead_of_reporting_full() { + // The defect this guards. `push` reads the claim word, and the room test + // then reads `head`. If the queue fills and drains in between, `head` + // passes the position that word carried and `position.wrapping_sub(head)` + // wraps to near `u32::MAX` -- so an EMPTY queue reports `Full`, and records + // a refusal for it. The compare-and-swap that would have caught the + // staleness is never reached, because the room test returns first. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let rx = Rc::new(rx); + let racing = advance_past(tx.clone(), Rc::clone(&rx), 4); + + let outcome = race_hooks::CLAIM.with(racing, || tx.push(99)); + + assert!( + outcome.is_ok(), + "the queue is empty when the claim is made, so the push must land: {outcome:?}" + ); + assert_eq!( + rx.pop(), + Ok(99), + "the item the retry claimed must actually be in the queue" + ); + assert_eq!( + tx.refused(), + 0, + "a retried claim is not backpressure and must not be counted as one" + ); +} + +#[test] +fn a_reservation_whose_claim_goes_stale_retries_instead_of_failing() { + // `reserve` shares the room test, so it shares the window: the same stale + // pair made an empty queue refuse a reservation. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let rx = Rc::new(rx); + let racing = advance_past(tx.clone(), Rc::clone(&rx), 4); + + let reservation = race_hooks::CLAIM.with(racing, || tx.reserve()); + + let reservation = reservation.expect("the queue is empty when the claim is made"); + reservation.send(7).expect("the consumer is still here"); + assert_eq!(rx.pop(), Ok(7)); +} + +#[test] +fn a_genuinely_full_queue_still_reports_full_through_the_window() { + // The other direction, so the retry cannot pass by never refusing. Nothing + // races here, so the claim word the room test rejected is still current and + // the refusal is authoritative. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + tx.push(1).expect("there is room"); + tx.push(2).expect("there is room"); + + let outcome = race_hooks::CLAIM.with(|| {}, || tx.push(3)); + + assert!( + matches!(outcome, Err(PushError::Full(3))), + "a full queue must still refuse, and hand the item back: {outcome:?}" + ); + assert_eq!(tx.refused(), 1, "a real refusal is still counted"); + assert_eq!(rx.pop(), Ok(1)); +} +// --------------------------------------------------------------------------- +// Through the traits, which is where this shape and `slotwise_mpsc` visibly differ. +// --------------------------------------------------------------------------- + +#[test] +fn the_shape_is_usable_through_the_reserving_trait() { + fn reserve_and_send

(producer: &P, item: P::Item) -> bool + where + P: Reserving + Bounded, + P::Item: Copy, + for<'a> P::Reservation<'a>: ReservationLike, + { + let before = producer.outstanding_reservations(); + let Some(slot) = producer.reserve() else { + return false; + }; + assert_eq!(producer.outstanding_reservations(), before + 1); + slot.deliver(item).is_ok() + } + + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!(reserve_and_send(&tx, 7)); + assert_eq!(rx.pop(), Ok(7)); +} + +/// The one operation the [`Reserving`] trait deliberately does not name. +/// +/// Redeeming consumes the reservation and hands back a shape-specific error, so +/// putting it on the trait would have meant an associated error type carried for +/// the sake of one method. The trait names how a claim is *obtained*, which is +/// the part a generic caller needs; a caller generic over redeeming as well can +/// say so itself, as this does. +trait ReservationLike { + type Error; + fn deliver(self, item: T) -> Result<(), Self::Error>; +} + +impl ReservationLike for Reservation { + type Error = crate::Disconnected; + + fn deliver(self, item: T) -> Result<(), Self::Error> { + self.send(item) + } +} + +impl ReservationLike for crate::spsc::Reservation<'_, T> { + type Error = crate::Disconnected; + + fn deliver(self, item: T) -> Result<(), Self::Error> { + self.send(item) + } +} + +#[test] +fn both_reserving_shapes_satisfy_the_trait() { + // The D-3 check, run for the `Reserving` trait: two implementations that do + // not resemble each other internally, one handing out a borrowed + // reservation and one an owned one, reached through the same generic code. + fn claim_one(producer: &P) -> Option> { + producer.reserve() + } + + let (spsc_tx, _spsc_rx) = crate::spsc::bounded::(4).expect("4 is valid for both"); + let (mpsc_tx, _mpsc_rx) = bounded::(4).expect("4 is valid for both"); + + assert!(claim_one(&spsc_tx).is_some()); + assert!(claim_one(&mpsc_tx).is_some()); + assert_eq!( + spsc_tx.outstanding_reservations(), + 0, + "the claim was dropped" + ); + assert_eq!(mpsc_tx.outstanding_reservations(), 0); +} + +// --------------------------------------------------------------------------- +// Teardown: what becomes of items nobody drained. +// +// The policy itself is covered in `crate::disposal`'s suite. What this shape +// adds is the interaction with reservations, which is where teardown matters +// most: a reservation exists because its message must not be lost, so a +// message redeemed into a queue that is then abandoned would be lost after +// all -- just later, and more quietly. +// --------------------------------------------------------------------------- + +/// Records that it was destroyed, so a test can tell "handed to the owner" from +/// "destructor run by whichever thread dropped last". +#[derive(Debug)] +struct Tracked { + id: u32, + destroyed: Arc, +} + +impl Drop for Tracked { + fn drop(&mut self) { + self.destroyed.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn undrained_items_reach_the_disposal_sink_instead_of_being_destroyed() { + let destroyed = Arc::new(AtomicUsize::new(0)); + let (undelivered, reaper) = std::sync::mpsc::channel(); + + { + let (tx, _rx) = bounded_with::( + 8, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("8 is a valid capacity"); + + for id in 0..5 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + + assert_eq!( + reaper.iter().map(|item| item.id).collect::>(), + vec![0, 1, 2, 3, 4] + ); + assert_eq!(destroyed.load(Ordering::Relaxed), 5); +} + +#[test] +fn a_reserved_message_abandoned_at_teardown_is_still_accounted_for() { + // **The case this shape exists to make safe.** A reservation is taken + // precisely because the message must not be lost. Redeeming it into a queue + // that is then torn down undrained would lose it after all, so the sink has + // to see it like any other survivor. + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, _rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + let slot = tx.reserve().expect("room"); + tx.push(1).expect("room"); + slot.send(99).expect("the room was ours"); + } + + assert_eq!( + reaper.iter().collect::>(), + vec![1, 99], + "a redeemed reservation is an ordinary queued item, and is accounted for as one" + ); +} + +#[test] +fn an_unredeemed_reservation_hands_nothing_to_the_sink() { + // A reservation holds *capacity*, not an item. There is nothing to dispose + // of, and reporting a phantom would be worse than reporting nothing -- + // the sink is the owner's accounting, and it must not lie in either + // direction. + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, _rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + let _slot = tx.reserve().expect("room"); + tx.push(1).expect("room"); + } + + assert_eq!( + reaper.iter().collect::>(), + vec![1], + "the abandoned reservation was capacity, not a message" + ); +} + +#[test] +fn a_queue_torn_down_by_a_reservation_still_reaches_the_sink() { + // A reservation counts as a producer, so it can be the last handle + // standing -- and then its drop is what tears the queue down. + let (undelivered, reaper) = std::sync::mpsc::channel(); + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + tx.push(1).expect("room"); + let slot = tx.reserve().expect("room"); + drop(tx); + drop(rx); + drop(slot); + + assert_eq!( + reaper.iter().collect::>(), + vec![1], + "whichever handle releases last must still account for the survivors" + ); +} + +#[test] +fn the_sink_sees_survivors_after_the_ring_has_wrapped() { + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + for round in 0..6 { + tx.push(round).expect("room"); + rx.pop().expect("an item"); + } + for round in 100..103 { + tx.push(round).expect("room"); + } + } + assert_eq!(reaper.iter().collect::>(), vec![100, 101, 102]); +} + +#[test] +fn without_a_sink_undrained_items_are_destroyed_in_place() { + let destroyed = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + for id in 0..3 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + assert_eq!(destroyed.load(Ordering::Relaxed), 3); +} + +// --------------------------------------------------------------------------- +// Observability. +// --------------------------------------------------------------------------- + +#[test] +fn refusals_are_counted_but_disconnections_are_not() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert!(tx.push(3).is_err()); + assert_eq!(tx.refused(), 1); + assert_eq!(rx.refused(), 1, "both handles report the same queue"); + + drop(rx); + assert!(matches!(tx.push(4), Err(PushError::Disconnected(4)))); + assert_eq!( + tx.refused(), + 1, + "the end of the stream is not backpressure and must not be counted as it" + ); +} + +#[test] +fn a_push_refused_because_a_slot_is_reserved_counts_as_a_refusal() { + // It is backpressure like any other from the caller's side: the queue had + // no room for *this* push, and the reason is the queue's business rather + // than the refused producer's. + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _slot = tx.reserve().expect("room"); + tx.push(1).expect("one slot is unreserved"); + + assert!(tx.push(2).is_err()); + assert_eq!(tx.refused(), 1); +} + +#[test] +fn high_water_is_untracked_by_default() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + assert_eq!(tx.high_water(), None); + assert_eq!(rx.high_water(), None); +} + +#[test] +fn high_water_records_the_peak_when_asked_for() { + let (tx, rx) = bounded_with::(8, Options::new().tracking_high_water()) + .expect("8 is a valid capacity"); + assert_eq!(tx.high_water(), Some(0)); + + for value in 0..5 { + tx.push(value).expect("room"); + } + assert_eq!(tx.high_water(), Some(5)); + + while rx.pop().is_ok() {} + assert_eq!(rx.high_water(), Some(5)); +} + +#[test] +fn an_unredeemed_reservation_does_not_count_towards_the_peak() { + // A reservation holds capacity, not an item. Counting it as depth would + // report a backlog that does not exist, and the whole point of the mark is + // to size a queue from evidence. + let (tx, _rx) = bounded_with::(8, Options::new().tracking_high_water()) + .expect("8 is a valid capacity"); + + let _slot = tx.reserve().expect("room"); + tx.push(1).expect("room"); + + assert_eq!( + tx.high_water(), + Some(1), + "one item is one item, whatever else is promised" + ); +} + +#[test] +fn the_ring_count_reports_syscalls_rather_than_signal_attempts() { + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + for value in 0..4 { + tx.push(value).expect("room"); + } + + assert_eq!( + rx.doorbell_rings(), + 1, + "the first push lit it; the other three had nothing to do" + ); +} + +#[test] +fn a_reserved_delivery_rings_like_any_other() { + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + let slot = tx.reserve().expect("room"); + slot.send(1).expect("the room was ours"); + + assert_eq!( + rx.doorbell_rings(), + 1, + "the message a reservation exists to protect must wake a parked consumer" + ); +} + +#[test] +fn the_debug_renderings_name_the_type_and_its_state() { + // See the same test in the other shapes. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + + let producer = format!("{tx:?}"); + assert!( + producer.contains("reserving_mpsc::Producer"), + "got {producer}" + ); + assert!(producer.contains('4'), "the capacity must show: {producer}"); + + let consumer = format!("{rx:?}"); + assert!( + consumer.contains("reserving_mpsc::Consumer"), + "got {consumer}" + ); + + let reservation = tx.reserve().expect("there is room"); + let rendered = format!("{reservation:?}"); + assert!( + rendered.contains("reserving_mpsc::Reservation"), + "got {rendered}" + ); +} + +// --------------------------------------------------------------------------- +// The gauges: `len` under a skewed pair of loads, and `remaining` against the +// reservations `len` deliberately excludes. +// --------------------------------------------------------------------------- + +#[test] +fn remaining_subtracts_outstanding_reservations() { + // The defect. `Bounded`'s default is `capacity - len`, and `len` excludes + // reservations by design, so an empty queue of four holding one reservation + // answered four -- promising room for a fourth item that `push` is + // guaranteed to refuse, because the reservation is holding the slot. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + + assert_eq!(tx.len(), 0, "a reservation is not an item"); + assert_eq!( + tx.remaining(), + 3, + "one of the four slots is spoken for by the reservation" + ); + assert_eq!( + rx.remaining(), + 3, + "both handles describe the same queue and must agree" + ); + + // And the number is honest: exactly three further pushes fit. + for i in 0..3 { + tx.push(i).expect("remaining() said there was room"); + } + assert_eq!(tx.remaining(), 0); + assert!(tx.is_full(), "no unreserved slot is left"); + assert!(matches!(tx.push(99), Err(PushError::Full(99)))); + + slot.send(7).expect("the consumer is still here"); + assert_eq!(rx.len(), 4, "the redeemed reservation is now an item"); +} + +#[test] +fn remaining_agrees_through_the_bounded_trait() { + // The override is on the trait impls, not only the inherent methods: a + // caller generic over `Bounded` is exactly who would be misled by the + // default, since it cannot reach `outstanding_reservations` to correct it. + fn room_through_trait(handle: &B) -> usize { + handle.remaining() + } + + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let _slot = tx.reserve().expect("an empty queue has room"); + + assert_eq!(room_through_trait(&tx), 3); + assert_eq!(room_through_trait(&rx), 3); +} + +#[test] +fn the_gauges_are_clamped_when_head_has_passed_the_sampled_position() { + // `len` and `remaining` each read the claim word and then `head`, which are + // two instants rather than one. If the consumer drains past the position + // the claim held, `head` overtakes it and `wrapping_sub` yields a number + // near `u32::MAX` -- a four-slot queue reporting four billion items, and + // four billion slots of room, straight out of a public metric. + // + // The skewed pair is written directly rather than raced for. The CLAIM hook + // opens the window inside `push`, but by the time `push` returns the two + // values agree again, so a test that called `len()` afterwards would assert + // nothing -- which is exactly what an earlier version of this test did, and + // a sabotage run caught it doing. + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + + tx.shared + .claim + .0 + .store(claim_word::(0, 1), Ordering::Release); + tx.shared.head.0.store(2, Ordering::Release); + + assert_eq!( + tx.len(), + tx.capacity(), + "a bounded queue must never report holding more than it can" + ); + assert_eq!( + tx.remaining(), + 0, + "the clamp must resolve towards full, which is the safe direction" + ); + assert!(tx.is_full()); + + // **Restored before the handles drop, and this is not tidiness.** Teardown + // walks from `head` to the claim position to dispose whatever is still + // held, so leaving `head` ahead sets that walk a `u32::MAX`-length loop and + // the test hangs rather than fails. Measured the hard way. + tx.shared.head.0.store(0, Ordering::Release); + tx.shared + .claim + .0 + .store(claim_word::(0, 0), Ordering::Release); +} + +#[test] +fn the_gauges_are_exact_when_the_two_loads_agree() { + // The guard must not have been bought by clamping everything: an ordinary + // reading still reports the true count and the true room. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert_eq!(tx.len(), 2); + assert_eq!(tx.remaining(), 2); + assert!(!tx.is_full()); + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(tx.len(), 1); + assert_eq!(tx.remaining(), 3); +} + +#[test] +fn publish_waits_for_a_head_that_has_freed_the_slot() { + // The `send`-path data race, asserted as the guarantee that closes it. + // + // Freeing a slot is the consumer's `head.store(Release)`, and on that path + // it is the *only* release it performs -- so a producer may write the slot + // only once an acquire load of `head` has actually observed that store. A + // single acquire load does not give that: it may legally return any earlier + // value in the modification order, and synchronizes only with the release + // whose value it in fact reads. `Reservation::send` is the exposed path, + // because it has no room check and a `Reservation` is `Send`, so the thread + // redeeming one need never have read `head` at all. + // + // A stale read cannot be raced for on a coherent machine, so the state one + // would observe is written instead: `head` far enough behind the claim + // position that this slot's previous occupant has not been freed. `publish` + // must then wait rather than write. + // + // **This replaces `the_high_water_mark_never_exceeds_the_capacity`**, which + // drove the same stale state to prove the high-water *clamp* bounded the + // over-report. Waiting for a fresh `head` removes the over-report at its + // source -- after the wait, `position - head < capacity`, so the depth is + // already bounded and the clamp cannot be reached from here. Asserting the + // fix is worth more than asserting a mitigation that is now unreachable. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + + // `send` claims position 0, so this leaves `position - head == 11` on a + // four-slot queue: a view in which the slot is not free. + tx.shared + .head + .0 + .store(POSITION_MASK - 10, Ordering::Release); + + // `Arc` rather than a `static`, for the reason `DropCounter` + // gives: tests share a process, so a module-scope flag would be visible to + // whichever test ran beside this one. + let sent = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&sent); + let sender = thread::spawn(move || { + slot.send(7).expect("the consumer is still here"); + flag.store(true, Ordering::Release); + }); + + // One-sided on purpose: a slow machine leaves it waiting and the assertion + // still holds. Only a `publish` that wrongly proceeded can fail it, and that + // one returns immediately. + thread::sleep(Duration::from_millis(50)); + assert!( + !sent.load(Ordering::Acquire), + "the slot was written while `head` still said its previous occupant was live" + ); + + // Free it, and the wait must end. Also restores a `head` the teardown walk + // can use: it steps from `head` to the claim position, and a head this far + // behind would set it a four-billion-step loop that hangs rather than fails. + tx.shared.head.0.store(0, Ordering::Release); + sender.join().expect("the sending thread must not panic"); + assert!( + sent.load(Ordering::Acquire), + "observing the freeing store must end the wait" + ); + + assert_eq!(rx.pop(), Ok(7), "the item itself must be unaffected"); +} + +#[test] +fn the_high_water_mark_still_reaches_a_genuine_peak() { + // The clamp must not have been bought by flattening the answer: filling the + // queue must still be reported as having filled it. + let (tx, rx) = bounded_with::(4, Options::new().tracking_high_water()) + .expect("4 is a valid capacity"); + + for i in 0..4 { + tx.push(i).expect("room"); + } + + assert_eq!( + tx.high_water(), + Some(4), + "the queue was filled, so the peak is its capacity" + ); + assert_eq!(rx.pop(), Ok(0)); +} + +#[test] +fn the_high_water_mark_is_untracked_by_default_on_this_shape() { + // `None` and `Some(0)` are different answers, and the default is the former. + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + assert_eq!(tx.high_water(), None); +} + +// --------------------------------------------------------------------------- +// The claim-word layouts. +// +// The apportionment is arithmetic over compile-time constants, so most of it is +// checked by `ClaimLayout::VALID` at build time. What a test is needed for is +// the part that is *behaviour*: that a narrow reservation half actually refuses +// at its ceiling, and that the ceiling no longer drags the capacity down with +// it. Neither is observable under `Balanced`, whose ceiling of 2^32 cannot be +// reached on any queue that fits in memory -- which is why these tests are +// written against `Perpetual`. +// --------------------------------------------------------------------------- + +#[test] +fn each_layout_divides_the_word_as_documented() { + // The documentation states these numbers to callers choosing between the + // layouts, so they are asserted rather than left to be re-derived by a + // reader who wants to check the table. + assert_eq!(::POSITION_BITS, 32); + assert_eq!(::MAX_RESERVED, u64::from(u32::MAX)); + + assert_eq!(::POSITION_BITS, 48); + assert_eq!(::MAX_RESERVED, 65_535); + + assert_eq!(::POSITION_BITS, 56); + assert_eq!(::MAX_RESERVED, 255); +} + +#[test] +fn a_layout_may_hold_more_slots_than_it_can_reserve() { + // **This is the decoupling, and it is the whole point of the change.** + // Before it, the reservation half had to be wide enough for the entire + // capacity, because every slot could be reserved at once -- so a layout + // with 255 reservations would have been limited to 255 slots, and a large + // capacity would have been unbuildable. A queue of 1024 slots on a layout + // that can reserve 255 of them is exactly what that rule forbade. + let capacity = 1024; + assert!( + capacity > ::MAX_RESERVED, + "the fixture must exceed the reservation ceiling or it tests nothing" + ); + + let (tx, rx) = bounded_as::(capacity as usize) + .expect("a capacity far below the layout's ceiling"); + for value in 0..capacity as u32 { + tx.push(value).expect("every slot is free"); + } + assert!(tx.is_full(), "all 1024 slots hold an item"); + for expected in 0..capacity as u32 { + assert_eq!(rx.pop(), Ok(expected)); + } +} + +#[test] +fn reservations_stop_at_the_layouts_ceiling_not_at_the_capacity() { + // The other half of the decoupling: the ceiling is real and refuses, rather + // than being a number that silently overflows into the position beside it. + let ceiling = ::MAX_RESERVED as usize; + let (tx, _rx) = + bounded_as::(1024).expect("a capacity far above the reservation ceiling"); + + let held: Vec<_> = (0..ceiling) + .map(|index| { + tx.reserve() + .unwrap_or_else(|| panic!("reservation {index} is within the ceiling")) + }) + .collect(); + assert_eq!(held.len(), ceiling); + assert_eq!(tx.outstanding_reservations(), ceiling); + + assert!( + tx.reserve().is_none(), + "the reservation past the ceiling must be refused even though 769 slots are still free" + ); + assert!( + !tx.is_full(), + "and the refusal must be the layout's ceiling rather than a full queue -- otherwise this \ + test would pass for the wrong reason" + ); + + // Releasing one makes room for exactly one more, so the ceiling is a live + // count rather than a latch. + drop( + held.into_iter() + .next_back() + .expect("the ceiling is not zero"), + ); + assert!( + tx.reserve().is_some(), + "a released reservation returns its place under the ceiling" + ); +} + +#[test] +fn a_reservation_on_a_deep_layout_still_delivers_its_message() { + // The layouts are not merely constants: each is a distinct instantiation of + // the whole protocol, so the capability the shape exists for is exercised + // on a non-default one rather than assumed to follow. + let (tx, rx) = bounded_as::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + tx.push(1).expect("room beyond the reservation"); + slot.send(99).expect("the consumer is still here"); + + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(99)); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +// --------------------------------------------------------------------------- +// The 128-bit layout. +// +// Gated with the feature that supplies it, so the default build compiles none +// of this and a `dwcas` build runs the same protocol tests against a word twice +// as wide. The point is that `Wide` is a full instantiation of the protocol +// rather than a constant: nothing about it is shared with the `u64` layouts +// below the `ClaimWord` trait, so it is exercised rather than assumed. +// --------------------------------------------------------------------------- + +#[cfg(feature = "dwcas")] +mod dwcas { + use super::*; + use crate::reserving_mpsc::Wide; + + #[test] + fn the_wide_layout_divides_a_128_bit_word() { + assert_eq!(::WORD_BITS, 128); + assert_eq!(::POSITION_BITS, 64); + assert_eq!( + ::POSITION_MASK, + u64::MAX, + "a 64-bit position occupies the whole of the u64 it is carried in, which is the case \ + the mask's shift cannot express and must special-case" + ); + assert_eq!( + ::MAX_RESERVED, + u64::from(u32::MAX), + "the field holds 64 bits but the count is reported as a u32, so the ceiling is the \ + narrower of the two rather than what the word could carry" + ); + } + + #[test] + fn the_wide_word_round_trips_both_halves() { + // The packing is the part that differs from the `u64` layouts, and a + // position at its maximum is where a carry into the count would show. + for &reserved in &[0_u32, 1, 1000, u32::MAX] { + for &position in &[0_u64, 1, 1000, u64::MAX - 1, u64::MAX] { + let word = claim_word::(reserved, position); + assert_eq!( + (reserved_of::(word), position_of::(word)), + (reserved, position), + "packing must be lossless in both halves of the wider word too" + ); + } + } + } + + #[test] + fn a_position_at_its_maximum_wraps_without_touching_the_count() { + let wrapped = claim_word::(7, advance::(u64::MAX)); + assert_eq!( + (reserved_of::(wrapped), position_of::(wrapped)), + (7, 0), + "the position laps within its own half rather than incrementing the count" + ); + } + + #[test] + fn the_wide_layout_delivers_items_and_reservations() { + let (tx, rx) = bounded_as::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + tx.push(1).expect("room beyond the reservation"); + slot.send(99).expect("the consumer is still here"); + + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(99)); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + } + + #[test] + fn the_wide_layout_refuses_when_full_and_recovers() { + let (tx, rx) = bounded_as::(2).expect("2 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one slot remains"); + assert!( + tx.push(3).is_err(), + "a full queue refuses rather than overwriting, whatever the word's width" + ); + assert_eq!(rx.pop(), Ok(1)); + tx.push(3).expect("the popped slot is free again"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!(rx.pop(), Ok(3)); + } +} + +// --------------------------------------------------------------------------- +// The surface added after comparing this crate against the published queue +// crates, repeated here for the reason `spsc`'s copy records: each shape +// implements `pop` and the `Bounded` accessors separately, so covering one says +// nothing about the others. These run under the default layout; the layout +// tests above cover the rest. +// --------------------------------------------------------------------------- + +#[test] +fn an_empty_queue_is_distinguishable_from_a_finished_one() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert_eq!( + rx.pop(), + Err(TryRecvError::Empty), + "empty with a producer alive is a reason to try again" + ); + + drop(tx); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "empty with every producer gone is a reason to stop" + ); +} + +#[test] +fn an_outstanding_reservation_holds_the_stream_open() { + // Specific to this shape: a reservation counts as a producer, so the stream + // has not ended while one is outstanding even though the handle is gone. + // `pop` must report `Empty` rather than `Disconnected`, or the consumer + // stops before the message the reservation exists to protect arrives. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + drop(tx); + + assert_eq!( + rx.pop(), + Err(TryRecvError::Empty), + "a reservation is a promise of a message still to come" + ); + + slot.send(7).expect("the consumer is still here"); + assert_eq!(rx.pop(), Ok(7)); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and only once it is redeemed is the stream finished" + ); +} + +#[test] +fn a_departed_producers_items_are_delivered_before_the_disconnection() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one item does not fill four slots"); + drop(tx); + + assert_eq!(rx.pop(), Ok(1), "the items come first"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and only then the end of the stream" + ); +} + +#[test] +fn is_full_agrees_across_the_trait_and_both_handles() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert!(!tx.is_full()); + assert!(!rx.is_full()); + + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one slot remains"); + + assert!(tx.is_full(), "the producer sees a full queue"); + assert!(rx.is_full(), "and so does the consumer"); + assert!( + Bounded::is_full(&tx), + "and so does a caller generic over the trait" + ); + assert!(Bounded::is_full(&rx)); + + assert_eq!(rx.pop(), Ok(1)); + assert!( + !tx.is_full(), + "and it is no longer full once a slot is freed" + ); +} + +#[test] +fn is_full_counts_a_reservation_as_occupancy() { + // Specific to this shape, and the reason `is_full` is derived from + // `remaining` rather than from `len`: a reservation withdraws capacity + // without becoming an item, so a queue whose every slot is spoken for is + // full even though it holds nothing. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + let _first = tx.reserve().expect("an empty queue has room"); + let _second = tx.reserve().expect("one slot remains"); + + assert!(tx.is_full(), "every slot is spoken for"); + assert!(rx.is_full()); + assert!( + rx.is_empty(), + "and yet it holds nothing -- which is why `is_full` is not `len == capacity`" + ); +} + +#[test] +fn try_iter_and_drain_are_the_same_iterator_and_need_no_import() { + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + for value in 0..4u32 { + tx.push(value).expect("four items fit in eight slots"); + } + let taken: Vec = rx.try_iter().collect(); + assert_eq!(taken, vec![0, 1, 2, 3]); + + for value in 4..6u32 { + tx.push(value).expect("room remains"); + } + let taken: Vec = rx.drain().collect(); + assert_eq!(taken, vec![4, 5]); +} diff --git a/crates/windows-waitable-queues/src/slotwise_mpsc.rs b/crates/windows-waitable-queues/src/slotwise_mpsc.rs new file mode 100644 index 00000000..89b8f219 --- /dev/null +++ b/crates/windows-waitable-queues/src/slotwise_mpsc.rs @@ -0,0 +1,1228 @@ +// Copyright (c) Mike Grier. + +//! The multi-producer, single-consumer bounded array queue. +//! +//! Any number of producers, one consumer, a fixed number of slots, and no +//! allocation after construction. It is the submission direction of a two-layer +//! ring, where many threads offer work and one domain thread takes it. +//! +//! # Vyukov's sequence protocol +//! +//! The obvious multi-producer array queue -- claim an index with a +//! fetch-and-add, write the slot, and let the consumer read it -- does not +//! work, because the consumer has no way to tell a slot that has been *claimed* +//! from one that has been *written*. A producer preempted between the two +//! leaves a hole, and a consumer reading through the hole reads uninitialized +//! memory. +//! +//! The remedy is a sequence number per slot, which carries both facts at once. +//! Slot `i` starts at sequence `i`, and thereafter: +//! +//! | Slot sequence, relative to a position `pos` | Meaning | +//! |---|---| +//! | `sequence == pos` | free, and this producer may claim it | +//! | `sequence == pos + 1` | written and published; the consumer may take it | +//! | `sequence < pos` | the queue is full at this position | +//! | `sequence > pos` | another producer got here first; re-read the tail | +//! +//! A producer claims a slot by advancing the shared tail with a +//! compare-and-swap, writes the item, and *publishes* it by storing +//! `pos + 1` into the slot's sequence with a release. The consumer takes a slot +//! only when it sees exactly that value, so a claimed-but-unwritten slot is +//! invisible to it. Taking an item frees the slot by storing +//! `pos + capacity`, which is the position the next lap will claim it at. +//! +//! **Lock-free, not wait-free.** A producer that loses its compare-and-swap +//! retries, and there is no bound on how many times it may lose. What is +//! guaranteed is that some producer always makes progress, and -- the property +//! that matters for an I/O submission path -- that a producer suspended by the +//! scheduler at any point blocks nobody but the consumer's view of the items +//! behind it, and never the other producers. +//! +//! **Bounded by construction, so backpressure is free.** A full queue is a slot +//! whose sequence has not come round, which costs one load to discover. There +//! is no separate count to maintain, no allocation to fail, and no policy knob: +//! the refusal *is* the backpressure. +//! +//! # The signatures, and what this shape validates +//! +//! [`spsc`](crate::spsc) wrote its intended trait signatures into its +//! documentation before its types existed, so that a second shape could be +//! checked against them rather than the traits being retrofitted to whichever +//! spelling came first. This is that second shape, and it matches: `push` and +//! `pop` take `&self`, the handles are split, and the error type is the shared +//! one. The traits themselves therefore ship with this module -- see +//! [`crate::traits`] and [D-3](../DESIGN-NOTES.md#d-3). +//! +//! Exactly one cell of `spsc`'s auto-trait table changes, which is what "the +//! multi-producer shape relaxes exactly one cell" was written to predict: +//! +//! | | [`Clone`] | [`Send`] | [`Sync`] | +//! |---|---|---|---| +//! | [`Producer`] | **yes** | yes, if `T: Send` | no | +//! | [`Consumer`] | no | yes, if `T: Send` | no | +//! +//! Producers multiply by cloning, not by sharing: each thread owns its own +//! handle. Keeping the handle `!Sync` is not a leftover from `spsc` -- it means +//! a producer handle is never touched by two threads at once, so nothing about +//! this queue's cardinality has to be remembered rather than checked. + +use core::cell::{Cell, UnsafeCell}; +use core::fmt; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + +/// A claim position, and the slot sequence numbers that are compared against +/// one. +/// +/// **64 bits on every target, deliberately, rather than `usize`.** The protocol +/// below rests on a producer's observation of a slot's sequence still being +/// true when its compare-exchange succeeds, and the exchange guards only the +/// tail. A producer suspended between the two resumes safely *because* the tail +/// cannot have returned to the value it read -- which holds only while the +/// counter cannot lap. +/// +/// With `usize` it can. On a 32-bit target the counter laps after 2^32 claims, +/// which at this crate's measured rates is a matter of minutes: the stalled +/// producer then sees the same tail bits, succeeds, and writes a slot that has +/// since been refilled from the previous lap of the ring. Every other guard in +/// this shape holds -- the position really is claimed by exactly one producer; +/// what fails is the older claim that the slot was free. +/// +/// 2^64 claims cannot be reached, so the lap cannot happen, and the argument is +/// restored on every target rather than only on the ones where `usize` happened +/// to be wide enough. The cost is confined to 32-bit, where the exchange becomes +/// a 64-bit one (`cmpxchg8b` on x86); on a 64-bit target this is exactly what +/// `usize` already was. +type Position = u64; +use std::io; +use std::os::windows::io::{BorrowedHandle, OwnedHandle}; +use std::sync::Arc; +use std::time::Duration; + +use crate::CacheAligned; +use crate::blocking::{self, Parked}; +use crate::capacity::{Bounds, MAX_ADMISSIBLE_CAPACITY, validate_capacity}; +use crate::disposal::Teardown; +use crate::doorbell::Doorbell; +use crate::error::{CapacityError, PushError, RecvError, RecvTimeoutError, TryRecvError}; +use crate::metrics::Metrics; +use crate::options::Options; + +/// What this shape accepts as a capacity. +/// +/// **The minimum is two, and it is a property of the sequence protocol rather +/// than a taste.** +/// A slot's sequence has to distinguish three states, and it does so by +/// counting: `pos` means free, `pos + 1` means published, and the consumer +/// frees it again by storing `pos + capacity`, the position the next lap will +/// claim it at. With one slot, `capacity == 1`, so "published at `pos`" and +/// "free at `pos + 1`" are the *same number* -- a producer would read the +/// sequence of the item it just pushed, conclude the slot was free, and +/// overwrite an item the consumer had not read. +/// +/// It is reported rather than worked around. The obvious workaround -- +/// allocating two slots and refusing the second -- would put a load of the +/// consumer's position back on the producer's hot path, which is exactly the +/// cost this protocol exists to avoid, and it would do so for every queue in +/// order to serve a capacity of one. A caller that genuinely wants a one-item +/// handoff wants [`spsc`](crate::spsc), which represents it exactly. +/// +/// The maximum is the widest any shape may be, because this one's positions are +/// full-width [`usize`] values with nothing packed beside them -- +/// [`reserving_mpsc`](crate::reserving_mpsc) pays for its reservations with a +/// far lower ceiling of 2^31. +/// +/// **Do not choose between the shapes on this.** That ceiling counts *slots +/// allocated at construction*, not items ever pushed, and a ring of 2^31 slots +/// is tens of gigabytes before it holds anything useful. The difference is real +/// and practically unreachable. +const BOUNDS: Bounds = Bounds { + min: 2, + max: MAX_ADMISSIBLE_CAPACITY, +}; + +/// Creates a multi-producer, single-consumer bounded array queue. +/// +/// One producer handle is returned; further producers are made by cloning it, +/// and the queue is disconnected when the last of them is dropped. +/// +/// `capacity` must be a power of two of at least two, and is the exact number +/// of items the queue holds -- not a hint, and not rounded. See +/// [`CapacityError`] for why a rejection is preferred to rounding. One slot is +/// not enough for this shape because its sequence protocol distinguishes +/// "published" from "free" by counting, and at `capacity == 1` those two states +/// are the same number; [`spsc`](crate::spsc) represents a one-item handoff +/// exactly. +/// +/// # Errors +/// +/// Returns [`CapacityError`] if `capacity` is zero, is not a power of two, is +/// less than two, or exceeds `2^(usize::BITS - 2)`. +/// +/// # Examples +/// +/// ``` +/// use windows_waitable_queues::{slotwise_mpsc, TryRecvError}; +/// +/// let (tx, rx) = slotwise_mpsc::bounded::(4)?; +/// let second = tx.clone(); +/// +/// tx.push(1).expect("a fresh queue has room"); +/// second.push(2).expect("a fresh queue has room"); +/// +/// assert_eq!(rx.pop(), Ok(1)); +/// assert_eq!(rx.pop(), Ok(2)); +/// assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub fn bounded(capacity: usize) -> Result<(Producer, Consumer), CapacityError> { + build(capacity, Options::new()) +} + +/// Creates a queue with something other than the default behaviour. +/// +/// Identical to [`bounded`] except for what [`Options`] asks for. +/// +/// **Note which switch costs this shape something.** +/// [`Options::tracking_high_water`] makes the producer read the consumer's +/// position on every push -- the single shared line this shape's push is built +/// to avoid touching. Off, which is the default, it costs one predictable +/// branch on a field that is written once at construction. +/// +/// That avoidance is what distinguishes the two multi-producer shapes, but +/// **it is not what makes either one faster**: measurement found this shape the +/// slower of the two under contention, by up to 6.4x. See the crate +/// documentation for the numbers and for how to choose. +/// +/// # Errors +/// +/// As [`bounded`]. +pub fn bounded_with( + capacity: usize, + options: Options, +) -> Result<(Producer, Consumer), CapacityError> { + build(capacity, options) +} +fn build( + capacity: usize, + options: Options, +) -> Result<(Producer, Consumer), CapacityError> { + validate_capacity(capacity, BOUNDS)?; + + let mut slots = Vec::with_capacity(capacity); + for index in 0..capacity { + slots.push(Slot { + sequence: AtomicU64::new(index as Position), + value: UnsafeCell::new(MaybeUninit::uninit()), + }); + } + + let shared = Arc::new(Shared { + teardown: Teardown::new(options.disposal), + metrics: Metrics::new(options.track_high_water), + slots: slots.into_boxed_slice(), + mask: capacity - 1, + capacity, + head: CacheAligned(AtomicU64::new(0)), + tail: CacheAligned(AtomicU64::new(0)), + producers: AtomicUsize::new(1), + consumer_live: AtomicBool::new(true), + doorbell: Doorbell::new(), + }); + + Ok(( + Producer { + shared: Arc::clone(&shared), + not_sync: PhantomData, + }, + Consumer { + shared, + not_sync: PhantomData, + }, + )) +} + +/// One cell of the ring: an item, and a sequence number that says what state +/// the cell is in. +struct Slot { + /// The state machine described in the [module documentation](self). + /// + /// Deliberately *inside* the slot rather than gathered into a separate + /// array. The producer that publishes a slot and the consumer that takes it + /// touch the sequence and the item together, so keeping them adjacent costs + /// one cache line instead of two. Slots do share lines with their + /// neighbours, and that is intended: the contention this shape must avoid + /// is on the two *positions*, which are padded apart below, not on the + /// slots, which different producers touch at different indices anyway. + sequence: AtomicU64, + value: UnsafeCell>, +} + +struct Shared { + /// What becomes of undrained items at teardown. + /// + /// Read only by [`Shared::drop`], which holds `&mut self`, so it needs no + /// synchronization and costs the hot paths nothing but its space. + teardown: Teardown, + /// The counters this queue keeps about itself. See [`crate::metrics`]. + metrics: Metrics, + slots: Box<[Slot]>, + mask: usize, + capacity: usize, + /// Where the consumer will next read. Written only by the consumer. + /// + /// Padded onto its own cache line by [`CacheAligned`], and the padding is + /// load-bearing rather than waste. Every successful push writes `tail` and + /// every successful pop writes `head`. Adjacent, they would share a line, + /// and each write would invalidate the other side's copy of a value it only + /// ever reads -- false sharing, which turns an uncontended queue into a + /// contended one while every individual load and store stays correct. That + /// is a cost with no symptom other than being slow, which is exactly the + /// kind that survives a code review. + head: CacheAligned, + /// The claim counter. Advanced by a compare-and-swap, by any producer. + /// + /// Padded for the reason given on [`Shared::head`], and it matters more + /// here than it does in `spsc`: this line is already the contended one, and + /// letting the consumer's writes land on it too would add the consumer to + /// the set of threads fighting over it. + tail: CacheAligned, + /// How many producer handles are alive. + /// + /// Reaching zero is the disconnection, and it is a count rather than a flag + /// because producers multiply by cloning. Not padded: it changes only when + /// a handle is created or destroyed, which is not a hot path. + producers: AtomicUsize, + consumer_live: AtomicBool, + /// Readiness as a waitable `HANDLE`. Costs nothing until somebody asks for + /// the handle, so a polling consumer never allocates a kernel object. + doorbell: Doorbell, +} + +// SAFETY: a slot is written by exactly one producer -- the one whose +// compare-and-swap claimed that position -- and read by exactly one consumer, +// which reads it only after observing the release store of `pos + 1` that +// publishes it. The write of the item therefore happens-before the read, and no +// two threads ever touch the same slot's contents at the same time. `T: Send` +// is required and sufficient because an item is moved between threads and never +// referenced from both. +// +// The `teardown` field is deliberately NOT covered by that argument, because it +// cannot be: it holds a boxed FnMut, which is Send but not Sync, so this +// impl is forcing Sync onto a field that does not have it. That is sound for +// a narrower reason -- the field is unreachable through a shared reference. It +// is private, no method reads it, and the only access is from Drop, which +// holds &mut self and runs when the last handle is already gone. So no two +// threads can reach it at all, concurrently or otherwise. +unsafe impl Sync for Shared {} +// SAFETY: as above; sending the shared state is sending the items it holds. +unsafe impl Send for Shared {} + +impl Shared { + /// The slot a position addresses. + /// + /// The cast cannot lose anything the mask would have kept: the mask is + /// `capacity - 1`, and a capacity fits a `usize` by construction, so every + /// bit above the mask is discarded either way. Narrowing first and masking + /// second is the same answer as masking first and narrowing second. + fn slot_index(&self, position: Position) -> usize { + (position as usize) & self.mask + } + + /// Items currently held, as a snapshot. + /// + /// **Counts slots a producer has claimed but not yet finished writing.** + /// The alternative -- counting only published items -- would need a walk of + /// the ring, and this number exists for metrics rather than for control + /// flow. It never under-reports, so it is safe in the direction that + /// matters for a backpressure gauge, and it is never used to decide whether + /// to wait: [`Consumer::arm`] asks [`Shared::has_ready_item`] instead, + /// which is the exact question `pop` answers. + /// + /// **Clamped to the capacity, because the two loads are not one instant.** + /// `tail` is read first; if the consumer then drains past the value it + /// held, `head` overtakes it and the wrapping subtraction yields a number + /// near [`Position::MAX`] -- a bounded queue claiming to hold more items + /// than it has slots. Over-reporting is the safe direction for this gauge + /// and under-reporting is not, so the skew is resolved towards "full" + /// rather than towards zero; what the clamp removes is only the impossible + /// value. + /// + /// The clamp is also what makes the narrowing cast exact: the result is at + /// most the capacity, which is a `usize` by construction. + /// The two orderings differ, because the two atomics do. + /// + /// `tail` never receives a release write at all -- the claim CAS in `push` + /// is deliberately `Relaxed/Relaxed`, and says so -- so every operation on + /// it is relaxed and an acquire load here would pair with nothing. + /// + /// `head` does carry a release store (in `pop`), so its loads are acquire + /// throughout. Not because this snapshot needs the edge -- nothing is + /// dereferenced on the strength of the number -- but because a relaxed load + /// mixed onto an atomic that also carries acquire/release operations is a + /// plain load, unanchored with respect to the ordered operations on the + /// same object and free to be moved. Mixing the two disciplines on one + /// atomic makes the source text stop describing what happens. + fn len(&self) -> usize { + let tail = self.tail.0.load(Ordering::Relaxed); + let head = self.head.0.load(Ordering::Acquire); + tail.wrapping_sub(head).min(self.capacity as Position) as usize + } + + /// Whether the consumer would find an item right now. + /// + /// The emptiness half of the arming protocol, and it asks precisely what + /// [`Consumer::pop`] asks: is the slot at the head position published? A + /// claimed-but-unpublished slot answers `false`, which is the right answer + /// -- the consumer may safely park on it, because the producer's publishing + /// release store is followed by a signal that will wake it. Using + /// [`Shared::len`] here instead would answer `true` and send the consumer + /// round a spin loop until that producer got scheduled again. + /// + /// The `Acquire` load is one half of the pair described on + /// [`Doorbell::signal`](crate::doorbell::Doorbell::signal): the producer + /// stores this sequence and then loads the doorbell state, while the + /// consumer stores the doorbell state and then loads this sequence. The + /// sequentially consistent fences on both sides are what stop both loads + /// from returning stale values. + fn has_ready_item(&self) -> bool { + // Acquire, matching every other load of `head`: it carries a release + // store, so a relaxed load here would be a plain load with no defined + // position relative to it. See `len` for the full argument. + let position = self.head.0.load(Ordering::Acquire); + let slot = &self.slots[self.slot_index(position)]; + slot.sequence.load(Ordering::Acquire) == position.wrapping_add(1) + } +} + +impl Drop for Shared { + fn drop(&mut self) { + // Every handle is gone, so no synchronization is needed and the + // positions can be read directly. A slot between the two positions + // still holds an item nobody took, and tearing the queue down must + // account for them rather than leak them. + // + // Each is *moved out* and handed to the teardown policy rather than + // destroyed where it lies. For the default policy the two are the same + // thing; for a queue whose items own handles they are not, and this is + // the only place that sees every survivor. See `crate::disposal`. + // + // The sequence is consulted per slot rather than assuming every + // position in the range holds an item. A producer cannot be mid-push + // here -- it would have to hold a handle, and there are none -- so in + // practice every one of them does; the check states the invariant the + // read depends on instead of leaving it to that argument. + let mask = self.mask; + let head = *self.head.0.get_mut(); + let tail = *self.tail.0.get_mut(); + let mut position = head; + while position != tail { + let published = position.wrapping_add(1); + let slot = &mut self.slots[(position as usize) & mask]; + if *slot.sequence.get_mut() == published { + // SAFETY: the slot's sequence says the producer finished + // writing it and the consumer never took it, so it holds an + // initialized item. It is read exactly once, because `position` + // advances every iteration and the slot is never read again. + let item = unsafe { slot.value.get_mut().assume_init_read() }; + self.teardown.dispose(item); + } + position = position.wrapping_add(1); + } + } +} + +/// A writing half of an [`slotwise_mpsc`](self) queue. +/// +/// [`Clone`], and that is the only difference from `spsc`'s producer: cloning +/// is how a second producer comes into existence, and the queue is disconnected +/// when the last clone is dropped. +/// +/// Not [`Sync`], so a handle is used by one thread at a time. Give each thread +/// its own clone rather than sharing one behind a reference. +pub struct Producer { + shared: Arc>, + /// Removes [`Sync`] without removing [`Send`]. A [`Cell`] is exactly that + /// shape, and no value of it is ever created. + not_sync: PhantomData>, +} + +impl Producer { + /// Appends an item. + /// + /// # Errors + /// + /// [`PushError::Full`] when the queue is at capacity, which is the + /// backpressure signal rather than a malfunction, and + /// [`PushError::Disconnected`] when the consumer is gone. Either way the + /// item comes back, so nothing is lost by the refusal. + pub fn push(&self, item: T) -> Result<(), PushError> { + // Relaxed: this load only proposes a position. The compare-and-swap + // below is what makes the claim, and it fails if the proposal was + // stale, so a stale read costs a retry rather than correctness. + let mut position = self.shared.tail.0.load(Ordering::Relaxed); + loop { + let slot = &self.shared.slots[self.shared.slot_index(position)]; + // Acquire: pairs with the consumer's release store when it frees a + // slot, so a slot it has finished with is visible as free here. + let sequence = slot.sequence.load(Ordering::Acquire); + // Signed, which is why the capacity is capped at half the range: + // both positions wrap, and only a difference smaller than half the + // range can be told apart from its complement. + let difference = sequence.wrapping_sub(position) as isize; + + if difference < 0 { + // The slot has not come round: the queue is full here, and + // because positions are claimed in order it is full outright. + // + // Report disconnection in preference to fullness: a full queue + // whose consumer is gone will never drain, and telling the + // caller to retry would be telling it to spin forever. + if !self.shared.consumer_live.load(Ordering::Acquire) { + // Not counted as a refusal: this is the end of the stream, + // not backpressure. + return Err(PushError::Disconnected(item)); + } + self.shared.metrics.record_refusal(); + return Err(PushError::Full(item)); + } + if difference > 0 { + // Another producer claimed this position between the load of + // the tail and now. Re-read rather than incrementing blindly: + // several producers may have got in. + // + // **A mutation run reports this branch as removable, and it is + // right.** Reversing the comparison makes the branch dead -- + // the negative case returned above -- so a stale position falls + // through to the exchange instead, which fails precisely + // because the position is stale and hands back the very tail + // this branch would have loaded. The two routes end in the same + // place. Kept because the difference is a failed + // read-modify-write on the one line every producer touches, + // taken on the contended path, in exchange for a load; and + // because saying "somebody got in" where it happens is worth + // more than leaving it to be re-derived from an exchange that + // fails for a reason nothing states. Measured rather than + // argued: the mutant survives thirty runs of the two + // many-producer tests, including the capacity-two one, without + // losing or duplicating an item. + position = self.shared.tail.0.load(Ordering::Relaxed); + continue; + } + if !self.shared.consumer_live.load(Ordering::Acquire) { + return Err(PushError::Disconnected(item)); + } + + // Relaxed on both sides is sufficient: this exchange orders nothing + // but the claim itself. The item's visibility comes from the + // release store that publishes the slot below, and the freedom to + // write the slot comes from the acquire load above. + match self.shared.tail.0.compare_exchange_weak( + position, + position.wrapping_add(1), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => position = actual, + } + } + + let slot = &self.shared.slots[self.shared.slot_index(position)]; + // SAFETY: this thread's compare-and-swap claimed `position`, and a + // position is claimed by exactly one producer. The consumer will not + // read the slot until the release store below publishes it, and the + // slot's sequence said it was free, so no initialized item is + // overwritten. + unsafe { + (*slot.value.get()).write(item); + } + + // **Guarded, and this branch is the whole reason high-water is a + // switch.** This shape's producer never reads `head`, which is what + // keeps its push off the one line every thread touches. Depth cannot be + // known without that read, so the read is taken only when somebody + // asked for the answer. + // + // Note what this property does *not* buy: measurement found this shape + // slower than `reserving_mpsc` under contention despite it, because the + // slot sequence a producer must read instead marches through memory + // while other producers write it. Staying off the shared line is why + // the two shapes are different, not why either is quick. + // + // Off, the cost is one predictable branch on a field written once at + // construction, so the line is shared but read-only -- the cheap kind. + // + // **Before the publication below, and that placement is load-bearing.** + // The subtraction is only non-negative while the consumer cannot have + // passed `position`, and what holds it back is precisely that + // `position` is not published yet. Taken afterwards, the consumer is + // free to drain past it between the two statements, `position - head` + // goes negative, and the wrapping turns it into a vast unsigned number + // that `fetch_max` then keeps forever -- a peak the queue never reached + // and could not reach. Measured before this moved: about one run in + // thirty reported a high-water mark of `usize::MAX`. + // + // A stale `head` is harmless in the other direction: it can only be + // older, which over-reports the depth by the number of items drained + // since, and that is still bounded by the capacity. + if self.shared.metrics.tracks_high_water() { + let head = self.shared.head.0.load(Ordering::Acquire); + let depth = position.wrapping_sub(head).wrapping_add(1); + // States the invariant that the placement above buys, and states it + // where it can fail rather than only in prose. A depth cannot + // exceed the capacity, so anything larger is the wrapped + // subtraction -- and without this the only witness is a + // `high_water` assertion at the end of one test, which caught the + // real defect about once in sixty runs. Here it fires in whichever + // push raced, in every test that tracks high water, with the + // offending value in hand. + debug_assert!( + depth <= self.shared.capacity as Position, + "depth {depth} exceeds capacity {}: the head was read after the \ + publication and the consumer drained past this position", + self.shared.capacity + ); + // The assertion above is a `debug_assert`, so the cast must be + // sound in release too. Saturating rather than `as`: a wrapped + // subtraction on a 32-bit target would otherwise truncate to an + // arbitrary small number and record a *lower* depth than the true + // one, which is the direction this gauge must never err in. + self.shared + .metrics + .record_depth(usize::try_from(depth).unwrap_or(usize::MAX)); + } + + // Release, and this is the publication: it must come after the write, + // and this is what forbids the compiler and the processor from moving + // it earlier. Until it lands, the consumer sees the slot as + // claimed-but-empty and skips it. + slot.sequence + .store(position.wrapping_add(1), Ordering::Release); + + // After the publication, never before: the doorbell says "there is + // something to take", and that must not become true before the item is + // actually takeable. A consumer woken early would find nothing, clear + // the doorbell, and go back to sleep on an item that is about to exist + // -- a lost wakeup manufactured by signalling too eagerly. + // + // Note that a producer may signal while an *earlier* position is still + // unpublished, so the consumer wakes and finds nothing. That is a + // spurious wakeup, which the protocol tolerates by construction: the + // producer holding the earlier slot signals in its turn. + self.shared.doorbell.signal(); + Ok(()) + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + /// + /// Includes slots claimed by a producer that has not finished writing, so + /// it never under-reports. Implemented by the internal `Shared::len`. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether the next push would be refused for want of room, as a snapshot. + /// + /// Advisory only, and more advisory here than in `spsc`: another producer + /// may take the last slot between this call and the push. Nothing is gained + /// by testing it beforehand, since [`Self::push`] reports the same + /// condition without the window; it is offered for metrics. + #[must_use] + pub fn is_full(&self) -> bool { + self.len() >= self.shared.capacity + } + + /// Whether the consumer has been dropped. + #[must_use] + pub fn is_disconnected(&self) -> bool { + !self.shared.consumer_live.load(Ordering::Acquire) + } +} + +impl Clone for Producer { + fn clone(&self) -> Self { + // Relaxed: the thread doing the cloning already holds a live handle, so + // the count cannot reach zero during this call and no other thread's + // decision depends on when this increment becomes visible. The + // `Release`/`Acquire` pairing that matters is in `Drop`, where the + // count reaching zero publishes everything every producer pushed. + self.shared.producers.fetch_add(1, Ordering::Relaxed); + Self { + shared: Arc::clone(&self.shared), + not_sync: PhantomData, + } + } +} + +// Hand-written rather than derived: deriving would demand `T: Debug`, which +// would make a handle to a queue of non-`Debug` items un-printable for no +// reason. The item type is not the handle's business, so the handle reports the +// queue's state instead. +impl fmt::Debug for Producer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("slotwise_mpsc::Producer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("producers", &self.shared.producers.load(Ordering::Relaxed)) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Producer { + fn drop(&mut self) { + // `AcqRel` carries both halves of what this decrement has to do. The + // release half publishes everything this producer pushed to whichever + // thread observes the count reaching zero, so a consumer that sees the + // disconnection can trust that draining to empty really has drained + // everything. The acquire half makes *this* thread -- when it is the + // one that drives the count to zero -- see the other producers' + // pushes, which is what makes the signal below meaningful. + if self.shared.producers.fetch_sub(1, Ordering::AcqRel) != 1 { + return; + } + + // Disconnection is a wakeup like any other, and the only one nobody + // else can deliver. A consumer blocked on the doorbell would otherwise + // wait forever for an item that can no longer be sent -- the queue + // would be correct and the program would still hang. + // + // Only the *last* producer rings: an earlier one leaving changes + // nothing a consumer could act on, and waking it to discover that would + // be a spurious wakeup per departing thread. + self.shared.doorbell.signal(); + } +} + +/// The reading half of an [`slotwise_mpsc`](self) queue. +/// +/// Neither [`Clone`] nor [`Sync`], which is what makes "single consumer" a fact +/// the compiler checks rather than a rule to remember. +pub struct Consumer { + shared: Arc>, + /// See [`Producer::not_sync`]. + not_sync: PhantomData>, +} + +impl Consumer { + /// Takes the oldest item. + /// + /// # Errors + /// + /// [`TryRecvError::Empty`] when nothing is queued right now, and + /// [`TryRecvError::Disconnected`] when every producer is gone *and* the + /// queue has been drained -- in that order, so the tail of a stream whose + /// producers have already departed is still delivered. See + /// [`Consumer::pop`](crate::Consumer::pop) for why that ordering is a + /// guarantee rather than an implementation detail. + pub fn pop(&self) -> Result { + match self.take() { + Some(item) => Ok(item), + // Only on the empty path, so a successful take never pays for this + // load. The queue must be observed empty *before* disconnection is + // reported, which is exactly what this ordering enforces. + None if self.is_disconnected() => Err(TryRecvError::Disconnected), + None => Err(TryRecvError::Empty), + } + } + + /// The take itself, without the disconnection question. + fn take(&self) -> Option { + // Acquire, matching every other load of `head`. Sole-writer coherence + // would suffice to read this thread's own latest value, but `head` also + // carries the release store below, and a relaxed load mixed onto such an + // atomic is a plain load the code generator may move. See `len`. + let position = self.shared.head.0.load(Ordering::Acquire); + let slot = &self.shared.slots[self.shared.slot_index(position)]; + // Acquire: pairs with the producer's release store, so an item it + // published is visible here. + let sequence = slot.sequence.load(Ordering::Acquire); + + // Anything other than "published at this position" means there is + // nothing to take: a lower sequence is a slot from the previous lap + // that nobody has claimed yet, and a claimed-but-unpublished slot + // carries the previous lap's sequence too. + if sequence != position.wrapping_add(1) { + return None; + } + + // SAFETY: the sequence says the producer that claimed this position + // finished writing it, and the release/acquire pair above makes that + // write visible here. This is the only consumer, and the slot is freed + // below, so the item is read exactly once. + let item = unsafe { (*slot.value.get()).assume_init_read() }; + + // The head moves before the slot is freed, and not after. A producer + // that sees the freed slot may push immediately; if it did so while + // `head` still named the old position, `len` would briefly report more + // items than the queue can hold. Both stores are `Release`, so neither + // may be reordered before the read of the item above, and the first may + // not be reordered after the second. + self.shared + .head + .0 + .store(position.wrapping_add(1), Ordering::Release); + + // Freeing the slot is a store of the position the *next* lap will claim + // it at, which is one whole capacity further on. Release, because it + // must not become visible before the item has been read out: a producer + // that saw it early would overwrite an item this thread had not + // finished taking. + slot.sequence.store( + position.wrapping_add(self.shared.capacity as Position), + Ordering::Release, + ); + Some(item) + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + /// + /// Includes slots claimed by a producer that has not finished writing, so + /// it never under-reports. Implemented by the internal `Shared::len`. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether a further best-effort push would be refused for want of room. + /// + /// The consumer's view of the question the producer answers, so a caller + /// holding only this handle need not import [`Bounded`](crate::Bounded). + #[must_use] + pub fn is_full(&self) -> bool { + crate::Bounded::is_full(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// The inherent form of [`Consumer::drain`](crate::Consumer::drain), so it + /// works without importing the trait. + pub fn drain(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// An alias for [`Self::drain`] under the name most of the ecosystem uses. + pub fn try_iter(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Whether every producer has been dropped. + /// + /// **A queue can be disconnected and still hold items**, because a producer + /// may push and then drop -- so this alone does not mean the stream is + /// finished, and acting on it while items remain would discard them. + /// [`Self::pop`] answers the composite question in the only order that + /// cannot lose the tail, and is what a drain loop should use. + /// + /// The release in the last producer's `Drop` is what makes every producer's + /// preceding pushes visible to a consumer that observes this. + #[must_use] + pub fn is_disconnected(&self) -> bool { + self.shared.producers.load(Ordering::Acquire) == 0 + } + + /// Borrows the queue's readiness as a waitable `HANDLE`. + /// + /// This is the point of the crate. The handle is a manual-reset event that + /// is signalled while the queue has something to take, so it can go into + /// `WaitForMultipleObjects` beside an I/O completion, a shutdown event, or + /// a timer -- a wait that no queue with a private parking primitive can + /// join. + /// + /// The event is created on the first call, so a consumer that only ever + /// polls with [`Self::pop`] is charged for no kernel object. + /// + /// The borrow is deliberate: the event belongs to the queue and must not be + /// closed. Use [`Self::doorbell_owned`] where ownership is required. + /// + /// # Waiting on it correctly + /// + /// **Do not simply wait and then drain.** Use [`Self::arm`] to decide + /// whether waiting is safe, or the wait can miss an item and block forever: + /// + /// ```no_run + /// # use windows_waitable_queues::{slotwise_mpsc, TryRecvError}; + /// # use windows_sys::Win32::System::Threading::{WaitForSingleObject, INFINITE}; + /// # use std::os::windows::io::AsRawHandle; + /// # fn demo(rx: &slotwise_mpsc::Consumer) -> std::io::Result<()> { + /// loop { + /// // Take everything available. The drain ends by saying *why*, so the + /// // end of the stream cannot be missed by forgetting to ask. + /// loop { + /// match rx.pop() { + /// Ok(item) => { let _ = item; } + /// Err(TryRecvError::Disconnected) => return Ok(()), + /// // Empty for now. A wildcard because `TryRecvError` is + /// // `#[non_exhaustive]`, as every error type here is. + /// Err(_) => break, + /// } + /// } + /// if !rx.arm()? { + /// continue; // Something arrived; waiting now would be wrong. + /// } + /// // Ask once more, because `arm` has just cleared the doorbell -- + /// // including the single ring the last producer's drop made, which is + /// // not coming again. Skipping this blocks forever on a finished + /// // stream, and a producer that pushed *and then* dropped in the + /// // window above is also caught here rather than discarded. + /// match rx.pop() { + /// Ok(item) => { let _ = item; continue; } + /// Err(TryRecvError::Disconnected) => return Ok(()), + /// Err(_) => {} + /// } + /// let handle = rx.doorbell()?; + /// // SAFETY: a live event handle borrowed for the call. + /// unsafe { WaitForSingleObject(handle.as_raw_handle(), INFINITE) }; + /// } + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn doorbell(&self) -> io::Result> { + self.shared.doorbell.handle() + } + + /// A duplicate of [`Self::doorbell`] that the caller owns. + /// + /// The duplicate names the same event, so signalling reaches both, and the + /// caller may close its copy whenever it likes. This is the form a + /// `ThreadpoolWait` needs, since arming one takes ownership of its target. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` or `DuplicateHandle`. + pub fn doorbell_owned(&self) -> io::Result { + self.shared.doorbell.owned() + } + + /// Clears the doorbell and reports whether a later push could be missed. + /// + /// `true` means the queue had nothing takeable after the doorbell was + /// cleared, so any later push is guaranteed to signal. `false` means + /// something arrived in the meantime: take it instead of waiting. + /// + /// **`true` is not by itself permission to wait indefinitely.** It answers + /// only whether a later *push* can be missed, and says nothing about the + /// end of the stream: with every producer gone it still returns `true`, + /// having just cleared the single ring their drop left behind. See + /// [`Waitable::arm`](crate::Waitable::arm) for the four-step protocol an + /// indefinite wait needs, and the example on [`Self::doorbell`] for it + /// written out. + /// + /// The order inside this method is the whole correctness argument, and it + /// is the reverse of the one that reads naturally. Checking first would + /// leave a window in which a push both signals and has its signal erased, + /// and the consumer would sleep on a queue that is not empty and will never + /// be signalled again. + /// + /// Clearing first splits every push into two cases, and this shape's + /// division is **not** the one `spsc` uses -- the difference is why + /// the internal `Doorbell::clear` had to be + /// corrected before this shape was sound: + /// + /// - **A push that publishes at the head before the clear** is found by the + /// check, so the caller does not wait. + /// - **Every other push** -- one that publishes after the clear, and one + /// that publishes at a *later position* before it -- leaves the check + /// finding nothing, and the caller waits. That is safe because the head + /// position is then still owed a publication, and `clear` guarantees the + /// doorbell can ring again when it comes. + /// + /// The second case is the one that has no counterpart in `spsc`, where any + /// push at all makes the check find something. It is why `clear` must reset + /// the event *before* clearing the flag that mirrors it, rather than + /// relying on this check to cover the window. + /// + /// This also creates the doorbell if it does not exist, which must happen + /// before the check for the same reason: a producer running while there is + /// no event skips signalling, so the check has to come after the event + /// exists to catch what that skip left behind. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn arm(&self) -> io::Result { + // Before the clear, and so before the check: see above. + self.shared.doorbell.handle()?; + self.shared.doorbell.clear(); + #[cfg(test)] + crate::race_hooks::ARM.run(); + // Deliberately not `is_empty`. The question is whether `pop` would find + // something, and a slot that a producer has claimed but not published + // is not something `pop` can find -- see `Shared::has_ready_item`. + Ok(!self.shared.has_ready_item()) + } + + /// The last take before reporting the end of the stream. + /// + /// Called only after [`Self::is_disconnected`] has returned `true`, which + /// makes the answer final rather than a snapshot: no producer remains to + /// add anything, so `None` here means empty forever. + /// + /// This exists as a named step, rather than as a bare `pop` inlined into + /// each caller, because it guards a race that is real and narrow: a + /// producer may push *and then* drop in the window between a receive's + /// first `pop` and its disconnection check. Reporting the disconnection + /// without this final take would silently discard an item that was + /// successfully sent. Being a separate function is what lets a test reach + /// it directly instead of hoping to schedule that window. + fn finish(&self) -> Option { + self.take() + } + + /// Takes the oldest item, blocking until one arrives. + /// + /// Parks on the doorbell rather than spinning, so a consumer with nothing + /// to do costs nothing. + /// + /// # Errors + /// + /// [`RecvError::Disconnected`] once every producer is gone *and* the queue + /// is drained -- items pushed before the last producer dropped are still + /// delivered. [`RecvError::Io`] if the doorbell cannot be created or waited + /// on. + pub fn recv(&self) -> Result { + blocking::recv(self) + } + + /// Takes the oldest item, blocking until one arrives or the deadline + /// passes. + /// + /// The timeout bounds the whole call, not each individual wait: a consumer + /// woken spuriously does not get a fresh budget. + /// + /// # Errors + /// + /// [`RecvTimeoutError::Timeout`] if the deadline passes with the queue + /// still empty, which is not a malfunction. Otherwise as [`Self::recv`]. + pub fn recv_timeout(&self, timeout: Duration) -> Result { + blocking::recv_timeout(self, timeout) + } +} + +impl Parked for Consumer { + type Item = T; + + fn pop(&self) -> Option { + Self::take(self) + } + + fn finish(&self) -> Option { + Self::finish(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } + + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } +} + +/// See [`Producer`]'s impl for why this is hand-written. +impl fmt::Debug for Consumer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("slotwise_mpsc::Consumer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("producers", &self.shared.producers.load(Ordering::Relaxed)) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Consumer { + fn drop(&mut self) { + self.shared.consumer_live.store(false, Ordering::Release); + } +} + +impl crate::Producer for Producer { + type Item = T; + + fn push(&self, item: T) -> Result<(), PushError> { + Self::push(self, item) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Consumer for Consumer { + type Item = T; + + fn pop(&self) -> Result { + Self::pop(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Bounded for Producer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } +} + +impl crate::Bounded for Consumer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } +} + +impl Shared { + /// The counters, as the [`Observable`](crate::Observable) trait reports + /// them. Written once so the two handles cannot drift apart. + fn refused(&self) -> u64 { + self.metrics.refused() + } + + fn doorbell_rings(&self) -> u64 { + self.doorbell.rings() + } + + fn high_water(&self) -> Option { + self.metrics.high_water() + } +} + +impl Producer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl Consumer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl crate::Observable for Producer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Observable for Consumer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Waitable for Consumer { + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } + + fn doorbell_owned(&self) -> io::Result { + Self::doorbell_owned(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/slotwise_mpsc/tests.rs b/crates/windows-waitable-queues/src/slotwise_mpsc/tests.rs new file mode 100644 index 00000000..d3e441fe --- /dev/null +++ b/crates/windows-waitable-queues/src/slotwise_mpsc/tests.rs @@ -0,0 +1,1456 @@ +// Copyright (c) Mike Grier. + +//! Tests for the MPSC bounded array queue. +//! +//! Every one runs in memory, and the whole file finishes in well under a +//! second. The multi-producer cases join every thread before asserting, so the +//! assertion runs after the peers have finished rather than after a guess about +//! how long they take. +//! +//! **They assert what the shape actually guarantees, and not more.** A +//! multi-producer queue promises that every item arrives exactly once and that +//! one producer's items keep that producer's order. It does *not* promise a +//! global interleaving, and a test that pinned one down would be asserting the +//! scheduler rather than the queue -- green today, red on a different machine, +//! and evidence of nothing either way. + +use super::{BOUNDS, Consumer, Producer, bounded, bounded_with, validate_capacity}; +use crate::Bounded; +use crate::error::TryRecvError; +use crate::race_hooks; +use crate::{Disposal, Options}; +use crate::{PushError, RecvError, RecvTimeoutError}; +use std::collections::BTreeMap; +use std::os::windows::io::AsRawHandle; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; + +use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; +use windows_sys::Win32::System::Threading::WaitForSingleObject; + +/// Counts its own drops, so a test can prove an item was destroyed rather than +/// leaked. `Arc` rather than a `static`, so tests that run +/// concurrently in one process cannot see each other's counts. +#[derive(Debug)] +struct DropCounter(Arc); + +impl Drop for DropCounter { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// Pushes with a spin on a full queue. +/// +/// Spinning rather than sleeping is right here: the consumer is draining +/// concurrently, so a full queue clears in nanoseconds, and a sleep would turn +/// a microsecond test into a millisecond one. +fn push_spinning(producer: &Producer, mut item: T) { + loop { + match producer.push(item) { + Ok(()) => return, + Err(PushError::Full(returned)) => { + item = returned; + std::hint::spin_loop(); + } + Err(PushError::Disconnected(_)) => panic!("the consumer is alive"), + } + } +} + +#[test] +fn a_pushed_item_comes_back_out() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + tx.push(42).expect("a fresh queue has room"); + assert_eq!(rx.pop(), Ok(42)); +} + +#[test] +fn an_empty_queue_pops_nothing() { + let (_tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + assert!(rx.is_empty()); + assert_eq!(rx.len(), 0); +} + +#[test] +fn items_come_out_in_the_order_they_went_in() { + let (tx, rx) = bounded::(8).expect("a power-of-two capacity"); + for value in 0..8 { + tx.push(value).expect("room for eight"); + } + let drained: Vec = rx.try_iter().collect(); + assert_eq!(drained, (0..8).collect::>()); +} + +#[test] +fn a_full_queue_refuses_and_hands_the_item_back() { + let (tx, rx) = bounded::(2).expect("a power-of-two capacity"); + // Both directions: an `is_full` that always says so refuses every push, and + // asserting only the positive case cannot tell the two apart. + assert!(!tx.is_full(), "an empty queue is not full"); + tx.push(1).expect("room"); + assert!(!tx.is_full(), "nor is a partly filled one"); + tx.push(2).expect("room"); + assert!(tx.is_full()); + + match tx.push(3) { + Err(PushError::Full(returned)) => assert_eq!( + returned, 3, + "the refused item must come back, or a caller cannot retry it" + ), + other => panic!("expected Full, got {other:?}"), + } + + // And the refusal did not disturb what was already there. + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(2)); +} + +#[test] +fn the_smallest_capacity_holds_exactly_two() { + // Two is this shape's floor rather than one, so the two-slot ring is the + // edge case that `spsc`'s one-slot ring is: every push after the first two + // is a refusal, and every pop frees exactly one slot. + let (tx, rx) = bounded::(BOUNDS.min).expect("the shape's own minimum must be accepted"); + tx.push(1).expect("room for two"); + tx.push(2).expect("room for two"); + assert!(matches!(tx.push(3), Err(PushError::Full(3)))); + + assert_eq!(rx.pop(), Ok(1)); + tx.push(3).expect("the slot was freed"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!(rx.pop(), Ok(3)); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn the_ring_wraps_many_times_without_losing_order() { + // Far more operations than slots, so every slot is reused repeatedly. This + // is the test that indicts the sequence arithmetic: a slot freed with the + // wrong number is either claimed a lap early -- overwriting a live item -- + // or never claimed again, and both show up here as a wrong value or a + // refusal rather than as a crash. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for round in 0..1000 { + tx.push(round).expect("the previous item was taken"); + assert_eq!(rx.pop(), Ok(round)); + } + assert!(rx.is_empty()); +} + +#[test] +fn a_partly_full_ring_wraps_correctly() { + // Keeps two items resident while cycling, so the head and the tail are + // never equal and never a whole lap apart -- the case a simple "empty when + // equal" test never reaches. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + tx.push(0).expect("room"); + tx.push(1).expect("room"); + for round in 2..500 { + tx.push(round) + .expect("room, because one is taken each round"); + assert_eq!(rx.pop(), Ok(round - 2)); + assert_eq!(rx.len(), 2); + } +} + +#[test] +fn len_tracks_pushes_and_pops() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert_eq!(tx.len(), 0); + tx.push(1).expect("room"); + assert_eq!(tx.len(), 1); + assert_eq!(rx.len(), 1, "both handles report the same queue"); + tx.push(2).expect("room"); + assert_eq!(tx.len(), 2); + rx.pop().expect("an item"); + assert_eq!(rx.len(), 1); + rx.pop().expect("an item"); + assert!(rx.is_empty()); +} + +#[test] +fn zero_sized_items_round_trip() { + // A ZST exercises the slot arithmetic with no bytes to copy, so a mistake + // cannot hide behind a memcpy that happens to do the right thing. + let (tx, rx) = bounded::<()>(2).expect("a power-of-two capacity"); + tx.push(()).expect("room"); + tx.push(()).expect("room"); + assert!(matches!(tx.push(()), Err(PushError::Full(())))); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn dropping_the_queue_drops_the_items_it_still_holds() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(8).expect("a power-of-two capacity"); + for _ in 0..5 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + assert_eq!(drops.load(Ordering::Relaxed), 0, "nothing dropped yet"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 5, + "every undrained item must be dropped, not leaked" + ); +} + +#[test] +fn dropping_the_queue_after_a_wrap_drops_only_what_is_resident() { + // The interesting case for the drop loop: both positions are far from zero + // and the live range straddles the end of the slot array, so a drop that + // iterated `0..len` instead of `head..tail` would destroy the wrong slots + // -- and would drop uninitialized memory. + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for _ in 0..6 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + rx.pop().expect("an item"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 6, + "the six taken were dropped" + ); + + // Now leave three resident, starting from a wrapped position. + for _ in 0..3 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + } + assert_eq!( + drops.load(Ordering::Relaxed), + 9, + "the three still resident must also be dropped" + ); +} + +// --------------------------------------------------------------------------- +// Capacity: the rule, and this shape's own floor. +// --------------------------------------------------------------------------- + +#[test] +fn a_zero_capacity_is_refused_because_it_could_never_accept_anything() { + let error = bounded::(0).expect_err("zero is not a usable capacity"); + assert_eq!(error.requested(), 0); + assert_eq!( + error.next_valid(), + Some(BOUNDS.min), + "the suggestion must be this shape's own floor, not a crate-wide one" + ); +} + +#[test] +fn a_capacity_of_one_is_refused_because_the_sequence_protocol_cannot_encode_it() { + // Not a taste, and not a copy of `spsc`'s rule: with one slot, "published + // at position p" and "free again at position p + capacity" are the same + // number, so a producer would read the sequence of the item it had just + // pushed, conclude the slot was free, and overwrite an unread item. + // + // `spsc` accepts one, which is why the minimum belongs to the shape rather + // than to the crate. Asserted here so that shipping a one-slot MPSC would + // be a deliberate change to this test rather than a silent regression. + let error = bounded::(1).expect_err("one slot cannot carry three states"); + assert_eq!(error.requested(), 1); + assert_eq!(error.min_valid(), 2); + assert_eq!( + error.next_valid(), + Some(2), + "the correction must be offered, since one is an entirely reasonable ask" + ); + assert_eq!( + error.previous_valid(), + None, + "and there is nothing valid below it to suggest" + ); +} + +#[test] +fn a_non_power_of_two_capacity_is_refused_with_both_neighbours() { + let error = bounded::(100).expect_err("100 is not a power of two"); + assert_eq!(error.requested(), 100); + assert_eq!( + (error.previous_valid(), error.next_valid()), + (Some(64), Some(128)), + "the error should make the correction obvious without arithmetic" + ); +} + +#[test] +fn every_power_of_two_capacity_from_the_floor_up_is_accepted() { + for shift in 1..16 { + let capacity = 1_usize << shift; + let (tx, rx) = bounded::(capacity).expect("a power of two at or above the floor"); + assert_eq!(tx.capacity(), capacity); + assert_eq!(rx.capacity(), capacity, "both handles agree"); + tx.push(shift).expect("a fresh queue has room"); + assert_eq!(rx.pop(), Ok(shift)); + } +} + +#[test] +fn a_capacity_above_half_the_address_space_is_refused() { + // Not because the allocation would fail first, but because the position + // arithmetic would become ambiguous across wraparound. Checked explicitly + // so the reason survives even though no machine could allocate it. + let error = bounded::(1_usize << (usize::BITS - 1)).expect_err("too large"); + assert!( + error.next_valid().is_none(), + "there is nothing larger to suggest" + ); +} + +#[test] +fn a_suggested_capacity_is_one_the_constructor_would_accept() { + // The suggestion exists so a caller can correct the call. One that is + // itself refused is worse than none, because the caller acts on it. + // + // Asks `validate_capacity` rather than `bounded`, and rather than + // re-listing the rules here. Calling `bounded` would be a truer test of the + // real path, but a suggestion near the bound is 2^62, and constructing that + // queue means asking for half the address space. + for requested in [0_usize, 1, 3, 100, 1000, usize::MAX / 2, usize::MAX] { + let Err(error) = validate_capacity(requested, BOUNDS) else { + continue; + }; + if let Some(previous) = error.previous_valid() { + assert!( + validate_capacity(previous, BOUNDS).is_ok(), + "previous_valid() for {requested} suggested {previous}, which is itself rejected" + ); + } + if let Some(next) = error.next_valid() { + assert!( + validate_capacity(next, BOUNDS).is_ok(), + "next_valid() for {requested} suggested {next}, which is itself rejected" + ); + } + } +} + +// --------------------------------------------------------------------------- +// Disconnection, in both directions. +// --------------------------------------------------------------------------- + +#[test] +fn a_consumer_that_is_gone_turns_a_push_into_a_disconnect() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert!(!tx.is_disconnected()); + drop(rx); + assert!(tx.is_disconnected()); + + match tx.push(1) { + Err(PushError::Disconnected(returned)) => assert_eq!(returned, 1), + other => panic!("expected Disconnected, got {other:?}"), + } +} + +#[test] +fn a_full_queue_whose_consumer_is_gone_reports_disconnected_not_full() { + // The distinction is the whole point of having two variants: Full invites a + // retry, and retrying this one would spin for ever. + let (tx, rx) = bounded::(2).expect("a power-of-two capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(rx); + + match tx.push(3) { + Err(PushError::Disconnected(_)) => {} + Err(PushError::Full(_)) => { + panic!("a full queue with no consumer will never drain, so Full would invite a spin") + } + Ok(()) => panic!("the queue was full"), + } +} + +#[test] +fn the_queue_is_disconnected_only_when_the_last_producer_goes() { + // The one place where multi-producer disconnection is genuinely different + // from single-producer disconnection, and where a flag rather than a count + // would be wrong: the first producer to leave must not end the stream for + // the others. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + let second = tx.clone(); + let third = second.clone(); + assert!(!rx.is_disconnected()); + + drop(tx); + assert!(!rx.is_disconnected(), "two producers remain"); + drop(second); + assert!(!rx.is_disconnected(), "one producer remains"); + drop(third); + assert!( + rx.is_disconnected(), + "and only now is the stream genuinely over" + ); +} + +#[test] +fn a_producer_that_is_gone_leaves_the_queued_items_takeable() { + // Disconnection must not discard what was already pushed, which is why the + // documented order is drain first and check afterwards. The clone matters: + // the items were pushed through a handle that no longer exists by the time + // the consumer looks. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + let second = tx.clone(); + tx.push(1).expect("room"); + second.push(2).expect("room"); + drop(tx); + drop(second); + + assert!(rx.is_disconnected()); + assert_eq!(rx.pop(), Ok(1), "a dropped producer does not discard"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn the_final_drain_returns_an_item_that_raced_the_disconnection() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The race `Consumer::finish` guards, reconstructed rather than waited for: + // a producer pushed and then dropped in the window between a receive's + // first `pop` and its disconnection check. At this point the queue reports + // disconnected *and* holds an item. + tx.push(1).expect("there is room"); + drop(tx); + assert!(rx.is_disconnected(), "the last producer is gone"); + + assert_eq!( + rx.finish(), + Some(1), + "the end of the stream must not discard an item that was sent before it" + ); + assert_eq!( + rx.finish(), + None, + "and once genuinely drained, the answer is final" + ); +} + +#[test] +fn the_final_drain_is_empty_when_nothing_was_sent() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + assert_eq!( + rx.finish(), + None, + "nothing was ever sent, so nothing is owed" + ); +} + +// --------------------------------------------------------------------------- +// Many producers, which is what this shape exists for. +// --------------------------------------------------------------------------- + +/// How many producer threads the concurrent tests use. +/// +/// Fixed rather than derived from the machine's core count, so a failure +/// reproduces on the machine that reported it. Four is enough to make the +/// compare-and-swap on the tail genuinely contended even on a two-core box, +/// because more threads than cores is exactly the case that interleaves a +/// producer between its claim and its publish. +const PRODUCERS: usize = 4; + +/// How many items each producer sends in the concurrent tests. +const PER_PRODUCER: usize = 500; + +/// Runs `PRODUCERS` threads against one queue and returns everything the +/// consumer saw, in arrival order, as `(producer, sequence)` pairs. +fn run_producers(capacity: usize) -> Vec<(usize, usize)> { + let (tx, rx) = bounded::<(usize, usize)>(capacity).expect("a valid capacity"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|producer| { + let handle = tx.clone(); + thread::spawn(move || { + for sequence in 0..PER_PRODUCER { + push_spinning(&handle, (producer, sequence)); + } + }) + }) + .collect(); + // The original handle would otherwise keep the queue connected for ever. + drop(tx); + + let mut received = Vec::with_capacity(PRODUCERS * PER_PRODUCER); + // Drains concurrently rather than after the join, which is the point: with + // a capacity far below the run length the producers block on a full queue + // and the consumer on an empty one, repeatedly and in both directions. + while let Ok(item) = rx.recv() { + received.push(item); + } + for thread in threads { + thread.join().expect("no producer may panic"); + } + received +} + +#[test] +fn a_clone_pushes_into_the_same_queue() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + let second = tx.clone(); + tx.push(1).expect("room"); + second.push(2).expect("room"); + + assert_eq!(rx.len(), 2, "one queue, not two"); + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(2)); +} + +#[test] +fn many_producers_deliver_every_item_exactly_once() { + let received = run_producers(64); + + assert_eq!( + received.len(), + PRODUCERS * PER_PRODUCER, + "no item may be lost, and none may be delivered twice" + ); + + let mut seen: BTreeMap> = BTreeMap::new(); + for (producer, sequence) in received { + seen.entry(producer).or_default().push(sequence); + } + assert_eq!(seen.len(), PRODUCERS, "every producer must be represented"); + for (producer, sequences) in seen { + assert_eq!( + sequences, + (0..PER_PRODUCER).collect::>(), + "producer {producer} must have every one of its items, exactly once, in its own order" + ); + } +} + +#[test] +fn many_producers_against_the_smallest_queue_still_deliver_everything() { + // The same run through a two-slot ring, so nearly every push is refused at + // least once and the tail's compare-and-swap is contended continuously. + // This is where a mis-ordered claim or a slot freed at the wrong sequence + // stops being theoretical. + let received = run_producers(BOUNDS.min); + + assert_eq!(received.len(), PRODUCERS * PER_PRODUCER); + let mut per_producer = [0_usize; PRODUCERS]; + for (producer, sequence) in received { + assert_eq!( + sequence, per_producer[producer], + "a producer's own items must arrive in that producer's order" + ); + per_producer[producer] += 1; + } + assert!(per_producer.iter().all(|count| *count == PER_PRODUCER)); +} + +#[test] +fn a_producer_can_be_moved_to_another_thread_and_cloned() { + // `Send` is what makes the split useful, and `Clone` is what makes this + // shape multi-producer. `!Sync` is asserted by the absence of any test that + // shares one handle across threads: the compiler refuses to write it. + fn assert_send() {} + fn assert_clone() {} + assert_send::>(); + assert_send::>(); + assert_clone::>(); + + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + let second = tx.clone(); + thread::spawn(move || { + second.push(7).expect("room"); + }) + .join() + .expect("the pushing thread"); + tx.push(8).expect("room"); + + assert_eq!(rx.pop(), Ok(7)); + assert_eq!(rx.pop(), Ok(8)); +} + +#[test] +fn items_cross_a_thread_boundary_intact() { + // The real test of the memory ordering. Boxed, so each item is a heap + // pointer the consumer must observe fully initialized -- a missing release + // on the publishing store would surface as a corrupt pointer rather than as + // a wrong integer. + const COUNT: usize = 20_000; + let (tx, rx) = bounded::>(64).expect("a power-of-two capacity"); + + let producer = thread::spawn(move || { + for value in 0..COUNT { + push_spinning(&tx, Box::new(value)); + } + }); + + let mut received = 0_usize; + while received < COUNT { + if let Ok(item) = rx.pop() { + assert_eq!(*item, received, "items must arrive in order and intact"); + received += 1; + } else { + std::hint::spin_loop(); + } + } + + producer.join().expect("the producer thread"); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +// --------------------------------------------------------------------------- +// The doorbell, joined to the queue. +// +// The tests below are about the *pairing* of the two; the doorbell's own +// behaviour as a kernel object is covered in `crate::doorbell`'s suite. +// --------------------------------------------------------------------------- + +/// Whether the queue's doorbell is signalled right now, asked of the kernel +/// rather than of the mirror flag. +/// +/// Uses a zero timeout, so it does not block, and the event is manual-reset, so +/// asking does not consume the answer. +fn doorbell_is_lit(consumer: &Consumer) -> bool { + let handle = consumer.doorbell().expect("the doorbell must be creatable"); + // SAFETY: a live event handle borrowed for the call; a zero timeout returns + // immediately and has no other precondition. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 0) }; + assert!( + result == WAIT_OBJECT_0 || result == WAIT_TIMEOUT, + "the wait must resolve to signalled or not, got {result:#x}" + ); + result == WAIT_OBJECT_0 +} + +#[test] +fn polling_never_creates_a_kernel_object() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The laziness claim, asserted rather than assumed: a consumer that only + // ever polls must not be charged for an event it never waits on. + let second = tx.clone(); + for value in 0..4 { + if value % 2 == 0 { + tx.push(value).expect("there is room"); + } else { + second.push(value).expect("there is room"); + } + } + while rx.pop().is_ok() {} + drop(tx); + drop(second); + while rx.pop().is_ok() {} + + assert!( + !rx.shared.doorbell.is_armed(), + "a poll-only consumer must allocate no kernel object, even when a producer disconnects" + ); +} + +#[test] +fn a_push_lights_the_doorbell() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!( + !doorbell_is_lit(&rx), + "an empty queue must not claim readiness" + ); + tx.push(1).expect("there is room"); + assert!(doorbell_is_lit(&rx), "a pushed item must be announced"); +} + +#[test] +fn the_doorbell_stays_lit_across_repeated_observation() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // Created before the push, which is not incidental: a push that runs while + // no event exists signals nothing, as + // `an_item_pushed_before_the_doorbell_existed_is_still_found` asserts. This + // test is about the level, so it starts from an armed doorbell. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + + // A level, not an edge. An auto-reset event would fail the second pass, and + // a consumer sharing the wait with other handles would lose the queue. + for observation in 1..=3 { + assert!( + doorbell_is_lit(&rx), + "observation {observation} must still see the level" + ); + } +} + +#[test] +fn arm_reports_unsafe_to_wait_while_items_remain() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("there is room"); + + assert!( + !rx.arm().expect("arming must succeed"), + "arming must refuse to bless a wait while an item is sitting there" + ); +} + +#[test] +fn arm_reports_safe_to_wait_when_empty() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // Created before the push, and that is the whole test. Without it the push + // takes `signal`'s "no event yet" path, the doorbell is never lit, and the + // assertion below that arming CLEARS it would hold trivially. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + assert!( + doorbell_is_lit(&rx), + "the doorbell must be lit before a test of clearing it can mean anything" + ); + assert_eq!(rx.pop(), Ok(1)); + + assert!( + rx.arm().expect("arming must succeed"), + "a drained queue is safe to wait on" + ); + assert!( + !doorbell_is_lit(&rx), + "arming must clear the doorbell, or the next wait returns at once forever" + ); +} + +#[test] +fn arm_relights_the_doorbell_for_a_later_push() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // As above: the first push must actually SET the mirror flag, or the claim + // that `clear` cleared it is a claim about a flag that was never set. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + assert!(doorbell_is_lit(&rx), "the first push must light it"); + assert_eq!(rx.pop(), Ok(1)); + assert!(rx.arm().expect("arming must succeed")); + + // The signal that must never be skipped: the doorbell was cleared, so the + // producer's mirror flag has to have been cleared with it. Pushed through a + // *different* handle, because the flag belongs to the queue rather than to + // whichever producer last rang. + let second = tx.clone(); + second.push(2).expect("there is room"); + assert!( + doorbell_is_lit(&rx), + "the first push after a clear must light the doorbell again" + ); +} + +#[test] +fn an_item_pushed_before_the_doorbell_existed_is_still_found() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The lazy-creation hole. This push signals nothing, because there is no + // event yet to signal -- `crate::doorbell`'s suite asserts that directly. + tx.push(1).expect("there is room"); + assert!(!rx.shared.doorbell.is_armed(), "no event exists yet"); + + // Arming creates the event and only then checks, so the item is found + // instead of waited on. Had the check come first, this would report "safe + // to wait" and the consumer would block on an item already queued. + assert!( + !rx.arm().expect("arming must succeed"), + "arming must not bless a wait over an item that predates the doorbell" + ); +} + +#[test] +fn the_real_arm_finds_an_item_that_lands_inside_its_window() { + // The deterministic indictment of the reversed order, driven through the + // REAL `Consumer::arm` rather than through a copy of it. + // + // The hook fires between `arm`'s clear and its readiness check -- precisely + // the window a producer must hit for the hazard to bite. With the correct + // order the check follows the push and finds it, so arming refuses to bless + // a wait. With the two statements swapped the check has already happened, + // arming returns "safe to wait", and the consumer parks on a queue holding + // an item whose signal the clear erased. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + // The hook owns the producer outright. Sharing one behind an `Arc` would be + // pointless: a `Producer` is deliberately `!Sync`. + let safe_to_wait = race_hooks::ARM.with( + move || { + tx.push(1).expect("there is room"); + }, + || rx.arm().expect("arming must succeed"), + ); + + assert!( + !safe_to_wait, + "an item landing between the clear and the check must be found, not waited past" + ); +} + +#[test] +fn the_real_arm_still_blesses_a_wait_when_its_window_stays_empty() { + // The complement, so the test above cannot pass by `arm` simply never + // blessing anything. + let (_tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + let safe_to_wait = race_hooks::ARM.with(|| {}, || rx.arm().expect("arming must succeed")); + assert!( + safe_to_wait, + "nothing arrived, so waiting is exactly what the consumer should do" + ); +} + +#[test] +fn the_owned_doorbell_outlives_the_consumers_use_of_it() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let owned = rx.doorbell_owned().expect("duplication must succeed"); + tx.push(1).expect("there is room"); + + // SAFETY: `owned` is a live event handle; a zero timeout returns at once. + let result = unsafe { WaitForSingleObject(owned.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_OBJECT_0, + "a caller holding its own duplicate must see the queue's signals" + ); +} + +// --------------------------------------------------------------------------- +// Blocking receive. +// --------------------------------------------------------------------------- + +#[test] +fn recv_returns_an_item_already_queued() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(7).expect("there is room"); + assert_eq!(rx.recv().expect("an item is queued"), 7); +} + +#[test] +fn recv_blocks_until_a_push_arrives() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let producer = thread::spawn(move || { + // A short sleep so the consumer is genuinely parked rather than racing + // to the first `pop`. Correctness does not depend on winning that race + // -- it depends on the wakeup arriving either way. + thread::sleep(Duration::from_millis(50)); + tx.push(99).expect("there is room"); + }); + + assert_eq!(rx.recv().expect("the producer pushes"), 99); + producer.join().expect("the producer must not panic"); +} + +#[test] +fn recv_reports_disconnection_once_drained() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "an empty queue with no producer is finished" + ); +} + +#[test] +fn recv_delivers_items_pushed_before_the_last_producer_dropped() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + tx.push(1).expect("there is room"); + second.push(2).expect("there is room"); + drop(tx); + drop(second); + + // Disconnection must not discard what was already sent. Testing the flag + // before draining is the mistake this guards. + assert_eq!(rx.recv().expect("item one is owed"), 1); + assert_eq!(rx.recv().expect("item two is owed"), 2); + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "and only then is it finished" + ); +} + +#[test] +fn a_blocked_recv_is_released_only_by_the_last_producer_dropping() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + + let producers = thread::spawn(move || { + // The first departure must NOT release the consumer, and the second + // must. Without a signal in the last producer's `Drop` this test hangs + // forever: the queue would be correct and the program still wedged. + thread::sleep(Duration::from_millis(30)); + drop(tx); + thread::sleep(Duration::from_millis(30)); + drop(second); + }); + + let started = Instant::now(); + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "the last producer leaving must wake a parked consumer" + ); + assert!( + started.elapsed() >= Duration::from_millis(50), + "and the FIRST producer leaving must not have released it" + ); + producers.join().expect("the producers must not panic"); +} + +#[test] +fn recv_timeout_gives_up_on_an_empty_live_queue() { + let (_tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let started = Instant::now(); + let result = rx.recv_timeout(Duration::from_millis(60)); + + assert!( + matches!(result, Err(RecvTimeoutError::Timeout)), + "an empty queue with a live producer times out rather than ending" + ); + assert!( + result.is_err_and(|error| error.is_retryable()), + "and a timeout is worth retrying, unlike the other two variants" + ); + assert!( + started.elapsed() >= Duration::from_millis(50), + "it must actually have waited rather than returned at once" + ); +} + +#[test] +fn recv_timeout_returns_an_item_that_arrives_in_time() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let producer = thread::spawn(move || { + thread::sleep(Duration::from_millis(30)); + tx.push(5).expect("there is room"); + }); + + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)) + .expect("the push lands well inside the deadline"), + 5 + ); + producer.join().expect("the producer must not panic"); +} + +#[test] +fn recv_timeout_reports_disconnection_rather_than_waiting_out_the_clock() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + + let started = Instant::now(); + let result = rx.recv_timeout(Duration::from_secs(30)); + + assert!( + matches!(result, Err(RecvTimeoutError::Disconnected)), + "a finished queue is finished, deadline or not" + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "and it must be reported at once rather than after the deadline" + ); +} + +#[test] +fn recv_timeout_does_not_panic_on_an_unrepresentable_deadline() { + // `Instant + Duration` panics when the sum is not representable, and + // `Duration::MAX` is an ordinary way to spell "effectively forever". The + // queue is disconnected up front so the call has a reason to return at all; + // the assertion is that it returns rather than aborting the process. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + + assert!( + matches!( + rx.recv_timeout(Duration::MAX), + Err(RecvTimeoutError::Disconnected) + ), + "an unrepresentable deadline must degrade to the untimed wait it asked for" + ); +} + +#[test] +fn a_blocking_consumer_receives_every_item_from_every_producer() { + // The whole mechanism under load, through the blocking path rather than by + // polling: a capacity far smaller than the run, so the producers block on a + // full queue and the consumer parks on an empty one, repeatedly. + let received = run_producers(16); + assert_eq!( + received.len(), + PRODUCERS * PER_PRODUCER, + "a parked consumer must miss nothing" + ); +} + +// --------------------------------------------------------------------------- +// Teardown: what becomes of items nobody drained. +// +// The policy itself is covered in `crate::disposal`'s suite. What is asserted +// here is that THIS shape's walk reaches it -- and this walk is the one that +// consults each slot's sequence rather than assuming the whole resident range +// is published, so it has a case the other shapes do not. +// --------------------------------------------------------------------------- + +/// Records that it was destroyed, so a test can tell "handed to the owner" from +/// "destructor run by whichever thread dropped last". +#[derive(Debug)] +struct Tracked { + id: u32, + destroyed: Arc, +} + +impl Drop for Tracked { + fn drop(&mut self) { + self.destroyed.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn undrained_items_reach_the_disposal_sink_instead_of_being_destroyed() { + let destroyed = Arc::new(AtomicUsize::new(0)); + let (undelivered, reaper) = std::sync::mpsc::channel(); + + { + let (tx, _rx) = bounded_with::( + 8, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("8 is a valid capacity"); + + for id in 0..5 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + + assert_eq!( + reaper.iter().map(|item| item.id).collect::>(), + vec![0, 1, 2, 3, 4], + "every undrained item must reach the sink, in queue order" + ); + assert_eq!(destroyed.load(Ordering::Relaxed), 5); +} + +#[test] +fn items_from_every_producer_reach_the_sink() { + // Multi-producer is what this shape is for, and teardown must not favour + // whichever handle happened to be dropped last. + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, _rx) = bounded_with::<(usize, usize)>( + 16, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("16 is a valid capacity"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|producer| { + let handle = tx.clone(); + thread::spawn(move || { + for sequence in 0..3 { + push_spinning(&handle, (producer, sequence)); + } + }) + }) + .collect(); + for thread in threads { + thread.join().expect("no producer may panic"); + } + } + + let mut per_producer = [0_usize; PRODUCERS]; + for (producer, _) in reaper.iter() { + per_producer[producer] += 1; + } + assert!( + per_producer.iter().all(|count| *count == 3), + "every producer's abandoned items must be accounted for, not just the last one's" + ); +} + +#[test] +fn the_sink_sees_survivors_after_the_ring_has_wrapped() { + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + for round in 0..6 { + tx.push(round).expect("room"); + rx.pop().expect("an item"); + } + for round in 100..103 { + tx.push(round).expect("room"); + } + } + assert_eq!( + reaper.iter().collect::>(), + vec![100, 101, 102], + "the survivors are the resident range, not the whole slot array" + ); +} + +#[test] +fn a_queue_torn_down_by_the_producer_still_reaches_the_sink() { + let (undelivered, reaper) = std::sync::mpsc::channel(); + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(rx); + drop(tx); + + assert_eq!(reaper.iter().collect::>(), vec![1, 2]); +} + +#[test] +fn without_a_sink_undrained_items_are_destroyed_in_place() { + let destroyed = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + for id in 0..3 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + assert_eq!(destroyed.load(Ordering::Relaxed), 3); +} + +// --------------------------------------------------------------------------- +// Observability. +// +// This shape's high-water is the one that costs something, so what matters +// here is that it is genuinely off unless asked for and genuinely right when +// it is. +// --------------------------------------------------------------------------- + +#[test] +fn refusals_are_counted_but_disconnections_are_not() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert!(tx.push(3).is_err()); + assert_eq!(tx.refused(), 1); + assert_eq!(rx.refused(), 1, "both handles report the same queue"); + + drop(rx); + assert!(matches!(tx.push(4), Err(PushError::Disconnected(4)))); + assert_eq!( + tx.refused(), + 1, + "the end of the stream is not backpressure and must not be counted as it" + ); +} + +#[test] +fn every_producer_counts_into_the_same_refusal_total() { + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let second = tx.clone(); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert!(tx.push(3).is_err()); + assert!(second.push(4).is_err()); + assert_eq!( + tx.refused(), + 2, + "refusals are a property of the queue, not of whichever handle saw them" + ); +} + +#[test] +fn high_water_is_untracked_by_default() { + // **This shape's default matters most**, because tracking makes its + // producer read the consumer's position on every push -- the single shared + // line the design avoids, and the reason `reserving_mpsc` exists. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert_eq!(tx.high_water(), None); + assert_eq!(rx.high_water(), None); +} + +#[test] +fn high_water_records_the_peak_when_asked_for() { + let (tx, rx) = bounded_with::(8, Options::new().tracking_high_water()) + .expect("8 is a valid capacity"); + assert_eq!(tx.high_water(), Some(0)); + + for value in 0..5 { + tx.push(value).expect("room"); + } + assert_eq!(tx.high_water(), Some(5)); + + while rx.pop().is_ok() {} + assert_eq!( + rx.high_water(), + Some(5), + "the mark is the deepest it got, not the depth right now" + ); +} + +#[test] +fn high_water_survives_contention_from_many_producers() { + // The peak has to be the real one, not whichever producer happened to + // write last. `record_depth` loads before it modifies, so this is the test + // that says the `fetch_max` behind that shortcut is doing its job. + const PER_PRODUCER: usize = 200; + let (tx, rx) = bounded_with::(64, Options::new().tracking_high_water()) + .expect("64 is a valid capacity"); + + let threads: Vec<_> = (0..PRODUCERS) + .map(|_| { + let handle = tx.clone(); + thread::spawn(move || { + for value in 0..PER_PRODUCER { + push_spinning(&handle, value); + } + }) + }) + .collect(); + drop(tx); + + let mut received = 0; + while rx.recv().is_ok() { + received += 1; + } + for thread in threads { + thread.join().expect("no producer may panic"); + } + + assert_eq!(received, PRODUCERS * PER_PRODUCER); + let peak = rx.high_water().expect("tracking was asked for"); + assert!( + (1..=64).contains(&peak), + "the peak must be a depth the queue could actually reach, got {peak}" + ); +} + +#[test] +fn the_ring_count_reports_syscalls_rather_than_signal_attempts() { + // The number the skip rule is measured by; see the same test on + // `reserving_mpsc` for the full argument. + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + for value in 0..4 { + tx.push(value).expect("room"); + } + + assert_eq!( + rx.doorbell_rings(), + 1, + "the first push lit it; the other three had nothing to do" + ); +} + +#[test] +fn a_poll_only_consumer_rings_no_doorbells() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + for value in 0..4 { + tx.push(value).expect("room"); + } + while rx.pop().is_ok() {} + assert_eq!(rx.doorbell_rings(), 0); +} + +#[test] +fn the_debug_renderings_name_the_type_and_its_state() { + // See the same test in the other shapes: a `Debug` returning `Ok(default)` + // renders nothing and passes any test that only checks it does not panic. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + + let producer = format!("{tx:?}"); + assert!( + producer.contains("slotwise_mpsc::Producer"), + "got {producer}" + ); + assert!(producer.contains('4'), "the capacity must show: {producer}"); + + let consumer = format!("{rx:?}"); + assert!( + consumer.contains("slotwise_mpsc::Consumer"), + "got {consumer}" + ); +} + +#[test] +fn len_is_clamped_when_head_has_passed_the_sampled_tail() { + // `len` reads `tail` and then `head`, which are two instants rather than + // one. If the consumer drains past the value `tail` held, `head` overtakes + // it and `tail.wrapping_sub(head)` yields a number near `usize::MAX` -- a + // four-slot queue reporting four billion items through a public metric. + // + // The skewed pair is written directly rather than raced for: it is a + // transient a reader observes, not a state the queue rests in, so a + // scheduler could only be asked to produce it by chance. Writing it makes + // the arithmetic the assertion is actually about deterministic. + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + + tx.shared.tail.0.store(1, Ordering::Release); + tx.shared.head.0.store(2, Ordering::Release); + + assert_eq!( + tx.len(), + tx.capacity(), + "a bounded queue must never report holding more than it can" + ); + assert_eq!( + crate::Bounded::remaining(&tx), + 0, + "the clamp must resolve towards full, which is the safe direction" + ); + + // **Restored before the handles drop, and this is not tidiness.** Teardown + // walks `head..tail` to dispose whatever the queue still holds, so leaving + // `head` ahead of `tail` sets that walk a `usize::MAX`-length loop and the + // test hangs instead of failing. Measured the hard way. + tx.shared.head.0.store(0, Ordering::Release); + tx.shared.tail.0.store(0, Ordering::Release); +} + +#[test] +fn len_is_exact_when_the_two_loads_agree() { + // The guard must not have been bought by clamping everything: an ordinary + // reading still reports the true count rather than the capacity. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert_eq!(tx.len(), 2); + assert_eq!(crate::Bounded::remaining(&tx), 2); + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(tx.len(), 1); +} + +// --------------------------------------------------------------------------- +// The public surface added after comparing this crate against the published +// queue crates: `pop` distinguishing empty from disconnected, `is_full` on the +// `Bounded` trait, and the iterator aliases. +// --------------------------------------------------------------------------- + +#[test] +fn an_empty_queue_is_distinguishable_from_a_finished_one() { + // **The distinction `pop` exists to make.** Under an `Option` return these + // two situations were the same value, and telling them apart needed a + // second call in an order the caller had to remember. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert_eq!( + rx.pop(), + Err(TryRecvError::Empty), + "empty with a producer alive is a reason to try again" + ); + + drop(tx); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "empty with every producer gone is a reason to stop" + ); +} + +#[test] +fn a_departed_producers_items_are_delivered_before_the_disconnection() { + // **The ordering guarantee, which is the whole reason this is not two + // separate questions.** A producer may push and then drop, so a queue can + // be disconnected and still owe items. Reporting the disconnection while + // items remain would lose the tail of the stream. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one item does not fill four slots"); + drop(tx); + + assert_eq!(rx.pop(), Ok(1), "the items come first"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and only then the end of the stream" + ); +} + +#[test] +fn is_full_agrees_across_the_trait_and_both_handles() { + // `is_full` was an inherent method on the producer and nowhere else, so + // generic code could not ask it and a consumer could not either. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert!(!tx.is_full()); + assert!(!rx.is_full()); + + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one slot remains"); + + assert!(tx.is_full(), "the producer sees a full queue"); + assert!(rx.is_full(), "and so does the consumer"); + assert!( + Bounded::is_full(&tx), + "and so does a caller generic over the trait" + ); + assert!(Bounded::is_full(&rx)); + + assert_eq!(rx.pop(), Ok(1)); + assert!( + !tx.is_full(), + "and it is no longer full once a slot is freed" + ); +} + +#[test] +fn try_iter_is_drain_under_the_name_the_ecosystem_uses() { + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + for value in 0..4u32 { + tx.push(value).expect("four items fit in eight slots"); + } + + // Both are inherent, so neither needs the trait imported -- which was the + // other half of this gap. + let taken: Vec = rx.try_iter().collect(); + assert_eq!(taken, vec![0, 1, 2, 3]); + + for value in 4..6u32 { + tx.push(value).expect("room remains"); + } + let taken: Vec = rx.drain().collect(); + assert_eq!(taken, vec![4, 5], "drain is the same iterator"); +} + +#[test] +fn the_iterators_stop_at_empty_rather_than_at_the_end_of_the_stream() { + // A statement about this instant, not about the stream: the iterator ends + // when nothing is queued *right now*, and a later push is still delivered. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + + assert_eq!(rx.try_iter().collect::>(), vec![1]); + + tx.push(2).expect("the queue drained, so there is room"); + assert_eq!( + rx.try_iter().collect::>(), + vec![2], + "the iterator is reusable because it does not consume the handle" + ); +} diff --git a/crates/windows-waitable-queues/src/spsc.rs b/crates/windows-waitable-queues/src/spsc.rs new file mode 100644 index 00000000..ca3e2a4f --- /dev/null +++ b/crates/windows-waitable-queues/src/spsc.rs @@ -0,0 +1,1193 @@ +// Copyright (c) Mike Grier. + +//! The single-producer, single-consumer bounded ring. +//! +//! The cheapest shape in the crate: neither side ever executes a +//! compare-and-swap, because each owns one of the two positions outright and +//! only *reads* the other's. It is the completion direction of a two-layer +//! ring, where one domain thread produces and one drainer consumes. +//! +//! # The signatures this module fixes for every later shape +//! +//! This is the first shape written, so its method signatures become the ones a +//! capability trait must be able to name. Written down before the type, per +//! [D-3](../DESIGN-NOTES.md#d-3), because a second shape that spells the +//! same operation differently cannot later be unified without breaking one of +//! them: +//! +//! ```text +//! trait Producer { +//! type Item; +//! fn push(&self, item: Self::Item) -> Result<(), PushError>; +//! fn is_disconnected(&self) -> bool; +//! } +//! +//! trait Consumer { +//! type Item; +//! fn pop(&self) -> Option; +//! fn is_disconnected(&self) -> bool; +//! } +//! +//! trait Bounded { +//! fn capacity(&self) -> usize; +//! fn len(&self) -> usize; +//! fn is_empty(&self) -> bool; +//! } +//! ``` +//! +//! **They have since shipped, and this is the sketch as written -- not as the +//! traits ended up.** [`slotwise_mpsc`](crate::slotwise_mpsc) was written +//! against it and matched it, which is the validation +//! [D-3](../DESIGN-NOTES.md#d-3) demanded before any trait was allowed to +//! exist. The sketch is left unamended because that is the whole of its value: +//! updating it would turn a record of what was predicted into a copy of what +//! was built, and the check it made possible could not be re-run. +//! +//! Two things have moved since, and [`crate::traits`] is authoritative for +//! both: +//! +//! - **[`Consumer::pop`](crate::Consumer::pop) returns +//! `Result`**, not `Option`. An empty queue and a +//! finished one are different answers demanding opposite reactions, and the +//! sketch could not say so. [`Consumer`](crate::Consumer) also grew `drain` +//! and `try_iter`. +//! - **[`Bounded`](crate::Bounded) also carries `remaining` and `is_full`.** +//! +//! [`Producer`](crate::Producer) is unchanged from the sketch. +//! +//! # Why the operations take `&self` +//! +//! `&mut self` would also make single-producer sound, and several SPSC crates +//! spell it that way. It is rejected here because it does not generalize: a +//! multi-producer shape must let several threads push through a shared handle, +//! which `&mut self` forbids. Since one spelling has to serve every shape, the +//! one that serves the widest is chosen. +//! +//! Cardinality is then carried by the auto traits instead, which is +//! [D-4](../DESIGN-NOTES.md#d-4): +//! +//! | | [`Clone`] | [`Send`] | [`Sync`] | +//! |---|---|---|---| +//! | [`Producer`] | no | yes, if `T: Send` | **no** | +//! | [`Consumer`] | no | yes, if `T: Send` | **no** | +//! +//! Not [`Sync`] is what makes "single" true: a handle that cannot be shared +//! between threads and cannot be duplicated is held by exactly one thread. The +//! compiler enforces it, so no documented precondition has to be remembered. A +//! multi-producer shape will relax exactly one cell of that table. + +use core::cell::{Cell, UnsafeCell}; +use core::fmt; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::io; +use std::os::windows::io::{BorrowedHandle, OwnedHandle}; +use std::sync::Arc; +use std::time::Duration; + +use crate::CacheAligned; +use crate::blocking::{self, Parked}; +use crate::capacity::{Bounds, MAX_ADMISSIBLE_CAPACITY, validate_capacity}; +use crate::disposal::Teardown; +use crate::doorbell::Doorbell; +use crate::error::{ + CapacityError, Disconnected, PushError, RecvError, RecvTimeoutError, TryRecvError, +}; +use crate::metrics::Metrics; +use crate::options::Options; + +/// What this shape accepts as a capacity. +/// +/// The minimum is one, and there is nothing to work around: a single slot is +/// either inside `[head, tail)` or outside it, and those are the only two +/// states this shape's positions have to distinguish. [`slotwise_mpsc`](crate::slotwise_mpsc) +/// needs two, because its slots carry a third state, and that difference is why +/// each shape names its own bounds rather than sharing one pair. +/// +/// The maximum is the widest any shape may be, because this one's positions are +/// full-width [`usize`] values with nothing packed beside them. +const BOUNDS: Bounds = Bounds { + min: 1, + max: MAX_ADMISSIBLE_CAPACITY, +}; + +/// Creates a single-producer, single-consumer bounded ring. +/// +/// `capacity` must be a power of two, and is the exact number of items the +/// queue holds -- not a hint, and not rounded. See [`CapacityError`] for why a +/// rejection is preferred to rounding. +/// +/// # Errors +/// +/// Returns [`CapacityError`] if `capacity` is zero, is not a power of two, or +/// exceeds `2^(usize::BITS - 2)`. +/// +/// # Examples +/// +/// ``` +/// use windows_waitable_queues::{spsc, TryRecvError}; +/// +/// let (tx, rx) = spsc::bounded::(2)?; +/// tx.push(7).expect("a fresh queue has room"); +/// assert_eq!(rx.pop(), Ok(7)); +/// assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub fn bounded(capacity: usize) -> Result<(Producer, Consumer), CapacityError> { + build(capacity, Options::new()) +} + +/// Creates a ring with something other than the default behaviour. +/// +/// Identical to [`bounded`] except for what [`Options`] asks for. See +/// [`Disposal`](crate::Disposal) for why undrained items need a decision made +/// here rather than at teardown, and +/// [`Options::tracking_high_water`] for the one switch that costs the push path +/// anything. +/// +/// # Errors +/// +/// As [`bounded`]. +/// +/// # Examples +/// +/// ``` +/// use std::sync::mpsc; +/// use windows_waitable_queues::{Disposal, Options, spsc}; +/// +/// let (undelivered, reaper) = mpsc::channel(); +/// let (tx, rx) = spsc::bounded_with::( +/// 4, +/// Options::new() +/// .disposal(Disposal::new(move |item| { +/// let _ = undelivered.send(item); +/// })) +/// .tracking_high_water(), +/// )?; +/// +/// tx.push(1).expect("a fresh queue has room"); +/// tx.push(2).expect("a fresh queue has room"); +/// assert_eq!(rx.high_water(), Some(2)); +/// +/// drop((tx, rx)); +/// assert_eq!(reaper.into_iter().collect::>(), vec![1, 2]); +/// # Ok::<(), windows_waitable_queues::CapacityError>(()) +/// ``` +pub fn bounded_with( + capacity: usize, + options: Options, +) -> Result<(Producer, Consumer), CapacityError> { + build(capacity, options) +} +fn build( + capacity: usize, + options: Options, +) -> Result<(Producer, Consumer), CapacityError> { + validate_capacity(capacity, BOUNDS)?; + + let mut slots = Vec::with_capacity(capacity); + slots.resize_with(capacity, || UnsafeCell::new(MaybeUninit::uninit())); + + let shared = Arc::new(Shared { + teardown: Teardown::new(options.disposal), + metrics: Metrics::new(options.track_high_water), + slots: slots.into_boxed_slice(), + mask: capacity - 1, + capacity, + head: CacheAligned(AtomicUsize::new(0)), + tail: CacheAligned(AtomicUsize::new(0)), + producer_live: AtomicBool::new(true), + consumer_live: AtomicBool::new(true), + reserved: AtomicUsize::new(0), + doorbell: Doorbell::new(), + }); + + Ok(( + Producer { + shared: Arc::clone(&shared), + not_sync: PhantomData, + }, + Consumer { + shared, + not_sync: PhantomData, + }, + )) +} + +struct Shared { + slots: Box<[UnsafeCell>]>, + mask: usize, + capacity: usize, + /// What becomes of undrained items at teardown. + /// + /// Read only by [`Shared::drop`], which holds `&mut self`, so it needs no + /// synchronization and costs the hot paths nothing but its space. + teardown: Teardown, + /// The counters this queue keeps about itself. See [`crate::metrics`]. + metrics: Metrics, + /// Where the consumer will next read. Owned by the consumer. + head: CacheAligned, + /// Where the producer will next write. Owned by the producer. + tail: CacheAligned, + producer_live: AtomicBool, + consumer_live: AtomicBool, + /// Slots claimed by a [`Reservation`] and not yet redeemed. + /// + /// **Written only by the producer's thread**, which is what makes + /// reservation nearly free in this shape: `reserve`, `Reservation::send` and + /// `Reservation::drop` all run on the single producer, so a plain load and + /// store suffice where [`reserving_mpsc`](crate::reserving_mpsc) needs a + /// compare-and-swap against a packed word. The line is exclusive to that + /// core, so the extra read on the push path costs essentially nothing. + /// + /// Atomic rather than a [`Cell`] only because the consumer reads it as a + /// metric, and a torn read of a metric is still undefined behaviour. + reserved: AtomicUsize, + /// Readiness as a waitable `HANDLE`. Costs nothing until somebody asks for + /// the handle, so a polling consumer never allocates a kernel object. + doorbell: Doorbell, +} + +// SAFETY: the two positions partition the slot array between the threads. A +// slot in `[head, tail)` is owned by the consumer and read exactly once; a slot +// outside it is owned by the producer and written exactly once. Each side +// publishes its position with a release store that the other acquires, so the +// write of an item happens-before the read of that item. `T: Send` is required +// and sufficient because an item is moved between the threads and never +// referenced from both. +// +// The `teardown` field is deliberately NOT covered by that argument, because it +// cannot be: it holds a boxed FnMut, which is Send but not Sync, so this +// impl is forcing Sync onto a field that does not have it. That is sound for +// a narrower reason -- the field is unreachable through a shared reference. It +// is private, no method reads it, and the only access is from Drop, which +// holds &mut self and runs when the last handle is already gone. So no two +// threads can reach it at all, concurrently or otherwise. +unsafe impl Sync for Shared {} +// SAFETY: as above; sending the shared state is sending the items it holds. +unsafe impl Send for Shared {} + +impl Shared { + /// Items currently held. + /// + /// Both loads are `Acquire` so that a caller on either side sees a value + /// consistent with the items it can actually observe. It is a snapshot the + /// moment it is returned: the peer may push or pop immediately afterwards, + /// which is why nothing here invites a check-then-act. + /// + /// **Clamped to the capacity**, for the reason the other shapes' gauges are: + /// `tail` is read before `head`, so a consumer draining past the sampled + /// value makes the wrapping subtraction produce a number near `usize::MAX`. + /// A bounded queue must never report holding more than it can. + fn len(&self) -> usize { + let tail = self.tail.0.load(Ordering::Acquire); + let head = self.head.0.load(Ordering::Acquire); + tail.wrapping_sub(head).min(self.capacity) + } + + /// How many further items a best-effort push could still place. + /// + /// **Not `capacity - len()`, which is what the [`Bounded`](crate::Bounded) + /// default computes and is wrong for this shape too.** A reservation + /// withdraws a slot without becoming an item, so after reserving every slot + /// the default still answers the full capacity while both `push` and + /// `reserve` refuse. + fn remaining(&self) -> usize { + let held = self.len(); + let reserved = self.reserved.load(Ordering::Relaxed); + self.capacity.saturating_sub(held.saturating_add(reserved)) + } + + /// Write an item into the slot at `tail` and publish it. + /// + /// Shared by [`Producer::push`] and [`Reservation::send`] so that the + /// ordering argument below is made once. The two differ only in how they + /// established that there is room -- one checked, the other was promised -- + /// and nothing downstream of that decision should be written twice. + /// + /// # Safety + /// + /// The caller must have established that the slot at `tail` is free: either + /// by the room check in `push`, or by holding a reservation. + unsafe fn publish(&self, tail: usize, item: T) { + // Free on this shape: the producer owns `tail` and already loaded + // `head` to decide there was room, so the depth is a subtraction of two + // values it is holding. The counter's line is producer-owned too, since + // nothing else writes it. + // Acquire, matching every other load of `head`: it carries a release + // store, and a relaxed load mixed onto such an atomic is a plain load + // with no defined position relative to the ordered operations on it. + self.metrics + .record_depth(tail.wrapping_sub(self.head.0.load(Ordering::Acquire)) + 1); + + // SAFETY: the caller's precondition says this slot holds no initialized + // item, so writing a `MaybeUninit` over it drops nothing. + unsafe { + (*self.slots[tail & self.mask].get()).write(item); + } + + // Release: publishes the slot write to the consumer's acquire load. The + // store must come after the write, and this is what forbids the + // compiler and the processor from moving it earlier. + self.tail.0.store(tail.wrapping_add(1), Ordering::Release); + + // After the release store, never before: the doorbell says "there is + // something to take", and that must not become true before the item is + // actually takeable. A consumer woken early would find the queue empty, + // clear the doorbell, and go back to sleep on an item that is about to + // exist -- a lost wakeup manufactured by signalling too eagerly. + // + // Cheap when it is redundant: `signal` returns without a syscall if the + // doorbell is already lit, so a producer running ahead of its consumer + // pays one atomic per push rather than one `SetEvent`. + self.doorbell.signal(); + } +} + +impl Drop for Shared { + fn drop(&mut self) { + // Both handles are gone, so no synchronization is needed and the + // positions can be read directly. Every slot in `[head, tail)` still + // holds an initialized item that nobody took, and tearing the queue + // down must account for them rather than leak them. + // + // Each is *moved out* and handed to the teardown policy rather than + // destroyed where it lies. For the default policy the two are the same + // thing; for a queue whose items own handles they are not, and this is + // the only place that sees every survivor. See `crate::disposal`. + let head = *self.head.0.get_mut(); + let tail = *self.tail.0.get_mut(); + let mask = self.mask; + let mut pos = head; + while pos != tail { + // SAFETY: `pos` is in `[head, tail)`, so this slot was written by + // the producer and never read by the consumer. It is read exactly + // once, because `pos` advances every iteration, and the slot is + // never read again afterwards. + let item = unsafe { (*self.slots[pos & mask].get()).assume_init_read() }; + self.teardown.dispose(item); + pos = pos.wrapping_add(1); + } + } +} + +/// The writing half of an [`spsc`](self) ring. +/// +/// Neither [`Clone`] nor [`Sync`], which is what makes "single producer" a fact +/// the compiler checks rather than a rule to remember. +pub struct Producer { + shared: Arc>, + /// Removes [`Sync`] without removing [`Send`]. A [`Cell`] is exactly that + /// shape, and no value of it is ever created. + not_sync: PhantomData>, +} + +impl Producer { + /// Appends an item, best-effort. + /// + /// **Cannot take a reserved slot.** A queue with one free slot and one + /// outstanding [`Reservation`] refuses this, which is the reservation doing + /// its job rather than a malfunction. + /// + /// # Errors + /// + /// [`PushError::Full`] when no unreserved room remains, which is the + /// backpressure signal rather than a malfunction, and + /// [`PushError::Disconnected`] when the consumer is gone. Either way the + /// item comes back, so nothing is lost by the refusal. + pub fn push(&self, item: T) -> Result<(), PushError> { + // Acquire, matching every other load of `tail`. Sole-writer coherence + // would suffice to read this thread's own latest value, but `tail` also + // carries a release store (in `publish`), and a relaxed load mixed onto + // an atomic that carries acquire/release operations is a plain load: + // unanchored with respect to those operations, and free to be moved by + // the optimizer or the processor. Uniform acquire is what keeps the + // source text describing what actually happens. + let tail = self.shared.tail.0.load(Ordering::Acquire); + // Acquire: pairs with the consumer's release store, so a slot it freed + // is visible as free here. + let head = self.shared.head.0.load(Ordering::Acquire); + // Relaxed, and this is the whole cost of reservation on this shape: the + // only writer of `reserved` is this thread, so the line is exclusive to + // this core and cannot hold a stale value of its own. + let reserved = self.shared.reserved.load(Ordering::Relaxed); + + // The sum cannot overflow: each term is at most the capacity, which is + // itself at most half of `usize::MAX`. + if tail.wrapping_sub(head) + reserved >= self.shared.capacity { + // Report disconnection in preference to fullness: a full queue + // whose consumer is gone will never drain, and telling the caller + // to retry would be telling it to spin forever. + if !self.shared.consumer_live.load(Ordering::Acquire) { + // Not counted as a refusal: this is the end of the stream, not + // backpressure, and folding the two together would make a + // shutting-down queue look like an overloaded one. + return Err(PushError::Disconnected(item)); + } + self.shared.metrics.record_refusal(); + return Err(PushError::Full(item)); + } + if !self.shared.consumer_live.load(Ordering::Acquire) { + return Err(PushError::Disconnected(item)); + } + + // SAFETY: `tail` is outside `[head, tail)`, so this slot is owned by + // the producer and holds no initialized item, and the room check above + // left it unclaimed by any reservation. + unsafe { + self.shared.publish(tail, item); + } + Ok(()) + } + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether the next best-effort push would be refused, as a snapshot. + /// + /// True when the queue is full *or* every remaining slot is reserved, since + /// those are indistinguishable to a best-effort caller. + /// + /// Advisory only. Nothing is gained by testing it before [`Self::push`], + /// which reports the same condition without the window in between; it is + /// offered for metrics rather than for control flow. + #[must_use] + pub fn is_full(&self) -> bool { + self.remaining() == 0 + } + + /// How many further items a best-effort push could still place, as a + /// snapshot. + /// + /// **Reservations are subtracted**, unlike `capacity() - len()`: a reserved + /// slot is spoken for, so counting it as room would promise a push that + /// [`push`](Self::push) is guaranteed to refuse. Advisory only, like every + /// other gauge here. + #[must_use] + pub fn remaining(&self) -> usize { + self.shared.remaining() + } + + /// Slots currently claimed by a [`Reservation`] and not yet redeemed. + #[must_use] + pub fn outstanding_reservations(&self) -> usize { + self.shared.reserved.load(Ordering::Relaxed) + } + + /// Claims one slot for a message that must not be lost. + /// + /// See [`Reserving::reserve`](crate::Reserving::reserve) for what a + /// reservation is for. The short form: failing here is cheap, because no + /// work has been started yet, whereas failing at delivery means blocking or + /// losing the message. + /// + /// **The reservation borrows this producer**, which is not an arbitrary + /// choice of ownership. This shape is sound because exactly one thread ever + /// writes the ring, and the producer handle is what makes that true -- it is + /// neither [`Clone`] nor [`Sync`]. An owned reservation could be moved to a + /// second thread while the producer stayed on the first, and then two + /// threads would be writing. Borrowing pins the producer for as long as any + /// reservation is outstanding, so the compiler enforces what the shape + /// requires. [`reserving_mpsc`](crate::reserving_mpsc), which has no such + /// constraint, hands out an owned reservation instead. + /// + /// **The refusal is asserted, not merely described.** A reservation cannot + /// be moved to another thread, because it borrows a producer that is not + /// [`Sync`], so `&Producer` is not [`Send`]: + /// + /// ```compile_fail + /// # use windows_waitable_queues::spsc; + /// let (tx, _rx) = spsc::bounded::(4).unwrap(); + /// let slot = tx.reserve().expect("room"); + /// // Rejected: moving this would put a second writer on the ring. + /// std::thread::spawn(move || { + /// slot.send(1).ok(); + /// }); + /// ``` + /// + /// The consumer handle *is* [`Send`], so a blocked receiver can still live + /// on another thread -- it is only the writing side that is pinned. + #[must_use = "a reservation withholds capacity from the best-effort path until it is used or dropped"] + pub fn reserve(&self) -> Option> { + // Acquire on both: each carries a release store, so a relaxed load on + // either would be a plain load, unanchored with respect to it. + let tail = self.shared.tail.0.load(Ordering::Acquire); + let head = self.shared.head.0.load(Ordering::Acquire); + let reserved = self.shared.reserved.load(Ordering::Relaxed); + + if tail.wrapping_sub(head) + reserved >= self.shared.capacity { + return None; + } + + // A plain store, where `reserving_mpsc` needs a compare-and-swap against + // a packed word: there is only one producer, so `reserve`, `push` and + // the redemption all run on this thread and cannot interleave with each + // other. That is the entire difference between the two shapes' + // reservation machinery, and it is why this one costs nothing. + self.shared.reserved.store(reserved + 1, Ordering::Relaxed); + Some(Reservation { producer: self }) + } + + /// Whether the consumer has been dropped. + #[must_use] + pub fn is_disconnected(&self) -> bool { + !self.shared.consumer_live.load(Ordering::Acquire) + } +} + +// Hand-written rather than derived: deriving would demand `T: Debug`, which +// would make a handle to a queue of non-`Debug` items un-printable for no +// reason. The item type is not the handle's business, so the handle reports the +// queue's state instead. +impl fmt::Debug for Producer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("spsc::Producer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Producer { + fn drop(&mut self) { + // Release: everything this producer pushed happens-before a consumer + // observing the disconnection, so a consumer that sees it can trust + // that draining to empty really has drained everything. + self.shared.producer_live.store(false, Ordering::Release); + + // Disconnection is a wakeup like any other, and the only one nobody + // else can deliver. A consumer blocked on the doorbell would otherwise + // wait forever for an item that can no longer be sent -- the queue + // would be correct and the program would still hang. + self.shared.doorbell.signal(); + } +} + +/// A slot claimed in advance, which [`Reservation::send`] redeems. +/// +/// Borrows the [`Producer`] that made it, so the producer cannot move to +/// another thread while a claim is outstanding. See [`Producer::reserve`] for +/// why that is a soundness requirement here and not merely a style choice. +/// +/// Dropping it returns the slot to the best-effort path. +#[must_use = "a reservation withholds capacity from the best-effort path until it is used or dropped"] +pub struct Reservation<'a, T> { + producer: &'a Producer, +} + +impl Reservation<'_, T> { + /// Delivers into the reserved slot. + /// + /// **This cannot fail for want of room**, which is the entire purpose: the + /// slot was withheld from the best-effort path from the moment the + /// reservation was taken. See [`Disconnected`] for why that is the only + /// error and why the type says so. + /// + /// # Errors + /// + /// [`Disconnected`] if the consumer is gone, carrying the item back so it + /// can be accounted for rather than silently dropped. + pub fn send(self, item: T) -> Result<(), Disconnected> { + let shared = &self.producer.shared; + if !shared.consumer_live.load(Ordering::Acquire) { + // Dropping `self` on the way out releases the slot, which is what + // should happen: this message is never being delivered. + return Err(Disconnected(item)); + } + + // Acquire, matching every other load of `tail`; see `push`. + let tail = shared.tail.0.load(Ordering::Acquire); + // SAFETY: the reservation guarantees a free slot -- the room check that + // granted it withheld one from the best-effort path, and this thread is + // the only one that could have consumed it since. + unsafe { + shared.publish(tail, item); + } + + // Released only now, after the slot it guaranteed has actually been + // used. No other thread pushes into this shape, so the moment between + // the publication and this store is invisible to anything that could + // act on it; the consumer may see the pair inconsistently, but only as + // a metric. + let reserved = shared.reserved.load(Ordering::Relaxed); + debug_assert!( + reserved >= 1, + "this reservation is outstanding, so the count cannot be zero" + ); + shared.reserved.store(reserved - 1, Ordering::Relaxed); + + // The slot has been given up above, so the `Drop` that would give it up + // a second time must not run. + core::mem::forget(self); + Ok(()) + } + + /// Whether the consumer has been dropped, so redeeming would fail. + #[must_use] + pub fn is_disconnected(&self) -> bool { + self.producer.is_disconnected() + } +} + +impl fmt::Debug for Reservation<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("spsc::Reservation") + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Reservation<'_, T> { + fn drop(&mut self) { + let reserved = self.producer.shared.reserved.load(Ordering::Relaxed); + debug_assert!( + reserved >= 1, + "this reservation is outstanding, so the count cannot be zero" + ); + self.producer + .shared + .reserved + .store(reserved - 1, Ordering::Relaxed); + } +} + +/// The reading half of an [`spsc`](self) ring. +/// +/// Neither [`Clone`] nor [`Sync`], which is what makes "single consumer" a fact +/// the compiler checks rather than a rule to remember. +pub struct Consumer { + shared: Arc>, + /// See [`Producer::not_sync`]. + not_sync: PhantomData>, +} + +impl Consumer { + /// Takes the oldest item. + /// + /// # Errors + /// + /// [`TryRecvError::Empty`] when nothing is queued right now, and + /// [`TryRecvError::Disconnected`] when every producer is gone *and* the + /// queue has been drained -- in that order, so the tail of a stream whose + /// producers have already departed is still delivered. See + /// [`Consumer::pop`](crate::Consumer::pop) for why that ordering is a + /// guarantee rather than an implementation detail. + pub fn pop(&self) -> Result { + match self.take() { + Some(item) => Ok(item), + // Only on the empty path, so a successful take never pays for this + // load. The queue must be observed empty *before* disconnection is + // reported, which is exactly what this ordering enforces. + None if self.is_disconnected() => Err(TryRecvError::Disconnected), + None => Err(TryRecvError::Empty), + } + } + + /// The take itself, without the disconnection question. + fn take(&self) -> Option { + // Acquire, matching every other load of `head`. Sole-writer coherence + // would suffice here, but `head` also carries the release store below; + // see `push` for why the two disciplines are not mixed on one atomic. + let head = self.shared.head.0.load(Ordering::Acquire); + // Acquire: pairs with the producer's release store, so an item it + // published is visible here. + let tail = self.shared.tail.0.load(Ordering::Acquire); + + if head == tail { + return None; + } + + // SAFETY: `head` is in `[head, tail)`, so the producer wrote this slot + // and released it. It is read exactly once, because `head` advances + // below before any other read can observe the slot as free. + let item = + unsafe { (*self.shared.slots[head & self.shared.mask].get()).assume_init_read() }; + + // Release: publishes the slot as free to the producer's acquire load. + // It must come after the read, or the producer could overwrite an item + // this thread has not finished taking. + self.shared + .head + .0 + .store(head.wrapping_add(1), Ordering::Release); + Some(item) + } + + /// The exact number of items this queue holds when full. + #[must_use] + pub fn capacity(&self) -> usize { + self.shared.capacity + } + + /// Items currently held, as a snapshot. + #[must_use] + pub fn len(&self) -> usize { + self.shared.len() + } + + /// Whether the queue holds nothing, as a snapshot. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether a further best-effort push would be refused for want of room. + /// + /// The consumer's view of the question the producer answers, so a caller + /// holding only this handle need not import [`Bounded`](crate::Bounded). + #[must_use] + pub fn is_full(&self) -> bool { + crate::Bounded::is_full(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// The inherent form of [`Consumer::drain`](crate::Consumer::drain), so it + /// works without importing the trait. + pub fn drain(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Takes items until the queue is momentarily empty. + /// + /// An alias for [`Self::drain`] under the name most of the ecosystem uses. + pub fn try_iter(&self) -> crate::Drain<'_, Self> { + crate::Consumer::drain(self) + } + + /// Whether the producer has been dropped. + /// + /// **A queue can be disconnected and still hold items**, because a producer + /// may push and then drop -- so this alone does not mean the stream is + /// finished, and acting on it while items remain would discard them. + /// [`Self::pop`] answers the composite question in the only order that + /// cannot lose the tail, and is what a drain loop should use. + /// + /// The release store in the producer's `Drop` is what makes its preceding + /// pushes visible to a consumer that observes this. + #[must_use] + pub fn is_disconnected(&self) -> bool { + !self.shared.producer_live.load(Ordering::Acquire) + } + + /// Borrows the queue's readiness as a waitable `HANDLE`. + /// + /// This is the point of the crate. The handle is a manual-reset event that + /// is signalled exactly while the queue has something to take, so it can go + /// into `WaitForMultipleObjects` beside an I/O completion, a shutdown + /// event, or a timer -- a wait that no queue with a private parking + /// primitive can join. + /// + /// The event is created on the first call, so a consumer that only ever + /// polls with [`Self::pop`] is charged for no kernel object. + /// + /// The borrow is deliberate: the event belongs to the queue and must not be + /// closed. Use [`Self::doorbell_owned`] where ownership is required. + /// + /// # Waiting on it correctly + /// + /// **Do not simply wait and then drain.** Use [`Self::arm`] to decide + /// whether waiting is safe, or the wait can miss an item and block forever: + /// + /// ```no_run + /// # use windows_waitable_queues::{spsc, TryRecvError}; + /// # use windows_sys::Win32::System::Threading::{WaitForSingleObject, INFINITE}; + /// # use std::os::windows::io::AsRawHandle; + /// # fn demo(rx: &spsc::Consumer) -> std::io::Result<()> { + /// loop { + /// // Take everything available. The drain ends by saying *why*, so the + /// // end of the stream cannot be missed by forgetting to ask. + /// loop { + /// match rx.pop() { + /// Ok(item) => { let _ = item; } + /// Err(TryRecvError::Disconnected) => return Ok(()), + /// // Empty for now. A wildcard because `TryRecvError` is + /// // `#[non_exhaustive]`, as every error type here is. + /// Err(_) => break, + /// } + /// } + /// if !rx.arm()? { + /// continue; // Something arrived; waiting now would be wrong. + /// } + /// // Ask once more, because `arm` has just cleared the doorbell -- + /// // including the single ring the last producer's drop made, which is + /// // not coming again. Skipping this blocks forever on a finished + /// // stream, and a producer that pushed *and then* dropped in the + /// // window above is also caught here rather than discarded. + /// match rx.pop() { + /// Ok(item) => { let _ = item; continue; } + /// Err(TryRecvError::Disconnected) => return Ok(()), + /// Err(_) => {} + /// } + /// let handle = rx.doorbell()?; + /// // SAFETY: a live event handle borrowed for the call. + /// unsafe { WaitForSingleObject(handle.as_raw_handle(), INFINITE) }; + /// } + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn doorbell(&self) -> io::Result> { + self.shared.doorbell.handle() + } + + /// A duplicate of [`Self::doorbell`] that the caller owns. + /// + /// The duplicate names the same event, so signalling reaches both, and the + /// caller may close its copy whenever it likes. This is the form a + /// `ThreadpoolWait` needs, since arming one takes ownership of its target. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` or `DuplicateHandle`. + pub fn doorbell_owned(&self) -> io::Result { + self.shared.doorbell.owned() + } + + /// Clears the doorbell and reports whether a later push could be missed. + /// + /// `true` means the queue was still empty after the doorbell was cleared, + /// so any later push is guaranteed to signal. `false` means something + /// arrived in the meantime: take it instead of waiting. + /// + /// **`true` is not by itself permission to wait indefinitely.** It answers + /// only whether a later *push* can be missed, and says nothing about the + /// end of the stream: with every producer gone it still returns `true`, + /// having just cleared the single ring their drop left behind. See + /// [`Waitable::arm`](crate::Waitable::arm) for the four-step protocol an + /// indefinite wait needs, and the example on [`Self::doorbell`] for it + /// written out. + /// + /// The order inside this method is the whole correctness argument, and it + /// is the reverse of the one that reads naturally. Clearing *first* and + /// checking emptiness *second* is what makes a lost wakeup impossible: an + /// item that arrives before the clear is found by the check, and an item + /// that arrives after the clear signals a doorbell that + /// the internal `clear` has left able to ring. + /// Checking first would leave a window in which a push both signals and has + /// its signal erased, and the consumer would sleep on a queue that is not + /// empty and will never be signalled again. + /// + /// The first of those two cases is stronger here than it is for + /// [`slotwise_mpsc`](crate::slotwise_mpsc): there is one producer and one position, so *any* + /// push before the clear makes this check find something. That is why this + /// shape never exhibited the doorbell defect `slotwise_mpsc` exposed, and why the + /// fix for it belongs to the doorbell rather than to either caller. + /// + /// This also creates the doorbell if it does not exist, which must happen + /// before the emptiness check for the same reason: a producer running while + /// there is no event skips signalling, so the check has to come after the + /// event exists to catch what that skip left behind. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + pub fn arm(&self) -> io::Result { + // Before the clear, and so before the check: see above. + self.shared.doorbell.handle()?; + self.shared.doorbell.clear(); + #[cfg(test)] + crate::race_hooks::ARM.run(); + Ok(self.is_empty()) + } + + /// The last take before reporting the end of the stream. + /// + /// Called only after [`Self::is_disconnected`] has returned `true`, which + /// makes the answer final rather than a snapshot: no producer remains to + /// add anything, so `None` here means empty forever. + /// + /// This exists as a named step, rather than as a bare `pop` inlined into + /// each caller, because it guards a race that is real and narrow: a + /// producer may push *and then* drop in the window between a receive's + /// first `pop` and its disconnection check. Reporting the disconnection + /// without this final take would silently discard an item that was + /// successfully sent. Being a separate function is what lets a test reach + /// it directly instead of hoping to schedule that window. + fn finish(&self) -> Option { + self.take() + } + + /// Takes the oldest item, blocking until one arrives. + /// + /// Parks on the doorbell rather than spinning, so a consumer with nothing + /// to do costs nothing. + /// + /// # Errors + /// + /// [`RecvError::Disconnected`] once the producer is gone *and* the queue is + /// drained -- items pushed before the producer dropped are still delivered. + /// [`RecvError::Io`] if the doorbell cannot be created or waited on. + pub fn recv(&self) -> Result { + blocking::recv(self) + } + + /// Takes the oldest item, blocking until one arrives or the deadline + /// passes. + /// + /// The timeout bounds the whole call, not each individual wait: a consumer + /// woken spuriously does not get a fresh budget. + /// + /// # Errors + /// + /// [`RecvTimeoutError::Timeout`] if the deadline passes with the queue + /// still empty, which is not a malfunction. Otherwise as [`Self::recv`]. + pub fn recv_timeout(&self, timeout: Duration) -> Result { + blocking::recv_timeout(self, timeout) + } +} + +impl Parked for Consumer { + type Item = T; + + fn pop(&self) -> Option { + Self::take(self) + } + + fn finish(&self) -> Option { + Self::finish(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } + + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } +} + +/// See [`Producer`]'s impl for why this is hand-written. +impl fmt::Debug for Consumer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("spsc::Consumer") + .field("capacity", &self.capacity()) + .field("len", &self.len()) + .field("disconnected", &self.is_disconnected()) + .finish() + } +} + +impl Drop for Consumer { + fn drop(&mut self) { + self.shared.consumer_live.store(false, Ordering::Release); + } +} + +impl crate::Producer for Producer { + type Item = T; + + fn push(&self, item: T) -> Result<(), PushError> { + Self::push(self, item) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Consumer for Consumer { + type Item = T; + + fn pop(&self) -> Result { + Self::pop(self) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Claim for Reservation<'_, T> { + type Item = T; + + fn send(self, item: T) -> Result<(), Disconnected> { + Self::send(self, item) + } + + fn is_disconnected(&self) -> bool { + Self::is_disconnected(self) + } +} + +impl crate::Reserving for Producer { + type Item = T; + type Reservation<'a> + = Reservation<'a, T> + where + Self: 'a; + + fn reserve(&self) -> Option> { + Self::reserve(self) + } + + fn outstanding_reservations(&self) -> usize { + Self::outstanding_reservations(self) + } +} + +impl crate::Bounded for Producer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + // Overridden, because the default `capacity - len` counts a reserved slot as + // room: a reservation withdraws capacity without becoming an item, so after + // reserving every slot the default would answer the full capacity while both + // `push` and `reserve` refuse. + fn remaining(&self) -> usize { + Self::remaining(self) + } +} + +impl crate::Bounded for Consumer { + fn capacity(&self) -> usize { + Self::capacity(self) + } + + fn len(&self) -> usize { + Self::len(self) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + // The consumer's view has to agree with the producer's: both describe the + // same queue, and a caller generic over `Bounded` should not get a different + // answer depending on which handle it holds. + fn remaining(&self) -> usize { + self.shared.remaining() + } +} + +impl Shared { + /// The counters, as the [`Observable`](crate::Observable) trait reports + /// them. Written once so the two handles cannot drift apart. + fn refused(&self) -> u64 { + self.metrics.refused() + } + + fn doorbell_rings(&self) -> u64 { + self.doorbell.rings() + } + + fn high_water(&self) -> Option { + self.metrics.high_water() + } +} + +impl Producer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl Consumer { + /// How many pushes have been refused for want of room. + #[must_use] + pub fn refused(&self) -> u64 { + self.shared.refused() + } + + /// How many times the doorbell has actually rung. + #[must_use] + pub fn doorbell_rings(&self) -> u64 { + self.shared.doorbell_rings() + } + + /// The deepest this queue has been, if tracking was asked for. + #[must_use] + pub fn high_water(&self) -> Option { + self.shared.high_water() + } +} + +impl crate::Observable for Producer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Observable for Consumer { + fn refused(&self) -> u64 { + Self::refused(self) + } + + fn doorbell_rings(&self) -> u64 { + Self::doorbell_rings(self) + } + + fn high_water(&self) -> Option { + Self::high_water(self) + } +} + +impl crate::Waitable for Consumer { + fn doorbell(&self) -> io::Result> { + Self::doorbell(self) + } + + fn doorbell_owned(&self) -> io::Result { + Self::doorbell_owned(self) + } + + fn arm(&self) -> io::Result { + Self::arm(self) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/spsc/tests.rs b/crates/windows-waitable-queues/src/spsc/tests.rs new file mode 100644 index 00000000..5be8c165 --- /dev/null +++ b/crates/windows-waitable-queues/src/spsc/tests.rs @@ -0,0 +1,1927 @@ +// Copyright (c) Mike Grier. + +//! Tests for the SPSC bounded ring. +//! +//! Every one runs in memory in microseconds. The cross-thread cases use a +//! joined thread rather than a sleep, so they are deterministic: the assertion +//! runs after the peer has finished, not after a guess about how long it takes. + +use super::{BOUNDS, Consumer, Producer, bounded, bounded_with, validate_capacity}; +use crate::Bounded; +use crate::error::TryRecvError; +use crate::race_hooks; +use crate::{Disposal, Options}; +use crate::{PushError, RecvError, RecvTimeoutError}; +use std::os::windows::io::AsRawHandle; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; + +use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; +use windows_sys::Win32::System::Threading::WaitForSingleObject; + +/// Counts its own drops, so a test can prove an item was destroyed rather than +/// leaked. `Arc` rather than a `static`, so tests that run +/// concurrently in one process cannot see each other's counts. +#[derive(Debug)] +struct DropCounter(Arc); + +impl Drop for DropCounter { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn a_pushed_item_comes_back_out() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + tx.push(42).expect("a fresh queue has room"); + assert_eq!(rx.pop(), Ok(42)); +} + +#[test] +fn an_empty_queue_pops_nothing() { + let (_tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + assert!(rx.is_empty()); + assert_eq!(rx.len(), 0); +} + +#[test] +fn items_come_out_in_the_order_they_went_in() { + let (tx, rx) = bounded::(8).expect("a power-of-two capacity"); + for value in 0..8 { + tx.push(value).expect("room for eight"); + } + let drained: Vec = rx.try_iter().collect(); + assert_eq!(drained, (0..8).collect::>()); +} + +#[test] +fn a_full_queue_refuses_and_hands_the_item_back() { + let (tx, rx) = bounded::(2).expect("a power-of-two capacity"); + // **Both directions.** Asserting only that a full queue says so is + // satisfied by an `is_full` that always says so -- which is a queue that + // refuses every push, and a mutation run found exactly that constant alive + // on all three shapes. + assert!(!tx.is_full(), "an empty queue is not full"); + tx.push(1).expect("room"); + assert!(!tx.is_full(), "nor is a partly filled one"); + tx.push(2).expect("room"); + assert!(tx.is_full()); + + match tx.push(3) { + Err(PushError::Full(returned)) => assert_eq!( + returned, 3, + "the refused item must come back, or a caller cannot retry it" + ), + other => panic!("expected Full, got {other:?}"), + } + + // And the refusal did not disturb what was already there. + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(rx.pop(), Ok(2)); +} + +#[test] +fn a_capacity_of_one_holds_exactly_one() { + let (tx, rx) = bounded::(1).expect("one is a power of two"); + tx.push(1).expect("room for one"); + assert!(matches!(tx.push(2), Err(PushError::Full(2)))); + assert_eq!(rx.pop(), Ok(1)); + tx.push(3).expect("the slot was freed"); + assert_eq!(rx.pop(), Ok(3)); +} + +#[test] +fn the_ring_wraps_many_times_without_losing_order() { + // Far more operations than slots, so every slot is reused repeatedly and a + // mistake in the masking or in the free-slot arithmetic shows up as a + // wrong value rather than as a crash. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for round in 0..1000 { + tx.push(round).expect("the previous item was taken"); + assert_eq!(rx.pop(), Ok(round)); + } + assert!(rx.is_empty()); +} + +#[test] +fn a_partly_full_ring_wraps_correctly() { + // Keeps two items resident while cycling, so head and tail are never equal + // and never a whole lap apart -- the case a simple "empty when equal" test + // never reaches. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + tx.push(0).expect("room"); + tx.push(1).expect("room"); + for round in 2..500 { + tx.push(round) + .expect("room, because one is taken each round"); + assert_eq!(rx.pop(), Ok(round - 2)); + assert_eq!(rx.len(), 2); + } +} + +#[test] +fn len_tracks_pushes_and_pops() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert_eq!(tx.len(), 0); + tx.push(1).expect("room"); + assert_eq!(tx.len(), 1); + assert_eq!(rx.len(), 1, "both handles report the same queue"); + tx.push(2).expect("room"); + assert_eq!(tx.len(), 2); + rx.pop().expect("an item"); + assert_eq!(rx.len(), 1); + rx.pop().expect("an item"); + assert!(rx.is_empty()); +} + +#[test] +fn dropping_the_queue_drops_the_items_it_still_holds() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(8).expect("a power-of-two capacity"); + for _ in 0..5 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + assert_eq!(drops.load(Ordering::Relaxed), 0, "nothing dropped yet"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 5, + "every undrained item must be dropped, not leaked" + ); +} + +#[test] +fn dropping_the_queue_after_a_wrap_drops_only_what_is_resident() { + // The interesting case for the drop loop: head and tail are both far from + // zero and the live range straddles the end of the slot array, so a drop + // that iterated `0..len` instead of `head..tail` would destroy the wrong + // slots -- and would drop uninitialized memory. + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + for _ in 0..6 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + rx.pop().expect("an item"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 6, + "the six taken were dropped" + ); + + // Now leave three resident, starting from a wrapped position. + for _ in 0..3 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + } + assert_eq!( + drops.load(Ordering::Relaxed), + 9, + "the three still resident must also be dropped" + ); +} + +#[test] +fn a_consumer_that_is_gone_turns_a_push_into_a_disconnect() { + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + assert!(!tx.is_disconnected()); + drop(rx); + assert!(tx.is_disconnected()); + + match tx.push(1) { + Err(PushError::Disconnected(returned)) => assert_eq!(returned, 1), + other => panic!("expected Disconnected, got {other:?}"), + } +} + +#[test] +fn a_full_queue_whose_consumer_is_gone_reports_disconnected_not_full() { + // The distinction is the whole point of having two variants: Full invites a + // retry, and retrying this one would spin for ever. + let (tx, rx) = bounded::(2).expect("a power-of-two capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(rx); + + match tx.push(3) { + Err(PushError::Disconnected(_)) => {} + Err(PushError::Full(_)) => { + panic!("a full queue with no consumer will never drain, so Full would invite a spin") + } + Ok(()) => panic!("the queue was full"), + } +} + +#[test] +fn a_producer_that_is_gone_leaves_the_queued_items_takeable() { + // Disconnection must not discard what was already pushed, which is why the + // documented order is drain first and check afterwards. + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(tx); + + assert!(rx.is_disconnected()); + assert_eq!(rx.pop(), Ok(1), "a dropped producer does not discard"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn a_zero_capacity_is_refused_because_it_could_never_accept_anything() { + let error = bounded::(0).expect_err("zero is not a usable capacity"); + assert_eq!(error.requested(), 0); + assert_eq!(error.next_valid(), Some(1)); +} + +#[test] +fn a_non_power_of_two_capacity_is_refused_with_both_neighbours() { + let error = bounded::(100).expect_err("100 is not a power of two"); + assert_eq!(error.requested(), 100); + assert_eq!( + (error.previous_valid(), error.next_valid()), + (Some(64), Some(128)), + "the error should make the correction obvious without arithmetic" + ); +} + +#[test] +fn every_power_of_two_capacity_up_to_a_reasonable_bound_is_accepted() { + for shift in 0..16 { + let capacity = 1_usize << shift; + let (tx, rx) = bounded::(capacity).expect("a power of two"); + assert_eq!(tx.capacity(), capacity); + assert_eq!(rx.capacity(), capacity, "both handles agree"); + tx.push(shift).expect("a fresh queue has room"); + assert_eq!(rx.pop(), Ok(shift)); + } +} + +#[test] +fn a_capacity_above_half_the_address_space_is_refused() { + // Not because the allocation would fail first, but because the position + // arithmetic would become ambiguous across wraparound. Checked explicitly + // so the reason survives even though no machine could allocate it. + let error = bounded::(1_usize << (usize::BITS - 1)).expect_err("too large"); + assert!( + error.next_valid().is_none(), + "there is nothing larger to suggest" + ); +} + +#[test] +fn zero_sized_items_round_trip() { + // A ZST exercises the slot arithmetic with no bytes to copy, so a mistake + // cannot hide behind a memcpy that happens to do the right thing. + let (tx, rx) = bounded::<()>(2).expect("a power-of-two capacity"); + tx.push(()).expect("room"); + tx.push(()).expect("room"); + assert!(matches!(tx.push(()), Err(PushError::Full(())))); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Ok(())); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); +} + +#[test] +fn items_cross_a_thread_boundary_in_order_and_intact() { + // The real test of the memory ordering. Each item carries a value derived + // from its index, so a torn or stale read is a wrong value rather than a + // silent pass. Boxed, so each item is a heap pointer the consumer must + // observe fully initialized -- a missing release would surface as a + // corrupt pointer rather than as a wrong integer. + const COUNT: usize = 20_000; + let (tx, rx) = bounded::>(64).expect("a power-of-two capacity"); + + let producer = std::thread::spawn(move || { + for value in 0..COUNT { + // Spin rather than sleep: the consumer is draining concurrently, + // so a full queue clears in nanoseconds. + let mut item = Box::new(value); + loop { + match tx.push(item) { + Ok(()) => break, + Err(PushError::Full(returned)) => { + item = returned; + std::hint::spin_loop(); + } + Err(PushError::Disconnected(_)) => panic!("the consumer is alive"), + } + } + } + }); + + let mut received = 0_usize; + while received < COUNT { + if let Ok(item) = rx.pop() { + assert_eq!(*item, received, "items must arrive in order and intact"); + received += 1; + } else { + std::hint::spin_loop(); + } + } + + producer.join().expect("the producer thread"); + assert_eq!(rx.pop(), Err(TryRecvError::Disconnected)); +} + +#[test] +fn a_producer_can_be_moved_to_another_thread() { + // `Send` is the property that makes the split useful, and it is worth + // pinning: a handle that could not move would force construction on the + // thread that ends up owning it. + fn assert_send() {} + assert_send::>(); + assert_send::>(); + + let (tx, rx) = bounded::(4).expect("a power-of-two capacity"); + std::thread::spawn(move || { + tx.push(7).expect("room"); + }) + .join() + .expect("the pushing thread"); + assert_eq!(rx.pop(), Ok(7)); +} + +// --------------------------------------------------------------------------- +// The doorbell, joined to the queue. +// +// The tests below are about the *pairing* of the two; the doorbell's own +// behaviour as a kernel object is covered in `crate::doorbell`'s suite. +// --------------------------------------------------------------------------- + +/// Whether the queue's doorbell is signalled right now, asked of the kernel +/// rather than of the mirror flag. +/// +/// Uses a zero timeout, so it is a state query and never blocks. The event is +/// manual-reset, so asking does not consume the answer. +fn doorbell_is_lit(consumer: &Consumer) -> bool { + let handle = consumer.doorbell().expect("the doorbell must be creatable"); + // SAFETY: a live event handle borrowed for the call; a zero timeout returns + // immediately and has no other precondition. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 0) }; + assert!( + result == WAIT_OBJECT_0 || result == WAIT_TIMEOUT, + "the wait must resolve to signalled or not, got {result:#x}" + ); + result == WAIT_OBJECT_0 +} + +#[test] +fn polling_never_creates_a_kernel_object() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The laziness claim, asserted rather than assumed: a consumer that only + // ever polls must not be charged for an event it never waits on. + for value in 0..4 { + tx.push(value).expect("there is room"); + } + while rx.pop().is_ok() {} + drop(tx); + while rx.pop().is_ok() {} + + assert!( + !rx.shared.doorbell.is_armed(), + "a poll-only consumer must allocate no kernel object" + ); +} + +#[test] +fn a_push_lights_the_doorbell() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!( + !doorbell_is_lit(&rx), + "an empty queue must not claim readiness" + ); + tx.push(1).expect("there is room"); + assert!(doorbell_is_lit(&rx), "a pushed item must be announced"); +} + +#[test] +fn the_doorbell_stays_lit_across_repeated_observation() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // Created before the push, which is not incidental: a push that runs while + // no event exists signals nothing, as + // `an_item_pushed_before_the_doorbell_existed_is_still_found` asserts. This + // test is about the level, so it starts from an armed doorbell. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + + // A level, not an edge. An auto-reset event would fail the second pass, and + // a consumer sharing the wait with other handles would lose the queue. + for observation in 1..=3 { + assert!( + doorbell_is_lit(&rx), + "observation {observation} must still see the level" + ); + } +} + +#[test] +fn arm_reports_unsafe_to_wait_while_items_remain() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("there is room"); + + assert!( + !rx.arm().expect("arming must succeed"), + "arming must refuse to bless a wait while an item is sitting there" + ); +} + +#[test] +fn arm_reports_safe_to_wait_when_empty() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // Created before the push, and that is the whole test. Without it the push + // takes `signal`'s "no event yet" path, the doorbell is never lit, and the + // assertion below that arming CLEARS it holds trivially -- it passed with + // `clear`'s `ResetEvent` deleted, which is how this was found. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + assert!( + doorbell_is_lit(&rx), + "the doorbell must be lit before a test of clearing it can mean anything" + ); + assert_eq!(rx.pop(), Ok(1)); + + assert!( + rx.arm().expect("arming must succeed"), + "a drained queue is safe to wait on" + ); + assert!( + !doorbell_is_lit(&rx), + "arming must clear the doorbell, or the next wait returns at once forever" + ); +} + +#[test] +fn arm_reports_safe_to_wait_on_an_empty_disconnected_queue() { + // **The exception `arm`'s contract has to state, and the reason the + // documented protocol needs a fourth step.** + // + // `arm` answers one question -- can a later *push* be missed -- and on a + // queue with no producers left the answer is trivially no, so it says + // `true`. Read as "safe to wait", which is what the contract used to say + // flatly, that is a permanent hang: the last producer's drop rings the + // doorbell exactly once, `arm` clears precisely that ring, and nothing + // remains to ring it again. + // + // `blocking::recv` has always had the missing step -- it checks + // disconnection and takes one last item before waiting. What was wrong was + // every *statement* of the protocol: the trait's contract, three shapes' + // method docs, three worked examples, and the README all described the + // three-step form a caller could follow into an indefinite wait. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + drop(tx); + + assert!( + rx.is_disconnected(), + "the producer is gone, so the stream has ended" + ); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and nothing is left to take" + ); + assert!( + rx.arm().expect("arming must succeed"), + "arm reports on missed pushes, not on the end of the stream -- so it \ + says `true` here, and a caller that treats that as permission to wait \ + indefinitely never wakes" + ); + assert!( + !doorbell_is_lit(&rx), + "and it has consumed the one-shot wakeup the producer's drop left, \ + which is what makes the wait permanent rather than merely long" + ); +} + +#[test] +fn arm_relights_the_doorbell_for_a_later_push() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + // As above: the first push must actually SET the mirror flag, or the claim + // that `clear` cleared it is a claim about a flag that was never set. + rx.doorbell().expect("the doorbell must be creatable"); + tx.push(1).expect("there is room"); + assert!(doorbell_is_lit(&rx), "the first push must light it"); + assert_eq!(rx.pop(), Ok(1)); + assert!(rx.arm().expect("arming must succeed")); + + // The signal that must never be skipped: the doorbell was cleared, so the + // producer's mirror flag has to have been cleared with it. + tx.push(2).expect("there is room"); + assert!( + doorbell_is_lit(&rx), + "the first push after a clear must light the doorbell again" + ); +} + +#[test] +fn an_item_pushed_before_the_doorbell_existed_is_still_found() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The lazy-creation hole. This push signals nothing, because there is no + // event yet to signal -- `crate::doorbell`'s suite asserts that directly. + tx.push(1).expect("there is room"); + assert!(!rx.shared.doorbell.is_armed(), "no event exists yet"); + + // Arming creates the event and only then checks emptiness, so the item is + // found instead of waited on. Had the check come first, this would report + // "safe to wait" and the consumer would block on an item already queued. + assert!( + !rx.arm().expect("arming must succeed"), + "arming must not bless a wait over an item that predates the doorbell" + ); +} + +// --------------------------------------------------------------------------- +// The lost-wakeup guard, verified by sabotage. +// +// `Consumer::arm` clears the doorbell and *then* checks emptiness. The reverse +// reads more naturally and is wrong. These two tests build the identical race +// against each order and assert that one produces a hang and the other does +// not, which is the only evidence that the order in `arm` is load-bearing +// rather than incidental. +// +// The race is driven deterministically on one thread rather than raced for on +// two: an interleaving that must be hit to prove a point is not one to leave to +// the scheduler. +// --------------------------------------------------------------------------- + +/// The sabotage: emptiness observed *before* the doorbell is cleared. +/// +/// Deliberately wrong, and called by nothing but the test that indicts it. +/// Mirrors [`Consumer::arm`] in every other respect, so the only difference +/// under test is the order of the two statements in the middle. +/// +/// `racing` runs in the window the wrong order opens -- between the emptiness +/// check and the clear. Passing it in makes the interleaving deterministic +/// rather than something two threads have to be lucky to produce. +fn arm_reversed_racing(consumer: &Consumer, racing: impl FnOnce()) -> bool { + consumer + .shared + .doorbell + .handle() + .expect("the doorbell must be creatable"); + let empty = consumer.is_empty(); + racing(); + consumer.shared.doorbell.clear(); + empty +} + +#[test] +fn reversing_the_clear_and_the_check_strands_an_item() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + // Step 1 of the protocol: the consumer drains and sees nothing. + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + + // The producer lands inside the window the wrong order opens: after the + // reversed check has already read `is_empty` as true, but before its clear + // erases the signal that push is about to raise. Driving it through + // `arm_reversed` keeps the sabotage a single definition rather than a + // paraphrase that could drift from the thing it is meant to indict. + let empty_before = arm_reversed_racing(&rx, || { + tx.push(1).expect("there is room"); + }); + + assert!( + empty_before, + "the reversed check saw an empty queue and would bless a wait" + ); + assert_eq!(rx.len(), 1, "yet the queue holds an item"); + assert!( + !doorbell_is_lit(&rx), + "and the doorbell is dark, so nothing will ever wake a waiter" + ); + + // Proof that this state really is a hang and not merely suspicious: a real + // wait against it times out. A generous 250 ms, because the assertion is + // "this never fires", not "this is slow". + let handle = rx.doorbell().expect("the doorbell must be creatable"); + // SAFETY: a live event handle borrowed for the call. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 250) }; + assert_eq!( + result, WAIT_TIMEOUT, + "a consumer that checked before clearing waits forever on a queue that \ + is not empty -- this is the lost wakeup, reproduced" + ); +} + +#[test] +fn clearing_before_the_check_finds_the_item_instead_of_waiting() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + assert_eq!(rx.pop(), Err(TryRecvError::Empty)); + + // The identical race, against the correct order. `arm` clears first, so the + // producer's push either lands before the clear and is caught by the check, + // or lands after it and lights a doorbell nobody is about to reset. + tx.push(1).expect("there is room"); + let safe_to_wait = rx.arm().expect("arming must succeed"); + + assert!( + !safe_to_wait, + "the check after the clear must see the item and refuse the wait" + ); + assert_eq!(rx.len(), 1, "the item is still there to be taken"); + assert_eq!(rx.pop(), Ok(1), "and taking it is what happens instead"); +} + +#[test] +fn a_push_after_arming_lights_a_doorbell_that_stays_lit() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert!( + rx.arm().expect("arming must succeed"), + "empty, so safe to wait" + ); + + // The other half of the window: a push landing after the clear. Nothing + // resets the doorbell between the signal and the wait, so the wait returns + // at once rather than blocking. + tx.push(1).expect("there is room"); + + let handle = rx.doorbell().expect("the doorbell must be creatable"); + // SAFETY: a live event handle borrowed for the call. + let result = unsafe { WaitForSingleObject(handle.as_raw_handle(), 250) }; + assert_eq!( + result, WAIT_OBJECT_0, + "a push after arming must wake a waiter immediately" + ); +} + +// --------------------------------------------------------------------------- +// Blocking receive. +// --------------------------------------------------------------------------- + +#[test] +fn recv_returns_an_item_already_queued() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(7).expect("there is room"); + assert_eq!(rx.recv().expect("an item is queued"), 7); +} + +#[test] +fn recv_blocks_until_a_push_arrives() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let producer = thread::spawn(move || { + // A short sleep so the consumer is genuinely parked rather than racing + // to the first `pop`. Correctness does not depend on winning that race + // -- it depends on the wakeup arriving either way. + thread::sleep(Duration::from_millis(50)); + tx.push(99).expect("there is room"); + }); + + assert_eq!(rx.recv().expect("the producer pushes"), 99); + producer.join().expect("the producer must not panic"); +} + +#[test] +fn recv_reports_disconnection_once_drained() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "an empty queue with no producer is finished" + ); +} + +#[test] +fn recv_delivers_items_pushed_before_the_producer_dropped() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("there is room"); + tx.push(2).expect("there is room"); + drop(tx); + + // Disconnection must not discard what was already sent. Testing the flag + // before draining is the mistake this guards. + assert_eq!(rx.recv().expect("item one is owed"), 1); + assert_eq!(rx.recv().expect("item two is owed"), 2); + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "and only then is it finished" + ); +} + +#[test] +fn a_blocked_recv_is_released_by_the_producer_dropping() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let producer = thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + drop(tx); + }); + + // Without a signal in the producer's `Drop` this hangs forever: the queue + // would be correct and the program would still be wedged. + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "dropping the producer must wake a parked consumer" + ); + producer.join().expect("the producer must not panic"); +} + +#[test] +fn recv_timeout_gives_up_on_an_empty_live_queue() { + let (_tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let started = Instant::now(); + let result = rx.recv_timeout(Duration::from_millis(60)); + + assert!( + matches!(result, Err(RecvTimeoutError::Timeout)), + "an empty queue with a live producer times out rather than ending" + ); + assert!( + result.is_err_and(|error| error.is_retryable()), + "and a timeout is worth retrying, unlike the other two variants" + ); + assert!( + started.elapsed() >= Duration::from_millis(50), + "it must actually have waited rather than returned at once" + ); +} + +#[test] +fn recv_timeout_returns_an_item_that_arrives_in_time() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let producer = thread::spawn(move || { + thread::sleep(Duration::from_millis(30)); + tx.push(5).expect("there is room"); + }); + + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)) + .expect("the push lands well inside the deadline"), + 5 + ); + producer.join().expect("the producer must not panic"); +} + +#[test] +fn recv_timeout_reports_disconnection_rather_than_waiting_out_the_clock() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + + let started = Instant::now(); + let result = rx.recv_timeout(Duration::from_secs(30)); + + assert!( + matches!(result, Err(RecvTimeoutError::Disconnected)), + "a finished queue is finished, deadline or not" + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "and it must be reported at once rather than after the deadline" + ); +} + +#[test] +fn a_blocking_consumer_receives_every_item_in_order() { + // The whole mechanism under load: a capacity far smaller than the run, so + // the producer blocks on a full queue and the consumer blocks on an empty + // one, repeatedly, in both directions. + const COUNT: u32 = 10_000; + let (tx, rx) = bounded::(16).expect("16 is a valid capacity"); + + let producer = thread::spawn(move || { + for value in 0..COUNT { + let mut item = value; + while let Err(PushError::Full(returned)) = tx.push(item) { + item = returned; + std::hint::spin_loop(); + } + } + }); + + for expected in 0..COUNT { + assert_eq!( + rx.recv().expect("the producer is still sending"), + expected, + "items must arrive exactly once and in order" + ); + } + producer.join().expect("the producer must not panic"); + + assert!( + matches!(rx.recv(), Err(RecvError::Disconnected)), + "and the stream ends cleanly once the producer is gone" + ); +} + +#[test] +fn the_owned_doorbell_outlives_the_consumers_use_of_it() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let owned = rx.doorbell_owned().expect("duplication must succeed"); + tx.push(1).expect("there is room"); + + // SAFETY: `owned` is a live event handle; a zero timeout returns at once. + let result = unsafe { WaitForSingleObject(owned.as_raw_handle(), 0) }; + assert_eq!( + result, WAIT_OBJECT_0, + "a caller holding its own duplicate must see the queue's signals" + ); +} + +#[test] +fn the_final_drain_returns_an_item_that_raced_the_disconnection() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + + // The race `Consumer::finish` guards, reconstructed rather than waited for: + // a producer pushed and then dropped in the window between a receive's + // first `pop` and its disconnection check. At this point the queue reports + // disconnected *and* holds an item. + tx.push(1).expect("there is room"); + drop(tx); + assert!(rx.is_disconnected(), "the producer is gone"); + + assert_eq!( + rx.finish(), + Some(1), + "the end of the stream must not discard an item that was sent before it" + ); + assert_eq!( + rx.finish(), + None, + "and once genuinely drained, the answer is final" + ); +} + +#[test] +fn the_final_drain_is_empty_when_nothing_was_sent() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + assert_eq!( + rx.finish(), + None, + "nothing was ever sent, so nothing is owed" + ); +} + +#[test] +fn recv_timeout_does_not_panic_on_an_unrepresentable_deadline() { + // `Instant + Duration` panics when the sum is not representable, and + // `Duration::MAX` is an ordinary way to spell "effectively forever". The + // queue is disconnected up front so the call has a reason to return at all; + // the assertion is that it returns rather than aborting the process. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + drop(tx); + + assert!( + matches!( + rx.recv_timeout(Duration::MAX), + Err(RecvTimeoutError::Disconnected) + ), + "an unrepresentable deadline must degrade to the untimed wait it asked for" + ); +} + +#[test] +fn recv_timeout_delivers_an_item_under_an_unrepresentable_deadline() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(3).expect("there is room"); + + // The degraded path must still be a working receive, not merely one that + // does not panic. + assert_eq!( + rx.recv_timeout(Duration::from_secs(u64::MAX)) + .expect("an item is queued"), + 3 + ); +} + +#[test] +fn a_suggested_capacity_is_one_the_constructor_would_accept() { + // The suggestion exists so a caller can correct the call. One that is + // itself refused is worse than none, because the caller acts on it. + // + // Asks `validate_capacity` rather than `bounded`, and rather than + // re-listing the rules here. Calling `bounded` would be a truer test of the + // real path, but a suggestion near the bound is 2^62, and constructing that + // queue means asking for half the address space -- the first version of + // this test aborted the process with a four-exabyte allocation failure. + for requested in [1_usize, 3, 100, 1000, 0, usize::MAX / 2, usize::MAX] { + let Err(error) = validate_capacity(requested, BOUNDS) else { + continue; + }; + if let Some(previous) = error.previous_valid() { + assert!( + validate_capacity(previous, BOUNDS).is_ok(), + "previous_valid() for {requested} suggested {previous}, which is itself rejected" + ); + } + if let Some(next) = error.next_valid() { + assert!( + validate_capacity(next, BOUNDS).is_ok(), + "next_valid() for {requested} suggested {next}, which is itself rejected" + ); + } + } +} + +#[test] +fn the_largest_request_is_clamped_rather_than_rounded() { + // Rounding `usize::MAX` down to the nearest power of two gives 2^63, which + // is larger than the largest representable capacity. Before this was fixed + // the suggestion was exactly that unusable value. + let error = + validate_capacity(usize::MAX, BOUNDS).expect_err("usize::MAX is not a valid capacity"); + let previous = error + .previous_valid() + .expect("there is a valid capacity below usize::MAX"); + + assert!( + previous <= error.max_valid(), + "the suggestion {previous} must not exceed the shape's own bound {}", + error.max_valid() + ); + assert!( + previous.is_power_of_two(), + "and it must still be a power of two" + ); + assert!( + validate_capacity(previous, BOUNDS).is_ok(), + "and must be accepted" + ); +} + +#[test] +fn the_real_arm_finds_an_item_that_lands_inside_its_window() { + // The deterministic indictment of the reversed order, driven through the + // REAL `Consumer::arm` rather than through a copy of it. + // + // The hook fires between `arm`'s clear and its emptiness check -- precisely + // the window a producer must hit for the hazard to bite. With the correct + // order the check follows the push and finds it, so arming refuses to bless + // a wait. With the two statements swapped the check has already happened, + // arming returns "safe to wait", and the consumer parks on a queue holding + // an item whose signal the clear erased. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + // The hook owns the producer outright. An `Arc` would be pointless here and + // clippy says so: a `Producer` is deliberately `!Sync`, so sharing one is + // exactly what the type system is built to prevent. + let safe_to_wait = race_hooks::ARM.with( + move || { + tx.push(1).expect("there is room"); + }, + || rx.arm().expect("arming must succeed"), + ); + + assert!( + !safe_to_wait, + "an item landing between the clear and the check must be found, not waited past" + ); +} + +#[test] +fn the_real_arm_still_blesses_a_wait_when_its_window_stays_empty() { + // The complement, so the test above cannot pass by `arm` simply never + // blessing a wait -- which would satisfy it while breaking every consumer. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + drop(tx); + + let safe_to_wait = race_hooks::ARM.with(|| {}, || rx.arm().expect("arming must succeed")); + + assert!( + safe_to_wait, + "an empty queue must still be safe to wait on, or the wait never happens at all" + ); +} + +// --------------------------------------------------------------------------- +// Reservation. +// +// The mechanism here is a plain counter written only by the producer's thread, +// where `reserving_mpsc` needs a compare-and-swap against a packed word. The +// two implementations share nothing, so the guarantee has to be asserted +// separately on each -- a point made empirically rather than by argument: the +// sabotage sweep found this whole section missing, because the reserving_mpsc +// tests covered the slotwise_mpsc path and left this one unguarded. +// --------------------------------------------------------------------------- + +/// Fills every slot the best-effort path is allowed to take, and reports how +/// many went in. +fn fill(producer: &Producer) -> usize { + let mut pushed = 0; + while producer.push(0).is_ok() { + pushed += 1; + } + pushed +} + +#[test] +fn a_reservation_withholds_a_slot_from_the_best_effort_path() { + let (tx, _rx) = bounded::(8).expect("8 is a valid capacity"); + assert_eq!( + fill(&tx), + 8, + "with nothing reserved, every slot is available" + ); + + let (tx, _rx) = bounded::(8).expect("8 is a valid capacity"); + let reservations: Vec<_> = (0..3).map(|_| tx.reserve().expect("room")).collect(); + assert_eq!(tx.outstanding_reservations(), 3); + assert_eq!( + fill(&tx), + 5, + "three reserved leaves five for the best-effort path" + ); + drop(reservations); +} + +#[test] +fn a_reserved_slot_is_delivered_into_a_queue_that_is_otherwise_full() { + // The contract in one test: reserve, let the best-effort path take + // everything it is allowed to, and redeem anyway. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("a fresh queue has room"); + + assert_eq!(fill(&tx), 3, "the reservation withheld exactly one slot"); + assert!(tx.is_full(), "and now nothing more may be pushed"); + + slot.send(99).expect("the room was already ours"); + + let drained: Vec = rx.try_iter().collect(); + assert_eq!( + drained, + vec![0, 0, 0, 99], + "the reserved item lands where it was redeemed, not where it was claimed" + ); +} + +#[test] +fn a_push_refused_for_a_reservation_is_still_reported_as_full() { + // A best-effort caller cannot tell "no slots" from "the only slot is + // reserved", and should not have to: both mean "no room for you". + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _slot = tx.reserve().expect("room"); + tx.push(1).expect("one slot is unreserved"); + + assert!( + matches!(tx.push(2), Err(PushError::Full(2))), + "the reserved slot is not available to the best-effort path" + ); +} + +#[test] +fn dropping_a_reservation_returns_the_slot() { + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + assert_eq!(tx.outstanding_reservations(), 1); + + drop(slot); + assert_eq!(tx.outstanding_reservations(), 0); + assert_eq!( + fill(&tx), + 4, + "a released reservation is capacity given back, not capacity lost" + ); +} + +#[test] +fn a_redeemed_reservation_does_not_also_release_its_slot() { + // The double-release bug `send` avoids by consuming `self` and suppressing + // the drop. If both ran, the count would underflow and the queue would + // over-admit for ever afterwards. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + for round in 0..10 { + let slot = tx.reserve().expect("room"); + assert_eq!(tx.outstanding_reservations(), 1); + slot.send(round).expect("the room was ours"); + assert_eq!( + tx.outstanding_reservations(), + 0, + "redeeming releases the claim exactly once" + ); + assert_eq!(rx.pop(), Ok(round)); + } + assert_eq!(fill(&tx), 4, "and the capacity is intact after ten cycles"); +} + +#[test] +fn reserving_fails_when_every_slot_is_spoken_for() { + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _first = tx.reserve().expect("room"); + let _second = tx.reserve().expect("room"); + assert!( + tx.reserve().is_none(), + "reservations are drawn from the same capacity as everything else" + ); + + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + fill(&tx); + assert!( + tx.reserve().is_none(), + "and a full queue has nothing left to promise" + ); +} + +#[test] +fn a_reservation_survives_many_wraps_of_the_ring() { + // The counter must be independent of the positions, so hundreds of laps + // beneath a held reservation must not disturb it. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + + for round in 0..500 { + tx.push(round).expect("three slots remain unreserved"); + assert_eq!(rx.pop(), Ok(round)); + assert_eq!(tx.outstanding_reservations(), 1, "round {round}"); + } + + slot.send(99).expect("still ours after five hundred laps"); + assert_eq!(rx.pop(), Ok(99)); +} + +#[test] +fn redeeming_into_a_departed_consumer_hands_the_item_back() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + drop(rx); + + assert!(slot.is_disconnected()); + let error = slot.send(7).expect_err("nobody is left to take it"); + assert_eq!( + error.into_inner(), + 7, + "an item important enough to reserve for must not be dropped silently" + ); +} + +#[test] +fn a_reserved_delivery_lights_the_doorbell() { + // A reserved send is a delivery like any other, so it must ring. If it did + // not, a consumer parked on the doorbell would sleep through precisely the + // message that was important enough to reserve a slot for. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + let slot = tx.reserve().expect("room"); + assert!(rx.arm().expect("arming must succeed"), "nothing yet"); + + slot.send(1).expect("the room was ours"); + assert!( + doorbell_is_lit(&rx), + "a reserved delivery must ring like any other" + ); + assert!( + !rx.arm().expect("arming must succeed"), + "and must be visible to the arming protocol" + ); +} + +#[test] +fn a_blocked_consumer_is_woken_by_a_reserved_delivery() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("room"); + + // **The reservation cannot cross a thread boundary, and the compiler says + // so**: it borrows a producer that is not `Sync`, so `&Producer` is not + // `Send` and even a scoped thread is refused. That is the borrow doing its + // job rather than an inconvenience -- see the `compile_fail` doctest on + // `Producer::reserve`, which asserts the refusal directly. + // + // So the *consumer* goes across instead. It is a separate handle and is + // `Send`, which is what makes this test expressible at all. + let receiver = thread::spawn(move || rx.recv()); + thread::sleep(Duration::from_millis(50)); + slot.send(7).expect("the consumer is alive"); + + assert_eq!( + receiver + .join() + .expect("the consumer must not panic") + .expect("the reservation is redeemed"), + 7, + "a parked consumer must be woken by a reserved delivery" + ); +} + +#[test] +fn an_abandoned_reservation_leaves_the_queue_usable() { + // A reservation that fails to be redeemed must not poison the capacity. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + for _ in 0..50 { + let slot = tx.reserve().expect("room"); + drop(slot); + } + assert_eq!(tx.outstanding_reservations(), 0); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert!(tx.push(3).is_err()); + assert_eq!(rx.pop(), Ok(1)); +} + +#[test] +fn dropping_the_queue_drops_a_reserved_item_it_still_holds() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(8).expect("a power-of-two capacity"); + let slot = tx.reserve().expect("room"); + for _ in 0..5 { + tx.push(DropCounter(Arc::clone(&drops))).expect("room"); + } + slot.send(DropCounter(Arc::clone(&drops))) + .expect("the room was ours"); + assert_eq!(drops.load(Ordering::Relaxed), 0, "nothing dropped yet"); + } + assert_eq!( + drops.load(Ordering::Relaxed), + 6, + "every undrained item must be dropped, including the reserved one" + ); +} + +// --------------------------------------------------------------------------- +// Teardown: what becomes of items nobody drained. +// +// The disposal policy's own behaviour is covered in `crate::disposal`'s suite. +// What is asserted here is that THIS shape's teardown walk actually reaches it +// -- each shape finds its survivors by walking its own layout, so covering one +// says nothing about the others. +// --------------------------------------------------------------------------- + +/// Records that it was destroyed, and where. +/// +/// The distinction the whole mechanism turns on is "handed to the owner" versus +/// "destructor run by whichever thread dropped last", so a test needs to be able +/// to tell those apart rather than merely count survivors. +#[derive(Debug)] +struct Tracked { + id: u32, + destroyed: Arc, +} + +impl Drop for Tracked { + fn drop(&mut self) { + self.destroyed.fetch_add(1, Ordering::Relaxed); + } +} + +#[test] +fn undrained_items_reach_the_disposal_sink_instead_of_being_destroyed() { + let destroyed = Arc::new(AtomicUsize::new(0)); + let (undelivered, reaper) = std::sync::mpsc::channel(); + + { + let (tx, _rx) = bounded_with::( + 8, + Options::new().disposal(Disposal::new(move |item| { + // Moved out of teardown rather than destroyed in it, which is + // the entire point: the owner now decides when and where. + let _ = undelivered.send(item); + })), + ) + .expect("8 is a valid capacity"); + + for id in 0..5 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + + let rescued: Vec = reaper.iter().map(|item| item.id).collect(); + assert_eq!( + rescued, + vec![0, 1, 2, 3, 4], + "every undrained item must reach the sink, in queue order" + ); + assert_eq!( + destroyed.load(Ordering::Relaxed), + 5, + "and be destroyed only once the owner has finished with them" + ); +} + +#[test] +fn only_the_undrained_items_reach_the_sink() { + // What the consumer already took is the consumer's, and must not be + // reported as abandoned. + let (undelivered, reaper) = std::sync::mpsc::channel(); + let destroyed = Arc::new(AtomicUsize::new(0)); + + { + let (tx, rx) = bounded_with::( + 8, + Options::new().disposal(Disposal::new(move |item: Tracked| { + let _ = undelivered.send(item.id); + })), + ) + .expect("8 is a valid capacity"); + + for id in 0..5 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + assert_eq!(rx.pop().expect("an item").id, 0); + assert_eq!(rx.pop().expect("an item").id, 1); + } + + assert_eq!( + reaper.iter().collect::>(), + vec![2, 3, 4], + "the two the consumer took are not abandoned items" + ); +} + +#[test] +fn an_empty_queue_hands_nothing_to_the_sink() { + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + tx.push(1).expect("room"); + assert_eq!(rx.pop(), Ok(1)); + } + assert_eq!( + reaper.iter().collect::>(), + Vec::::new(), + "a queue drained to empty has nothing to account for" + ); +} + +#[test] +fn the_sink_sees_survivors_after_the_ring_has_wrapped() { + // The teardown walk is over a wrapped range, which is where an index error + // would show up as the wrong items rather than as a crash. + let (undelivered, reaper) = std::sync::mpsc::channel(); + { + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + for round in 0..6 { + tx.push(round).expect("room"); + rx.pop().expect("an item"); + } + for round in 100..103 { + tx.push(round).expect("room"); + } + } + assert_eq!( + reaper.iter().collect::>(), + vec![100, 101, 102], + "the survivors are the resident range, not the whole slot array" + ); +} + +#[test] +fn a_queue_torn_down_by_the_producer_still_reaches_the_sink() { + // Which handle happens to die last is not knowable in advance, and the + // guarantee must not depend on it. Here the consumer goes first, so the + // producer's drop is what tears the queue down. + let (undelivered, reaper) = std::sync::mpsc::channel(); + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + tx.push(1).expect("room"); + tx.push(2).expect("room"); + drop(rx); + drop(tx); + + assert_eq!( + reaper.iter().collect::>(), + vec![1, 2], + "teardown accounts for the survivors whichever handle releases last" + ); +} + +#[test] +fn a_queue_torn_down_on_another_thread_still_reaches_the_sink() { + // The dropping thread is whichever one happens to release last, which is + // exactly why disposal cannot be left to it implicitly. + let (undelivered, reaper) = std::sync::mpsc::channel(); + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item| { + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + tx.push(1).expect("room"); + drop(rx); + + thread::spawn(move || drop(tx)) + .join() + .expect("the dropping thread must not panic"); + + assert_eq!(reaper.iter().collect::>(), vec![1]); +} + +#[test] +fn without_a_sink_undrained_items_are_destroyed_in_place() { + // The default, asserted rather than assumed -- it is the behaviour every + // existing caller has, and the reason a queue of `u32` need not think about + // any of this. + let destroyed = Arc::new(AtomicUsize::new(0)); + { + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + for id in 0..3 { + tx.push(Tracked { + id, + destroyed: Arc::clone(&destroyed), + }) + .expect("room"); + } + } + assert_eq!( + destroyed.load(Ordering::Relaxed), + 3, + "with no sink there is nowhere else for them to go" + ); +} + +// --------------------------------------------------------------------------- +// The hazard itself, stated as a test rather than as a paragraph. +// +// The claim disposal exists to make good on is not "the sink receives the +// items" -- that is the mechanism. The claim is that **a destructor which +// blocks does not run on whichever thread happened to release the last +// handle**, because that thread may be a pool callback that must not block. So +// these two assert where the destructor actually runs, with a control proving +// the test can tell the difference. +// --------------------------------------------------------------------------- + +/// Records the thread its destructor ran on. +#[derive(Debug)] +struct ThreadWitness(Arc>>); + +impl Drop for ThreadWitness { + fn drop(&mut self) { + self.0 + .lock() + .expect("no test holds this poisoned") + .push(std::thread::current().id()); + } +} + +#[test] +fn a_sink_keeps_the_destructor_off_the_thread_that_tore_the_queue_down() { + let ran_on = Arc::new(std::sync::Mutex::new(Vec::new())); + let (undelivered, reaper) = std::sync::mpsc::channel(); + + let (tx, rx) = bounded_with::( + 4, + Options::new().disposal(Disposal::new(move |item: ThreadWitness| { + // The sink's whole job: move it somewhere a thread that may block + // will find it. Nothing here runs the destructor. + let _ = undelivered.send(item); + })), + ) + .expect("4 is a valid capacity"); + + tx.push(ThreadWitness(Arc::clone(&ran_on))).expect("room"); + + // Tear the queue down somewhere that is emphatically not this thread, + // standing in for the pool callback that must not block. + let teardown_thread = thread::spawn(move || { + drop(rx); + drop(tx); + std::thread::current().id() + }) + .join() + .expect("the tearing-down thread must not panic"); + + assert!( + ran_on.lock().expect("not poisoned").is_empty(), + "the destructor must not have run yet: the item is the owner's now, and \ + the thread that dropped the queue has already moved on" + ); + + // The owner takes delivery here, and *this* is where the destructor runs. + let rescued = reaper.recv().expect("the sink was handed the survivor"); + drop(rescued); + + let ran_on = ran_on.lock().expect("not poisoned"); + assert_eq!(ran_on.len(), 1); + assert_ne!( + ran_on[0], teardown_thread, + "a blocking destructor must not run on the thread that released the last handle" + ); + assert_eq!( + ran_on[0], + std::thread::current().id(), + "it runs where the owner chose to take delivery" + ); +} + +#[test] +fn without_a_sink_the_destructor_does_run_on_the_thread_that_tore_the_queue_down() { + // The control. Without it the test above could pass for the wrong reason -- + // it would look identical if destructors simply never ran anywhere + // observable. This is also the honest statement of the default: it is not + // that nothing blocks, it is that the blocking lands on a thread nobody + // chose. + let ran_on = Arc::new(std::sync::Mutex::new(Vec::new())); + + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(ThreadWitness(Arc::clone(&ran_on))).expect("room"); + + let teardown_thread = thread::spawn(move || { + drop(rx); + drop(tx); + std::thread::current().id() + }) + .join() + .expect("the tearing-down thread must not panic"); + + let ran_on = ran_on.lock().expect("not poisoned"); + assert_eq!(ran_on.len(), 1, "the item was destroyed at teardown"); + assert_eq!( + ran_on[0], teardown_thread, + "and with no sink it was destroyed on whichever thread released last, \ + which is exactly the behaviour a disposal sink exists to replace" + ); +} + +// --------------------------------------------------------------------------- +// Observability. +// +// The counters' arithmetic is covered in `crate::metrics`. What is asserted +// here is that this shape *feeds* them from the right places -- and, for the +// doorbell, that the number reports syscalls rather than signal attempts, +// which is what makes the skip rule measurable rather than assumed. +// --------------------------------------------------------------------------- + +#[test] +fn refusals_are_counted_but_disconnections_are_not() { + // The two are different facts and must not be summed. A full queue is + // backpressure; a departed consumer is the end of the stream, and a queue + // shutting down should not read as an overloaded one. + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert_eq!(tx.refused(), 0); + + tx.push(1).expect("room"); + tx.push(2).expect("room"); + assert!(tx.push(3).is_err()); + assert!(tx.push(4).is_err()); + assert_eq!(tx.refused(), 2, "two pushes were refused for want of room"); + assert_eq!(rx.refused(), 2, "and both handles report the same queue"); + + drop(rx); + assert!(matches!(tx.push(5), Err(PushError::Disconnected(5)))); + assert_eq!( + tx.refused(), + 2, + "a push refused because the consumer is gone is not a loss to backpressure" + ); +} + +#[test] +fn high_water_is_untracked_by_default() { + // Off unless asked for, because it is the one metric that cannot be made + // free. `None` rather than `Some(0)` so a caller cannot mistake "nobody was + // counting" for "it never filled". + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert_eq!(tx.high_water(), None); + assert_eq!(rx.high_water(), None); +} + +#[test] +fn high_water_records_the_peak_when_asked_for() { + let (tx, rx) = bounded_with::(8, Options::new().tracking_high_water()) + .expect("8 is a valid capacity"); + assert_eq!(tx.high_water(), Some(0), "counting, and nothing seen yet"); + + for value in 0..5 { + tx.push(value).expect("room"); + } + assert_eq!(tx.high_water(), Some(5)); + + // Draining does not lower it: the peak is a fact about the past. + while rx.pop().is_ok() {} + assert_eq!(rx.len(), 0); + assert_eq!( + rx.high_water(), + Some(5), + "the mark is the deepest it got, not the depth right now" + ); + + // And a smaller later burst does not replace it. + tx.push(0).expect("room"); + tx.push(1).expect("room"); + assert_eq!(tx.high_water(), Some(5)); +} + +#[test] +fn high_water_counts_reserved_deliveries_like_any_other() { + let (tx, rx) = bounded_with::(4, Options::new().tracking_high_water()) + .expect("4 is a valid capacity"); + + let slot = tx.reserve().expect("room"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + slot.send(3).expect("the room was ours"); + + assert_eq!( + tx.high_water(), + Some(3), + "a redeemed reservation is an ordinary queued item, and counts as depth like one" + ); + assert_eq!(rx.len(), 3); +} + +#[test] +fn a_poll_only_consumer_rings_no_doorbells() { + // The laziness being visible rather than a gap: a consumer that never asks + // for the handle never creates the event, so there is nothing to ring. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + for value in 0..4 { + tx.push(value).expect("room"); + } + while rx.pop().is_ok() {} + + assert_eq!( + rx.doorbell_rings(), + 0, + "no kernel object was created, so no signal was ever issued" + ); +} + +#[test] +fn the_ring_count_reports_syscalls_rather_than_signal_attempts() { + // **The number the skip rule is measured by.** Four pushes against a + // doorbell nobody clears is one real `SetEvent` and three skips, because a + // manual-reset event does not count and setting an already-set one changes + // nothing. If this ever reported four, the skip would have stopped + // happening -- which is exactly what the sabotage entry for it asserts. + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + for value in 0..4 { + tx.push(value).expect("room"); + } + + assert_eq!( + rx.doorbell_rings(), + 1, + "the first push lit it; the other three had nothing to do" + ); +} + +#[test] +fn clearing_the_doorbell_makes_the_next_push_ring_again() { + // The complement: the count must not be stuck at one. Each drain-and-arm + // cycle costs exactly one more ring, which is the shape a parked consumer + // actually produces. + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + rx.doorbell().expect("the doorbell must be creatable"); + + for round in 1..=3 { + tx.push(round).expect("room"); + tx.push(round).expect("room"); + assert_eq!( + rx.doorbell_rings(), + round as u64, + "round {round}: one ring per cycle, not one per push" + ); + while rx.pop().is_ok() {} + assert!(rx.arm().expect("arming must succeed")); + } +} + +#[test] +fn the_debug_renderings_name_the_type_and_its_state() { + // A `Debug` that writes nothing satisfies any test which only checks that + // formatting does not panic, and a mutation run found exactly that constant + // alive on every handle in this crate. These are the diagnostic surface a + // reader reaches for when a queue is stuck, so an empty rendering is the + // moment it is least affordable. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + + let producer = format!("{tx:?}"); + assert!(producer.contains("spsc::Producer"), "got {producer}"); + assert!(producer.contains('4'), "the capacity must show: {producer}"); + + let consumer = format!("{rx:?}"); + assert!(consumer.contains("spsc::Consumer"), "got {consumer}"); + + let reservation = tx.reserve().expect("there is room"); + let rendered = format!("{reservation:?}"); + assert!(rendered.contains("spsc::Reservation"), "got {rendered}"); +} + +// --------------------------------------------------------------------------- +// The gauges: reservations withdraw capacity without becoming items, and the +// two position loads are not one instant. +// --------------------------------------------------------------------------- + +#[test] +fn remaining_subtracts_outstanding_reservations() { + // The defect. `Bounded`'s default is `capacity - len`, and a reservation + // withdraws a slot without becoming an item -- so with every slot reserved + // the default answered the full capacity while both `push` and `reserve` + // refuse. This shape reserves too, which is exactly what made it easy to + // miss when the sibling was fixed. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let slot = tx.reserve().expect("an empty queue has room"); + + assert_eq!(tx.len(), 0, "a reservation is not an item"); + assert_eq!(tx.remaining(), 3, "one of the four slots is spoken for"); + assert_eq!( + crate::Bounded::remaining(&rx), + 3, + "both handles describe the same queue and must agree" + ); + + for i in 0..3 { + tx.push(i).expect("remaining() said there was room"); + } + assert_eq!(tx.remaining(), 0); + assert!(tx.is_full()); + assert!(matches!(tx.push(99), Err(PushError::Full(99)))); + + slot.send(7).expect("the consumer is still here"); + assert_eq!(rx.len(), 4, "the redeemed reservation is now an item"); +} + +#[test] +fn remaining_is_zero_when_every_slot_is_reserved() { + // The case the finding named directly: reserve everything, and the queue is + // empty of items yet has no room at all. + let (tx, _rx) = bounded::(2).expect("2 is a valid capacity"); + let _first = tx.reserve().expect("room"); + let _second = tx.reserve().expect("room"); + + assert_eq!(tx.len(), 0, "no item has been sent"); + assert_eq!( + tx.remaining(), + 0, + "every slot is spoken for, so nothing further fits" + ); + assert!(tx.is_full()); + assert!(tx.reserve().is_none(), "and no further slot can be claimed"); +} + +#[test] +fn remaining_agrees_through_the_bounded_trait() { + // The override is on the trait impls, not only the inherent methods: a + // caller generic over `Bounded` is exactly who would be misled by the + // default, since it cannot reach `outstanding_reservations` to correct it. + fn room_through_trait(handle: &B) -> usize { + handle.remaining() + } + + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + let _slot = tx.reserve().expect("an empty queue has room"); + + assert_eq!(room_through_trait(&tx), 3); + assert_eq!(room_through_trait(&rx), 3); +} + +#[test] +fn len_is_clamped_when_head_has_passed_the_sampled_tail() { + // `len` reads `tail` and then `head`, which are two instants rather than + // one. If the consumer drains past the value `tail` held, `head` overtakes + // it and the wrapping subtraction yields a number near `usize::MAX`. + // + // The skewed pair is written directly rather than raced for: it is a + // transient a reader observes, not a state the queue rests in. + let (tx, _rx) = bounded::(4).expect("4 is a valid capacity"); + + tx.shared.tail.0.store(1, Ordering::Release); + tx.shared.head.0.store(2, Ordering::Release); + + assert_eq!( + tx.len(), + tx.capacity(), + "a bounded queue must never report holding more than it can" + ); + assert_eq!(tx.remaining(), 0, "the clamp resolves towards full"); + + // Restored before the handles drop: teardown walks `head..tail`, and an + // inverted pair sets it a `usize::MAX`-length loop that hangs rather than + // fails. + tx.shared.head.0.store(0, Ordering::Release); + tx.shared.tail.0.store(0, Ordering::Release); +} + +#[test] +fn the_gauges_are_exact_when_nothing_is_reserved_or_skewed() { + // The guard must not have been bought by clamping or subtracting always. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("room"); + tx.push(2).expect("room"); + + assert_eq!(tx.len(), 2); + assert_eq!(tx.remaining(), 2); + assert!(!tx.is_full()); + assert_eq!(rx.pop(), Ok(1)); + assert_eq!(tx.len(), 1); + assert_eq!(tx.remaining(), 3); +} + +// --------------------------------------------------------------------------- +// The surface added after comparing this crate against the published queue +// crates. Repeated per shape rather than tested once on one of them: each shape +// is a separate implementation of `pop` and of the `Bounded` accessors, so a +// test on a sibling says nothing about this one. A mutation run found exactly +// that -- the versions written only for `slotwise_mpsc` left this shape's +// `pop` guard and `is_full` uncovered. +// --------------------------------------------------------------------------- + +#[test] +fn an_empty_queue_is_distinguishable_from_a_finished_one() { + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + assert_eq!( + rx.pop(), + Err(TryRecvError::Empty), + "empty with a producer alive is a reason to try again" + ); + + drop(tx); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "empty with every producer gone is a reason to stop" + ); +} + +#[test] +fn a_departed_producers_items_are_delivered_before_the_disconnection() { + // A producer may push and then drop, so a queue can be disconnected and + // still owe items. Reporting the end of the stream while any remain would + // lose the tail. + let (tx, rx) = bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one item does not fill four slots"); + drop(tx); + + assert_eq!(rx.pop(), Ok(1), "the items come first"); + assert_eq!(rx.pop(), Ok(2)); + assert_eq!( + rx.pop(), + Err(TryRecvError::Disconnected), + "and only then the end of the stream" + ); +} + +#[test] +fn is_full_agrees_across_the_trait_and_both_handles() { + let (tx, rx) = bounded::(2).expect("2 is a valid capacity"); + assert!(!tx.is_full()); + assert!(!rx.is_full()); + + tx.push(1).expect("an empty queue has room"); + tx.push(2).expect("one slot remains"); + + assert!(tx.is_full(), "the producer sees a full queue"); + assert!(rx.is_full(), "and so does the consumer"); + assert!( + Bounded::is_full(&tx), + "and so does a caller generic over the trait" + ); + assert!(Bounded::is_full(&rx)); + + assert_eq!(rx.pop(), Ok(1)); + assert!( + !tx.is_full(), + "and it is no longer full once a slot is freed" + ); +} + +#[test] +fn try_iter_and_drain_are_the_same_iterator_and_need_no_import() { + let (tx, rx) = bounded::(8).expect("8 is a valid capacity"); + for value in 0..4u32 { + tx.push(value).expect("four items fit in eight slots"); + } + let taken: Vec = rx.try_iter().collect(); + assert_eq!(taken, vec![0, 1, 2, 3]); + + for value in 4..6u32 { + tx.push(value).expect("room remains"); + } + let taken: Vec = rx.drain().collect(); + assert_eq!(taken, vec![4, 5]); +} diff --git a/crates/windows-waitable-queues/src/traits.rs b/crates/windows-waitable-queues/src/traits.rs new file mode 100644 index 00000000..049a0e8a --- /dev/null +++ b/crates/windows-waitable-queues/src/traits.rs @@ -0,0 +1,458 @@ +// Copyright (c) Mike Grier. + +//! The capability traits, each naming one thing a queue can do. +//! +//! # Narrow, on the `std::io` model +//! +//! There is deliberately no single `WaitableQueue` trait. `std::io` does not +//! have one `Io` trait either; it has [`Read`](std::io::Read), +//! [`Write`](std::io::Write) and [`Seek`](std::io::Seek), and a type implements +//! the subset it genuinely has. The same choice is forced here rather than +//! merely preferred, because a fat trait is *unimplementable* by shapes this +//! crate plans to ship: a queue that is never waited on has no doorbell to +//! return, and an unbounded one has no capacity to report. Recorded as +//! [D-2](../DESIGN-NOTES.md#d-2). +//! +//! What that buys is a consumer generic over exactly what it needs. A drainer +//! that parks on a queue asks for [`Consumer`] and [`Waitable`], and stays +//! usable against a shape that has never heard of reservation or loss +//! reporting. +//! +//! # Why they arrive with the second shape and not the first +//! +//! A trait written against one implementation designs in a vacuum: every +//! signature that type happens to have looks like a requirement, and nothing +//! tests whether the abstraction is the right one. So the trait *shape* was +//! fixed in prose when `spsc` was written -- the signatures were spelled out in +//! its module documentation before the type existed -- and the traits +//! themselves waited for `slotwise_mpsc` to exist to be checked against. That is +//! [D-3](../DESIGN-NOTES.md#d-3), and the check it demands is not rhetorical: +//! `slotwise_mpsc` is a lock-free multi-producer array queue with no structural +//! resemblance to `spsc` beyond its interface, so a signature that fitted only +//! the first shape would have failed here rather than in a consumer's code. +//! +//! # The name a trait shares with a handle +//! +//! [`Producer`] and [`Consumer`] are also the names of the concrete handles in +//! [`spsc`](crate::spsc) and [`slotwise_mpsc`](crate::slotwise_mpsc). That is deliberate: the +//! trait is named for the role, the handle is named for the role, and the +//! handle plays the role. `std` does the same thing with `fmt::Write` and +//! `io::Write`, and the module path disambiguates. Importing the traits +//! anonymously -- `use windows_waitable_queues::{Bounded as _, Consumer as _}` +//! -- avoids the question entirely when only the methods are wanted. + +use std::io; +use std::os::windows::io::{BorrowedHandle, OwnedHandle}; + +use crate::error::{Disconnected, PushError, TryRecvError}; + +/// The writing end of a queue. +pub trait Producer { + /// What this queue carries. + type Item; + + /// Appends an item. + /// + /// Takes `&self` rather than `&mut self`, which is what lets a + /// multi-producer shape share one handle's operation across threads. A + /// single-producer shape gets its guarantee from not being [`Sync`] + /// instead, so nothing is given up by the weaker receiver. + /// + /// # Errors + /// + /// [`PushError::Full`] when the queue is at capacity, which is the + /// backpressure signal rather than a malfunction, and + /// [`PushError::Disconnected`] when every consumer is gone. Either way the + /// item comes back, so nothing is lost by the refusal. + fn push(&self, item: Self::Item) -> Result<(), PushError>; + + /// Whether every consumer is gone, so nothing will ever take an item again. + fn is_disconnected(&self) -> bool; +} + +/// The reading end of a queue. +pub trait Consumer { + /// What this queue carries. + type Item; + + /// Takes the oldest item. + /// + /// # Errors + /// + /// [`TryRecvError::Empty`] when nothing is queued right now, which is a + /// statement about this instant rather than about the stream, and + /// [`TryRecvError::Disconnected`] when every producer is gone *and* the + /// queue has been drained. + /// + /// **The order of those two is a guarantee, not an implementation + /// detail.** A producer may push and then drop, so a queue can be + /// disconnected and still hold items; reporting disconnection before the + /// items are handed over would lose the tail of the stream. An earlier + /// version of this trait returned `Option` and asked the caller to pair it + /// with [`Consumer::is_disconnected`] in that order. Settling it here + /// removes a protocol nothing could enforce. + fn pop(&self) -> Result; + + /// Whether every producer is gone. + /// + /// **A queue can be disconnected and still hold items**, so this alone does + /// not mean the stream is finished. [`Consumer::pop`] answers the composite + /// question directly, and is what a drain loop should use. + fn is_disconnected(&self) -> bool; + + /// Takes items until the queue is momentarily empty. + /// + /// Ends at the first [`TryRecvError`] of either kind, which is a statement + /// about this instant and not about the stream: a producer may push again + /// immediately afterwards. It is the "take everything available" step of + /// the arming protocol, not a way to consume a queue to its end. + /// + /// Named for what it does. [`Consumer::try_iter`] is the same iterator + /// under the name most of the ecosystem uses. + fn drain(&self) -> Drain<'_, Self> + where + Self: Sized, + { + Drain { consumer: self } + } + + /// Takes items until the queue is momentarily empty. + /// + /// An alias for [`Consumer::drain`], because `try_iter` is the name + /// `crossbeam-channel`, `flume`, `std::sync::mpsc`, and `concurrent-queue` + /// all use for this, and a reader arriving from any of them should not have + /// to discover that this crate spells it differently. + fn try_iter(&self) -> Drain<'_, Self> + where + Self: Sized, + { + self.drain() + } +} + +/// Takes items from a [`Consumer`] until it is momentarily empty. +/// +/// Created by [`Consumer::drain`]. +#[derive(Debug)] +pub struct Drain<'a, C> { + consumer: &'a C, +} + +impl Iterator for Drain<'_, C> { + type Item = C::Item; + + fn next(&mut self) -> Option { + self.consumer.pop().ok() + } +} + +/// A queue that holds a fixed number of items and says how many. +/// +/// Implemented by both ends, because both have a use for it: a producer reads +/// it to report backpressure, and a consumer to report depth. +pub trait Bounded { + /// The exact number of items this queue holds when full. + /// + /// Not a hint and not rounded -- it is the number the caller asked for. + fn capacity(&self) -> usize; + + /// Items currently held, as a snapshot. + /// + /// A snapshot the moment it is returned: the other end may push or pop + /// immediately afterwards, which is why nothing here invites a + /// check-then-act. Use it for metrics, not for control flow. + fn len(&self) -> usize; + + /// Whether the queue holds nothing, as a snapshot. + fn is_empty(&self) -> bool; + + /// How many more items would fit, as a snapshot. + /// + /// Saturating rather than wrapping, because a shape may count a slot that a + /// producer has claimed but not yet finished writing, and a momentary + /// overshoot should read as "no room" rather than as a very large number. + /// + /// **A [`Reserving`] shape must override this.** A reservation withdraws + /// capacity *without* becoming an item, so it does not appear in + /// [`len`](Self::len) -- and this default therefore reports room that + /// `push` is guaranteed to refuse. Both shipped reserving shapes override + /// it, on the producer and the consumer alike, and a new one that forgets to + /// will report a queue with every slot reserved as entirely empty of + /// commitments. + fn remaining(&self) -> usize { + self.capacity().saturating_sub(self.len()) + } + + /// Whether a further best-effort push would be refused for want of room. + /// + /// Derived from [`remaining`](Self::remaining) rather than declared, so a + /// shape that overrides `remaining` -- as every [`Reserving`] one must -- + /// gets a consistent answer without restating it. Reservations therefore + /// count as occupancy here, which is the useful reading: a queue whose every + /// slot is spoken for is full whether or not the items have arrived yet. + fn is_full(&self) -> bool { + self.remaining() == 0 + } +} +/// A claimed slot, which is redeemed or released but never ignored. +/// +/// Bound onto [`Reserving::Reservation`] so a caller generic over the trait can +/// actually *discharge* what it claims. Without it `reserve` hands back a value +/// with no usable operations: it can be dropped, and nothing else. +pub trait Claim { + /// What the queue this claim came from carries. + type Item; + + /// Delivers into the reserved slot. + /// + /// Consumes the claim, because the slot it names is used exactly once. + /// + /// # Errors + /// + /// [`Disconnected`] if every consumer has gone, carrying the item back so a + /// caller can account for it rather than losing it. + fn send(self, item: Self::Item) -> Result<(), Disconnected>; + + /// Whether every consumer is gone. + /// + /// Offered on the claim itself, not only on the producer that made it: a + /// reservation may outlive the moment the producer was last consulted, and + /// [`reserving_mpsc`](crate::reserving_mpsc)'s is [`Send`], so it may be + /// redeemed on a thread holding no producer handle to ask. + fn is_disconnected(&self) -> bool; +} + +/// A producer that can claim a slot in advance, so that a later delivery cannot +/// be refused for want of room. +/// +/// # What a reservation is for +/// +/// A bounded queue refuses when it is full, and that refusal is the +/// backpressure it exists to provide. But not everything travelling a queue can +/// survive being refused the same way. A telemetry sample lost to a full queue +/// is a gap in a chart; an I/O completion lost to a full queue is a caller +/// waiting forever for something that already happened. +/// +/// Rather than sort that out per message at the point of delivery -- where the +/// queue is already full and the decision is already too late -- reliability +/// becomes a property of **capacity claimed in advance**. The slot is taken +/// before the work that will fill it is allowed to start, so by the time there +/// is something to deliver, the room is already the holder's. One line covers +/// it: *reserved is guaranteed, unreserved is best-effort.* +/// +/// The same discipline, reached independently, is what +/// `windows-file-watcher`'s notification queue runs on. +/// +/// # Why this is a trait a shape may lack +/// +/// [`slotwise_mpsc`](crate::slotwise_mpsc) deliberately does **not** implement this, and that is +/// the clearest illustration of why the capability traits are narrow +/// ([D-2](../DESIGN-NOTES.md#d-2)). Honouring a reservation means knowing how +/// many slots remain, which costs a producer a read of the consumer's position +/// on every push -- a single line every thread touches. `slotwise_mpsc`'s push avoids +/// that read by design, so it cannot answer the question, and +/// [`reserving_mpsc`](crate::reserving_mpsc) exists beside it for callers who +/// would rather pay than lose a message. +/// +/// A fat trait would have forced that cost on both, or excluded the reservation +/// from the contract entirely. Narrow traits let the two ship as peers. +pub trait Reserving { + /// What this queue carries. + type Item; + + /// The claim, which is redeemed or released but never ignored. + /// + /// **Generic over a lifetime because the two shapes genuinely differ**, and + /// that difference is the trait being validated by two implementations + /// rather than shaped around one ([D-3](../DESIGN-NOTES.md#d-3)). + /// [`reserving_mpsc`](crate::reserving_mpsc) hands out an owned, [`Send`] + /// reservation, because its use case is to claim a slot when an operation is + /// submitted and redeem it from whichever thread the completion arrives on. + /// [`spsc`](crate::spsc) hands out one that borrows the producer, because + /// there the producer handle *is* the single-producer guarantee: an owned + /// reservation could outlive it on another thread, and then two threads + /// would be writing the ring. + type Reservation<'a>: Claim + where + Self: 'a; + + /// Claims one slot, or reports that none is available. + /// + /// **This is the fallible half, and deliberately so.** Failing here is + /// cheap: no work has been started and nothing needs delivering, so a + /// caller can wait, shed load, or refuse the request upstream. That is the + /// whole trade -- the failure is moved from the moment of delivery, when + /// the only remaining options are to block or to lose the message, to the + /// moment of admission, when there are still good ones. + /// + /// A claim held is capacity withdrawn from every other producer, so hold it + /// for as long as correctness needs and no longer. Dropping it returns the + /// slot. + #[must_use = "a reservation withholds capacity from every other producer until it is used or dropped"] + fn reserve(&self) -> Option>; + + /// How many slots are currently claimed and not yet redeemed. + /// + /// A snapshot, and offered for metrics rather than for control flow: + /// [`Reserving::reserve`] answers "can I have one" without the window that + /// testing this first would open. + fn outstanding_reservations(&self) -> usize; +} + +/// What a queue can report about its own history. +/// +/// # Why depth is not here +/// +/// [D-2](../DESIGN-NOTES.md#d-2)'s sketch of this trait listed "depth, +/// high-water, doorbells actually rung", and depth has been left off +/// deliberately. [`Bounded::len`] already reports it, computed on demand from +/// positions the queue keeps anyway. Naming it again here would give one number +/// two spellings and two places to drift apart, which is the restatement +/// problem this workspace has paid for before. What belongs here is only what +/// has to be **accumulated** -- facts about the past that the queue's current +/// state cannot reconstruct. +/// +/// # Implemented by both ends +/// +/// A producer wants to know how often it was refused; a consumer wants to know +/// how deep the backlog got and how often it was actually woken. Both are +/// asking about the same queue, so both handles answer. +pub trait Observable { + /// How many pushes have been refused for want of room. + /// + /// **This is the loss count**, and it is the part of the file watcher's + /// coalesced loss latch that generalises: a latch can only coalesce losses + /// that are *idempotent*, which is a property of the payload rather than of + /// the queue, but counting them needs nothing of the payload at all. See + /// [D-19](../DESIGN-NOTES.md#d-19). + /// + /// Counts refusals for **room** only. A push refused because every consumer + /// is gone is the end of the stream rather than a loss, and folding the two + /// together would make a shutting-down queue look like an overloaded one. + fn refused(&self) -> u64; + + /// How many times the doorbell has actually rung. + /// + /// Counts `SetEvent` calls rather than signal attempts, and the difference + /// between the two *is* the skip optimisation. That is what makes this the + /// number worth reporting: the skip rule becomes measurable rather than + /// assumed, and turning the skip off has to move it. + /// + /// A queue whose consumer only ever polls never creates its doorbell, so + /// this stays zero -- which is the laziness being visible rather than a + /// gap. + fn doorbell_rings(&self) -> u64; + + /// The deepest the queue has been, if it is being tracked. + /// + /// `None` means nobody was counting, which is **not** the same answer as + /// `Some(0)`. Tracking is off unless + /// [`Options::tracking_high_water`](crate::Options::tracking_high_water) + /// asked for it, because a peak has to observe every change and that is the + /// one metric here which cannot be made free. + /// + /// # It is an upper bound on the peak, not the peak exactly + /// + /// The depth is sampled by a producer at publication, from its own position + /// and a load of the consumer's, and those are two readings rather than one + /// instant. The consumer may have drained between them, so the sample can + /// exceed the depth that held when the item landed. + /// + /// The error is **one-directional and bounded**: a stale read of the + /// consumer's position can only be *older*, which over-reports by the number + /// of items drained since, and the result is clamped to the capacity. So + /// this never reads below the true peak, and never above the queue's own + /// size. + /// + /// That is the useful direction for the question this answers -- whether a + /// capacity was ever close to exhausted -- and it is why the cheap sample is + /// preferred to an exact one. Counting exactly would mean a read-modify-write + /// on a line shared by every producer *and* the consumer, at every push and + /// every pop; this crate pads its positions apart specifically to keep that + /// line out of the hot path. + fn high_water(&self) -> Option; +} + +/// A queue whose readiness can be waited on as a Windows `HANDLE`. +/// +/// This is the capability the crate is named for, and the reason it exists +/// rather than deferring to an established concurrent-queue crate: a `HANDLE` +/// goes into `WaitForMultipleObjects` beside an I/O completion, a timer, or a +/// shutdown event, and a private parking primitive goes nowhere. +/// +/// **Not necessarily queue-specific.** "Hands out a `HANDLE` you can wait on" +/// is equally a property of an event, a timer, or a completion port. If a +/// second kind of thing wants to implement it, this trait moves to a lower +/// crate and this one depends on it; that move is planned rather than a +/// surprise, which is why it is said here. +pub trait Waitable { + /// Borrows the queue's readiness as a waitable `HANDLE`. + /// + /// The event is created on the first call, so a consumer that only ever + /// polls is charged for no kernel object. + /// + /// The borrow is deliberate: the event belongs to the queue and must not be + /// closed. Use [`Waitable::doorbell_owned`] where ownership is required. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + fn doorbell(&self) -> io::Result>; + + /// A duplicate of [`Waitable::doorbell`] that the caller owns. + /// + /// The duplicate names the same event, so signalling reaches both. This is + /// the form a `ThreadpoolWait` needs, since arming one takes ownership of + /// its target. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` or `DuplicateHandle`. + fn doorbell_owned(&self) -> io::Result; + + /// Clears the doorbell and reports whether a later push could be missed. + /// + /// `true` means the queue had nothing to take *after* the doorbell was + /// cleared, so any later push is guaranteed to signal. `false` means + /// something arrived in the meantime: take it instead of waiting. + /// + /// **Waiting without arming is a permanent hang, not an occasional missed + /// wakeup.** The full argument is in + /// [D-9](../DESIGN-NOTES.md#d-9); the short form is that clearing must come + /// before the emptiness check, which is the reverse of the order that reads + /// naturally. + /// + /// # `true` is not by itself permission to wait + /// + /// It answers exactly one question -- *can a later push be missed* -- and + /// says nothing about the end of the stream. On a queue whose producers are + /// all gone the answer to that question is trivially no, so this returns + /// `true`; but the last producer's drop rings the doorbell **once**, this + /// call clears precisely that ring, and nothing remains to ring it again. A + /// caller that waits indefinitely on the strength of `true` alone therefore + /// never wakes. + /// + /// So an indefinite wait needs four steps, not three: + /// + /// 1. take everything available; + /// 2. `arm`, and if it returns `false`, start again -- something arrived; + /// 3. **check [`Consumer::is_disconnected`], and if the producers are + /// gone, take one last time before reporting the end of the stream.** + /// That last take is not belt-and-braces: a producer may push *and then* + /// drop in the window between step 1 and this check, and skipping it + /// discards an item that was successfully sent; + /// 4. only now, wait. + /// + /// This is what [`Consumer::recv`](crate::Consumer) already does; the steps + /// are spelled out because a caller driving the handle itself -- through a + /// `ThreadpoolWait`, or a `WaitForMultipleObjects` over several queues -- + /// cannot delegate to it. + /// + /// # Errors + /// + /// Returns the error from `CreateEventW` on the first call. + fn arm(&self) -> io::Result; +} + +#[cfg(test)] +mod tests; diff --git a/crates/windows-waitable-queues/src/traits/tests.rs b/crates/windows-waitable-queues/src/traits/tests.rs new file mode 100644 index 00000000..9bdf739f --- /dev/null +++ b/crates/windows-waitable-queues/src/traits/tests.rs @@ -0,0 +1,641 @@ +// Copyright (c) Mike Grier. + +//! Tests for the capability traits. +//! +//! # What these are actually for +//! +//! Two things, and an earlier version of this header dismissed the second. +//! +//! The first is the claim [D-3](../../DESIGN-NOTES.md#d-3) makes: that the +//! traits are a real abstraction over more than one implementation, so a caller +//! can be written against them without knowing which shape it has. The evidence +//! is a set of generic functions with no knowledge of any shape, exercised +//! against all of them. If a trait were shaped around one -- the failure D-3 +//! exists to prevent -- these would not compile against the others, which is a +//! stronger check than any assertion in the bodies. +//! +//! # The delegating impls are checked here too, and that is not redundant +//! +//! This header used to say testing them "would only assert that a delegating +//! trait impl delegates", and left them alone on that reasoning. A mutation run +//! falsified it: **79 of the 128 surviving mutants in the three shapes were in +//! trait impls**, because every test called the inherent method, which shadows +//! the trait one. `::len` could return `0` unconditionally +//! and the whole suite stayed green. +//! +//! The impls are hand-written forwarders, so they are a second statement of +//! each shape's contract -- and a second statement is exactly the thing this +//! repository does not leave unchecked. They are also the *only* surface a +//! generic consumer touches, which is the surface D-2 says the crate is for. +//! +//! So the generic helpers below assert against **known queue state** rather +//! than against the inherent methods. Comparing the two views would prove they +//! agree while leaving both free to be wrong together; asserting a queue filled +//! to four reports a length of four fails a forwarder that returns a constant, +//! and fails an inherent method that does, and does not care which one broke. + +use crate::{ + Bounded, Claim, Consumer, Observable, Options, Producer, PushError, Waitable, reserving_mpsc, + slotwise_mpsc, spsc, +}; + +/// Fills a queue through nothing but the [`Producer`] and [`Bounded`] traits, +/// and reports what the refusal said. +/// +/// Deliberately generic over two unrelated types with two unrelated internal +/// protocols. Both `where` bounds are load-bearing: this is a caller that needs +/// to push *and* to know the bound, and D-2's whole argument is that it should +/// be able to ask for exactly those two things. +fn fill_to_capacity

(producer: &P) -> PushError +where + P: Producer + Bounded, +{ + assert!(producer.is_empty(), "a fresh queue holds nothing"); + assert_eq!( + producer.remaining(), + producer.capacity(), + "and all of its room is available" + ); + + for value in 0..producer.capacity() { + let value = u32::try_from(value).expect("the test capacities are small"); + producer.push(value).expect("there is room"); + } + + assert_eq!(producer.len(), producer.capacity()); + assert_eq!( + producer.remaining(), + 0, + "a full queue has no room, which is what the default method must compute" + ); + producer + .push(u32::MAX) + .expect_err("a full queue must refuse") +} + +/// Drains a queue through nothing but the [`Consumer`] trait, using the +/// provided `drain` method rather than a hand-written `while let` loop. +fn drain_all(consumer: &C) -> Vec +where + C: Consumer, +{ + let drained: Vec = consumer.drain().collect(); + assert!( + consumer.drain().next().is_none(), + "draining must leave the queue empty" + ); + drained +} + +/// Parks-or-proceeds through nothing but the [`Waitable`] trait. +/// +/// This is the arming protocol as a *consumer* would write it, which is the +/// case D-2 names: a drainer needs `Consumer` and `Waitable` and nothing else, +/// and must not be coupled to reservation or loss reporting to get them. +fn arm_and_report(consumer: &C) -> bool +where + C: Consumer + Waitable, +{ + consumer.doorbell().expect("the doorbell must be creatable"); + consumer + .doorbell_owned() + .expect("the duplicate must be creatable"); + consumer.arm().expect("arming must succeed") +} + +#[test] +fn both_shapes_satisfy_the_producer_and_bounded_traits() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid for both shapes"); + let (mpsc_tx, mpsc_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid for both shapes"); + + assert!( + matches!(fill_to_capacity(&spsc_tx), PushError::Full(u32::MAX)), + "the generic filler must work against the ring" + ); + assert!( + matches!(fill_to_capacity(&mpsc_tx), PushError::Full(u32::MAX)), + "and against the sequence-protocol queue, unchanged" + ); + + assert_eq!(drain_all(&spsc_rx), vec![0, 1, 2, 3]); + assert_eq!(drain_all(&mpsc_rx), vec![0, 1, 2, 3]); +} + +#[test] +fn both_shapes_report_disconnection_through_the_traits() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid for both shapes"); + let (mpsc_tx, mpsc_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid for both shapes"); + + fn producer_sees_it>(producer: &P) -> bool { + producer.is_disconnected() + } + fn consumer_sees_it>(consumer: &C) -> bool { + consumer.is_disconnected() + } + + assert!(!producer_sees_it(&spsc_tx)); + assert!(!producer_sees_it(&mpsc_tx)); + assert!(!consumer_sees_it(&spsc_rx)); + assert!(!consumer_sees_it(&mpsc_rx)); + + // **The consumer's own view, after the producers go.** Asserting only the + // `false` before disconnection left a forwarder returning a constant + // `false` alive on both shapes -- the direction that matters, because a + // consumer that never learns the stream ended waits forever. + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + assert!(!consumer_sees_it(&res_rx)); + + drop(spsc_tx); + drop(mpsc_tx); + drop(res_tx); + assert!( + consumer_sees_it(&spsc_rx), + "the consumer must see the producer go" + ); + assert!(consumer_sees_it(&mpsc_rx)); + assert!(consumer_sees_it(&res_rx)); + + // And the converse direction, on fresh queues so the drops above do not + // decide the answer. All three shapes, because each producer's + // `is_disconnected` is its own forwarder. + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (mpsc_tx, mpsc_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + assert!(!producer_sees_it(&res_tx), "still connected"); + + drop(spsc_rx); + drop(mpsc_rx); + drop(res_rx); + assert!(producer_sees_it(&spsc_tx)); + assert!( + producer_sees_it(&mpsc_tx), + "one consumer gone is every consumer gone, for both shapes" + ); + assert!(producer_sees_it(&res_tx), "and for the reserving shape"); +} + +#[test] +fn both_reserving_shapes_report_outstanding_claims_through_the_trait() { + // `spsc` implements `Reserving` too, with a borrowing reservation where + // `reserving_mpsc` hands out an owned one -- which is the two-implementor + // evidence D-3 asks for, and was untested until a mutation run said so. + fn claim_then_release

(producer: &P) -> (usize, usize, usize) + where + P: crate::Reserving, + { + let before = producer.outstanding_reservations(); + let reservation = producer.reserve().expect("a fresh queue has room"); + let held = producer.outstanding_reservations(); + drop(reservation); + (before, held, producer.outstanding_reservations()) + } + + let (spsc_tx, _rx) = spsc::bounded::(4).expect("4 is valid"); + assert_eq!( + claim_then_release(&spsc_tx), + (0, 1, 0), + "the borrowing reservation is counted while it is held" + ); + + let (res_tx, _rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + assert_eq!( + claim_then_release(&res_tx), + (0, 1, 0), + "and so is the owned one, through the same trait" + ); +} + +#[test] +fn both_shapes_satisfy_the_waitable_trait() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid for both shapes"); + let (mpsc_tx, mpsc_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid for both shapes"); + + assert!(arm_and_report(&spsc_rx), "an empty ring is safe to wait on"); + assert!( + arm_and_report(&mpsc_rx), + "and so is an empty sequence-protocol queue" + ); + + spsc_tx.push(1).expect("there is room"); + mpsc_tx.push(1).expect("there is room"); + assert!( + !arm_and_report(&spsc_rx), + "and neither blesses a wait over an item" + ); + assert!(!arm_and_report(&mpsc_rx)); +} + +#[test] +fn the_multi_producer_shape_is_usable_through_the_producer_trait_from_a_clone() { + // The trait was written against handles that are not `Clone` and handles + // that are, and `push` taking `&self` is what lets it span both. Had the + // first shape shipped `push(&mut self)` -- which single-producer soundness + // would have permitted -- this could not compile. + let (tx, rx) = slotwise_mpsc::bounded::(4).expect("4 is a valid capacity"); + let second = tx.clone(); + + fn push_one>(producer: &P, value: u32) { + producer.push(value).expect("there is room"); + } + + push_one(&tx, 1); + push_one(&second, 2); + assert_eq!(drain_all(&rx), vec![1, 2]); +} + +#[test] +fn drain_stops_at_the_current_end_rather_than_at_the_end_of_the_stream() { + // `drain` is the "take everything available" step of the arming protocol, + // not a way to consume a queue to its end. A caller that read it as the + // latter would drop items pushed afterwards, so the distinction is asserted + // rather than left to the documentation. + let (tx, rx) = slotwise_mpsc::bounded::(4).expect("4 is a valid capacity"); + tx.push(1).expect("there is room"); + + assert_eq!(drain_all(&rx), vec![1]); + tx.push(2).expect("there is room"); + assert_eq!( + drain_all(&rx), + vec![2], + "the queue was momentarily empty, not finished" + ); +} + +/// Every `Bounded` reading, against a queue whose contents are known. +/// +/// Deliberately not compared against the inherent methods: see the header. A +/// filled queue of four *is* four items long, whichever implementation is +/// asked, so this fails a forwarder that returns a constant without needing to +/// know that a forwarder exists. +fn bounded_readings_match_known_state

(producer: &P, capacity: usize) +where + P: crate::Producer + Bounded, +{ + assert_eq!(producer.capacity(), capacity, "capacity as constructed"); + assert_eq!(producer.len(), 0, "a fresh queue is empty"); + assert!(producer.is_empty()); + assert_eq!(producer.remaining(), capacity); + + producer.push(1).expect("there is room"); + assert_eq!(producer.len(), 1, "one push is one item"); + assert!(!producer.is_empty(), "and one item is not empty"); + assert_eq!(producer.remaining(), capacity - 1); + assert_eq!( + producer.capacity(), + capacity, + "capacity does not move when the contents do" + ); + + for value in 1..capacity { + producer + .push(u32::try_from(value).expect("small")) + .expect("there is room"); + } + assert_eq!(producer.len(), capacity, "filled to the brim"); + assert_eq!(producer.remaining(), 0); + assert!(!producer.is_empty()); +} + +/// Every `Observable` reading, against known state. +/// +/// The three counters answer different questions and are asserted separately +/// on purpose: an implementation that returned the same number for all of them +/// would satisfy any test that only checked one had moved. +fn observable_readings_match_known_state

(producer: &P, capacity: usize) +where + P: crate::Producer + Observable, +{ + assert_eq!(producer.refused(), 0, "nothing has been refused yet"); + assert_eq!( + producer.doorbell_rings(), + 0, + "a doorbell nobody asked for has never rung" + ); + + for value in 0..capacity { + producer + .push(u32::try_from(value).expect("small")) + .expect("there is room"); + } + assert_eq!( + producer.refused(), + 0, + "filling a queue exactly refuses nothing" + ); + + producer.push(u32::MAX).expect_err("the queue is full"); + assert_eq!(producer.refused(), 1, "and one refusal is counted"); + producer.push(u32::MAX).expect_err("still full"); + assert_eq!(producer.refused(), 2, "each refusal counts separately"); +} + +/// The same `Bounded` readings, from the **consumer** handle. +/// +/// A separate helper because both handles implement the trait separately, and +/// each impl is its own forwarder: exercising only the producer's left every +/// consumer-side reading unverified, which is exactly what the first pass at +/// this file did and what a second mutation run caught. +fn consumer_bounded_readings_match_known_state(consumer: &C, producer: &P, capacity: usize) +where + C: Consumer + Bounded, + P: crate::Producer, +{ + assert_eq!(consumer.capacity(), capacity, "capacity as constructed"); + assert_eq!(consumer.len(), 0, "a fresh queue is empty"); + assert!(consumer.is_empty()); + assert_eq!(consumer.remaining(), capacity); + + producer.push(1).expect("there is room"); + producer.push(2).expect("there is room"); + assert_eq!(consumer.len(), 2, "the consumer sees what was pushed"); + assert!(!consumer.is_empty()); + assert_eq!(consumer.remaining(), capacity - 2); + + assert_eq!(consumer.pop(), Ok(1)); + assert_eq!(consumer.len(), 1, "and sees the depth fall as it drains"); + assert_eq!(consumer.pop(), Ok(2)); + assert!(consumer.is_empty(), "drained back to empty"); + assert_eq!(consumer.remaining(), capacity); +} + +/// The `Observable` counters from the **consumer** handle, which reports the +/// same shared state the producer does. +fn consumer_observable_readings_match_known_state(consumer: &C, producer: &P, capacity: usize) +where + C: Consumer + Observable, + P: crate::Producer, +{ + assert_eq!(consumer.refused(), 0, "nothing refused yet"); + assert_eq!( + consumer.doorbell_rings(), + 0, + "and the doorbell has not rung" + ); + + for value in 0..capacity { + producer + .push(u32::try_from(value).expect("small")) + .expect("there is room"); + } + producer.push(u32::MAX).expect_err("full"); + assert_eq!( + consumer.refused(), + 1, + "a refusal is shared state, visible from either end" + ); +} + +/// `high_water` from either end, tracked and untracked. +fn high_water_readings_match_known_state(subject: &O, expected: Option) { + assert_eq!(subject.high_water(), expected); +} + +#[test] +fn every_shape_reports_its_bounds_through_the_bounded_trait() { + // All three, including `reserving_mpsc`, which this file did not mention at + // all and which carried the largest share of the surviving mutants. + let (spsc_tx, _spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, _slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, _res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + bounded_readings_match_known_state(&spsc_tx, 4); + bounded_readings_match_known_state(&slot_tx, 4); + bounded_readings_match_known_state(&res_tx, 4); +} + +#[test] +fn every_shape_counts_refusals_through_the_observable_trait() { + let (spsc_tx, _spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, _slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, _res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + observable_readings_match_known_state(&spsc_tx, 4); + observable_readings_match_known_state(&slot_tx, 4); + observable_readings_match_known_state(&res_tx, 4); +} + +#[test] +fn high_water_distinguishes_untracked_from_a_tracked_zero() { + // `None` and `Some(0)` are different answers -- nobody counted, against + // counted and never grew -- and a forwarder returning either constant + // would satisfy a test that only looked at one configuration. + fn peak(producer: &P) -> Option { + producer.high_water() + } + + let (untracked, _rx) = spsc::bounded::(4).expect("4 is valid"); + assert_eq!(peak(&untracked), None, "tracking is off by default"); + + let (tracked, _rx) = + spsc::bounded_with::(4, Options::new().tracking_high_water()).expect("4 is valid"); + assert_eq!(peak(&tracked), Some(0), "counted, and never grown"); + + tracked.push(1).expect("there is room"); + tracked.push(2).expect("there is room"); + assert_eq!(peak(&tracked), Some(2), "the peak follows the depth up"); +} + +#[test] +fn the_reserving_shape_is_usable_through_the_reserving_trait() { + // `Reserving` has one implementor here, so this cannot show the trait spans + // shapes the way the others do. What it does show is that the trait is + // usable without naming the concrete type -- and it covers the forwarders, + // which is where the mutants survived. + // + // **Only claiming and releasing are exercised, because that is all the + // trait offers.** `Reservation<'a>` is declared with no bound, so a caller + // generic over `Reserving` can obtain a reservation and drop it and nothing + // else: `commit` is inherent to each shape's own type and is unreachable + // from here. That is a gap in the trait rather than in this test, and it is + // raised as such rather than papered over by reaching for the concrete + // type, which would stop testing the trait at all. + let (tx, rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + fn claim_then_release

(producer: &P) -> (usize, usize) + where + P: crate::Reserving, + { + let before = producer.outstanding_reservations(); + let reservation = producer.reserve().expect("a fresh queue has room"); + let held = producer.outstanding_reservations(); + drop(reservation); + (before, held) + } + + let (before, held) = claim_then_release(&tx); + assert_eq!(before, 0, "a fresh queue has nothing outstanding"); + assert_eq!(held, 1, "an open reservation is outstanding"); + assert_eq!( + tx.outstanding_reservations(), + 0, + "and dropping it returns the slot" + ); + + // The released slot is genuinely usable again, so `outstanding_reservations` + // reporting zero is not merely a constant that happens to read right. + tx.push(7).expect("the released slot is available"); + assert_eq!(rx.pop(), Ok(7)); +} + +#[test] +fn every_shape_reports_its_bounds_from_the_consumer_end_too() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + consumer_bounded_readings_match_known_state(&spsc_rx, &spsc_tx, 4); + consumer_bounded_readings_match_known_state(&slot_rx, &slot_tx, 4); + consumer_bounded_readings_match_known_state(&res_rx, &res_tx, 4); +} + +#[test] +fn every_shape_counts_refusals_from_the_consumer_end_too() { + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + consumer_observable_readings_match_known_state(&spsc_rx, &spsc_tx, 4); + consumer_observable_readings_match_known_state(&slot_rx, &slot_tx, 4); + consumer_observable_readings_match_known_state(&res_rx, &res_tx, 4); +} + +#[test] +fn every_shape_reports_high_water_from_either_end() { + // Both handles and all three shapes, tracked and untracked. `None` and + // `Some(n)` are different answers, so a forwarder returning either constant + // has to fail one of these configurations. + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + high_water_readings_match_known_state(&spsc_tx, None); + high_water_readings_match_known_state(&spsc_rx, None); + high_water_readings_match_known_state(&slot_tx, None); + high_water_readings_match_known_state(&slot_rx, None); + high_water_readings_match_known_state(&res_tx, None); + high_water_readings_match_known_state(&res_rx, None); + + // And with tracking on, the peak is visible from both ends and follows the + // depth up rather than sitting at a constant. + let options = || Options::new().tracking_high_water(); + let (tx, rx) = spsc::bounded_with::(4, options()).expect("4 is valid"); + let (stx, srx) = slotwise_mpsc::bounded_with::(4, options()).expect("4 is valid"); + let (rtx, rrx) = reserving_mpsc::bounded_with::(4, options()).expect("4 is valid"); + + high_water_readings_match_known_state(&tx, Some(0)); + high_water_readings_match_known_state(&rx, Some(0)); + high_water_readings_match_known_state(&stx, Some(0)); + high_water_readings_match_known_state(&srx, Some(0)); + high_water_readings_match_known_state(&rtx, Some(0)); + high_water_readings_match_known_state(&rrx, Some(0)); + + for value in 0..3 { + tx.push(value).expect("there is room"); + } + high_water_readings_match_known_state(&tx, Some(3)); + high_water_readings_match_known_state(&rx, Some(3)); + + for value in 0..3 { + stx.push(value).expect("there is room"); + rtx.push(value).expect("there is room"); + } + high_water_readings_match_known_state(&stx, Some(3)); + high_water_readings_match_known_state(&srx, Some(3)); + high_water_readings_match_known_state(&rtx, Some(3)); + high_water_readings_match_known_state(&rrx, Some(3)); +} + +#[test] +fn every_shape_counts_a_doorbell_ring_that_actually_happened() { + // `doorbell_rings` counts real `SetEvent` calls, so it stays zero until a + // consumer has armed and a producer has pushed against that armed state. + // Asserting only the zero -- which the refusal tests above do -- leaves a + // forwarder returning a constant zero alive. + fn rings(subject: &O) -> u64 { + subject.doorbell_rings() + } + + fn ring_once(producer: &P, consumer: &C) + where + P: crate::Producer, + C: Consumer + Waitable + Observable, + { + assert_eq!(rings(consumer), 0, "nothing has rung yet"); + assert!( + consumer.arm().expect("arming must succeed"), + "empty, so safe" + ); + producer.push(1).expect("there is room"); + assert!( + rings(consumer) >= 1, + "a push against an armed doorbell must ring it" + ); + } + + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + let (slot_tx, slot_rx) = slotwise_mpsc::bounded::(4).expect("4 is valid"); + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + + ring_once(&spsc_tx, &spsc_rx); + ring_once(&slot_tx, &slot_rx); + ring_once(&res_tx, &res_rx); + + // Visible from the producer end as well, which is its own forwarder. + assert!(rings(&spsc_tx) >= 1); + assert!(rings(&slot_tx) >= 1); + assert!(rings(&res_tx) >= 1); +} + +#[test] +fn a_generic_caller_can_claim_check_and_redeem() { + // **The whole point of the bound.** This function names no concrete shape + // and still completes the operation `Reserving` exists for. Without the + // bound on `Reservation<'a>` it does not compile at all: `reserve` hands + // back a value whose only available operation is `drop`. + fn claim_and_send

(producer: &P, item: u32) -> Result<(), crate::Disconnected> + where + P: crate::Reserving, + { + let claim = producer.reserve().expect("a fresh queue has room"); + assert!(!claim.is_disconnected(), "the consumer is still there"); + claim.send(item) + } + + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + claim_and_send(&spsc_tx, 7).expect("delivery must succeed"); + assert_eq!(spsc_rx.pop(), Ok(7)); + + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + claim_and_send(&res_tx, 9).expect("delivery must succeed"); + assert_eq!(res_rx.pop(), Ok(9)); + + // And the failure path is reachable generically too, which it was not + // before: a claim held past the consumer's exit reports it and hands the + // item back rather than losing it. + // + // **Both shapes, and both answers.** Asserting the connected case above and + // the disconnected case on only one shape leaves an `is_disconnected` stuck + // at `false` alive on the other -- which a mutation run found, and which is + // the reading that loses an item, since a caller checking it would deliver + // into a queue nobody will ever drain. + fn claim_survives_the_consumer

(producer: &P, item: u32) + where + P: crate::Reserving, + { + let claim = producer.reserve().expect("a fresh queue has room"); + assert!( + claim.is_disconnected(), + "the consumer is already gone, and the claim must say so" + ); + let returned = claim.send(item).expect_err("no consumer remains"); + assert_eq!(returned.into_inner(), item, "the item must come back"); + } + + let (spsc_tx, spsc_rx) = spsc::bounded::(4).expect("4 is valid"); + drop(spsc_rx); + claim_survives_the_consumer(&spsc_tx, 11); + + let (res_tx, res_rx) = reserving_mpsc::bounded::(4).expect("4 is valid"); + drop(res_rx); + claim_survives_the_consumer(&res_tx, 13); +} diff --git a/release-please-config.json b/release-please-config.json index 24017120..23467714 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -51,6 +51,10 @@ "package-name": "windows-topology-sys", "component": "windows-topology-sys" }, + "crates/windows-waitable-queues": { + "package-name": "windows-waitable-queues", + "component": "windows-waitable-queues" + }, "crates/wtf-string": { "package-name": "wtf-string", "component": "wtf-string"