diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..e1b41e3ae8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,12 +176,16 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), + # the renumbered t/351-356 family, t/357 multi-xmax alias floor. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by + # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy + # verified against origin branches 2026-07-08). + - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index d6152a4e28..24be2345d8 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -58,7 +58,8 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" -#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ #include "cluster/cluster_reconfig.h" /* apply_clean_leave + record/is_clean_departed + join_in_progress */ @@ -121,6 +122,8 @@ cluster_clean_leave_shmem_init(void) pg_atomic_init_u32(&cl_state->commit_point_observed, 0); pg_atomic_init_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_init_u32(&cl_state->committed_marker_durable, 0); + /* spec-2.29a r3 (evidence latch). */ + cl_state->committed_confirmed_epoch = 0; /* Hardening v1.0.2 (P1 preflight / P2 nonce). */ pg_atomic_init_u64(&cl_state->leave_attempt_nonce, 0); pg_atomic_init_u32(&cl_state->preflight_pending, 0); @@ -381,6 +384,11 @@ cluster_clean_leave_check_pending_in_proc_interrupts(void) * disabled survivor MUST reply LEAVE_DRAIN_NAK rather than go silent, else the * leaving node waits for an ACK that never comes and false-times-out. */ + +/* spec-2.29a ②b: forward decls — the bind sites (below) snapshot the + * others-dead set that the coherence gate (defined near drive_drain) compares. */ +static void cl_others_dead_snapshot(int32 leaving, uint8 *out); + static void cl_announce_handler(const ClusterICEnvelope *env, const void *payload) { @@ -509,6 +517,9 @@ cl_announce_handler(const ClusterICEnvelope *env, const void *payload) * fails the commit closed instead of committing on a stale membership view. */ cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes the leaving node) + * the coherence gate compares against. */ + cl_others_dead_snapshot(leaving, cl_state->leave_baseline_others_dead); pg_atomic_write_u32(&cl_state->survivor_acked, 0); pg_atomic_write_u32(&cl_state->commit_ready_received, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); @@ -546,11 +557,15 @@ cl_commit_ready_handler(const ClusterICEnvelope *env, const void *payload) } /* - * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate): - * the survivor coordinator has made the COMMITTED marker majority-durable and - * signals that the durable truth-source now exists, so this leaving node may - * proceed to COMMITTED and exit. Only acted on if it is about OUR leave; idem- - * potent (the coordinator re-sends each tick until we are gone). + * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate; + * spec-2.29a r3 evidence latch): the survivor coordinator has made the COMMITTED + * marker majority-durable and attests that the durable truth-source now exists. + * This confirmation is the leaver's own-commit MARKER EVIDENCE: the barrier tick + * latches on it (and only on it) and proceeds to COMMITTED/exit. Identity is + * checked by the pure evidence gate (self-addressed + currently leaving + + * per-attempt nonce + committed epoch past the bound baseline) — any mismatch is + * dropped fail-closed. Idempotent (the coordinator re-sends each tick until we + * are gone). */ static void cl_committed_handler(const ClusterICEnvelope *env, const void *payload) @@ -562,17 +577,19 @@ cl_committed_handler(const ClusterICEnvelope *env, const void *payload) return; if (!cluster_clean_leave_announce_payload_valid(p)) return; - if (p->leaving_node_id != cluster_node_id) - return; /* only the leaving node consumes its own COMMITTED confirmation */ - /* Hardening v1.0.2 (P2): bind to THIS attempt — a stale LEAVE_COMMITTED from a - * prior same-epoch attempt must not prematurely confirm the current one. Only - * meaningful once we are the leaver actively awaiting confirmation. */ - if (p->leave_nonce != pg_atomic_read_u64(&cl_state->leave_attempt_nonce)) + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + if (!cluster_clean_leave_committed_evidence_matches( + p->leaving_node_id, p->leave_nonce, p->leave_epoch, cluster_node_id, + cl_state->leaving_node_id, pg_atomic_read_u64(&cl_state->leave_attempt_nonce), + cl_state->leave_epoch)) { + LWLockRelease(&cl_state->lock); return; - if (cl_state->leaving_node_id != cluster_node_id) - return; /* we are not currently leaving */ - + } + /* the committed epoch E is recorded BEFORE the evidence flag is published, + * inside the same critical section the barrier tick reads it back under. */ + cl_state->committed_confirmed_epoch = p->leave_epoch; pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 1); + LWLockRelease(&cl_state->lock); } /* @@ -745,6 +762,125 @@ cluster_clean_leave_ic_send_ack(int32 dest_node_id, int32 leaving_node_id, uint6 /* qvotec-side per-process handshake cursors (mirror the fence-marker ones). */ static uint64 cl_qvotec_last_processed_marker_seq = 0; static uint64 cl_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync cl_lmon_marker_async; +static ClusterLeaveIntentMarker cl_lmon_marker; +static int cl_lmon_marker_phase = 0; +static int32 cl_lmon_marker_leaving = -1; +static uint64 cl_lmon_marker_epoch = 0; +static bool cl_lmon_marker_submitted = false; + +static void cl_build_marker(ClusterLeaveIntentMarker *m, uint8 marker_phase, int32 leaving, + uint64 epoch); + +typedef enum ClusterLeaveAsyncMarkerResult { + CL_LEAVE_ASYNC_PENDING = 0, + CL_LEAVE_ASYNC_ACKED, + CL_LEAVE_ASYNC_FAILED +} ClusterLeaveAsyncMarkerResult; + +static ClusterMarkerAsyncKind +cl_marker_phase_kind(int phase) +{ + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED; + return CLUSTER_MARKER_KIND_UNKNOWN; +} + +static void +cl_release_lmon_marker_stage(void) +{ + cluster_marker_async_release_stage(&cl_lmon_marker_async); + memset(&cl_lmon_marker, 0, sizeof(cl_lmon_marker)); + cl_lmon_marker_phase = 0; + cl_lmon_marker_leaving = -1; + cl_lmon_marker_epoch = 0; + cl_lmon_marker_submitted = false; +} + +static bool +cl_start_lmon_marker_stage(const ClusterLeaveIntentMarker *m, int phase, int32 leaving, + uint64 epoch) +{ + if (cl_lmon_marker_async.has_staged_event) + return false; + cl_lmon_marker = *m; + cl_lmon_marker_phase = phase; + cl_lmon_marker_leaving = leaving; + cl_lmon_marker_epoch = epoch; + cl_lmon_marker_async.has_staged_event = true; + cl_lmon_marker_submitted = false; + return true; +} + +static ClusterLeaveAsyncMarkerResult +cl_poll_lmon_marker_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_LEAVE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + if (!cl_lmon_marker_async.has_staged_event) + return CL_LEAVE_ASYNC_FAILED; + + now = GetCurrentTimestamp(); + kind = cl_marker_phase_kind(cl_lmon_marker_phase); + if (!cl_lmon_marker_submitted) { + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, kind, + cl_lmon_marker_leaving, now)) + return CL_LEAVE_ASYNC_PENDING; + cl_lmon_marker_submitted = true; + return CL_LEAVE_ASYNC_PENDING; + } + + pr = cluster_clean_leave_poll_marker_async(&cl_lmon_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return CL_LEAVE_ASYNC_PENDING; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, cl_lmon_marker_leaving, elapsed_us); + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + + cluster_reconfig_note_marker_slow_ack(kind, cl_lmon_marker_leaving, elapsed_us); + if (result != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + return CL_LEAVE_ASYNC_ACKED; +} + +static void +cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) +{ + ClusterLeaveIntentMarker cm; + ClusterLeaveAsyncMarkerResult ar; + + if (cl_lmon_marker_async.has_staged_event) { + if (cl_lmon_marker_phase != CLUSTER_LEAVE_MARKER_PHASE_COMMITTED + || cl_lmon_marker_leaving != leaving || cl_lmon_marker_epoch != committed_epoch) + return; + } else { + cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); + (void)cl_start_lmon_marker_stage(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + committed_epoch); + } + + ar = cl_poll_lmon_marker_stage(); + if (ar == CL_LEAVE_ASYNC_ACKED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, committed_epoch); + cl_release_lmon_marker_stage(); + } else if (ar == CL_LEAVE_ASYNC_FAILED) { + ereport(LOG, (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); + } +} ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) @@ -787,6 +923,38 @@ cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) } } +bool +cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (cl_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cl_state->marker_request_seq, + &cl_state->marker_completion_seq)) + return false; + + cl_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &cl_state->marker_request_seq, + &cl_state->marker_completion_seq, cl_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cl_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cl_state->marker_completion_seq, &cl_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512) { @@ -818,6 +986,7 @@ cluster_clean_leave_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&cl_state->marker_completion_seq, cl_qvotec_inflight_marker_seq); cl_qvotec_last_processed_marker_seq = cl_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -1277,6 +1446,8 @@ cl_request_body(void) cl_state->leaving_node_id = cluster_node_id; cl_state->leave_epoch = baseline_epoch; cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes self, the leaver). */ + cl_others_dead_snapshot(cluster_node_id, cl_state->leave_baseline_others_dead); cl_state->barrier_deadline_us = (uint64)GetCurrentTimestamp() + (uint64)cluster_clean_leave_drain_timeout_ms * 1000ULL; /* @@ -1304,6 +1475,7 @@ cl_request_body(void) pg_atomic_write_u32(&cl_state->commit_point_observed, 0); pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); + cl_state->committed_confirmed_epoch = 0; /* spec-2.29a r3: per-attempt evidence */ LWLockRelease(&cl_state->lock); cl_set_phase(CLUSTER_LEAVE_REQUESTED); @@ -1410,6 +1582,49 @@ cluster_clean_leave_request(void) * re-checks version coherence (CL-I3: an external epoch bump = a real death * intruded -> escalate) and the mixed-mode NAK (clean abort). */ + +/* + * spec-2.29a ②b: snapshot the CSSD dead set EXCLUDING the leaving node. The + * coherence gate compares this "others-dead" set rather than the scalar global + * dead_generation, so the leaving node's own expected alive->DEAD transition + * (it stops heart-beating once its drain finishes) never falsely escalates the + * leave. A third-party death still changes this set and escalates (CL-I3). + */ +static void +cl_others_dead_snapshot(int32 leaving, uint8 *out /* CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES */) +{ + int i; + + memset(out, 0, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == leaving) + continue; /* the leaving node's own DEAD is expected, not incoherence */ + if (i >= CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES * 8) + break; + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (cluster_cssd_get_peer_state(i) == CLUSTER_CSSD_PEER_DEAD) + out[i / 8] |= (uint8)(1u << (i % 8)); + } +} + +/* + * spec-2.29a ②b: the coherence gate used by every clean-leave step. Coherent + * iff the epoch has not moved AND no THIRD-PARTY death changed the others-dead + * set since the leave was bound. cl_state->leaving_node_id names the node + * whose own DEAD is excluded. + */ +static bool +cl_coherent(uint64 baseline_epoch) +{ + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + return cluster_clean_leave_version_coherent( + baseline_epoch, cluster_epoch_get_current(), cl_state->leave_baseline_others_dead, + now_others_dead, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); +} + void cluster_clean_leave_drive_drain(void) { @@ -1470,16 +1685,14 @@ cluster_clean_leave_drive_drain(void) /* * CL-I3 coherence gate, re-checked before every drain step. Uses the * dead_gen-aware helper, not an epoch-only compare: CSSD increments the - * dead_generation the moment it declares a peer dead, which is STRICTLY - * BEFORE the reconfig coordinator bumps the membership epoch. Checking - * dead_gen too lets us escalate in that earlier window instead of draining - * into a death that has not yet reached the epoch. At commit the guarded - * CAS in apply_clean_leave_as_coordinator is the final authority. + * others-dead set the moment it declares a THIRD-PARTY peer dead, which is + * STRICTLY BEFORE the reconfig coordinator bumps the membership epoch. + * Checking that set too lets us escalate in that earlier window instead of + * draining into a death that has not yet reached the epoch. The leaving + * node's OWN DEAD is excluded (spec-2.29a ②b). At commit the guarded CAS + * in apply_clean_leave_as_coordinator is the final authority. */ -#define CL_COHERENT() \ - cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), \ - cl_state->leave_baseline_dead_gen, \ - cluster_cssd_get_dead_generation()) +#define CL_COHERENT() cl_coherent(baseline_epoch) /* REQUESTED -> QUIESCING: abort local writable backends (53R62). */ if (!CL_COHERENT()) { @@ -1551,60 +1764,72 @@ cl_leaving_barrier_tick(void) { uint64 baseline_epoch = cl_state->leave_epoch; int32 coordinator; + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool committed_evidence; + bool others_dead_unchanged; + bool dead_gen_unchanged; /* - * 1. observe the commit. The survivor coordinator runs the actual two-phase - * commit and publishes the CLEAN_LEAVE event into ITS OWN reconfig state — - * the leaving node's last_applied never carries it. What DOES propagate to - * the leaving node is the membership epoch (every IC envelope piggybacks it). - * So the leaving node detects its own commit by the epoch advancing past the - * bound baseline. Discriminate from a real death intruding (CL-I3) by the - * CSSD dead_generation: a cooperative leave does NOT mark anyone CSSD-dead, so - * an unchanged dead_generation at the bump means OUR clean-leave committed; a - * changed one means a real death (escalate). + * 1. observe the commit — EVIDENCE over inference (spec-2.29a r3). The + * survivor coordinator runs the actual two-phase commit, publishes the + * CLEAN_LEAVE event into ITS OWN reconfig state, drives the COMMITTED marker + * to voting-disk majority-durability, and only then sends the nonce-bound + * LEAVE_COMMITTED (re-sent each tick while we are alive). That confirmation + * — validated by the pure identity gate in cl_committed_handler — is the ONLY + * basis on which this node latches its own commit: latch <=> a valid + * COMMITTED marker for THIS leave attempt exists. + * + * Two inference generations preceded this and each failed one way (see the + * predicate comment in cluster_clean_leave_policy.c): "epoch advanced + + * others-dead bitmap unchanged" could mis-latch a REFUSED leave after a + * third-party false-DEAD rebound (r2 P2-1 wedge); adding the monotone scalar + * dead_generation conjunct then false-escalated a healthy committed leave + * whenever a transient third-party flap advanced the leaver's local + * dead_generation during the leave window (nightly t/331 C1/C4). Marker + * evidence is immune to both: flaps cannot erase a durable marker, and a + * refused leave never produces one — no latch, so the barrier deadline below + * still bounds the wait (fail-closed escalation, unchanged semantics). * - * Hardening v1.0.4 (P2): this "epoch bump + dead_gen unchanged == OUR commit" - * inference is sound ONLY because there is no OTHER dead_gen-unchanged reconfig - * in flight. A spec-5.15 online JOIN also bumps the epoch with dead_gen - * unchanged and would be mis-observed here as the leave's commit (then the node - * stops escalating but never gets the real LEAVE_COMMITTED -> BARRIER_WAIT - * hang). That collision is removed STRUCTURALLY by the one-membership-reconfig- - * at-a-time serialization: the join driver does not bump the epoch while a clean - * leave is active (cluster_clean_leave_in_progress gates drive_joins + - * commit_member), and the leave does not start while a join is pending - * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * dead_gen-unchanged bump during a leave can only be the leave's own commit. + * The coherence observations are still taken, but ONLY for the flap-noise + * LOG at the latch (the predicate contract pins that they never affect the + * verdict). A real third-party death intruding mid-leave is refused on the + * survivor side (cl_coherent pre-check + guarded CAS, CL-I3), so no evidence + * arrives here and the deadline escalates this leaver — same outcome as the + * old immediate escalate arm, now bounded by the barrier deadline instead. + * + * Latching collapses the old two-step gate: the evidence IS the P1-V0.7 + * durable-truth-source confirmation, so the leave can no longer be + * un-committed (deadline must not escalate past here, Hardening v1.0.1 + * P1-1) AND the node may exit (COMMITTED) in the same tick. */ - if (cluster_epoch_get_current() > baseline_epoch) { - if (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen) { - /* - * Commit point observed (our clean-leave epoch was published; no real - * death intruded). The leave can no longer be un-committed, so from - * here the barrier deadline must NOT escalate (Hardening v1.0.1 P1-1). - */ - pg_atomic_write_u32(&cl_state->commit_point_observed, 1); - LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); - cl_state->leave_epoch = cluster_epoch_get_current(); /* the committed epoch E */ - LWLockRelease(&cl_state->lock); + committed_evidence = (pg_atomic_read_u32(&cl_state->committed_durable_confirmed) != 0); + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(committed_evidence, others_dead_unchanged, + dead_gen_unchanged)) { + uint64 committed_epoch; + + pg_atomic_write_u32(&cl_state->commit_point_observed, 1); + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + committed_epoch = cl_state->committed_confirmed_epoch; /* the committed epoch E */ + cl_state->leave_epoch = committed_epoch; + LWLockRelease(&cl_state->lock); - /* - * P1-V0.7 exit gate: reach COMMITTED ("may exit") ONLY after the - * coordinator confirms the COMMITTED marker is majority-durable - * (LEAVE_COMMITTED). Until then stay in BARRIER_WAIT and re-tick — the - * durable truth-source must exist before this node departs, else a - * survivor restart could not rebuild the clean-departed fact. - */ - if (pg_atomic_read_u32(&cl_state->committed_durable_confirmed)) { - cl_set_phase(CLUSTER_LEAVE_COMMITTED); - CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); - ereport(LOG, - (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " - "majority-durable; this node has drained and may exit", - (unsigned long long)cl_state->leave_epoch))); - } - } else { - cl_escalate(); /* a real death changed the version mid-leave (CL-I3) */ - } + if (!others_dead_unchanged || !dead_gen_unchanged) + ereport(LOG, (errmsg("cluster clean-leave: third-party liveness flap observed at the " + "commit latch (others-dead %s, dead_generation %s); durable " + "COMMITTED marker evidence overrides it", + others_dead_unchanged ? "unchanged" : "changed", + dead_gen_unchanged ? "unchanged" : "advanced"))); + + cl_set_phase(CLUSTER_LEAVE_COMMITTED); + CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); + ereport(LOG, (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " + "majority-durable; this node has drained and may exit", + (unsigned long long)committed_epoch))); return; } @@ -1615,8 +1840,12 @@ cl_leaving_barrier_tick(void) } /* 3. fail-closed deadline — ONLY before the commit point (P1-1: a committed - * leave is never un-committed; post-commit we wait for the durable marker, - * bounded by disk health, and never escalate). */ + * leave is never un-committed). With the r3 evidence latch this arm is what + * bounds EVERY no-evidence outcome: a refused leave, a foreign death that + * moved the version (the coordinator then refuses, CL-I3), or a lost/never- + * sent confirmation all end here instead of hanging in BARRIER_WAIT. The + * commit_point_observed guard is belt-and-braces: the latch above transitions + * out of BARRIER_WAIT in the same tick it sets the flag. */ if (!pg_atomic_read_u32(&cl_state->commit_point_observed) && (uint64)GetCurrentTimestamp() > cl_state->barrier_deadline_us) { cl_escalate(); @@ -1640,79 +1869,97 @@ cl_coordinator_commit(int32 leaving) uint64 new_epoch; ClusterLeaveIntentMarker m; + if (cl_lmon_marker_async.has_staged_event) { + ClusterLeaveAsyncMarkerResult ar = cl_poll_lmon_marker_stage(); + int phase = cl_lmon_marker_phase; + uint64 staged_epoch = cl_lmon_marker_epoch; + + if (ar == CL_LEAVE_ASYNC_PENDING) + return; + if (ar == CL_LEAVE_ASYNC_FAILED) + return; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { + cl_release_lmon_marker_stage(); + + /* + * spec-2.29a review r1 P1 — CL-I3 re-check at the staged-ACK + * handoff. The COMMITTING marker wait now spans LMON ticks, so a + * real death can bump CSSD dead_generation INSIDE the wait window + * while the fail-stop epoch has not yet advanced (the >=3-node + * window of Hardening v1.0.1 P1-2). The guarded CAS inside + * apply_clean_leave_as_coordinator only catches the epoch move; + * re-run the same dead_gen-aware coherence check the non-staged + * pre-check uses, at the last observable point before the commit + * applies. On failure the leave does not commit and the leaving + * node escalates to fail-stop (identical to the pre-check path). + */ + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) " + "across the COMMITTING marker wait for node %d; not committing " + "(escalate to fail-stop, CL-I3)", + leaving))); + return; + } + + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); + if (new_epoch == 0) { + ereport( + LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); + return; + } + cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + new_epoch); + (void)cl_poll_lmon_marker_stage(); + ereport(LOG, + (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", + leaving, (unsigned long long)new_epoch))); + return; + } + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, staged_epoch); + cl_release_lmon_marker_stage(); + return; + } + cl_release_lmon_marker_stage(); + return; + } + /* CL-I3 pre-check: refuse to commit a clean leave on a version a real death * already bumped — the leaving node will then observe a non-CLEAN_LEAVE event * and escalate. This is a cheap early-out; the authoritative guard is the * guarded CAS inside apply_clean_leave_as_coordinator below, which closes the * check-then-bump TOCTOU at >=3 nodes. */ /* - * CL-I3 commit-handoff coherence (epoch AND dead_gen): refuse to commit if a - * real death intruded since this survivor started tracking the leave — either - * the death's fail-stop already bumped the epoch, OR (the >=3-node window, - * Hardening v1.0.1 P1-2) CSSD bumped its dead_generation but the fail-stop - * epoch has NOT yet advanced, which an epoch-only check (and the guarded CAS - * below) would miss and commit on a stale membership view. Uses the same - * unit-tested version_coherent helper as drive_drain so the dead_gen-aware - * coherence holds through the commit handoff; the leaving node then observes - * the eventual foreign event and escalates. + * CL-I3 commit-handoff coherence (epoch AND others-dead): refuse to commit if + * a THIRD-PARTY death intruded since this survivor started tracking the leave + * — either the death's fail-stop already bumped the epoch, OR (the >=3-node + * window, Hardening v1.0.1 P1-2) CSSD added it to the others-dead set but the + * fail-stop epoch has NOT yet advanced, which an epoch-only check (and the + * guarded CAS below) would miss and commit on a stale membership view. Uses + * the same others-dead coherence helper as drive_drain (spec-2.29a ②b: the + * leaving node's own expected DEAD is excluded so it never falsely escalates); + * the leaving node then observes the eventual foreign event and escalates. */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { - ereport(LOG, (errmsg("cluster clean-leave: version moved (epoch or dead_generation) before " - "committing node %d; not committing (escalate to fail-stop, CL-I3)", - leaving))); + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) before " + "committing node %d; not committing (escalate to fail-stop, CL-I3)", + leaving))); return; } /* (1) COMMITTING(E) marker (coordinator's own slot, before the bump; NOT a * trust basis). Not durable -> do not commit. */ cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, baseline_epoch + 1); - if (cluster_clean_leave_submit_marker(&m) != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster clean-leave: COMMITTING marker not durable for node %d; " - "not committing (the leaving node will retry/escalate)", - leaving))); - return; - } - - /* (2) the real commit point: guarded epoch bump (only if still baseline) + - * publish CLEAN_LEAVE + record clean-departed. Returns 0 if a concurrent - * reconfig moved the epoch off the baseline after the COMMITTING marker — - * fail-closed, do NOT write the COMMITTED marker (CL-I3). */ - new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); - if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); - return; /* CL-I3: stale baseline (or cluster disabled / bad id) */ - } - - /* - * (3) COMMITTED(E) marker (after the bump; the ONLY rebuild trust basis). - * Post-commit it MUST become majority-durable — the durable truth-source must - * exist BEFORE the leaving node departs (P1-V0.7 exit gate; §2.5). Try once - * here. If it reaches majority, mark durable + tell the leaving node it may - * exit (LEAVE_COMMITTED). If NOT, do not give up: committed_marker_durable - * stays 0, cl_survivor_tick retries the marker every tick, and the leaving - * node stays in BARRIER_WAIT (no LEAVE_COMMITTED) until it is durable — it - * never departs without a durable truth-source. The leave is already - * committed (epoch bumped); we never revert. - */ - cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); - if (cluster_clean_leave_submit_marker(&m) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - cl_send_committed(leaving, new_epoch); - } else { - ereport( - LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the COMMITTED " - "marker is not yet majority-durable; retrying each tick (leaving node waits)", - leaving, (unsigned long long)new_epoch))); - } - - ereport(LOG, (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", - leaving, (unsigned long long)new_epoch))); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, + baseline_epoch + 1); + (void)cl_poll_lmon_marker_stage(); } /* survivor (incl. coordinator) side of another node's leave. */ @@ -1773,13 +2020,9 @@ cl_survivor_tick(int32 leaving) if (coordinator == cluster_node_id && cluster_reconfig_is_clean_departed(leaving)) { uint64 committed_epoch = cluster_reconfig_get_clean_departed_epoch(leaving); - if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) { - ClusterLeaveIntentMarker cm; + if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) + cl_drive_committed_marker_stage(leaving, committed_epoch); - cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); - if (cluster_clean_leave_submit_marker(&cm) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - } /* Re-send LEAVE_COMMITTED every tick once durable until the leaver is gone * (best-effort IC): the leaving node will not depart until it receives one, * and step 3 holds the slot until it is CSSD-dead, so delivery is assured. */ diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 0a4eeebd04..91a2c4d768 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -136,17 +136,112 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid) /* * cluster_clean_leave_version_coherent -- the bound leave is still coherent - * only if neither the cluster epoch nor the CSSD dead_generation moved since - * the leave was bound. Any external bump (a real death intruding mid-drain) - * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean - * leave on a stale version, which would let a destructive sweep run on a newer - * version and double-grant). + * only if neither the cluster epoch nor the OTHERS-dead set (every dead node + * except the leaving node itself) moved since the leave was bound. Any + * external membership change — a third-party death intruding mid-drain, or a + * third-party fail-stop that already bumped the epoch — makes it incoherent → + * caller must ABORTED_ESCALATE (never complete a clean leave on a stale + * membership view, which would let a destructive sweep run on a newer view + * and double-grant, CL-I3). + * + * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD + * dead_generation, which also counts the leaving node's OWN alive→DEAD + * transition (it stops heart-beating once its drain finishes — the EXPECTED + * terminal state of a clean leave). Under the async COMMITTING marker that + * spans several LMON ticks, a slow environment lets that transition land + * inside the wait window and the scalar check wrongly escalated an otherwise + * healthy leave. Comparing the others-dead bitmap (which excludes the + * leaving node) removes exactly that false positive while keeping every + * third-party incoherence escalation (see the reflexive-case matrix in + * spec-2.29a §②b 8.A argument). */ bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen) + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes) +{ + if (bound_epoch != current_epoch) + return false; + if (bound_others_dead == NULL || current_others_dead == NULL) + return false; /* fail-closed: cannot prove coherence without both views */ + return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; +} + +/* + * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick + * latches "my clean-leave committed" on direct EVIDENCE, never on inference + * (spec-2.29a r3). The evidence is committed_marker_evidence: the survivor + * coordinator made the COMMITTED marker for THIS leave attempt majority- + * durable on the voting disk and attested it with a nonce-bound + * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_ + * matches before the flag this argument mirrors is ever set). Latching + * suppresses the barrier-deadline escalation and lets the leaver exit, so + * the verdict must be exactly: latch <=> evidence. + * + * History of the two inference failure modes this replaces: + * - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead + * bitmap unchanged" inference could mis-latch a REFUSED leave after a + * third-party false-DEAD -> ALIVE rebound restored the (non-monotone) + * bitmap, suppressing the deadline escalation forever. + * - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation + * conjunct is monotone the other way — a third-party transient flap on + * the leaver's local CSSD view advances it forever, so a healthy + * committed leave was refused until the deadline escalated it. + * Marker evidence is immune to both: a refused leave never gets a COMMITTED + * marker (no latch -> bounded deadline escalation), and a flap cannot make + * durable evidence disappear (latch -> no false escalation). + * + * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence + * observations. They are deliberately kept in the signature as contract + * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the + * flap-immunity on them — and the runtime uses them only for the flap-noise + * LOG at the latch point (observability, never control flow). + */ +bool +cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged, + bool dead_gen_unchanged) +{ + (void)others_dead_unchanged; /* observability-only input (see above) */ + (void)dead_gen_unchanged; /* observability-only input (see above) */ + return committed_marker_evidence; +} + +/* + * cluster_clean_leave_committed_evidence_matches -- may the leaving node + * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave + * attempt? (spec-2.29a r3; the payload's magic/version/CRC were already + * checked by cluster_clean_leave_announce_payload_valid.) + * + * Identity is bound three ways, all fail-closed: + * - payload_leaving_node == self_node: the confirmation is addressed to + * this node's own leave (LEAVE_COMMITTED is point-to-point, but a + * misrouted frame must still not latch). + * - current_leaving_node == self_node: this node IS currently the leaver + * (not idle, not a survivor of someone else's leave). + * - payload_nonce == current_attempt_nonce: the per-attempt nonce + * (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale + * LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a + * PREVIOUS leave of the same node can never false-latch a new attempt. + * - payload_epoch > bound_leave_epoch: the committed epoch E the + * coordinator attests must lie past the baseline this leave bound + * (sanity; the commit is a guarded CAS off that baseline). + */ +bool +cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce, + uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, + uint64 current_attempt_nonce, + uint64 bound_leave_epoch) { - return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen; + if (payload_leaving_node != self_node) + return false; + if (current_leaving_node != self_node) + return false; + if (payload_nonce != current_attempt_nonce) + return false; + if (payload_epoch <= bound_leave_epoch) + return false; + return true; } diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..bcebe9a058 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -544,6 +544,10 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_slow_iter_count", + fmt_int64((int64)cluster_lmon_slow_iter_count())); } /* @@ -1475,6 +1479,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_reconfig_get_join_timeout_count())); emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count", fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count())); + emit_row(rsinfo, "reconfig", "marker_slow_ack_count", + fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count())); + emit_row(rsinfo, "reconfig", "marker_timeout_count", + fmt_int64((int64)cluster_reconfig_get_marker_timeout_count())); } /* diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..d85321d2b3 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1638,6 +1638,20 @@ cluster_grd_recovery_in_progress(void) return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE; } +/* + * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares + * against. Exposed for diagnostics + the IDLE baseline-hold unit test (the + * async pre-bump staging window must not let this re-capture a post-bump + * value; see the IDLE branch of cluster_grd_recovery_lmon_tick). + */ +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch); +} + /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module * call sites (S4 stale mapping in cluster_lock_acquire.c; GCS block * fail-closed guard in cluster_gcs_block.c). */ @@ -2108,9 +2122,19 @@ cluster_grd_recovery_lmon_tick(void) * fires, making old_epoch already == the post-bump epoch and * wedging WAIT_EPOCH forever (cur <= old). The last stable * idle epoch is the reliable "before reconfig" value. + * + * spec-2.29a nightly-regression fix: the coordinator's own + * async pre-bump stages (fail-stop / node-remove / join + * Phase-1) bump the epoch several ticks before publishing the + * event. During that window this IDLE tick must NOT re-capture + * the already-bumped epoch as the baseline, or the P0 accept + * that follows the publish reads old == cur and wedges + * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback + * needed. Hold the last stable pre-reconfig baseline instead. */ - pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, - cluster_epoch_get_current()); + if (!cluster_reconfig_has_pending_prebump_stage()) + pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, + cluster_epoch_get_current()); return; } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..5a53b070a9 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -307,6 +307,7 @@ int cluster_phase4_timeout = 30; * LMON main-loop tick / WaitLatch timeout in milliseconds. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */ int cluster_lck_main_loop_interval = 1000; @@ -2573,6 +2574,15 @@ cluster_init_guc(void) &cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.lmon_slow_iteration_warn_ms", + gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."), + gettext_noop("A value greater than zero logs LMON main-loop iterations above this " + "duration; the lmon_slow_iter_count counter is maintained even when " + "logging is disabled with 0."), + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, + NULL); + DefineCustomIntVariable( "cluster.lck_main_loop_interval", gettext_noop("LCK main-loop tick interval in milliseconds."), diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..332a4e97ff 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = { /* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */ { .name = "cluster-qvotec-poll-pre" }, { .name = "cluster-qvotec-poll-post" }, + { .name = "cluster-qvotec-marker-service-hold" }, { .name = "cluster-voting-disk-write-fail" }, { .name = "cluster-quorum-loss-broadcast" }, { .name = "cluster-collision-detect" }, diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..1af59978cc 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -681,6 +681,61 @@ cluster_lmon_main_loop_iters(void) return result; } +uint64 +cluster_lmon_last_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->last_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_max_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->max_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_slow_iter_count(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->slow_iter_count; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +void +cluster_lmon_marker_complete_wakeup(void) +{ + struct Latch *latch; + + if (cluster_lmon_state == NULL) + return; + + latch = cluster_lmon_state->lmon_latch; + if (latch != NULL) + SetLatch(latch); +} + /* ============================================================ * LMON main entry (AuxiliaryProcessMain dispatch target). @@ -720,12 +775,30 @@ lmon_publish_status(ClusterLmonStatus status) cluster_lmon_state->ready_at = 0; cluster_lmon_state->last_liveness_tick_at = 0; cluster_lmon_state->main_loop_iters = 0; + cluster_lmon_state->last_iter_us = 0; + cluster_lmon_state->max_iter_us = 0; + cluster_lmon_state->slow_iter_count = 0; + cluster_lmon_state->lmon_latch = NULL; } else if (status == CLUSTER_LMON_READY) { cluster_lmon_state->ready_at = now; + cluster_lmon_state->lmon_latch = MyLatch; + } else if (status == CLUSTER_LMON_EXITED) { + cluster_lmon_state->lmon_latch = NULL; } LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_clear_latch(int code, Datum arg) +{ + if (cluster_lmon_state == NULL) + return; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->lmon_latch = NULL; + LWLockRelease(&cluster_lmon_state->lwlock); +} + static bool lmon_shutdown_requested(void) @@ -760,6 +833,44 @@ lmon_advance_liveness_tick(void) LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_record_iteration(TimestampTz iter_started_at) +{ + TimestampTz now = GetCurrentTimestamp(); + uint64 elapsed_us; + bool slow; + bool should_log = false; + static TimestampTz last_slow_log_at = 0; + + if (now <= iter_started_at) + elapsed_us = 0; + else + elapsed_us = (uint64)(now - iter_started_at); + + slow = (cluster_lmon_slow_iteration_warn_ms >= 0) + && elapsed_us >= (uint64)cluster_lmon_slow_iteration_warn_ms * 1000ULL; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->last_iter_us = elapsed_us; + if (elapsed_us > cluster_lmon_state->max_iter_us) + cluster_lmon_state->max_iter_us = elapsed_us; + if (slow) + cluster_lmon_state->slow_iter_count++; + LWLockRelease(&cluster_lmon_state->lwlock); + + if (slow && cluster_lmon_slow_iteration_warn_ms > 0 + && (last_slow_log_at == 0 + || now - last_slow_log_at >= INT64CONST(60) * INT64CONST(1000000))) { + last_slow_log_at = now; + should_log = true; + } + + if (should_log) + ereport(LOG, (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); +} + void LmonMain(void) @@ -811,6 +922,7 @@ LmonMain(void) /* Publish SPAWNING (records pid + spawned_at). */ lmon_publish_status(CLUSTER_LMON_SPAWNING); + on_shmem_exit(lmon_clear_latch, (Datum)0); /* Sprint B inject: ready-publish (test slow startup / phase 1 wait timeout). */ CLUSTER_INJECTION_POINT("cluster-lmon-ready-publish"); @@ -938,6 +1050,7 @@ LmonMain(void) int n_events; long wait_ms; TimestampTz now; + TimestampTz iter_started_at; int32 i; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1063,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1414,6 +1528,7 @@ LmonMain(void) wait_ms = 1; } + lmon_record_iteration(iter_started_at); n_events = WaitEventSetWait(wes, wait_ms, ev, lengthof(ev), WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT); @@ -1588,6 +1703,7 @@ LmonMain(void) /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ for (;;) { int rc; + TimestampTz iter_started_at; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1715,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1677,6 +1794,7 @@ LmonMain(void) CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); + lmon_record_iteration(iter_started_at); rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, cluster_lmon_main_loop_interval, WAIT_EVENT_CLUSTER_BGPROC_LMON_MAIN_LOOP); diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 8825ac6828..39e31c4b72 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -57,6 +57,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" #include "cluster/cluster_membership.h" #include "cluster/cluster_node_remove.h" #include "cluster/cluster_pcm_lock.h" @@ -162,6 +163,14 @@ cluster_node_remove_get_state(ClusterNodeRemoveState *out) static uint64 nr_qvotec_last_processed_marker_seq = 0; static uint64 nr_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync nr_marker_async; +static ClusterRemovalMarker nr_marker_stage; +static int nr_marker_phase = 0; +static int32 nr_marker_node_id = -1; +static uint64 nr_marker_epoch = 0; +static uint64 nr_marker_removed_incarnation = 0; +static uint64 nr_marker_removal_event_id = 0; +static bool nr_marker_submitted = false; ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) @@ -195,6 +204,38 @@ cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) } } +bool +cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (nr_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&nr_state->marker_request_seq, + &nr_state->marker_completion_seq)) + return false; + + nr_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &nr_state->marker_request_seq, + &nr_state->marker_completion_seq, nr_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (nr_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, &nr_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out) { @@ -223,6 +264,7 @@ cluster_node_remove_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&nr_state->marker_completion_seq, nr_qvotec_inflight_marker_seq); nr_qvotec_last_processed_marker_seq = nr_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -241,24 +283,124 @@ cluster_node_remove_publish_qvotec_latch(struct Latch *latch) on_shmem_exit(nr_clear_qvotec_latch, (Datum)0); } -/* build + submit one durable removal marker; returns true iff majority-durable. */ +static ClusterMarkerAsyncKind +nr_marker_phase_kind(int phase) +{ + switch (phase) { + case CLUSTER_REMOVAL_MARKER_REMOVING: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING; + case CLUSTER_REMOVAL_MARKER_SHRUNK: + return CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK; + case CLUSTER_REMOVAL_MARKER_REMOVED: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED; + default: + return CLUSTER_MARKER_KIND_UNKNOWN; + } +} + +static void +nr_release_marker_stage(void) +{ + cluster_marker_async_release_stage(&nr_marker_async); + memset(&nr_marker_stage, 0, sizeof(nr_marker_stage)); + nr_marker_phase = 0; + nr_marker_node_id = -1; + nr_marker_epoch = 0; + nr_marker_removed_incarnation = 0; + nr_marker_removal_event_id = 0; + nr_marker_submitted = false; +} + +static bool +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, + uint64 removal_event_id) +{ + return nr_marker_async.has_staged_event && nr_marker_phase == phase + && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch + && nr_marker_removed_incarnation == removed_incarnation + && nr_marker_removal_event_id == removal_event_id; +} + +/* build + poll one durable removal marker; returns true iff majority-durable. */ static bool nr_write_marker(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, uint64 removal_event_id) { ClusterRemovalMarker m; + TimestampTz now; + uint32 result = CLUSTER_REMOVAL_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + now = GetCurrentTimestamp(); + kind = nr_marker_phase_kind(phase); + + if (nr_marker_async.has_staged_event + && !nr_marker_stage_matches(phase, node_id, remove_epoch, removed_incarnation, + removal_event_id)) { + if (!nr_marker_submitted) { + if (cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, + nr_marker_phase_kind(nr_marker_phase), + nr_marker_node_id, now)) + nr_marker_submitted = true; + return false; + } + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } else if (pr == CLUSTER_MARKER_POLL_ACKED) { + cluster_reconfig_note_marker_slow_ack(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } + return false; + } + + if (!nr_marker_async.has_staged_event) { + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; + m.version = CLUSTER_REMOVAL_MARKER_VERSION; + m.phase = (uint16)phase; + m.removed_node_id = node_id; + m.remove_epoch = remove_epoch; + m.removed_incarnation = removed_incarnation; + m.removal_event_id = removal_event_id; + cluster_removal_marker_compute_crc(&m); + + nr_marker_stage = m; + nr_marker_phase = phase; + nr_marker_node_id = node_id; + nr_marker_epoch = remove_epoch; + nr_marker_removed_incarnation = removed_incarnation; + nr_marker_removal_event_id = removal_event_id; + nr_marker_async.has_staged_event = true; + nr_marker_async.staged_expect_epoch = remove_epoch; + nr_marker_submitted = false; + } - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; - m.version = CLUSTER_REMOVAL_MARKER_VERSION; - m.phase = (uint16)phase; - m.removed_node_id = node_id; - m.remove_epoch = remove_epoch; - m.removed_incarnation = removed_incarnation; - m.removal_event_id = removal_event_id; - cluster_removal_marker_compute_crc(&m); - - return cluster_node_remove_submit_marker(&m) == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; + if (!nr_marker_submitted) { + if (!cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, kind, + node_id, now)) + return false; + nr_marker_submitted = true; + return false; + } + + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return false; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return false; + } + + cluster_reconfig_note_marker_slow_ack(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return result == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; } @@ -543,31 +685,43 @@ cluster_node_remove_drive(void) (void)baseline_dead_gen; /* contest is now signalled by out_contest, not derived */ - /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ - (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, - last_incarnation, removal_event_id); - /* - * spec-6.15 D5c (appendix B.3): durably retire the removed node's - * xid stripe slot BEFORE the removal point of no return below. - * Ordering is the identity-reuse defence: once removal commits, a - * later fresh join of the same node_id (new incarnation = a new - * durable owner) must land on a retired slot and refuse (53RB1), - * never resume the old owner's congruence class. Not durable yet - * -> stay FENCE_ARMING and retry next tick (fail-closed; same - * shape as a transient fence/quorum failure). A never-activated - * cluster returns true (nothing to retire). + * spec-2.29a review r1 P2: while the staged fence marker from a + * previous FENCE_ARMING tick is still in flight, skip the pre-work + * below — the epoch was already self-bumped at stage-entry, so + * re-running it would key a SECOND REMOVING marker to the post-bump + * epoch and re-submit the stripe retire during the very contention + * window the async stage relieves. Fall through straight to + * apply(), which polls the stage. */ - if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { - ereport(LOG, (errmsg("cluster node removal: stripe slot retire for node %d not durable " - "yet — retrying before the removal commit", - node_id))); - return; + if (!cluster_reconfig_node_removed_fence_stage_pending()) { + /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ + (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, + last_incarnation, removal_event_id); + + /* + * spec-6.15 D5c (appendix B.3): durably retire the removed node's + * xid stripe slot BEFORE the removal point of no return below. + * Ordering is the identity-reuse defence: once removal commits, a + * later fresh join of the same node_id (new incarnation = a new + * durable owner) must land on a retired slot and refuse (53RB1), + * never resume the old owner's congruence class. Not durable yet + * -> stay FENCE_ARMING and retry next tick (fail-closed; same + * shape as a transient fence/quorum failure). A never-activated + * cluster returns true (nothing to retire). + */ + if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { + ereport(LOG, + (errmsg("cluster node removal: stripe slot retire for node %d not durable " + "yet — retrying before the removal commit", + node_id))); + return; + } + /* D5d: carry the retire to WAL consumers (standby stops gap- + * filling the dead class). The voting-disk retire above is the + * correctness anchor; emission failure is tolerated (logged). */ + cluster_xid_stripe_emit_retire_wal(node_id); } - /* D5d: carry the retire to WAL consumers (standby stops gap- - * filling the dead class). The voting-disk retire above is the - * correctness anchor; emission failure is tolerated (logged). */ - cluster_xid_stripe_emit_retire_wal(node_id); /* * The commit point: guarded epoch bump + fence-arm (majority-durable, at diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 8eb25379a7..916a629399 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -94,8 +94,9 @@ #include "cluster/cluster_elog.h" /* CLUSTER_LOG (best-effort logging) */ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ -#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ -#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ +#include "cluster/cluster_inject.h" +#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ +#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D2: fence marker scan + token refresh */ @@ -883,6 +884,10 @@ qvotec_poll_once(void) * carried forward every poll like the fence marker (R12), and acked majority- * durable from the self-slot write tally. */ have_removal_submit = cluster_node_remove_qvotec_poll_pending(&removal_submit_marker); + if ((have_submit || have_leave_submit || have_join_submit || have_removal_submit) + && cluster_cr_injection_armed("cluster-qvotec-marker-service-hold", NULL)) + return; + memset(&apply_lease_request, 0, sizeof(apply_lease_request)); memset(&apply_lease_winner, 0, sizeof(apply_lease_winner)); apply_lease_winner.owner_node_id = -1; diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 1f79a6c9b9..ccaf5991c1 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -77,6 +77,7 @@ #include "cluster/cluster_clean_leave.h" /* v1.0.4 — cluster_clean_leave_in_progress (serialize) */ #include "cluster/cluster_guc.h" /* cluster_enabled, cluster_online_join */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_voting_disk_io.h" /* spec-5.15 D4 — region-3 join-marker slot I/O */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D4 — durable fence marker submit */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum */ @@ -114,6 +115,38 @@ StaticAssertDecl(CLUSTER_RECONFIG_TOUCH_KIND_COUNT == CLUSTER_TOUCH_KIND_COUNT, */ static ClusterReconfigState *ReconfigShmem = NULL; +typedef struct ClusterReconfigFenceStage { + ClusterMarkerAsync async; + ReconfigEvent event; + ClusterFenceMarker marker; + int32 node_id; + uint64 last_incarnation; + uint64 removal_event_id; + bool submitted; +} ClusterReconfigFenceStage; + +typedef struct ClusterReconfigJoinPrepareStage { + ClusterMarkerAsync async; + ReconfigEvent event; + uint8 join_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 joiner_incarnations[CLUSTER_MAX_NODES]; + int next_node; + bool submitted; +} ClusterReconfigJoinPrepareStage; + +typedef struct ClusterReconfigJoinCommitStage { + ClusterMarkerAsync async; + ClusterJoinCommitMarker marker; + int32 node_id; + uint64 admitted_incarnation; + bool submitted; +} ClusterReconfigJoinCommitStage; + +static ClusterReconfigFenceStage failstop_fence_stage; +static ClusterReconfigFenceStage node_removed_fence_stage; +static ClusterReconfigJoinPrepareStage join_prepare_stage; +static ClusterReconfigJoinCommitStage join_commit_stage; + /* ============================================================ * Shmem region lifecycle. @@ -144,6 +177,7 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->apply_counter, 0); pg_atomic_init_u64(&ReconfigShmem->dedup_skip_counter, 0); pg_atomic_init_u64(&ReconfigShmem->procsig_broadcast_count, 0); + pg_atomic_init_u32(&ReconfigShmem->prebump_sync_active, 0); /* spec-2.29a r2 t/274 */ /* spec-5.14 D6 — touched_peers observability counters. */ pg_atomic_init_u64(&ReconfigShmem->touched_abort_count, 0); pg_atomic_init_u64(&ReconfigShmem->touched_stamp_count, 0); @@ -211,6 +245,8 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->join_reject_count, 0); pg_atomic_init_u64(&ReconfigShmem->join_timeout_count, 0); pg_atomic_init_u64(&ReconfigShmem->clean_departed_cleared_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_slow_ack_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_timeout_count, 0); } /* @@ -306,6 +342,28 @@ cluster_reconfig_publish_event(const ReconfigEvent *evt) } +static void +cluster_reconfig_log_failstop_epoch_bump(const ReconfigEvent *evt) +{ + char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; + int off = 0; + int n; + + Assert(evt != NULL); + + dead[0] = '\0'; + for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { + if (evt->dead_bitmap[n / 8] & (1 << (n % 8))) + off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); + } + + ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " + "(coordinator node %d, dead node(s) {%s})", + (unsigned long long)evt->old_epoch, (unsigned long long)evt->new_epoch, + (int)evt->coordinator_node_id, dead))); +} + + /* ============================================================ * Counter accessors (Step 2 + Step 3 SRF support). * ============================================================ @@ -1032,6 +1090,42 @@ cluster_reconfig_get_clean_departed_cleared_count(void) : pg_atomic_read_u64(&ReconfigShmem->clean_departed_cleared_count); } +void +cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem == NULL || elapsed_us < 1000000ULL) + return; + + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); + ereport(LOG, (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +void +cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem != NULL) + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); + ereport(LOG, (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_slow_ack_count); +} + +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_timeout_count); +} + /* * cluster_reconfig_join_in_progress -- Hardening v1.0.4 (spec-5.13 clean-leave x * spec-5.15 online-join serialization, P1-1/P2): is a membership JOIN currently in @@ -1148,6 +1242,422 @@ cluster_reconfig_join_publish_proven(uint64 admitted_epoch) return advanced >= ((members / 2u) + 1u); } +static void +cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) +{ + cluster_marker_async_release_stage(&stage->async); + stage->submitted = false; + memset(&stage->event, 0, sizeof(stage->event)); + memset(&stage->marker, 0, sizeof(stage->marker)); + stage->node_id = -1; + stage->last_incarnation = 0; + stage->removal_event_id = 0; +} + +/* + * spec-2.29a nightly-regression fix — pre-bump staging window guard. + * + * The three pre-bump coordinator paths (fail-stop fence, node-remove, + * join Phase-1) advance the membership epoch at stage-entry but only + * publish the reconfig event once the voting-disk marker ACKs, which now + * spans several LMON ticks instead of the pre-async single-tick spin. + * Between the bump and the publish the GRD recovery IDLE tick would + * re-capture recovery_event_old_epoch as the post-bump value, so its P0 + * accept later reads old == cur and wedges WAIT_EPOCH forever (the + * spec-4.6a section 0 shape, here triggered by the coordinator on ITSELF + * — even in a 2-node cluster with no IC piggyback). While any pre-bump + * stage is live the GRD IDLE tick must hold its last stable (genuine + * pre-reconfig) baseline instead of re-capturing. join Phase-2 COMMITTED + * does not pre-bump, so it is intentionally excluded. + */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + /* spec-2.29a r2 t/274: the shmem bit covers a BACKEND-context coordinator's + * bump→publish window (LMON cannot see that process's local stage); the + * three LMON-local staged flags cover the LMON-driven paths. */ + if (ReconfigShmem != NULL && pg_atomic_read_u32(&ReconfigShmem->prebump_sync_active) != 0) + return true; + return failstop_fence_stage.async.has_staged_event + || node_removed_fence_stage.async.has_staged_event + || join_prepare_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, + int32 target_node, TimestampTz now) +{ + if (stage->submitted) + return true; + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, target_node, + now)) + return false; + stage->submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_failstop_fence_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!failstop_fence_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, now)) + return true; + + pr = cluster_write_fence_poll_marker_async(&failstop_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_log_failstop_epoch_bump(&failstop_fence_stage.event); + cluster_reconfig_broadcast_local_procsig(); + } else { + ereport(LOG, + (errmsg("cluster reconfig: fence marker did not reach a voting-disk majority " + "for epoch %llu; not publishing reconfig event (write-fenced, will retry)", + (unsigned long long)failstop_fence_stage.event.new_epoch))); + } + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; +} + +static uint64 +cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 removal_event_id, + uint64 last_incarnation) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!node_removed_fence_stage.async.has_staged_event) + return 0; + if (node_removed_fence_stage.node_id != removed_node_id + || node_removed_fence_stage.removal_event_id != removal_event_id) + return 0; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&node_removed_fence_stage, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, now)) + return 0; + + pr = cluster_write_fence_poll_marker_async(&node_removed_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return 0; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, removed_node_id, + elapsed_us); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " + "voting-disk majority for epoch %llu; not publishing removal " + "(write-fenced, will retry)", + removed_node_id, + (unsigned long long)node_removed_fence_stage.event.new_epoch))); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); + cluster_reconfig_publish_event(&node_removed_fence_stage.event); + cluster_reconfig_record_removed(removed_node_id, node_removed_fence_stage.event.new_epoch, + false); + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_shrink_to_removed(removed_node_id, last_incarnation); + LWLockRelease(&ReconfigShmem->lock); + + { + uint64 new_epoch = node_removed_fence_stage.event.new_epoch; + + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return new_epoch; + } +} + +static void +cluster_reconfig_release_join_prepare_stage(void) +{ + cluster_marker_async_release_stage(&join_prepare_stage.async); + memset(&join_prepare_stage.event, 0, sizeof(join_prepare_stage.event)); + memset(join_prepare_stage.join_bitmap, 0, sizeof(join_prepare_stage.join_bitmap)); + memset(join_prepare_stage.joiner_incarnations, 0, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; +} + +/* (spec-2.29a review r1: the former abort_join_prepare_stage — revert + * JOINING→DEAD + clear pending on PREPARE failure — was removed. Q5=A makes + * PREPARE strictly best-effort: TIMEOUT / non-ACK advances the queue and the + * staged JOIN_PENDING still publishes, so no revert path exists; reverting + * would let compute_join_bitmap re-detect the joiner and re-bump the epoch + * every deadline period — the P1-1 bump storm.) */ + +static bool +cluster_reconfig_submit_join_prepare_current(TimestampTz now) +{ + ClusterJoinCommitMarker m; + int i; + + for (i = join_prepare_stage.next_node; i < CLUSTER_MAX_NODES; i++) { + if (dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + break; + } + join_prepare_stage.next_node = i; + if (i >= CLUSTER_MAX_NODES) { + cluster_reconfig_publish_event(&join_prepare_stage.event); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + cluster_reconfig_broadcast_local_procsig(); + cluster_reconfig_release_join_prepare_stage(); + return true; + } + + if (join_prepare_stage.submitted) + return true; + + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_JCMK_MAGIC; + m.version = CLUSTER_JCMK_VERSION; + m.node_id = i; + m.phase = CLUSTER_JCMK_PHASE_PREPARE; + m.admitted_incarnation = join_prepare_stage.joiner_incarnations[i]; + m.generation = join_prepare_stage.joiner_incarnations[i]; + m.admitted_epoch = join_prepare_stage.event.new_epoch; + cluster_join_marker_compute_crc(&m); + + if (!cluster_reconfig_submit_join_marker_async(&join_prepare_stage.async, i, &m, + CLUSTER_MARKER_KIND_JOIN_PREPARE, now)) + return false; + join_prepare_stage.submitted = true; + return true; +} + +/* + * spec-2.29a review r1 P2 — node-remove driver pre-work gate. + * + * While the staged node-removed fence marker from a previous FENCE_ARMING + * tick is still in flight, the driver must not re-run its pre-work + * (REMOVING marker write / stripe retire): the epoch has already been + * self-bumped by the stage-entry, so a re-run would key a SECOND REMOVING + * marker to the post-bump epoch and add voting-disk writes during the + * exact contention window the async stage exists to relieve. + */ +bool +cluster_reconfig_node_removed_fence_stage_pending(void) +{ + return node_removed_fence_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_poll_join_prepare_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + int target; + + if (!join_prepare_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_prepare_stage.submitted) { + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; + } + + target = join_prepare_stage.next_node; + pr = cluster_reconfig_poll_join_marker_async(&join_prepare_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + + /* + * spec-2.29a Q5=A (review r1 P1): PREPARE is best-effort in OUTCOME — the + * pre-async code ignored the submit result and always published + * JOIN_PENDING (only the Phase-2 COMMITTED marker is a commit point, P1-r5). + * A TIMEOUT / non-ACK PREPARE therefore advances the queue instead of + * aborting: aborting would revert the joiner JOINING→DEAD, and the next + * tick's compute_join_bitmap would re-detect it CSSD-alive and RE-BUMP the + * epoch — exactly the per-tick epoch-bump storm the P1-1 staged record + * forbids. Draining to publish keeps the joiner on a stable JOINING + * (one Phase-1 bump total, regardless of PREPARE outcomes). + */ + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + } else { + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach " + "a voting-disk majority; continuing best-effort (COMMITTED is the " + "commit point)", + target))); + } + + join_prepare_stage.submitted = false; + join_prepare_stage.next_node++; + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; +} + +static void +cluster_reconfig_release_join_commit_stage(void) +{ + cluster_marker_async_release_stage(&join_commit_stage.async); + memset(&join_commit_stage.marker, 0, sizeof(join_commit_stage.marker)); + join_commit_stage.node_id = -1; + join_commit_stage.admitted_incarnation = 0; + join_commit_stage.submitted = false; +} + +static bool +cluster_reconfig_publish_join_commit(int32 node_id, uint64 admitted_incarnation, + uint64 expected_epoch) +{ + uint64 old_epoch, new_epoch; + XLogRecPtr lsn; + ReconfigEvent evt; + uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint64 incs[CLUSTER_MAX_NODES]; + + if (cluster_epoch_get_current() + 1 != expected_epoch) + return false; + + CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + + cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); + if (new_epoch != expected_epoch) + return false; + lsn = GetXLogInsertRecPtr(); + cluster_epoch_set_changed_at_lsn((uint64)lsn); + cluster_gcs_block_on_epoch_advance(new_epoch); + cluster_sinval_reset_all_on_reconfig(); + cluster_tt_status_flush_all((uint32)new_epoch); + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); + cluster_membership_record_admitted(node_id, admitted_incarnation); + ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); + LWLockRelease(&ReconfigShmem->lock); + + if (cluster_reconfig_is_clean_departed(node_id)) + pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); + cluster_reconfig_clear_clean_departed(node_id); + + dead_bitmap_set_bit(jb, node_id); + memset(incs, 0, sizeof(incs)); + incs[node_id] = admitted_incarnation; + + memset(&evt, 0, sizeof(evt)); + evt.event_id = cluster_reconfig_compute_event_id_v2( + RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); + evt.coordinator_node_id = cluster_node_id; + evt.old_epoch = old_epoch; + evt.new_epoch = new_epoch; + memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + evt.applied_at = GetCurrentTimestamp(); + evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); + evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; + cluster_reconfig_publish_event(&evt); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); + + return true; +} + +static bool +cluster_reconfig_poll_join_commit_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + uint64 admitted_incarnation = 0; + uint64 admitted_generation = 0; + ClusterMarkerPollResult pr; + + if (!join_commit_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_commit_stage.submitted) { + if (!cluster_reconfig_submit_join_marker_async( + &join_commit_stage.async, join_commit_stage.node_id, &join_commit_stage.marker, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, now)) + return true; + join_commit_stage.submitted = true; + return true; + } + + pr = cluster_reconfig_poll_join_marker_async(&join_commit_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " + "voting-disk majority; not committing (will retry)", + join_commit_stage.node_id))); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + if (!cluster_reconfig_get_observed_slot(join_commit_stage.node_id, &admitted_incarnation, + &admitted_generation) + || admitted_incarnation != join_commit_stage.admitted_incarnation + || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, + admitted_generation) + != CLUSTER_JOIN_ACCEPT + || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, + join_commit_stage.admitted_incarnation, + join_commit_stage.async.staged_expect_epoch)) { + pg_atomic_fetch_add_u64(&ReconfigShmem->join_reject_count, 1); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_release_join_commit_stage(); + return true; +} + /* * spec-5.15 D4 — coordinator-side join driver (called from the tick when * online_join is on and self is the min-MEMBER coordinator). Phase-1: fresh @@ -1166,6 +1676,10 @@ cluster_reconfig_drive_joins(int coordinator) if (state == NULL) return; + if (cluster_reconfig_poll_join_prepare_stage()) + return; + if (cluster_reconfig_poll_join_commit_stage()) + return; /* Phase-1 detection + a snapshot of the current pending set, under the lock * (compute_join_bitmap reads membership_state). */ @@ -1182,8 +1696,7 @@ cluster_reconfig_drive_joins(int coordinator) (void)cluster_reconfig_get_observed_slot(i, &joiner_incarnations[i], NULL); } cluster_reconfig_apply_join_as_coordinator(join_bitmap, coordinator, joiner_incarnations); - /* enter the JOIN_PENDING transition fail-closed on every local backend. */ - cluster_reconfig_broadcast_local_procsig(); + return; } /* Phase-2: commit pending joins that have converged. The just-added Phase-1 @@ -1646,6 +2159,7 @@ cluster_reconfig_lmon_tick(void) uint64 cssd_dead_generation; uint64 event_id; int i; + bool failstop_stage_handled = false; /* L20: runtime feature flag check first line. */ if (!cluster_enabled) @@ -1665,6 +2179,7 @@ cluster_reconfig_lmon_tick(void) self_id = cluster_node_id; if (self_id < 0 || self_id >= CLUSTER_MAX_NODES) return; /* defensive: bad self id, cannot participate */ + failstop_stage_handled = cluster_reconfig_poll_failstop_fence_stage(); /* * §3.1 + F11: build the raw CSSD DEAD bitmap, filtering out un-declared @@ -1886,7 +2401,7 @@ cluster_reconfig_lmon_tick(void) * stabilizing the survivor base), THEN the join edge. Each is an independent * ReconfigEvent; neither early-returns past the other. */ - if (!dead_bitmap_is_zero(dead_bitmap)) { + if (!failstop_stage_handled && !dead_bitmap_is_zero(dead_bitmap)) { CLUSTER_INJECTION_POINT("cluster-reconfig-decide-coordinator"); /* §3.2 P1.2: event_id from dead_bitmap + dead_generation snapshot. */ @@ -2024,6 +2539,16 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( CLUSTER_INJECTION_POINT("cluster-reconfig-epoch-bump-pre"); + /* + * spec-2.29a r2 t/274: mark the pre-bump→publish window BEFORE bumping the + * epoch, so a concurrent LMON GRD IDLE tick holds its WAIT_EPOCH baseline + * while this (possibly backend-context, synchronous) path bumps but has not + * yet published. Cleared at every return below. Harmless on the LMON / + * fence-off paths (they either hand off to the local staged flag or publish + * within this same call with no intervening tick). + */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 1); + /* D18: atomic CAS-loop increment. Returns pre/post snapshots. */ cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); @@ -2123,13 +2648,44 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " - "majority for epoch %llu; not publishing reconfig event " - "(write-fenced, will retry)", - (unsigned long long)new_epoch))); - return; /* fail-closed: epoch bumped, event NOT published, recovery NOT started */ + /* + * The async stage is process-local by design and is driven only by LMON + * ticks. Test-only backend callers, such as + * cluster_reconfig_inject_dead_node_test(), would otherwise stage the marker + * in their own process and return with no LMON-visible pending publish + * record. Keep those non-LMON paths on the old bounded wait: they are not + * transport-liveness actors, so this does not reintroduce the BUG-C1 LMON + * park. + */ + if (MyBackendType != B_LMON) { + ClusterFenceMarkerSubmitResult result; + + result = cluster_write_fence_submit_marker(&marker); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " + "majority for epoch %llu; not publishing reconfig event " + "(write-fenced, will retry)", + (unsigned long long)new_epoch))); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; + } + + cluster_reconfig_publish_event(&evt); + cluster_reconfig_log_failstop_epoch_bump(&evt); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; } + + failstop_fence_stage.event = evt; + failstop_fence_stage.marker = marker; + failstop_fence_stage.node_id = coordinator_node_id; + failstop_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + coordinator_node_id, GetCurrentTimestamp()); + /* LMON path: the local staged flag now covers the pending window. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; /* fail-closed until the staged marker is majority-durable */ } cluster_reconfig_publish_event(&evt); @@ -2143,21 +2699,10 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * fixed stack buffer (worst case: 128 node ids x 4 chars) keeps the LMON * tick free of allocator traffic. */ - { - char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; - int off = 0; - int n; - - dead[0] = '\0'; - for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { - if (dead_bitmap[n / 8] & (1 << (n % 8))) - off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); - } - ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " - "(coordinator node %d, dead node(s) {%s})", - (unsigned long long)old_epoch, (unsigned long long)new_epoch, - (int)coordinator_node_id, dead))); - } + cluster_reconfig_log_failstop_epoch_bump(&evt); + + /* spec-2.29a r2 t/274: fence-off path published within this call. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); } @@ -2287,6 +2832,9 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 return 0; if (removed_node_id < 0 || removed_node_id >= CLUSTER_MAX_NODES) return 0; + if (node_removed_fence_stage.async.has_staged_event) + return cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); CLUSTER_INJECTION_POINT("cluster-node-remove-shrink-committing"); @@ -2340,13 +2888,25 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 * a concurrent dead set, if any, is carried by its own fail-stop fence. */ memcpy(marker.fenced_dead_bitmap, removed_with_n, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " - "voting-disk majority for epoch %llu; not publishing removal " - "(write-fenced, will retry)", - removed_node_id, (unsigned long long)new_epoch))); - return 0; /* fail-closed: epoch bumped, removal NOT published, driver retries */ - } + node_removed_fence_stage.event.event_id + = cluster_reconfig_compute_removal_event_id(removed_with_n, removal_event_id); + node_removed_fence_stage.event.coordinator_node_id = cluster_node_id; + node_removed_fence_stage.event.old_epoch = old_epoch; + node_removed_fence_stage.event.new_epoch = new_epoch; + memcpy(node_removed_fence_stage.event.dead_bitmap, empty_dead, + CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + node_removed_fence_stage.event.applied_at = GetCurrentTimestamp(); + node_removed_fence_stage.event.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + node_removed_fence_stage.event.cssd_dead_generation = cssd_dead_generation; + node_removed_fence_stage.event.reconfig_kind = RECONFIG_KIND_NODE_REMOVED; + node_removed_fence_stage.marker = marker; + node_removed_fence_stage.node_id = removed_node_id; + node_removed_fence_stage.last_incarnation = last_incarnation; + node_removed_fence_stage.removal_event_id = removal_event_id; + node_removed_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); + return 0; /* fail-closed until the staged fence marker ACKs */ } CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); @@ -2431,6 +2991,42 @@ cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMa } } +bool +cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now) +{ + int wait_ms; + + if (ReconfigShmem == NULL || m == NULL || a == NULL) + return false; + if (target_node < 0 || target_node >= CLUSTER_MAX_NODES) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&ReconfigShmem->join_marker_request_seq, + &ReconfigShmem->join_marker_completion_seq)) + return false; + + ReconfigShmem->join_marker_target_node_id = target_node; + ReconfigShmem->join_pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit( + a, &ReconfigShmem->join_marker_request_seq, &ReconfigShmem->join_marker_completion_seq, + ReconfigShmem->join_qvotec_latch, now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (ReconfigShmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &ReconfigShmem->join_marker_completion_seq, + &ReconfigShmem->join_marker_result, now, out_result, + out_elapsed_us); +} + bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512) { @@ -2464,6 +3060,7 @@ cluster_reconfig_join_qvotec_complete(bool acked) pg_atomic_write_u64(&ReconfigShmem->join_marker_completion_seq, join_qvotec_inflight_marker_seq); join_qvotec_last_processed_marker_seq = join_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -2614,27 +3211,6 @@ cluster_reconfig_apply_join_as_coordinator( } LWLockRelease(&ReconfigShmem->lock); - /* Durable PREPARE marker per joiner (records the presented incarnation; does - * NOT seed — only COMMITTED is a basis). Best-effort: PREPARE failure does - * not block the JOIN_PENDING publish (the COMMITTED marker in Phase-2 is the - * commit point that must be majority-durable). */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - ClusterJoinCommitMarker m; - - if (!dead_bitmap_test_bit(join_bitmap, i)) - continue; - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_JCMK_MAGIC; - m.version = CLUSTER_JCMK_VERSION; - m.node_id = i; - m.phase = CLUSTER_JCMK_PHASE_PREPARE; - m.admitted_incarnation = joiner_incarnations[i]; - m.generation = joiner_incarnations[i]; /* monotonic per node (read-newest intent) */ - m.admitted_epoch = new_epoch; - cluster_join_marker_compute_crc(&m); - (void)cluster_reconfig_submit_join_marker(i, &m); - } - memset(&evt, 0, sizeof(evt)); evt.event_id = cluster_reconfig_compute_event_id_v2(RECONFIG_KIND_JOIN_PENDING, empty_dead, join_bitmap, @@ -2647,20 +3223,21 @@ cluster_reconfig_apply_join_as_coordinator( evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; evt.cssd_dead_generation = cssd_dead_generation; evt.reconfig_kind = RECONFIG_KIND_JOIN_PENDING; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + + join_prepare_stage.event = evt; + memcpy(join_prepare_stage.join_bitmap, join_bitmap, sizeof(join_prepare_stage.join_bitmap)); + memcpy(join_prepare_stage.joiner_incarnations, joiner_incarnations, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; + join_prepare_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_join_prepare_stage(); } bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) { ClusterJoinCommitMarker m; - uint64 old_epoch, new_epoch; - XLogRecPtr lsn; - ReconfigEvent evt; - uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint64 incs[CLUSTER_MAX_NODES]; if (!cluster_enabled || ReconfigShmem == NULL) return false; @@ -2704,60 +3281,17 @@ cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) m.commit_nonce = ((uint64)cluster_node_id << 56) ^ (uint64)GetCurrentTimestamp(); cluster_join_marker_compute_crc(&m); - if (cluster_reconfig_submit_join_marker(node_id, &m) != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " - "voting-disk majority; not committing (will retry)", - node_id))); + if (join_commit_stage.async.has_staged_event) return false; - } - - CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + join_commit_stage.marker = m; + join_commit_stage.node_id = node_id; + join_commit_stage.admitted_incarnation = admitted_incarnation; + join_commit_stage.async.has_staged_event = true; + join_commit_stage.async.staged_expect_epoch = m.admitted_epoch; + join_commit_stage.submitted = false; + (void)cluster_reconfig_poll_join_commit_stage(); - /* - * ② publish: bump JOIN_COMMITTED epoch + state MEMBER + last_admitted + clear - * pending + clear clean_departed[node] (INV-J10). Strict order — the publish - * (epoch + state MEMBER) precedes the joiner opening its write gate (D5). - */ - cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); - lsn = GetXLogInsertRecPtr(); - cluster_epoch_set_changed_at_lsn((uint64)lsn); - cluster_gcs_block_on_epoch_advance(new_epoch); - cluster_sinval_reset_all_on_reconfig(); - cluster_tt_status_flush_all((uint32)new_epoch); - - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); - cluster_membership_record_admitted(node_id, admitted_incarnation); - ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); - LWLockRelease(&ReconfigShmem->lock); - - /* INV-J10: clear the in-shmem clean_departed suppression so a clean-left node - * that just rejoined has its later real fail-stop honored again (the durable - * supersede across restart is resolved by the seed, RC-5). */ - if (cluster_reconfig_is_clean_departed(node_id)) - pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); - cluster_reconfig_clear_clean_departed(node_id); - - dead_bitmap_set_bit(jb, node_id); - memset(incs, 0, sizeof(incs)); - incs[node_id] = admitted_incarnation; - - memset(&evt, 0, sizeof(evt)); - evt.event_id = cluster_reconfig_compute_event_id_v2( - RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); - evt.coordinator_node_id = cluster_node_id; - evt.old_epoch = old_epoch; - evt.new_epoch = new_epoch; - memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - evt.applied_at = GetCurrentTimestamp(); - evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; - evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); - evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); - - return true; + return false; } diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index 2baebb13bd..8917959359 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -40,6 +40,7 @@ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_voting_disks */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_qvotec.h" /* ClusterVotingSlot (marker layout asserts) */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_voting_disk_io.h" /* D6 direct durable marker read */ @@ -769,6 +770,37 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m) } } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (cluster_write_fence_shmem == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq)) + return false; + + memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); + return cluster_marker_async_submit( + a, &cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq, cluster_write_fence_shmem->qvotec_latch, + now, (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cluster_write_fence_shmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cluster_write_fence_shmem->marker_completion_seq, + &cluster_write_fence_shmem->marker_result, now, out_result, + out_elapsed_us); +} + /* * cluster_write_fence_clear_qvotec_latch / _publish_qvotec_latch -- qvotec publishes * its MyLatch at startup so LMON can wake it; auto-cleared at proc_exit so a stale @@ -835,6 +867,7 @@ cluster_write_fence_qvotec_complete(bool acked) pg_atomic_write_u64(&cluster_write_fence_shmem->marker_completion_seq, qvotec_inflight_marker_seq); qvotec_last_processed_marker_seq = qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index e49ecfc194..6a4808f793 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -65,6 +65,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ #define CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES 16 /* Default drain deadline; aligns with the feature-082 30s barrier. */ @@ -149,11 +151,17 @@ typedef struct ClusterLeaveState { int32 leaving_node_id; /* -1 if none */ uint32 _pad0; uint64 leave_epoch; /* epoch this leave is bound to (CL-I3 immutable) */ - uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound; - * an unchanged value at the epoch bump means OUR - * clean-leave committed (no real death intruded) */ + uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound + * (retained for observability; the coherence gate + * now uses leave_baseline_others_dead, spec-2.29a + * ②b — the scalar counted the leaving node's own + * expected DEAD and falsely escalated) */ uint64 barrier_deadline_us; /* fail-closed deadline (drain_timeout_ms) */ uint8 ack_bitmap[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; /* survivor acks */ + /* spec-2.29a ②b: CSSD dead set (EXCLUDING the leaving node) snapshotted when + * the leave was bound; the coherence gate escalates only if a THIRD-PARTY + * death changed this set mid-drain. */ + uint8 leave_baseline_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; pg_atomic_uint64 ges_drained_count; /* shards/grants drained (observability) */ pg_atomic_uint64 gcs_flushed_count; /* dirty/X pages flushed */ pg_atomic_uint64 shards_remastered; /* shards moved off leaving node */ @@ -198,6 +206,14 @@ typedef struct ClusterLeaveState { pg_atomic_uint32 commit_point_observed; pg_atomic_uint32 committed_durable_confirmed; pg_atomic_uint32 committed_marker_durable; + /* spec-2.29a r3 (evidence latch): the committed epoch E the coordinator's + * LEAVE_COMMITTED attested (the epoch stamped into the majority-durable + * COMMITTED marker). Written by the leaving node's confirmation handler + * under `lock` BEFORE committed_durable_confirmed is set; consumed once by + * the barrier tick when the evidence latches (so the leaver records the + * exact committed epoch instead of inferring it from its possibly-stale + * local epoch view). Guarded by `lock`. */ + uint64 committed_confirmed_epoch; /* * Hardening v1.0.2 fields. @@ -335,10 +351,33 @@ extern bool cluster_clean_leave_should_abort_writable(bool in_transaction, bool /* version-coherent leave (U3 / CL-I3 / L235): the leave is still coherent only * if neither the cluster epoch nor the CSSD dead_generation moved since the - * leave was bound; any external bump (a real death intruding) => not coherent - * => the caller must ABORTED_ESCALATE. */ + * leave was bound; any external membership change (a third-party death, or a + * third-party fail-stop that already bumped the epoch) => not coherent => the + * caller must ABORTED_ESCALATE. spec-2.29a ②b: the dead set compared here + * EXCLUDES the leaving node itself (others-dead bitmap), so the leaving node's + * own expected alive→DEAD transition never falsely escalates the leave. */ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen); + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes); + +/* spec-2.29a r3: the leaving node's barrier-tick own-commit latch — evidence + * over inference. Latches iff the durable COMMITTED marker for THIS leave + * attempt was confirmed (nonce-bound LEAVE_COMMITTED attestation); the two + * coherence observations are contract inputs the verdict must ignore (a + * third-party transient flap must neither refuse an evidenced latch — the + * t/331 C1/C4 false-escalation — nor latch anything without evidence — the + * r2 P2-1 refused-leave mis-latch wedge). */ +extern bool cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, + bool others_dead_unchanged, + bool dead_gen_unchanged); + +/* spec-2.29a r3: identity gate for a LEAVE_COMMITTED confirmation — accept it + * as marker evidence only for THIS node's CURRENT leave attempt (self- + * addressed + currently leaving + per-attempt nonce match + committed epoch + * past the bound baseline); fail-closed on any mismatch. */ +extern bool cluster_clean_leave_committed_evidence_matches( + int32 payload_leaving_node, uint64 payload_nonce, uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, uint64 current_attempt_nonce, uint64 bound_leave_epoch); /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the @@ -408,6 +447,14 @@ extern void cluster_clean_leave_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); +extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, + const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512); extern void cluster_clean_leave_qvotec_complete(bool acked); extern void cluster_clean_leave_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..08d475a9d6 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -660,6 +660,7 @@ typedef struct ClusterGrdRecoveryCounters { } ClusterGrdRecoveryCounters; extern void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out); +extern uint64 cluster_grd_recovery_event_old_epoch(void); /* spec-2.29a WAIT_EPOCH baseline */ /* spec-4.6 P0#2 — pre-remaster stale-epoch sweep SCOPED to the affected * (dead-master) shards; affected_shards is a PGRAC_GRD_SHARD_COUNT-bit diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..1b3a06016f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -469,6 +469,7 @@ extern int cluster_phase4_timeout; * range: [100, 60000] (millisecond) */ extern int cluster_lmon_main_loop_interval; +extern int cluster_lmon_slow_iteration_warn_ms; /* diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..e1099ece40 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -100,6 +100,7 @@ #include "datatype/timestamp.h" #include "storage/lwlock.h" +struct Latch; /* * ClusterLmonStatus -- HC2 SSOT for LMON lifecycle state. @@ -137,6 +138,10 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 max_iter_us; /* high-water iteration wall time */ + uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ + struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ } ClusterLmonSharedState; @@ -188,6 +193,10 @@ extern TimestampTz cluster_lmon_spawned_at(void); extern TimestampTz cluster_lmon_ready_at(void); extern TimestampTz cluster_lmon_last_liveness_tick_at(void); extern int64 cluster_lmon_main_loop_iters(void); +extern uint64 cluster_lmon_last_iter_us(void); +extern uint64 cluster_lmon_max_iter_us(void); +extern uint64 cluster_lmon_slow_iter_count(void); +extern void cluster_lmon_marker_complete_wakeup(void); /* * Status enum -> canonical lowercase string ("not_started", "spawning", diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h new file mode 100644 index 0000000000..56cd836bf5 --- /dev/null +++ b/src/include/cluster/cluster_marker_async.h @@ -0,0 +1,218 @@ +/*------------------------------------------------------------------------- + * + * cluster_marker_async.h + * Small process-local async FSM for qvotec marker submit mailboxes. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_marker_async.h + * + * NOTES + * The mailbox remains the existing request_seq/completion_seq/result + * slot. This wrapper changes only the waiting shape: submit publishes + * a request and returns; the owning LMON tick polls completion without + * sleeping (spec-2.29a — the pre-async 2ms pg_usleep spin inside the + * LMON tick starved the CSSD heartbeat relay and caused false-DEAD + * storms during cold formation, BUG-C1). + * + * The staged has_staged_event flag is the P1-1 bump-once contract: a + * pre-bump caller (fail-stop fence / node-remove / join Phase-1) must + * not re-enter its epoch-bump path while a stage is live. + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_MARKER_ASYNC_H +#define CLUSTER_MARKER_ASYNC_H + +#include "c.h" + +#include + +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "storage/latch.h" + +typedef enum ClusterMarkerAsyncState { + CLUSTER_MARKER_ASYNC_IDLE = 0, + CLUSTER_MARKER_ASYNC_SUBMITTED +} ClusterMarkerAsyncState; + +typedef enum ClusterMarkerPollResult { + CLUSTER_MARKER_POLL_IDLE = 0, + CLUSTER_MARKER_POLL_PENDING, + CLUSTER_MARKER_POLL_ACKED, + CLUSTER_MARKER_POLL_TIMEOUT +} ClusterMarkerPollResult; + +typedef enum ClusterMarkerAsyncKind { + CLUSTER_MARKER_KIND_UNKNOWN = 0, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + CLUSTER_MARKER_KIND_JOIN_PREPARE, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED +} ClusterMarkerAsyncKind; + +typedef struct ClusterMarkerAsync { + ClusterMarkerAsyncState state; + uint64 inflight_seq; + uint64 deadline_us; + TimestampTz submitted_at; + ClusterMarkerAsyncKind kind; + int32 target_node; + + /* + * Caller-owned staged publish/commit state. While true, the owner must not + * re-enter the pre-bump path; it either submits/polls this record, publishes + * after ACK, or releases it after failure/timeout. + */ + bool has_staged_event; + uint64 staged_expect_epoch; +} ClusterMarkerAsync; + +static inline const char * +cluster_marker_async_kind_name(ClusterMarkerAsyncKind kind) +{ + switch (kind) { + case CLUSTER_MARKER_KIND_FENCE_FAILSTOP: + return "fence_failstop"; + case CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED: + return "fence_node_removed"; + case CLUSTER_MARKER_KIND_JOIN_PREPARE: + return "join_prepare"; + case CLUSTER_MARKER_KIND_JOIN_COMMITTED: + return "join_committed"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING: + return "clean_leave_committing"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED: + return "clean_leave_committed"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING: + return "node_remove_removing"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK: + return "node_remove_shrunk"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED: + return "node_remove_removed"; + case CLUSTER_MARKER_KIND_UNKNOWN: + default: + return "unknown"; + } +} + +static inline void +cluster_marker_async_init(ClusterMarkerAsync *a) +{ + memset(a, 0, sizeof(*a)); + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->target_node = -1; +} + +static inline bool +cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) +{ + return a != NULL && a->state == CLUSTER_MARKER_ASYNC_SUBMITTED; +} + +static inline bool +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq) +{ + return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); +} + +static inline bool +cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, + TimestampTz now, uint64 timeout_us, ClusterMarkerAsyncKind kind, + int32 target_node) +{ + uint64 seq; + + Assert(a != NULL); + Assert(request_seq != NULL); + Assert(completion_seq != NULL); + + if (a->state == CLUSTER_MARKER_ASYNC_SUBMITTED) + return true; + if (cluster_marker_async_mailbox_busy(request_seq, completion_seq)) + return false; + + pg_write_barrier(); + seq = pg_atomic_add_fetch_u64(request_seq, 1); + if (qvotec_latch != NULL) + SetLatch(qvotec_latch); + + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->inflight_seq = seq; + a->deadline_us = (uint64)now + timeout_us; + a->submitted_at = now; + a->kind = kind; + a->target_node = target_node; + return true; +} + +static inline ClusterMarkerPollResult +cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + uint64 elapsed; + + Assert(a != NULL); + Assert(completion_seq != NULL); + Assert(result_slot != NULL); + + if (out_result != NULL) + *out_result = 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (a->state != CLUSTER_MARKER_ASYNC_SUBMITTED) + return CLUSTER_MARKER_POLL_IDLE; + + if (pg_atomic_read_u64(completion_seq) == a->inflight_seq) { + pg_read_barrier(); + if (out_result != NULL) + *out_result = pg_atomic_read_u32(result_slot); + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_ACKED; + } + + if ((uint64)now >= a->deadline_us) { + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_TIMEOUT; + } + + return CLUSTER_MARKER_POLL_PENDING; +} + +static inline void +cluster_marker_async_release_stage(ClusterMarkerAsync *a) +{ + if (a == NULL) + return; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->inflight_seq = 0; + a->deadline_us = 0; + a->submitted_at = 0; + a->kind = CLUSTER_MARKER_KIND_UNKNOWN; + a->target_node = -1; + a->has_staged_event = false; + a->staged_expect_epoch = 0; +} + +#endif /* CLUSTER_MARKER_ASYNC_H */ diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index ce6bbd7b19..ef722e3f62 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -74,6 +74,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + #include "cluster/cluster_write_fence.h" /* CLUSTER_FENCE_MARKER_BYTES (marker offset) */ /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ @@ -415,6 +417,14 @@ extern void cluster_node_remove_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); +extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, + const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out); extern void cluster_node_remove_qvotec_complete(bool acked); extern void cluster_node_remove_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index e34c3a1942..1aa106f690 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -56,7 +56,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ struct Latch; /* spec-5.15 D4 — join-marker qvotec mailbox latch (pointer only) */ @@ -225,6 +226,16 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 apply_counter; /* total events observed */ pg_atomic_uint64 dedup_skip_counter; /* duplicate event_id skipped */ pg_atomic_uint64 procsig_broadcast_count; /* PROCSIG broadcast tally */ + /* spec-2.29a r2 t/274: a backend-context coordinator (e.g. the fail-stop + * inject test) advances the epoch then SYNCHRONOUSLY waits for the fence + * marker before publishing. During that (seconds-long) wait the epoch is + * bumped but the reconfig event is not yet published, so a concurrently + * running LMON GRD IDLE tick would re-capture the post-bump epoch as its + * WAIT_EPOCH baseline and wedge (old==cur). The LMON-process-local staged + * flags cannot see a backend's pre-bump window, so publish it in shmem: + * set before the bump, cleared after publish / on failure. The GRD IDLE + * baseline-hold guard reads this OR the local stage. */ + pg_atomic_uint32 prebump_sync_active; /* spec-5.14 D6 — touched_peers fail-stop observability (Q8 A': these * live in this existing region; the region count is unchanged). */ @@ -316,6 +327,8 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 join_reject_count; pg_atomic_uint64 join_timeout_count; pg_atomic_uint64 clean_departed_cleared_count; + pg_atomic_uint64 marker_slow_ack_count; + pg_atomic_uint64 marker_timeout_count; /* * spec-5.15 D1 — per declared node, the freshest voting-slot incarnation + @@ -406,6 +419,12 @@ extern void cluster_reconfig_shmem_register(void); */ extern void cluster_reconfig_get_last_event(ReconfigEvent *out); +/* spec-2.29a: true while a pre-bump coordinator stage (fail-stop fence / + * node-remove / join Phase-1) has bumped the epoch but not yet published its + * reconfig event -- the GRD recovery IDLE tick must hold its baseline instead + * of re-capturing the post-bump epoch (else WAIT_EPOCH wedges). */ +extern bool cluster_reconfig_has_pending_prebump_stage(void); + /* ============================================================ * Coordinator path APIs (Step 2 D2 wiring). @@ -580,10 +599,24 @@ extern bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarn */ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); +extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now); +extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512); extern void cluster_reconfig_join_qvotec_complete(bool acked); extern void cluster_reconfig_publish_join_qvotec_latch(struct Latch *latch); +extern void cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern void cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern uint64 cluster_reconfig_get_marker_slow_ack_count(void); +extern uint64 cluster_reconfig_get_marker_timeout_count(void); + /* ============================================================ * spec-5.15 D5 — joiner-side write gate + admission (INV-J9 / §2.4 / §2.7). * ============================================================ @@ -756,5 +789,11 @@ extern uint64 cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_n uint64 last_incarnation, bool *out_contest); +/* spec-2.29a review r1 P2: true while the staged node-removed fence marker is + * in flight -- the node-remove driver skips its FENCE_ARMING pre-work + * (REMOVING marker / stripe retire) instead of re-keying it to the live + * (already self-bumped) epoch. */ +extern bool cluster_reconfig_node_removed_fence_stage_pending(void); + #endif /* CLUSTER_RECONFIG_H */ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index 7543664e72..ac5a965882 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -39,6 +39,8 @@ #ifndef CLUSTER_WRITE_FENCE_H #define CLUSTER_WRITE_FENCE_H +#include "cluster/cluster_marker_async.h" + /* * cluster_write_fence_decide -- the PURE cooperative write-fence judge (spec-4.12 * D3). A shared-storage write is allowed iff enforcement is on AND the local @@ -517,6 +519,14 @@ extern void cluster_write_fence_note_baseline_stale(void); */ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); +extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern void cluster_write_fence_publish_qvotec_latch(struct Latch *latch); extern bool cluster_write_fence_qvotec_poll_pending(ClusterFenceMarker *out); extern void cluster_write_fence_qvotec_complete(bool acked); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..2035b40072 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'pg_stat_cluster_injections returns 159 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '159 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..6d53bad1df 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -63,8 +63,8 @@ 'postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'all 55 categories appear (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); # ---------- @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..42f8817b91 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L9 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..eab706fd4c 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '159', + 'L15 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..57a321a3a6 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..f2cac828a8 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L12a pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..2c41642a72 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- @@ -168,8 +168,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12 pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12 pg_cluster_state has 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -232,7 +232,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..62f66a1aa8 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -162,8 +162,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L6a pg_stat_cluster_injections has 159 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', @@ -189,8 +189,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L7b pg_cluster_state has 55 distinct categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L7b pg_cluster_state has 56 distinct categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -209,7 +209,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..3672d99bce 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', 'M1 159 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '318', + 'M5 inject category has 159×2 = 318 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); @@ -395,8 +395,8 @@ is($node->safe_psql('postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'O2 pg_cluster_state has all 55 categories (spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'O2 pg_cluster_state has all 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE value IS NULL}), diff --git a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl index f12abe0f14..9bc0e1fccf 100644 --- a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl +++ b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl @@ -325,8 +325,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); my $tt_categories = $pair->node0->safe_psql('postgres', q{ SELECT string_agg(c, ',' ORDER BY c) diff --git a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl index 4e65595eff..56d8cd35f9 100644 --- a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl +++ b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl @@ -296,8 +296,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L14a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L14a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl index 80447a5161..8f4f1c5ce7 100644 --- a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl +++ b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl @@ -287,7 +287,7 @@ # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', 'L13a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', 'L13a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl new file mode 100644 index 0000000000..005c0d4a4b --- /dev/null +++ b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl @@ -0,0 +1,191 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 363_cluster_2_29a_marker_async_liveness.pl +# BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. +# +# The hold injection freezes qvotec marker completion until released. Before +# BUG-C1, LMON synchronously waited inside marker submit, stopped sending HELLOs, +# and peers could false-DEAD it during a long marker write. This TAP holds the +# marker service for >=90s while an online join wants a PREPARE marker and checks: +# L1 hold injection is registered. +# L2 a true leave/restart creates a join marker backlog. +# L3 during a 95s hold, LMON keeps iterating and both live nodes keep each +# other CSSD-alive (no false-DEAD). +# L4 marker timeout/backlog is observable and does not publish admission. +# L5 releasing the hold lets the join converge. +# L6 an actual stopped node still becomes DEAD (no true-DEAD regression). +# +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterPair; +use Test::More; +use Time::HiRes qw(usleep); + +sub poll_until +{ + my ($node, $query, $expected, $timeout_s, $label) = @_; + $expected //= 't'; + $timeout_s //= 30; + my $deadline = time + $timeout_s; + my $last = ''; + while (time < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) } // ''; + return 1 if defined $last && $last eq $expected; + usleep(200_000); + } + diag("poll_until timeout ($label): last='$last' expected='$expected'"); + return 0; +} + +sub state_int +{ + my ($node, $category, $key) = @_; + my $v = $node->safe_psql('postgres', + "SELECT COALESCE((SELECT value::bigint FROM pg_cluster_state " + . "WHERE category = '$category' AND key = '$key'), 0)"); + return $v + 0; +} + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +my @conf = ( + 'autovacuum = off', + 'jit = off', + 'cluster.online_join = on', + 'cluster.join_convergence_timeout_ms = 180000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 8', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.lmon_slow_iteration_warn_ms = 0', +); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'marker_async_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [@conf]); +$pair->start_pair; +usleep(3_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-qvotec-marker-service-hold'}), + '1', + 'L1 hold injection point is registered'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 30, 'node1 initially member'), + 'L1 node1 initially MEMBER'); + +my $epoch0 = $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->stop; +ok(poll_until($node0, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead'), + 'L2 node0 detects the stopped node1 as dead'); +ok(poll_until($node0, + q{SELECT state <> 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'node1 demoted from member'), + 'L2 node1 is demoted from MEMBER before rejoin'); + +set_injection($node0, 'cluster-qvotec-marker-service-hold'); +my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); +my $epoch_hold_start = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->start; +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 CSSD alive after restart'), + 'L2 node1 is live again while marker service is held'); + +sleep 95; + +my $lmon_iters_after = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_after = state_int($node0, 'reconfig', 'marker_timeout_count'); + +cmp_ok($lmon_iters_after, '>', $lmon_iters_before + 20, + 'L3 node0 LMON keeps iterating during >=90s qvotec marker hold'); +ok(poll_until($node1, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 5, 'node1 still sees node0 alive after hold'), + 'L3 node1 still sees node0 CSSD-alive after the hold (no false-DEAD)'); +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 5, 'node0 still sees node1 alive after hold'), + 'L3 node0 still sees node1 CSSD-alive after the hold'); + +cmp_ok($timeouts_after, '>', $timeouts_before, + 'L4 marker timeout/backlog counter advances while qvotec marker service is held'); +isnt($node0->safe_psql('postgres', + q{SELECT state FROM pg_cluster_membership WHERE node_id = 1}), + 'member', + 'L4 held/timeout marker does not publish node1 admission'); + +# P1-1 bump-once (spec-2.29a review r1): the whole >=95s hold must cost at most +# ONE join Phase-1 epoch bump (staged record; PREPARE timeouts drain best-effort +# and never revert the joiner, so compute_join_bitmap cannot re-detect + re-bump +# per deadline period). Pre-fix abort+revert behavior re-bumped ~once per 3.5s +# deadline (~27 bumps in this window); a <=2 bound separates cleanly without +# being flaky against one incidental episode. +my $epoch_hold_end = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; +cmp_ok($epoch_hold_end - $epoch_hold_start, '<=', 2, + 'L4 epoch advance across the hold window is bounded (no per-deadline bump storm)'); + +set_injection($node0, ''); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 joins after hold release'), + 'L5 node0 admits node1 after hold release'); +ok(poll_until($node1, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 self member after hold release'), + 'L5 node1 self-state reaches MEMBER after hold release'); +cmp_ok($node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0, + '>', $epoch0, + 'L5 membership epoch advances after rejoin'); + +$node1->stop; +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead after final stop'), + 'L6 true DEAD detection still works after async marker liveness fix'); +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'membership dead after final stop'), + 'L6 membership fail-stop path still demotes a truly stopped peer'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..0d753ce2b6 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -41,7 +41,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ - test_cluster_epoch test_cluster_fence test_cluster_reconfig \ + test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory \ test_cluster_tt_slot_allocator test_cluster_itl_reader_real_triple \ test_cluster_itl_cleanout test_cluster_visibility_inject test_cluster_itl_cleanout_perf \ diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index b703798b38..04363fd736 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -159,14 +159,104 @@ UT_TEST(test_phase_fsm) UT_TEST(test_version_coherent) { - /* nothing moved -> coherent */ - UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, 3, 3)); - /* epoch bumped by an external death -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 3)); - /* dead_generation moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, 3, 4)); - /* both moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 4)); + /* spec-2.29a ②b: coherence = epoch unchanged AND the others-dead bitmap + * (dead set EXCLUDING the leaving node) unchanged. The reflexive-case + * matrix from the spec §②b 8.A argument, exercised on the pure predicate. */ + uint8 od_none[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + uint8 od_third[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + const int n = CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES; + + od_third[0] = 0x04; /* a THIRD-PARTY node (e.g. node 2) became DEAD */ + + /* (i) nothing moved (leaving node's own DEAD is already excluded upstream, + * so its transition does not appear here) -> coherent */ + UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, od_none, od_none, n)); + /* (iii) epoch bumped by an external fail-stop -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_none, n)); + /* (ii) a third-party death entered the others-dead set, epoch not yet + * bumped (the P1-b window) -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, od_third, n)); + /* (iv) both moved -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_third, n)); + /* fail-closed on a missing view */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, NULL, od_none, n)); + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, NULL, n)); +} + + +/* ============================================================ + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r3, evidence + * over inference) + * ============================================================ */ + +UT_TEST(test_own_commit_latched) +{ + /* The latch verdict is EVIDENCE-only: the first argument is "the durable + * COMMITTED marker for THIS leave attempt was confirmed" (the coordinator's + * nonce-bound LEAVE_COMMITTED attestation). The two coherence observations + * (others-dead bitmap / scalar dead_generation) stay in the signature as + * contract inputs the verdict must IGNORE — a third-party transient + * false-DEAD flap on the leaver's local CSSD view advances the (monotone) + * dead_generation and can transiently disturb the bitmap, and neither may + * refuse a latch that marker evidence backs (the t/331 C1/C4 false- + * escalation), nor may any combination latch without evidence (the r2 P2-1 + * refused-leave mis-latch hang). */ + + /* (a) evidence present, no flap noise -> latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); + + /* (d) PINNING LEG (t/331 C1/C4 regression): a third-party flap advanced the + * scalar dead_generation (it never rebounds) while the bitmap rebounded to + * its bound value — WITH marker evidence the leave still latches; the flap + * must not false-escalate a committed leave. */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, false)); + + /* (d) flap currently visible in the others-dead bitmap too -> still latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, true)); + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, false)); + + /* (c) no evidence -> never latch, whatever the coherence observations say + * (a refused leave never produces a COMMITTED marker, so the barrier + * deadline escalation stays armed and bounds the wait — the r2 P2-1 + * mis-latch wedge stays closed). */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, false)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, false)); +} + + +/* ============================================================ + * U3c — LEAVE_COMMITTED evidence identity gate (spec-2.29a r3) + * ============================================================ */ + +UT_TEST(test_committed_evidence_matches) +{ + /* args: payload_leaving_node, payload_nonce, payload_epoch, + * self_node, current_leaving_node, current_attempt_nonce, + * bound_leave_epoch */ + + /* (a) confirmation for THIS node's CURRENT attempt, committed epoch past + * the bound baseline -> evidence accepted */ + UT_ASSERT(cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 1, 42, 7)); + + /* (b) stale identity: a LEAVE_COMMITTED (and through it a COMMITTED + * marker) from a PREVIOUS leave attempt of the same node — nonce differs + * -> fail-closed, no latch */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 41, 8, 1, 1, 42, 7)); + + /* (b) misrouted: confirmation addressed to a different leaving node */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(2, 42, 8, 1, 1, 42, 7)); + + /* (b) this node is not currently leaving (idle: leaving_node_id == -1, + * or tracking someone else's leave) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, -1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 2, 42, 7)); + + /* (b) committed epoch not past the bound baseline (attested epoch must be + * the guarded-CAS successor of the baseline this leave bound) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 7, 1, 1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 0, 1, 1, 42, 7)); } @@ -382,10 +472,12 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(8); + UT_PLAN(10); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); + UT_RUN(test_own_commit_latched); + UT_RUN(test_committed_evidence_matches); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..b0a4fe5746 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2638,6 +2638,21 @@ cluster_lmon_main_loop_iters(void) { return 0; } +uint64 +cluster_lmon_last_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_max_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_slow_iter_count(void) +{ + return 0; +} int cluster_lmon_status(void) { @@ -3377,6 +3392,16 @@ cluster_reconfig_get_clean_departed_cleared_count(void) { return 0; } +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return 0; +} +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return 0; +} void cluster_touched_peers_self_hex(char *buf, Size buflen) { diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..77e7862d6f 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -383,6 +383,16 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a nightly-regression fix: controllable pre-bump-stage flag so the + * GRD IDLE baseline-hold can be unit-driven (default false = pre-fix + * re-capture behavior; existing tests are unaffected). */ +static bool ut_mock_pending_prebump = false; +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return ut_mock_pending_prebump; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -4044,6 +4054,43 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) } +/* ============================================================ + * spec-2.29a nightly-regression fix — GRD IDLE baseline hold while a + * pre-bump coordinator stage is in flight. + * + * The async fence/join/node-remove marker staging bumps the epoch + * several ticks before publishing. While that window is open the GRD + * IDLE tick must NOT re-capture the post-bump epoch as its WAIT_EPOCH + * baseline, or the P0 accept that follows reads old == cur and wedges. + * ============================================================ */ + +UT_TEST(test_recovery_idle_holds_baseline_during_prebump_stage) +{ + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_pending_prebump = false; + + /* Idle tick with epoch 5 captures baseline 5 (no staging; get_last_event + * returns event_id 0 so the tick stays IDLE and re-captures). */ + ut_mock_epoch = 5; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* A pre-bump stage goes live and the coordinator bumps the epoch to 6 + * before publishing. The IDLE tick must HOLD baseline 5. */ + ut_mock_pending_prebump = true; + ut_mock_epoch = 6; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* Marker ACKs, stage clears; a subsequent idle tick resumes tracking. */ + ut_mock_pending_prebump = false; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 6); + + cluster_enabled = false; +} + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4102,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4205,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_idle_holds_baseline_during_prebump_stage); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..af5f119123 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -355,6 +355,13 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a: no pre-bump staging in the starvation fixture (no-op false). */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return false; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..15bc7894b1 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -178,6 +178,7 @@ cluster_postmaster_start_lmon(void) * MyLatch + cluster_inject framework. Stubs cover them all -- * runtime LmonMain is not exercised. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; #include "cluster/cluster_inject.h" int cluster_injection_armed_count = 0; @@ -590,6 +591,9 @@ pg_usleep(long microsec pg_attribute_unused()) /* Sprint B: Latch / WaitLatch / ResetLatch stubs (LmonMain runtime * is not invoked at unit-test level). */ struct Latch *MyLatch = NULL; +void +SetLatch(struct Latch *latch pg_attribute_unused()) +{} int WaitLatch(struct Latch *latch pg_attribute_unused(), int wakeEvents pg_attribute_unused(), long timeout pg_attribute_unused(), uint32 wait_event_info pg_attribute_unused()) @@ -600,6 +604,11 @@ void ResetLatch(struct Latch *latch pg_attribute_unused()) {} +#include "storage/ipc.h" +void +on_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), Datum arg pg_attribute_unused()) +{} + /* cluster_lmon.c references MyBackendType (set by LmonMain). */ #include "miscadmin.h" BackendType MyBackendType = B_INVALID; @@ -804,9 +813,21 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_size); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_init); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_register); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_last_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_max_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_slow_iter_count); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_marker_complete_wakeup); UT_ASSERT_NOT_NULL((void *)LmonMain); } +UT_TEST(test_lmon_iteration_counters_null_safe) +{ + UT_ASSERT_EQ((unsigned long long)cluster_lmon_last_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_max_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_slow_iter_count(), 0ULL); + cluster_lmon_marker_complete_wakeup(); +} + /* ============================================================ * Test runner @@ -815,12 +836,13 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(6); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_iteration_counters_null_safe); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c new file mode 100644 index 0000000000..b90fa463a2 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -0,0 +1,246 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_marker_async.c + * Unit tests for the process-local qvotec marker async FSM. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_marker_async.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_marker_async.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +static int ut_set_latch_count = 0; + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +void +SetLatch(Latch *latch pg_attribute_unused()) +{ + ut_set_latch_count++; +} + +static void +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot) +{ + cluster_marker_async_init(a); + pg_atomic_init_u64(request_seq, 0); + pg_atomic_init_u64(completion_seq, 0); + pg_atomic_init_u32(result_slot, 0); + ut_set_latch_count = 0; +} + +UT_TEST(test_init_defaults_idle) +{ + ClusterMarkerAsync a; + + cluster_marker_async_init(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); +} + +UT_TEST(test_submit_publishes_one_request) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 7); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(pg_atomic_read_u64(&completion_seq), 0); + UT_ASSERT_EQ(ut_set_latch_count, 1); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.deadline_us, 6000); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_FAILSTOP); + UT_ASSERT_EQ(a.target_node, 7); +} + +UT_TEST(test_poll_pending_before_completion) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 99; + uint64 elapsed = 99; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_PREPARE, 2)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 1500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_PENDING); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(result, 0); + UT_ASSERT_EQ(elapsed, 0); +} + +UT_TEST(test_poll_ack_returns_result_and_elapsed) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 0; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, 3)); + pg_atomic_write_u32(&result_slot, 42); + pg_atomic_write_u64(&completion_seq, 1); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_ACKED); + UT_ASSERT_EQ(result, 42); + UT_ASSERT_EQ(elapsed, 1500); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_poll_timeout_releases_submit_state) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 1000, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, 4)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2000, NULL, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_TIMEOUT); + UT_ASSERT_EQ(elapsed, 1000); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_mailbox_busy_does_not_publish) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + pg_atomic_write_u64(&request_seq, 2); + pg_atomic_write_u64(&completion_seq, 1); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, 5); + + UT_ASSERT(!ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 2); + UT_ASSERT_EQ(ut_set_latch_count, 0); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_reentrant_submit_does_not_bump_again) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, 6)); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1200, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 99); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED); + UT_ASSERT_EQ(a.target_node, 6); +} + +UT_TEST(test_release_stage_clears_staged_record) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + a.has_staged_event = true; + a.staged_expect_epoch = 123; + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED, 8)); + + cluster_marker_async_release_stage(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); + UT_ASSERT_EQ(a.staged_expect_epoch, 0); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_init_defaults_idle); + UT_RUN(test_submit_publishes_one_request); + UT_RUN(test_poll_pending_before_completion); + UT_RUN(test_poll_ack_returns_result_and_elapsed); + UT_RUN(test_poll_timeout_releases_submit_state); + UT_RUN(test_mailbox_busy_does_not_publish); + UT_RUN(test_reentrant_submit_does_not_bump_again); + UT_RUN(test_release_stage_clears_staged_record); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 62b2099932..3400d68962 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -212,6 +212,14 @@ void cluster_elog_init(void) {} +#include "cluster/cluster_inject.h" +bool +cluster_cr_injection_armed(const char *name pg_attribute_unused(), + uint64 *out_param pg_attribute_unused()) +{ + return false; +} + /* Step 3 D7 stubs: signal/ps_display/procsignal symbols not linked * here (cluster_qvotec.c references them for ClusterQvotecMain; * unit test never invokes Main, just address-takes for T-6). */ diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..6d6fcbfd0f 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,14 @@ UT_DEFINE_GLOBALS(); +/* spec-2.29a: cluster_reconfig.c gates the async marker stage on + * MyBackendType == B_LMON. The fixture exercises the pre-2.29a bounded-wait + * semantics (write_fence submit stub returns FAILED synchronously), so run as + * a non-LMON backend; the async FSM itself is covered by + * test_cluster_marker_async.c. */ +#include "miscadmin.h" +BackendType MyBackendType = B_INVALID; + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. @@ -357,6 +365,48 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +/* spec-2.29a review r1 P1-c: controllable async-marker stubs so the tick-level + * P1-1 invariants (bump-once while PENDING, node-remove zero-false-contest, + * publish deferred to ACK) can be hard-asserted. Defaults preserve the + * pre-existing inert behavior (submit rejected, poll idle). */ +static bool ut_fence_async_submit_ok = false; +static ClusterMarkerPollResult ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; +static uint32 ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +static int ut_fence_async_submit_calls = 0; +static int ut_fence_async_poll_calls = 0; + +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m pg_attribute_unused(), + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now pg_attribute_unused()) +{ + ut_fence_async_submit_calls++; + if (!ut_fence_async_submit_ok) + return false; + /* mirror the real wrapper's FSM effect so is_submitted() sees SUBMITTED */ + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->kind = kind; + a->target_node = target_node; + return true; +} +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now pg_attribute_unused(), + uint32 *out_result, uint64 *out_elapsed_us) +{ + ut_fence_async_poll_calls++; + if (out_result != NULL) + *out_result = ut_fence_async_poll_result; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_ACKED + || ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_TIMEOUT) + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return ut_fence_async_poll_pr; +} +void +cluster_lmon_marker_complete_wakeup(void) +{} /* spec-3.1 D7 stub: cluster_reconfig_apply_epoch_bump_as_coordinator * calls cluster_tt_status_flush_all. Fixture has no TT overlay shmem; @@ -962,6 +1012,142 @@ UT_TEST(test_reconfig_lmon_tick_survivor_does_not_advance_epoch) } +/* ============================================================ + * T-reconfig-2.29a — P1-1 staged-record hard assertions (spec-2.29a + * review r1 P1-c). + * + * (a) fail-stop fence stage: while the marker is PENDING across ticks + * the epoch is bumped EXACTLY ONCE and nothing publishes; the + * staged event publishes only on ACKED+ACK. + * (b) node-remove stage: a tick re-entry while the fence marker is + * PENDING returns 0 with *out_contest == false (no false contest + * against our own already-advanced baseline) and no re-bump. + * ============================================================ */ + +UT_TEST(test_reconfig_failstop_fence_stage_bump_once_while_pending) +{ + uint64 epoch_after_bump; + uint64 apply_before; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; + ut_dead_generation = 1; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + apply_before = cluster_reconfig_get_apply_counter(); + + /* tick #1: stage-entry — bump once, submit, DO NOT publish. */ + cluster_reconfig_lmon_tick(); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT(epoch_after_bump > 0); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* ticks #2/#3: marker PENDING — the bump path must NOT re-enter + * (P1-1 invariant 0: zero re-bump, zero publish, zero side-effects). */ + cluster_reconfig_lmon_tick(); + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* tick #4: ACKED + ACK — the STAGED event publishes; still no re-bump. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)(apply_before + 1)); + + /* restore fixture defaults for the neighboring tests */ + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + +UT_TEST(test_reconfig_node_remove_stage_no_false_contest) +{ + uint64 baseline; + uint64 epoch_after_bump; + uint64 ret; + bool contest; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + baseline = cluster_epoch_get_current(); + + /* call #1: guarded CAS advances baseline→baseline+1, stages the fence + * marker, returns 0 (not yet committed), contest MUST be false. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT_EQ((unsigned long long)epoch_after_bump, (unsigned long long)(baseline + 1)); + + /* call #2 (tick re-entry with the SAME stale baseline while PENDING): + * without the staged record this would re-run the baseline CAS against + * our own advanced epoch and report a FALSE contest. P1-1: it must + * poll the stage instead — 0, contest==false, no second bump. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + /* call #3: ACKED + ACK — removal publishes at the STAGED epoch. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, (unsigned long long)epoch_after_bump); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + /* ============================================================ * T-reconfig-8 — ProcessInterrupts I6 guard (D4). * @@ -1575,6 +1761,8 @@ main(void) /* T-reconfig-3 — lmon_tick dedup. */ UT_RUN(test_reconfig_lmon_tick_dedups_same_event_id); UT_RUN(test_reconfig_lmon_tick_refires_on_dead_gen_bump); + UT_RUN(test_reconfig_failstop_fence_stage_bump_once_while_pending); + UT_RUN(test_reconfig_node_remove_stage_no_false_contest); /* T-reconfig-4 / 4b — Q2 A'' + I2 + L20 + F11 + empty-dead-set gates. */ UT_RUN(test_reconfig_lmon_tick_skips_when_not_in_quorum);