fix(cluster): async qvotec marker submission — LMON liveness (spec-2.29a, BUG-C1 false-DEAD)

.github/workflows/nightly.yml

@@ -176,12 +176,16 @@ jobs:
           # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external).
           - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false }
           # spec-6.12/6.15 cross-node perf + correctness family (t/346-350:
-          # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and
-          # the renumbered t/351-356 family + t/357 multi-xmax alias floor.
+          # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash),
+          # the renumbered t/351-356 family, t/357 multi-xmax alias floor.
           # Closes the 346+ shard hole (every new t/ file must land in a shard
           # the same commit; see docs/code-review-checklist.md).
           - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false }
           - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false }
+          # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by
+          # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy
+          # verified against origin branches 2026-07-08).
+          - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false }
     steps:
       - name: Checkout
         uses: actions/checkout@v4

src/backend/cluster/cluster_clean_leave_policy.c

@@ -136,17 +136,112 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid)
 
 /*
  * cluster_clean_leave_version_coherent -- the bound leave is still coherent
- * only if neither the cluster epoch nor the CSSD dead_generation moved since
- * the leave was bound.  Any external bump (a real death intruding mid-drain)
- * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean
- * leave on a stale version, which would let a destructive sweep run on a newer
- * version and double-grant).
+ * only if neither the cluster epoch nor the OTHERS-dead set (every dead node
+ * except the leaving node itself) moved since the leave was bound.  Any
+ * external membership change — a third-party death intruding mid-drain, or a
+ * third-party fail-stop that already bumped the epoch — makes it incoherent →
+ * caller must ABORTED_ESCALATE (never complete a clean leave on a stale
+ * membership view, which would let a destructive sweep run on a newer view
+ * and double-grant, CL-I3).
+ *
+ * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD
+ * dead_generation, which also counts the leaving node's OWN alive→DEAD
+ * transition (it stops heart-beating once its drain finishes — the EXPECTED
+ * terminal state of a clean leave).  Under the async COMMITTING marker that
+ * spans several LMON ticks, a slow environment lets that transition land
+ * inside the wait window and the scalar check wrongly escalated an otherwise
+ * healthy leave.  Comparing the others-dead bitmap (which excludes the
+ * leaving node) removes exactly that false positive while keeping every
+ * third-party incoherence escalation (see the reflexive-case matrix in
+ * spec-2.29a §②b 8.A argument).
  */
 bool
 cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch,
-									 uint64 bound_dead_gen, uint64 current_dead_gen)
+									 const uint8 *bound_others_dead,
+									 const uint8 *current_others_dead, int nbytes)
+{
+	if (bound_epoch != current_epoch)
+		return false;
+	if (bound_others_dead == NULL || current_others_dead == NULL)
+		return false; /* fail-closed: cannot prove coherence without both views */
+	return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0;
+}
+
+/*
+ * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick
+ * latches "my clean-leave committed" on direct EVIDENCE, never on inference
+ * (spec-2.29a r3).  The evidence is committed_marker_evidence: the survivor
+ * coordinator made the COMMITTED marker for THIS leave attempt majority-
+ * durable on the voting disk and attested it with a nonce-bound
+ * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_
+ * matches before the flag this argument mirrors is ever set).  Latching
+ * suppresses the barrier-deadline escalation and lets the leaver exit, so
+ * the verdict must be exactly: latch <=> evidence.
+ *
+ * History of the two inference failure modes this replaces:
+ *   - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead
+ *     bitmap unchanged" inference could mis-latch a REFUSED leave after a
+ *     third-party false-DEAD -> ALIVE rebound restored the (non-monotone)
+ *     bitmap, suppressing the deadline escalation forever.
+ *   - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation
+ *     conjunct is monotone the other way — a third-party transient flap on
+ *     the leaver's local CSSD view advances it forever, so a healthy
+ *     committed leave was refused until the deadline escalated it.
+ * Marker evidence is immune to both: a refused leave never gets a COMMITTED
+ * marker (no latch -> bounded deadline escalation), and a flap cannot make
+ * durable evidence disappear (latch -> no false escalation).
+ *
+ * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence
+ * observations.  They are deliberately kept in the signature as contract
+ * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the
+ * flap-immunity on them — and the runtime uses them only for the flap-noise
+ * LOG at the latch point (observability, never control flow).
+ */
+bool
+cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged,
+									   bool dead_gen_unchanged)
+{
+	(void)others_dead_unchanged; /* observability-only input (see above) */
+	(void)dead_gen_unchanged;	 /* observability-only input (see above) */
+	return committed_marker_evidence;
+}
+
+/*
+ * cluster_clean_leave_committed_evidence_matches -- may the leaving node
+ * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave
+ * attempt?  (spec-2.29a r3; the payload's magic/version/CRC were already
+ * checked by cluster_clean_leave_announce_payload_valid.)
+ *
+ *	Identity is bound three ways, all fail-closed:
+ *	  - payload_leaving_node == self_node: the confirmation is addressed to
+ *	    this node's own leave (LEAVE_COMMITTED is point-to-point, but a
+ *	    misrouted frame must still not latch).
+ *	  - current_leaving_node == self_node: this node IS currently the leaver
+ *	    (not idle, not a survivor of someone else's leave).
+ *	  - payload_nonce == current_attempt_nonce: the per-attempt nonce
+ *	    (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale
+ *	    LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a
+ *	    PREVIOUS leave of the same node can never false-latch a new attempt.
+ *	  - payload_epoch > bound_leave_epoch: the committed epoch E the
+ *	    coordinator attests must lie past the baseline this leave bound
+ *	    (sanity; the commit is a guarded CAS off that baseline).
+ */
+bool
+cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce,
+											   uint64 payload_epoch, int32 self_node,
+											   int32 current_leaving_node,
+											   uint64 current_attempt_nonce,
+											   uint64 bound_leave_epoch)
 {
-	return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen;
+	if (payload_leaving_node != self_node)
+		return false;
+	if (current_leaving_node != self_node)
+		return false;
+	if (payload_nonce != current_attempt_nonce)
+		return false;
+	if (payload_epoch <= bound_leave_epoch)
+		return false;
+	return true;
 }
 
 

src/backend/cluster/cluster_debug.c

@@ -544,6 +544,10 @@ dump_lmon(ReturnSetInfo *rsinfo)
 
 	iters = cluster_lmon_main_loop_iters();
 	emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters));
+	emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us()));
+	emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us()));
+	emit_row(rsinfo, "lmon", "lmon_slow_iter_count",
+			 fmt_int64((int64)cluster_lmon_slow_iter_count()));
 }
 
 /*
@@ -1475,6 +1479,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo)
 			 fmt_int64((int64)cluster_reconfig_get_join_timeout_count()));
 	emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count",
 			 fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count()));
+	emit_row(rsinfo, "reconfig", "marker_slow_ack_count",
+			 fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count()));
+	emit_row(rsinfo, "reconfig", "marker_timeout_count",
+			 fmt_int64((int64)cluster_reconfig_get_marker_timeout_count()));
 }
 
 /*

src/backend/cluster/cluster_grd.c

@@ -1638,6 +1638,20 @@ cluster_grd_recovery_in_progress(void)
 	return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE;
 }
 
+/*
+ * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares
+ * against.  Exposed for diagnostics + the IDLE baseline-hold unit test (the
+ * async pre-bump staging window must not let this re-capture a post-bump
+ * value; see the IDLE branch of cluster_grd_recovery_lmon_tick).
+ */
+uint64
+cluster_grd_recovery_event_old_epoch(void)
+{
+	if (cluster_grd_state == NULL)
+		return 0;
+	return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch);
+}
+
 /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module
  * call sites (S4 stale mapping in cluster_lock_acquire.c;  GCS block
  * fail-closed guard in cluster_gcs_block.c). */
@@ -2108,9 +2122,19 @@ cluster_grd_recovery_lmon_tick(void)
 			 * fires, making old_epoch already == the post-bump epoch and
 			 * wedging WAIT_EPOCH forever (cur <= old).  The last stable
 			 * idle epoch is the reliable "before reconfig" value.
+			 *
+			 * spec-2.29a nightly-regression fix:  the coordinator's own
+			 * async pre-bump stages (fail-stop / node-remove / join
+			 * Phase-1) bump the epoch several ticks before publishing the
+			 * event.  During that window this IDLE tick must NOT re-capture
+			 * the already-bumped epoch as the baseline, or the P0 accept
+			 * that follows the publish reads old == cur and wedges
+			 * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback
+			 * needed.  Hold the last stable pre-reconfig baseline instead.
 			 */
-			pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch,
-								cluster_epoch_get_current());
+			if (!cluster_reconfig_has_pending_prebump_stage())
+				pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch,
+									cluster_epoch_get_current());
 			return;
 		}
 

src/backend/cluster/cluster_guc.c

@@ -307,6 +307,7 @@ int cluster_phase4_timeout = 30;
  * LMON main-loop tick / WaitLatch timeout in milliseconds.
  */
 int cluster_lmon_main_loop_interval = 1000;
+int cluster_lmon_slow_iteration_warn_ms = 1000;
 
 /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */
 int cluster_lck_main_loop_interval = 1000;
@@ -2573,6 +2574,15 @@ cluster_init_guc(void)
 		&cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL,
 		NULL);
 
+	DefineCustomIntVariable(
+		"cluster.lmon_slow_iteration_warn_ms",
+		gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."),
+		gettext_noop("A value greater than zero logs LMON main-loop iterations above this "
+					 "duration; the lmon_slow_iter_count counter is maintained even when "
+					 "logging is disabled with 0."),
+		&cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL,
+		NULL);
+
 	DefineCustomIntVariable(
 		"cluster.lck_main_loop_interval",
 		gettext_noop("LCK main-loop tick interval in milliseconds."),

src/backend/cluster/cluster_inject.c

@@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = {
 	/* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */
 	{ .name = "cluster-qvotec-poll-pre" },
 	{ .name = "cluster-qvotec-poll-post" },
+	{ .name = "cluster-qvotec-marker-service-hold" },
 	{ .name = "cluster-voting-disk-write-fail" },
 	{ .name = "cluster-quorum-loss-broadcast" },
 	{ .name = "cluster-collision-detect" },