From cc578d065662855efb9a6f7f6bd891bb8e1d0ec6 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 12:37:53 +0800 Subject: [PATCH 01/38] feat(cluster): undo-block resource identity + owner-as-master routing Name an undo block (including the segment TT header block) as a first-class cluster resource: (owner_node, undo_segment, block_no, generation) encoded into the 16-byte ClusterResId wire format as class 0xF9. Undo is the first owner-as-master resid class: the resource master IS the owning instance (cluster_undo_resid_master), never a GRD shard-hash node. The GRD hash-master lookup now fails closed (Assert + new SQLSTATE 53R9Q) if an undo resid reaches it, and the generation predicate (segment wrap_count) lets callers fail closed on recycled-segment stale references. Pure identity layer only: no grant/PI/serving data plane, no write path change, no on-disk format or wire ABI change (ClusterResId stays 16 bytes; new class byte value only). - new cluster_undo_resid.h: class byte 0xF9 + collision StaticAssert net (9 header-visible classes + LOCKTAG_LAST_TYPE) + field mapping - new cluster_undo_resid.c: encode/decode/is_undo/master/ generation_matches pure layer (standalone-linkable) - cluster_grd.c: fail-closed undo-class guard at cluster_grd_lookup_master entry - errcodes.txt: 53R9Q ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED - tests: test_cluster_undo_resid (8 cases: round-trip, class byte, discriminator, 16B wire lock, owner bounds, owner-as-master, generation); guard-fire leg in test_cluster_grd via an ereport trampoline; t/006 SQLSTATE spot-check line Spec: spec-5.22a-undo-block-resource-identity.md --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_grd.c | 19 ++ src/backend/cluster/cluster_undo_resid.c | 134 ++++++++++ src/backend/utils/errcodes.txt | 9 + src/include/cluster/cluster_undo_resid.h | 149 +++++++++++ src/test/cluster_tap/t/006_errcodes.pl | 2 + src/test/cluster_unit/Makefile | 18 +- src/test/cluster_unit/test_cluster_grd.c | 78 +++++- .../cluster_unit/test_cluster_undo_resid.c | 245 ++++++++++++++++++ 9 files changed, 646 insertions(+), 9 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_resid.c create mode 100644 src/include/cluster/cluster_undo_resid.h create mode 100644 src/test/cluster_unit/test_cluster_undo_resid.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..c62e189809 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -183,6 +183,7 @@ OBJS = \ cluster_visibility_resolve.o \ cluster_visibility_verdict.o \ cluster_undo_record.o \ + cluster_undo_resid.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..836d825df4 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -53,6 +53,7 @@ #include "cluster/cluster_epoch.h" /* spec-4.6 D1 — accepted epoch reads */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — reconfig event consume */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 — unfreeze gate */ +#include "cluster/cluster_undo_resid.h" /* spec-5.22a D1-5 — undo-class hash-route guard */ #include "storage/procsignal.h" /* spec-4.6 D3 — redeclare broadcast */ #include "storage/sinvaladt.h" /* spec-4.6 D3 — BackendIdGetProc */ #include "utils/timestamp.h" /* spec-4.6 D1 — barrier deadline */ @@ -993,6 +994,24 @@ cluster_grd_lookup_master(const ClusterResId *resid) uint32 shard_id; int32 master; + /* + * PGRAC: spec-5.22a D1-5 -- undo resids are owner-as-master resources; + * their master is the encoded owner (cluster_undo_resid_master), never a + * shard-hash node. A hash-derived master would place the undo authority + * at a node that does not own the undo, so no caller may hash-route the + * undo class. Fail closed: no data-plane path legitimately reaches here + * with an undo resid. + */ + if (resid != NULL && resid->type == CLUSTER_UNDO_RESID_TYPE) { + Assert(false); + ereport( + ERROR, + (errcode(ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED), + errmsg("undo resource id must not be routed through the GRD hash master lookup"), + errhint( + "Undo resources are owner-as-master; route via cluster_undo_resid_master()."))); + } + Assert(cluster_grd_state != NULL); shard_id = cluster_grd_shard_for_resource(resid); diff --git a/src/backend/cluster/cluster_undo_resid.c b/src/backend/cluster/cluster_undo_resid.c new file mode 100644 index 0000000000..ddb0af925c --- /dev/null +++ b/src/backend/cluster/cluster_undo_resid.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_resid.c + * Shared-undo block resource identity + owner-as-master routing -- + * pure layer (spec-5.22a D1). + * + * This file ships the backend-pure layer: the undo resid + * encoder/decoder, the class discriminator, the owner-as-master + * routing function, and the anti-ABA generation predicate. None of + * these touch elog / shmem / locks, so the cluster_unit test links the + * object standalone. The data plane that consumes this identity + * (grant / PI / block serving / recovery materialization / retention) + * lands with later deliverables. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_resid.c + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22a-undo-block-resource-identity.md (D1, §2.2 / §3.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_scn.h" /* SCN_NODE_ID_VALID */ +#include "cluster/cluster_undo_resid.h" + +/* + * cluster_undo_resid_encode -- build the undo-block resource id. + * + * field3 is the segment reuse generation (the segment header wrap_count), + * so a recycled segment lands a distinct resource (ABA defence), + * mirroring the HW relfilenode field. field4 is the owning instance: + * the undo authority lives at the owner, so the owner is part of the + * identity, not a routing afterthought. + */ +void +cluster_undo_resid_encode(int32 owner_node, uint32 undo_segment, uint32 block_no, uint32 generation, + ClusterResId *dst) +{ + Assert(dst != NULL); + if (dst == NULL) + return; + Assert(SCN_NODE_ID_VALID(owner_node)); + + dst->field1 = undo_segment; + dst->field2 = block_no; + dst->field3 = generation; + dst->field4 = (uint16)owner_node; + dst->type = CLUSTER_UNDO_RESID_TYPE; + dst->lockmethodid = DEFAULT_LOCKMETHOD; +} + +/* + * cluster_undo_resid_decode -- split an undo resid back into its fields. + * + * The caller must pass an undo-class resid (Assert enforced); the + * decoder is the wire-ABI boundary, so it never guesses at foreign + * classes. + */ +void +cluster_undo_resid_decode(const ClusterResId *rid, int32 *owner_node, uint32 *undo_segment, + uint32 *block_no, uint32 *generation) +{ + Assert(rid != NULL); + if (rid == NULL || owner_node == NULL || undo_segment == NULL || block_no == NULL + || generation == NULL) + return; + Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + + *owner_node = (int32)rid->field4; + *undo_segment = rid->field1; + *block_no = rid->field2; + *generation = rid->field3; +} + +/* + * cluster_undo_resid_is_undo -- class discriminator. + */ +bool +cluster_undo_resid_is_undo(const ClusterResId *rid) +{ + Assert(rid != NULL); + if (rid == NULL) + return false; + + return rid->type == CLUSTER_UNDO_RESID_TYPE; +} + +/* + * cluster_undo_resid_master -- owner-as-master routing. + * + * Returns the encoded owner_node directly: the undo authority lives at + * the owning instance, so the master is part of the identity and is + * NEVER derived from a shard hash. A hash-derived master would place + * the authority at a node that does not own the undo, which is exactly + * the misrouting the GRD-side guard fails closed on. + */ +int32 +cluster_undo_resid_master(const ClusterResId *rid) +{ + Assert(rid != NULL); + if (rid == NULL) + return -1; + Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + + return (int32)rid->field4; +} + +/* + * cluster_undo_resid_generation_matches -- anti-ABA check. + * + * false means the reference predates a whole-segment recycle (the + * segment header wrap_count moved on) and the caller MUST fail closed; + * it must never be treated as a match. + */ +bool +cluster_undo_resid_generation_matches(const ClusterResId *rid, uint32 expected_generation) +{ + Assert(rid != NULL); + if (rid == NULL) + return false; + Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + + return rid->field3 == expected_generation; +} diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be8af8e758..190686996b 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -745,6 +745,15 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # retried; never write dependent bytes to storage. 53R9P E ERRCODE_CLUSTER_SMART_FUSION_DEP_LOST cluster_smart_fusion_dep_lost +# spec-5.22a D1: undo resids are owner-as-master resources (the resource +# master IS the owning instance); routing one through the GRD shard-hash +# master lookup would place the authority at a node that does not own the +# undo. No caller may hash-route an undo resid; the hash lookup fails +# closed here (route via cluster_undo_resid_master instead). Q is the next +# free slot in the 53R9 family (A..P allocated; X taken by the clean-page +# X-transfer band below). +53R9Q E ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED cluster_undo_resid_hash_routed + # spec-5.2a D6: clean-page X-transfer enabler terminal fail-closed. A clean # (sequence) page X-transfer could not complete safely and there is no proven- # safe fallback: a 3-node third-party master clean transfer (out of the 2-node diff --git a/src/include/cluster/cluster_undo_resid.h b/src/include/cluster/cluster_undo_resid.h new file mode 100644 index 0000000000..cb5060dc10 --- /dev/null +++ b/src/include/cluster/cluster_undo_resid.h @@ -0,0 +1,149 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_resid.h + * Shared-undo block resource identity + owner-as-master routing + * contract -- spec-5.22a D1. + * + * Names an undo block (including the segment TT header block) as a + * first-class cluster resource: (owner_node, undo_segment, block_no, + * generation) encoded into the 16-byte ClusterResId wire format. Undo + * is the first owner-as-master resid class: the resource master IS the + * owning instance (cluster_undo_resid_master returns owner_node), never + * a GRD shard-hash master. Hash-routing an undo resid through + * cluster_grd_lookup_master / cluster_gcs_lookup_master is a fail-closed + * error: the undo authority lives at the owner and a hash-derived master + * would bypass it. + * + * generation carries the segment reuse generation (the segment header + * wrap_count): a recycled segment reuses (undo_segment, block_no) for + * different content, so a generation mismatch means a stale reference + * and the caller must fail closed. The owner membership epoch (owner + * incarnation) is deliberately NOT part of the tag: it is a grant-time + * ownership-validity attribute negotiated on the authority path, not a + * static identity field. + * + * This header declares the PURE layer only (no elog / shmem / lock; + * standalone-linkable for cluster_unit). The data plane that consumes + * this identity (grant / PI / block serving / recovery materialization / + * retention) lands with later deliverables. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_undo_resid.h + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22a-undo-block-resource-identity.md (D1, §2.2 / §3.1) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_UNDO_RESID_H +#define CLUSTER_UNDO_RESID_H + +#include "cluster/cluster_cf_enqueue.h" /* CLUSTER_CF_RESID_TYPE (collision check) */ +#include "cluster/cluster_dl.h" /* CLUSTER_DL_RESID_TYPE (collision check) */ +#include "cluster/cluster_grd.h" /* ClusterResId */ +#include "cluster/cluster_hw.h" /* CLUSTER_HW_RESID_TYPE (collision check) */ +#include "cluster/cluster_ir.h" /* CLUSTER_IR_RESID_TYPE (collision check) */ +#include "cluster/cluster_ko.h" /* CLUSTER_KO_RESID_TYPE (collision check) */ +#include "cluster/cluster_oid_lease.h" /* CLUSTER_OID_RESID_TYPE (collision check) */ +#include "cluster/cluster_relmap_lock.h" /* CLUSTER_RELMAP_RESID_TYPE (collision check) */ +#include "cluster/cluster_sequence.h" /* CLUSTER_SQ_RESID_TYPE (collision check) */ +#include "cluster/cluster_ts.h" /* CLUSTER_TT_RESID_TYPE (collision check) */ +#include "storage/lock.h" /* LOCKTAG_LAST_TYPE, DEFAULT_LOCKMETHOD */ + +/* + * CLUSTER_UNDO_RESID_TYPE -- undo-block resource-id namespace marker. + * 0xF9 is the next free slot after the contiguous 0xF0-0xF8 run. Must be + * above every PG LockTagType and distinct from every existing resid class. + * + * NB: 0xF3 is double-booked today by CLUSTER_DL_RESID_TYPE (cluster_dl.h) + * and the backend-local CLUSTER_RAW_LAYOUT_RESID_TYPE + * (cluster_shared_fs_block_device.c), which this cross-assert net cannot + * name. 0xF9 collides with neither; cleaning up the 0xF3 double-booking + * is a registered follow-up outside this header. + */ +#define CLUSTER_UNDO_RESID_TYPE 0xF9 + +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE > LOCKTAG_LAST_TYPE, + "undo resid namespace must not collide with any PG LockTagType"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_SQ_RESID_TYPE, + "undo and SQ resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_CF_RESID_TYPE, + "undo and CF resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_HW_RESID_TYPE, + "undo and HW resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_DL_RESID_TYPE, + "undo and DL resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_TT_RESID_TYPE, + "undo and TT (tablespace-DDL) resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_IR_RESID_TYPE, + "undo and IR resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_KO_RESID_TYPE, + "undo and KO resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_OID_RESID_TYPE, + "undo and OID-lease resid namespaces must be distinct"); +StaticAssertDecl(CLUSTER_UNDO_RESID_TYPE != CLUSTER_RELMAP_RESID_TYPE, + "undo and RELMAP resid namespaces must be distinct"); + +/* + * ClusterResId field mapping for the undo class (16 bytes, NOT memcpy -- + * cluster_undo_resid_encode/decode are the wire-ABI boundary): + * + * field1 = undo_segment (per-instance undo segment number) + * field2 = block_no (block number within the segment; block 0 + * is the segment TT header block -- DATA and + * TT blocks share this one class) + * field3 = generation (segment reuse generation == the segment + * header wrap_count; anti-ABA guard against + * whole-segment recycling) + * field4 = owner_node (owning instance node id; uint16 on the + * wire, valid range [0, SCN_MAX_VALID_NODE_ID]) + * type = CLUSTER_UNDO_RESID_TYPE + * lockmethodid = DEFAULT_LOCKMETHOD + * + * The owner membership epoch (owner incarnation) is NOT in the tag; it is + * negotiated at grant/serve time on the authority path. + */ + +/* + * cluster_undo_resid_encode -- build the undo-block resource id. + */ +extern void cluster_undo_resid_encode(int32 owner_node, uint32 undo_segment, uint32 block_no, + uint32 generation, ClusterResId *dst); + +/* + * cluster_undo_resid_decode -- split an undo resid back into its fields. + * Must only be called on a resid whose type is CLUSTER_UNDO_RESID_TYPE. + */ +extern void cluster_undo_resid_decode(const ClusterResId *rid, int32 *owner_node, + uint32 *undo_segment, uint32 *block_no, uint32 *generation); + +/* + * cluster_undo_resid_is_undo -- class discriminator (type == 0xF9). + */ +extern bool cluster_undo_resid_is_undo(const ClusterResId *rid); + +/* + * cluster_undo_resid_master -- owner-as-master routing: returns the + * encoded owner_node directly, NEVER a hash-derived master. Undo + * resources route exclusively through this function; the GRD/GCS + * hash-master lookups reject the undo class (fail closed). + */ +extern int32 cluster_undo_resid_master(const ClusterResId *rid); + +/* + * cluster_undo_resid_generation_matches -- anti-ABA check: false means the + * reference is stale (the segment was recycled) and the caller MUST fail + * closed; it must never be treated as a match. + */ +extern bool cluster_undo_resid_generation_matches(const ClusterResId *rid, + uint32 expected_generation); + +#endif /* CLUSTER_UNDO_RESID_H */ diff --git a/src/test/cluster_tap/t/006_errcodes.pl b/src/test/cluster_tap/t/006_errcodes.pl index 075953a411..c233781d3e 100644 --- a/src/test/cluster_tap/t/006_errcodes.pl +++ b/src/test/cluster_tap/t/006_errcodes.pl @@ -106,6 +106,8 @@ sub raise_unknown "cluster_reconfig_in_progress -> 53R60"); is(raise_and_get_sqlstate('cluster_backup_incomplete'), '53RAD', "cluster_backup_incomplete -> 53RAD"); +is(raise_and_get_sqlstate('cluster_undo_resid_hash_routed'), '53R9Q', + "cluster_undo_resid_hash_routed -> 53R9Q"); is(raise_and_get_sqlstate('cluster_adg_apply_lag_excessive'), '57R06', "cluster_adg_apply_lag_excessive -> 57R06"); is(raise_and_get_sqlstate('cluster_adg_standby_unresolvable'), '57R07', diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..063e5aa67a 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -86,7 +86,8 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_membership \ test_cluster_node_remove \ test_cluster_hang_acceptance \ - test_cluster_xid_stripe + test_cluster_xid_stripe \ + test_cluster_undo_resid # Path to the cluster_version object (no PG deps, safe to link standalone). CLUSTER_VERSION_O = $(top_builddir)/src/backend/cluster/cluster_version.o @@ -178,7 +179,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1828,3 +1829,16 @@ test_cluster_xid_stripe: test_cluster_xid_stripe.c unit_test.h \ $(CC) $(CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_XID_STRIPE_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# spec-5.22a D1: test_cluster_undo_resid — shared-undo block resource +# identity pure layer (undo resid class byte + encode/decode + +# owner-as-master routing + anti-ABA generation predicate). Links +# cluster_hw.o for the is_undo cross-class check (real HW encoder). The +# pure layer has no PG-backend dependencies, so nothing is stubbed beyond +# the Assert ExceptionalCondition hook. +CLUSTER_UNDO_RESID_O = $(top_builddir)/src/backend/cluster/cluster_undo_resid.o +test_cluster_undo_resid: test_cluster_undo_resid.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_HW_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_HW_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..73f280fbd5 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -56,6 +56,7 @@ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ +#include "cluster/cluster_undo_resid.h" /* spec-5.22a D1-5 — undo-class hash-route guard */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ #include "port/atomics.h" @@ -85,34 +86,55 @@ bool IsUnderPostmaster = false; +/* spec-5.22a D1-5 verifies a fail-closed guard is reached. Setjmp-based + * trampoline (mirrors test_cluster_fence.c): when armed, an Assert trip + * (assert builds) or an ereport(ERROR) (production builds) jumps back to + * the test instead of aborting the binary. For elevel < ERROR the stubs + * keep the historical silent-no-op behaviour. */ +static sigjmp_buf ut_trap_jump; +static bool ut_trap_jump_armed = false; +static int ut_ereport_last_errcode = 0; +static int ut_current_elevel = 0; + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), int lineNumber pg_attribute_unused()) { + if (ut_trap_jump_armed) + siglongjmp(ut_trap_jump, 2); abort(); } bool -errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +errstart(int elevel, const char *d pg_attribute_unused()) { - return false; + ut_current_elevel = elevel; + /* PG: ERROR = 21 (elog.h). >= ERROR runs the ereport body so the + * errcode stub can capture the SQLSTATE before errfinish jumps. */ + return elevel >= 21; } bool -errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +errstart_cold(int elevel, const char *d) { - return false; + return errstart(elevel, d); } void errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), const char *fn pg_attribute_unused()) -{} +{ + if (ut_current_elevel >= 21 && ut_trap_jump_armed) + siglongjmp(ut_trap_jump, 1); + /* Otherwise (LOG/NOTICE/no jump armed): silent return. */ +} int -errcode(int s pg_attribute_unused()) +errcode(int s) { + if (ut_trap_jump_armed) + ut_ereport_last_errcode = s; return 0; } @@ -1983,6 +2005,47 @@ UT_TEST(test_grd_lookup_master_gen_q3c_verbatim) mock_lms_shard_master_generation = 0; } +/* spec-5.22a D1-5: the GRD shard-hash master lookup fail-closes on an + * undo-class resid. Undo is owner-as-master (the master IS the encoded + * owner); a hash-derived master would bypass the owner authority, so no + * caller may hash-route it. Assert builds trip the guard's Assert first + * (trampoline rc 2); production builds reach the ereport(ERROR) (rc 1) + * whose SQLSTATE must be 53R9Q. */ +UT_TEST(test_grd_lookup_master_rejects_undo_resid) +{ + int32 nodes2[] = { 0, 1 }; + ClusterResId undo; + int rc; + + cluster_grd_shmem_init(); + set_mock_declared(2, nodes2); + cluster_grd_master_map_init(); + + /* hand-built undo-class resid: only the type byte matters to the + * guard (the undo encoder object is deliberately not linked here) */ + memset(&undo, 0, sizeof(undo)); + undo.field1 = 7; /* undo_segment */ + undo.field2 = 129; /* block_no */ + undo.field3 = 3; /* generation */ + undo.field4 = 1; /* owner_node */ + undo.type = CLUSTER_UNDO_RESID_TYPE; + undo.lockmethodid = DEFAULT_LOCKMETHOD; + + ut_ereport_last_errcode = 0; + rc = sigsetjmp(ut_trap_jump, 0); + if (rc == 0) { + ut_trap_jump_armed = true; + (void)cluster_grd_lookup_master(&undo); + /* reaching here means the undo resid was hash-routed */ + ut_trap_jump_armed = false; + UT_ASSERT(false); + } + ut_trap_jump_armed = false; + UT_ASSERT(rc == 1 || rc == 2); + if (rc == 1) /* ereport path (production builds) */ + UT_ASSERT_EQ(ut_ereport_last_errcode, ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED); +} + UT_TEST(test_grd_shard_phase_accessors) { int32 nodes2[] = { 0, 1 }; @@ -4055,7 +4118,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4091,6 +4154,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_grd_remaster_multi_death_and_sparse); UT_RUN(test_grd_remaster_no_survivor_fail_closed); UT_RUN(test_grd_lookup_master_gen_q3c_verbatim); + UT_RUN(test_grd_lookup_master_rejects_undo_resid); UT_RUN(test_grd_shard_phase_accessors); UT_RUN(test_grd_d2_redeclare_scan_completion_gate); diff --git a/src/test/cluster_unit/test_cluster_undo_resid.c b/src/test/cluster_unit/test_cluster_undo_resid.c new file mode 100644 index 0000000000..9f85cf3328 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_undo_resid.c @@ -0,0 +1,245 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_undo_resid.c + * Unit tests for the shared-undo block resource identity pure layer + * (spec-5.22a D1). + * + * U2 pins the undo resid class byte (0xF9) above every PG LockTagType + * and distinct from every header-visible resid class (SQ 0xF0 / CF 0xF1 + * / HW 0xF2 / DL 0xF3 / TT 0xF4 / IR 0xF5 / KO 0xF6 / OID 0xF7 / + * RELMAP 0xF8), plus the backend-local raw-layout 0xF3 value that the + * header cross-assert net cannot see. The real owner-as-master data + * plane (grant / PI / shipping) is D2+; this file pins the identity + * contract only. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_undo_resid.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22a-undo-block-resource-identity.md (D1, §4.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_undo_resid.h" +#include "storage/lock.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} + +/* ====================================================================== + * U2 -- class byte: 0xF9, above PG lock types, no collision with any + * existing resid class (spec-5.22a §2.1) + * ====================================================================== */ +UT_TEST(test_undo_resid_class_byte) +{ + /* 0xF9 is the next free slot after the contiguous 0xF0-0xF8 run */ + UT_ASSERT_EQ(CLUSTER_UNDO_RESID_TYPE, 0xF9); + UT_ASSERT(CLUSTER_UNDO_RESID_TYPE > LOCKTAG_LAST_TYPE); + + /* distinct from every header-visible resid class */ + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_SQ_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_CF_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_HW_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_DL_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_TT_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_IR_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_KO_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_OID_RESID_TYPE); + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, CLUSTER_RELMAP_RESID_TYPE); + + /* + * Namespace audit beyond the header net: CLUSTER_RAW_LAYOUT_RESID_TYPE + * (0xF3) is a backend-local define in cluster_shared_fs_block_device.c, + * not header-visible, so the header StaticAssert net cannot name it. + * Pin the raw value here so the undo class also stays clear of it. + * NB: 0xF3 itself is double-booked today (DL + raw layout); cleanup is + * a registered follow-up outside this spec. + */ + UT_ASSERT_NE(CLUSTER_UNDO_RESID_TYPE, 0xF3); +} + +/* ====================================================================== + * U1 -- encode/decode round-trip: all four identity dimensions survive + * ====================================================================== */ +UT_TEST(test_undo_resid_encode_decode_roundtrip) +{ + ClusterResId r; + int32 owner_node; + uint32 undo_segment; + uint32 block_no; + uint32 generation; + + memset(&r, 0xEE, sizeof(r)); + cluster_undo_resid_encode(2, 7, 129, 3, &r); + + UT_ASSERT_EQ(r.field1, 7); /* undo_segment */ + UT_ASSERT_EQ(r.field2, 129); /* block_no */ + UT_ASSERT_EQ(r.field3, 3); /* generation */ + UT_ASSERT_EQ(r.field4, 2); /* owner_node */ + UT_ASSERT_EQ(r.type, CLUSTER_UNDO_RESID_TYPE); + UT_ASSERT_EQ(r.lockmethodid, DEFAULT_LOCKMETHOD); + + owner_node = -1; + undo_segment = 0xDEADBEEF; + block_no = 0xDEADBEEF; + generation = 0xDEADBEEF; + cluster_undo_resid_decode(&r, &owner_node, &undo_segment, &block_no, &generation); + UT_ASSERT_EQ(owner_node, 2); + UT_ASSERT_EQ(undo_segment, 7); + UT_ASSERT_EQ(block_no, 129); + UT_ASSERT_EQ(generation, 3); +} + +/* block 0 (the segment TT header block) is a valid block_no: DATA and TT + * blocks share the one undo class */ +UT_TEST(test_undo_resid_tt_header_block_zero) +{ + ClusterResId r; + int32 owner_node; + uint32 undo_segment; + uint32 block_no; + uint32 generation; + + cluster_undo_resid_encode(1, 42, 0, 9, &r); + cluster_undo_resid_decode(&r, &owner_node, &undo_segment, &block_no, &generation); + UT_ASSERT_EQ(block_no, 0); + UT_ASSERT_EQ(undo_segment, 42); +} + +/* ====================================================================== + * U3 -- class discriminator: true for undo, false for other classes + * ====================================================================== */ +UT_TEST(test_undo_resid_is_undo) +{ + ClusterResId undo_r; + ClusterResId hw_r; + ClusterResId cf_r; + RelFileLocator rloc; + + cluster_undo_resid_encode(0, 1, 2, 0, &undo_r); + UT_ASSERT(cluster_undo_resid_is_undo(&undo_r)); + + /* a real HW resid (encoder linked) is not an undo resid */ + rloc.spcOid = 1663; + rloc.dbOid = 5; + rloc.relNumber = 16384; + cluster_hw_resid_encode(rloc, MAIN_FORKNUM, &hw_r); + UT_ASSERT(!cluster_undo_resid_is_undo(&hw_r)); + + /* CF resid constructed by hand (the CF encoder lives in a backend + * object; only the type byte matters to the discriminator) */ + memset(&cf_r, 0, sizeof(cf_r)); + cf_r.type = CLUSTER_CF_RESID_TYPE; + cf_r.lockmethodid = DEFAULT_LOCKMETHOD; + UT_ASSERT(!cluster_undo_resid_is_undo(&cf_r)); +} + +/* ====================================================================== + * U6 -- wire ABI: ClusterResId stays 16 bytes + * ====================================================================== */ +UT_TEST(test_undo_resid_wire_abi_16_bytes) +{ + UT_ASSERT_EQ(sizeof(ClusterResId), 16); +} + +/* ====================================================================== + * U7 -- owner_node boundary encoding (0 / 15 / SCN_MAX_VALID_NODE_ID) + * ====================================================================== */ +UT_TEST(test_undo_resid_owner_bounds) +{ + static const int32 owners[] = { 0, 15, SCN_MAX_VALID_NODE_ID }; + int i; + + for (i = 0; i < (int)lengthof(owners); i++) { + ClusterResId r; + int32 owner_node = -1; + uint32 undo_segment; + uint32 block_no; + uint32 generation; + + cluster_undo_resid_encode(owners[i], 3, 5, 1, &r); + cluster_undo_resid_decode(&r, &owner_node, &undo_segment, &block_no, &generation); + UT_ASSERT_EQ(owner_node, owners[i]); + } +} + +/* ====================================================================== + * U4 -- owner-as-master routing: the master IS the encoded owner_node, + * never a hash-derived node (spec-5.22a §3.1) + * ====================================================================== */ +UT_TEST(test_undo_resid_master_is_owner) +{ + ClusterResId r; + + cluster_undo_resid_encode(3, 11, 200, 5, &r); + UT_ASSERT_EQ(cluster_undo_resid_master(&r), 3); + + /* the master must not vary with the non-owner identity dimensions + * (a shard-hash master would) */ + cluster_undo_resid_encode(3, 9999, 123456, 42, &r); + UT_ASSERT_EQ(cluster_undo_resid_master(&r), 3); + + cluster_undo_resid_encode(0, 1, 1, 1, &r); + UT_ASSERT_EQ(cluster_undo_resid_master(&r), 0); + + cluster_undo_resid_encode(SCN_MAX_VALID_NODE_ID, 1, 1, 1, &r); + UT_ASSERT_EQ(cluster_undo_resid_master(&r), SCN_MAX_VALID_NODE_ID); +} + +/* ====================================================================== + * U5 -- anti-ABA generation predicate: mismatch means stale reference + * (caller must fail closed, never treat as a match) + * ====================================================================== */ +UT_TEST(test_undo_resid_generation_matches) +{ + ClusterResId r; + + cluster_undo_resid_encode(1, 7, 129, 3, &r); + UT_ASSERT(cluster_undo_resid_generation_matches(&r, 3)); + UT_ASSERT(!cluster_undo_resid_generation_matches(&r, 2)); + UT_ASSERT(!cluster_undo_resid_generation_matches(&r, 4)); + + /* generation 0 (never-reused segment) matches only 0 */ + cluster_undo_resid_encode(1, 7, 129, 0, &r); + UT_ASSERT(cluster_undo_resid_generation_matches(&r, 0)); + UT_ASSERT(!cluster_undo_resid_generation_matches(&r, 1)); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_undo_resid_class_byte); + UT_RUN(test_undo_resid_encode_decode_roundtrip); + UT_RUN(test_undo_resid_tt_header_block_zero); + UT_RUN(test_undo_resid_is_undo); + UT_RUN(test_undo_resid_wire_abi_16_bytes); + UT_RUN(test_undo_resid_owner_bounds); + UT_RUN(test_undo_resid_master_is_owner); + UT_RUN(test_undo_resid_generation_matches); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From 925be67d8c13f5b06ba526c0e5e49a4c4ad2a4d1 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:41:27 +0800 Subject: [PATCH 02/38] feat(cluster): undo-block owner-as-master routing layer (spec-5.22b D2-1) D2-1 wires the D1 undo-block resource identity into the GCS data plane's routing half: two predicates that keep undo resources off the GRD/GCS hash-master path. Their authority lives at the owning instance, so the master IS the encoded owner_node (cluster_undo_resid_master), never a shard hash -- the D1-5 guard already fails closed (53R9Q) on any undo resid that reaches cluster_grd_lookup_master / cluster_gcs_lookup_master. - cluster_undo_gcs.{c,h} (new): cluster_undo_block_lookup_master returns the owner_node; cluster_undo_block_master_is_self is the local fast-path gate (owner_node == cluster_node_id). The owner-incarnation epoch self-check that L364 requires before serving from the local fast path is deferred to the grant/acquire path (D2-3), where the co-sampled live-authority triple is available, so this routing layer stays pure (node-id only) and cluster_unit links it standalone. - src/backend/cluster/Makefile: cluster_undo_gcs.o added to OBJS. - test_cluster_undo_gcs (new, U1/U2): lookup_master returns owner (not a hash); master_is_self true for owner==self, false for a foreign owner. Links cluster_version.o + cluster_undo_resid.o + cluster_undo_gcs.o with a test-owned cluster_node_id stub. Zero behaviour change: no consumer routes undo through these predicates yet (undo_gcs_coherence data plane is D2-3+), so this mirrors D1's land-then-wire shape. Spec: spec-5.22b-undo-block-gcs-integration.md --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_undo_gcs.c | 70 +++++++++++ src/include/cluster/cluster_undo_gcs.h | 69 ++++++++++ src/test/cluster_unit/Makefile | 16 ++- src/test/cluster_unit/test_cluster_undo_gcs.c | 119 ++++++++++++++++++ 5 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_gcs.c create mode 100644 src/include/cluster/cluster_undo_gcs.h create mode 100644 src/test/cluster_unit/test_cluster_undo_gcs.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index c62e189809..33f337ddeb 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -184,6 +184,7 @@ OBJS = \ cluster_visibility_verdict.o \ cluster_undo_record.o \ cluster_undo_resid.o \ + cluster_undo_gcs.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c new file mode 100644 index 0000000000..bd6d953116 --- /dev/null +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_gcs.c + * Shared-undo block GCS integration -- owner-as-master routing + + * coherent grant / PI data plane (spec-5.22b D2). + * + * D2-1 (this increment) ships owner-as-master routing: the two routing + * predicates that keep undo resources off the GRD/GCS hash-master path + * (their authority lives at the owning instance). The grant / PI / + * serve-gate / physical-migration legs (D2-2 .. D2-5) land here as they + * are implemented. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_gcs.c + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22b-undo-block-gcs-integration.md (D2, §2.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_mode.h" /* cluster_node_id */ +#include "cluster/cluster_undo_gcs.h" +#include "cluster/cluster_undo_resid.h" + +/* + * cluster_undo_block_lookup_master -- owner-as-master routing entry. + * + * The master of an undo resource is the encoded owner_node (D1's + * cluster_undo_resid_master), never a shard hash: the undo authority lives + * at the owning instance. This is the single legal master-lookup entry for + * the undo class; cluster_grd_lookup_master / cluster_gcs_lookup_master fail + * closed on it (D1-5 guard, 53R9Q). + */ +int32 +cluster_undo_block_lookup_master(const ClusterResId *undo_resid) +{ + Assert(undo_resid != NULL); + Assert(cluster_undo_resid_is_undo(undo_resid)); + + return cluster_undo_resid_master(undo_resid); +} + +/* + * cluster_undo_block_master_is_self -- local fast-path routing gate. + * + * true iff this instance owns the undo resource (owner_node == + * cluster_node_id), so the owner can read/write its own undo without a + * network grant. The owner-incarnation epoch self-check that L364 requires + * before actually serving from the local fast path is applied on the + * grant/acquire path (D2-3), where the co-sampled live-authority triple is + * available; this predicate is the pure node-id half. + */ +bool +cluster_undo_block_master_is_self(const ClusterResId *undo_resid) +{ + Assert(undo_resid != NULL); + Assert(cluster_undo_resid_is_undo(undo_resid)); + + return cluster_undo_resid_master(undo_resid) == cluster_node_id; +} diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h new file mode 100644 index 0000000000..f330bbc960 --- /dev/null +++ b/src/include/cluster/cluster_undo_gcs.h @@ -0,0 +1,69 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_gcs.h + * Shared-undo block GCS integration -- owner-as-master routing + + * coherent grant / PI data plane (spec-5.22b D2). + * + * D1 (cluster_undo_resid.h) named an undo block as a first-class cluster + * resource whose master IS the owning instance. D2 wires that identity + * into the real data plane: owner-as-master routing (this file's D2-1 + * layer), reader S-grant / writer X-grant with PI invalidation, a + * remaster serve-gate, and the physical migration of undo segments onto + * shared storage. + * + * D2-1 (owner-as-master routing, this increment) declares the two + * routing predicates. An undo resource NEVER hash-routes: its master is + * the encoded owner_node (cluster_undo_resid_master), and the GRD/GCS + * hash-master lookups fail closed on the undo class (D1-5 guard, 53R9Q). + * master==self selects the local fast path (no network grant); the + * owner-incarnation epoch self-check that L364 requires before serving + * from that fast path is applied on the grant/acquire path (D2-3), where + * the co-sampled live-authority triple is available -- this pure routing + * predicate is the node-id half only, so cluster_unit links it + * standalone. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_undo_gcs.h + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22b-undo-block-gcs-integration.md (D2, §2.1) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_UNDO_GCS_H +#define CLUSTER_UNDO_GCS_H + +#include "cluster/cluster_grd.h" /* ClusterResId */ + +/* + * cluster_undo_block_lookup_master -- owner-as-master routing entry. + * + * Returns the encoded owner_node (via cluster_undo_resid_master); an undo + * resource is NEVER hash-routed. cluster_grd_lookup_master / + * cluster_gcs_lookup_master fail closed on the undo class (D1-5 guard, + * ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED), so this is the single legal + * master-lookup entry for undo resources. + */ +extern int32 cluster_undo_block_lookup_master(const ClusterResId *undo_resid); + +/* + * cluster_undo_block_master_is_self -- local fast-path routing gate. + * + * true iff this instance is the owning master of the undo resource + * (owner_node == cluster_node_id), i.e. the owner reading/writing its own + * undo can take the local path without a network grant. This is the pure + * routing half; the owner-incarnation epoch self-check that guards actually + * serving from the local fast path (L364) lands on the grant/acquire path + * (D2-3), not here. + */ +extern bool cluster_undo_block_master_is_self(const ClusterResId *undo_resid); + +#endif /* CLUSTER_UNDO_GCS_H */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 063e5aa67a..93b4c74a8d 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -87,7 +87,8 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_node_remove \ test_cluster_hang_acceptance \ test_cluster_xid_stripe \ - test_cluster_undo_resid + test_cluster_undo_resid \ + test_cluster_undo_gcs # Path to the cluster_version object (no PG deps, safe to link standalone). CLUSTER_VERSION_O = $(top_builddir)/src/backend/cluster/cluster_version.o @@ -179,7 +180,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1842,3 +1843,14 @@ test_cluster_undo_resid: test_cluster_undo_resid.c unit_test.h \ $(CC) $(CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_HW_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# spec-5.22b D2-1: test_cluster_undo_gcs — owner-as-master routing layer +# (cluster_undo_block_lookup_master / _master_is_self). Links the D1 undo +# resid pure object plus the D2 routing object; the test owns the +# cluster_node_id stub so the routing object links without the GUC layer. +CLUSTER_UNDO_GCS_O = $(top_builddir)/src/backend/cluster/cluster_undo_gcs.o +test_cluster_undo_gcs: test_cluster_undo_gcs.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c new file mode 100644 index 0000000000..4f81d986a8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -0,0 +1,119 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_undo_gcs.c + * Unit tests for the shared-undo block GCS integration data plane + * (spec-5.22b D2). + * + * D2-1 (this increment) pins owner-as-master routing: the master of an + * undo resource IS the encoded owner_node (never a shard hash), and the + * master==self predicate selects the local fast path. The grant / PI / + * physical-migration legs (D2-2 .. D2-5) add their own truth-table cases + * to this file as they land. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_undo_gcs.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22b-undo-block-gcs-integration.md (D2, §4.1) + * + * cluster_undo_gcs.o references the cluster_node_id global (self node); + * the test supplies its own definition so the routing object links + * standalone without the full GUC layer. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_undo_gcs.h" +#include "cluster/cluster_undo_resid.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* + * Stub self-node global. cluster_undo_gcs.o resolves cluster_node_id from + * cluster_guc.c in a real backend; here the test owns it so master_is_self + * can be driven against a known self node. + */ +int cluster_node_id = 0; + +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} + +/* ====================================================================== + * U1 -- owner-as-master routing: lookup_master returns the encoded + * owner_node, never a hash-derived node (spec-5.22b §2.1, Q1) + * ====================================================================== */ +UT_TEST(test_undo_gcs_lookup_master_is_owner) +{ + ClusterResId r; + + cluster_undo_resid_encode(2, 7, 129, 3, &r); + UT_ASSERT_EQ(cluster_undo_block_lookup_master(&r), 2); + + /* the master must not vary with the non-owner identity dimensions + * (a shard-hash master would move with segment/block/generation) */ + cluster_undo_resid_encode(2, 9999, 424242, 17, &r); + UT_ASSERT_EQ(cluster_undo_block_lookup_master(&r), 2); + + cluster_undo_resid_encode(0, 1, 1, 0, &r); + UT_ASSERT_EQ(cluster_undo_block_lookup_master(&r), 0); + + cluster_undo_resid_encode(SCN_MAX_VALID_NODE_ID, 1, 1, 1, &r); + UT_ASSERT_EQ(cluster_undo_block_lookup_master(&r), SCN_MAX_VALID_NODE_ID); +} + +/* ====================================================================== + * U2 -- master_is_self: owner_node == cluster_node_id selects the local + * fast path (true); a foreign owner routes remote (false) (spec-5.22b §2.1) + * ====================================================================== */ +UT_TEST(test_undo_gcs_master_is_self) +{ + ClusterResId r; + + cluster_node_id = 2; + cluster_undo_resid_encode(2, 7, 129, 3, &r); /* owner == self */ + UT_ASSERT(cluster_undo_block_master_is_self(&r)); + + cluster_undo_resid_encode(3, 7, 129, 3, &r); /* owner 3 != self 2 */ + UT_ASSERT(!cluster_undo_block_master_is_self(&r)); + + cluster_node_id = 0; + cluster_undo_resid_encode(0, 1, 1, 0, &r); /* owner == self */ + UT_ASSERT(cluster_undo_block_master_is_self(&r)); + + cluster_undo_resid_encode(1, 1, 1, 0, &r); /* owner 1 != self 0 */ + UT_ASSERT(!cluster_undo_block_master_is_self(&r)); + + cluster_node_id = SCN_MAX_VALID_NODE_ID; + cluster_undo_resid_encode(SCN_MAX_VALID_NODE_ID, 4, 4, 4, &r); + UT_ASSERT(cluster_undo_block_master_is_self(&r)); +} + +int +main(void) +{ + UT_PLAN(2); + UT_RUN(test_undo_gcs_lookup_master_is_owner); + UT_RUN(test_undo_gcs_master_is_self); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From 9bba70300a6642c1ed332ffdb7d4ede152bc0b64 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 19:59:51 +0800 Subject: [PATCH 03/38] feat(cluster): undo-block shared-storage physical data plane (spec-5.22b D2-2) D2-2 gives an undo segment an optional physical home on the shared cluster_fs root under GCS, breaking the DataDir-local hardcode in the undo I/O path. The migration is gated on a new GUC (cluster.undo_gcs_coherence, default off), so with the default the whole change is inert and every path resolves to the local DataDir byte-for-byte as before -- mirroring D1/D2-1's land-then-wire shape. Path intent (the core contract): - ClusterUndoPathIntent {RUNTIME_SHARED, MATERIALIZED_LOCAL} names which home a segment resolves to. cluster_undo_intent_for_owner(owner) derives it -- an own-instance owner is RUNTIME_SHARED, a foreign owner (in D2 only ever a dead-origin copy recovery rebuilt in the local DataDir) is MATERIALIZED_LOCAL. Static inline in cluster_undo_alloc.h so the ~30 call sites add no link dependency. - cluster_undo_path_uses_shared_root(intent, peer_mode, coherence) is the single pure decision function: RUNTIME_SHARED resolves under the shared root only when peer-mode AND coherence are on; MATERIALIZED_LOCAL always stays local (P1-3 hard contract -- the dead-origin by-xid resolve path must never move to shared storage). Threading + migration: - intent is a new first parameter on cluster_undo_path_resolve and the four undo smgr APIs (read/write_block, read/write_header_bytes) and is carried in the get_segment_fd fd-cache key. - the redo write surface (cluster_undo_xlog.c build_undo_segment_path / redo_open_segment / redo_stamp_slot) delegates to the SAME cluster_undo_path_resolve, so runtime and redo cannot split-brain an own-instance segment onto different roots. - the shared home resolves via cluster_shared_fs_undo_path_resolve / _instance_dir_resolve (owner-partitioned instance_ under cluster.shared_data_dir). Undo is outside the RelFileLocator namespace, so this is a path resolver, not a shared_fs vtable callback. - ensure-instance-subdir gains a dual branch (shared root -> pg_mkdir_p on the shared tree; local -> the existing mkdir) so a coherence-on tree is physically complete. Supporting: - new GUC cluster.undo_gcs_coherence (bool, default off, PGC_SIGHUP); check hook rejects turning it on while cluster.shared_data_dir is unset (GUC_check_errdetail). - D4 tripwire: cluster_undo_path_resolve asserts (intent == RUNTIME_SHARED) == (owner == self), guarding the future dead-owner foreign-runtime-shared serve case. - drop a now-redundant local extern of cluster_undo_segments_max_per_instance (the new cluster_guc.h include already declares it) to clear a shadowed-declaration finding. - cluster_undo_alloc.h now includes cluster/cluster_scn.h (latent). Default path (coherence off) is byte-for-byte the pre-D2-2 behaviour. Tests: test_cluster_undo_gcs U3-U11 (shared-root branch per mode, materialized-never-migrates, intent derivation); intent parameter threaded through test_cluster_undo_buf / test_cluster_tt_durable stubs. cluster_unit 159/159, PG 219/219, cluster_regress 13/13, and the undo/recovery/CR TAP set (018/070/213/215/247/248/253) all green with coherence off. Spec: spec-5.22b-undo-block-gcs-integration.md --- src/backend/cluster/cluster_cr_server.c | 3 +- src/backend/cluster/cluster_guc.c | 52 +++++++++ src/backend/cluster/cluster_tt_durable.c | 38 ++++--- src/backend/cluster/cluster_tt_recovery.c | 9 +- src/backend/cluster/cluster_undo_gcs.c | 33 ++++++ src/backend/cluster/cluster_undo_record.c | 15 ++- .../cluster/storage/cluster_shared_fs.c | 57 ++++++++++ .../cluster/storage/cluster_undo_alloc.c | 91 +++++++++++++--- .../cluster/storage/cluster_undo_buf.c | 11 +- .../cluster/storage/cluster_undo_smgr.c | 45 +++++--- .../cluster/storage/cluster_undo_xlog.c | 102 ++++++++++++------ src/include/cluster/cluster_guc.h | 5 + src/include/cluster/cluster_undo_gcs.h | 28 ++++- src/include/cluster/cluster_undo_smgr.h | 30 ++++-- .../cluster/storage/cluster_shared_fs.h | 32 ++++++ .../cluster/storage/cluster_undo_alloc.h | 64 ++++++++++- .../cluster_unit/test_cluster_shared_fs.c | 29 ++++- .../cluster_unit/test_cluster_tt_durable.c | 10 +- src/test/cluster_unit/test_cluster_undo_buf.c | 14 ++- src/test/cluster_unit/test_cluster_undo_gcs.c | 72 ++++++++++++- 20 files changed, 628 insertions(+), 112 deletions(-) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 7aafcf3e5a..034a76eec1 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -335,7 +335,8 @@ lms_undo_fetch_serve(ClusterLmsCrSlot *slot) /* Serve only SELF-owned undo: the owner derives from this node's own * id, never from the wire (a forged request cannot redirect the read). */ - return cluster_undo_smgr_read_block(slot->undo_segment_id, (uint8)(cluster_node_id + 1), + return cluster_undo_smgr_read_block(cluster_undo_intent_for_owner((uint8)(cluster_node_id + 1)), + slot->undo_segment_id, (uint8)(cluster_node_id + 1), slot->undo_block_no, slot->result_page); } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..2f7978eeac 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -119,6 +119,11 @@ bool cluster_past_image = false; /* spec-6.12i: active-runtime cross-instance recycled-slot visibility * resolution via undo-block CF fetch (default OFF = 53R97). */ bool cluster_crossnode_runtime_visibility = false; +/* spec-5.22b D2-2: shared-undo GCS coherence master switch (default OFF = + * undo stays on the local DataDir, inert -- 6.12i unmastered fetch path; ON = + * own-instance runtime AND redo undo migrate to the shared cluster_fs root + * under owner-as-master GCS grant/PI). PGC_SIGHUP. */ +bool cluster_undo_gcs_coherence = false; /* spec-6.15 D1: xid space segmentation -- striped allocation (default * OFF = vanilla dense per-node xid allocation). */ bool cluster_xid_striping = false; @@ -985,6 +990,31 @@ check_cluster_shared_data_dir(char **newval, void **extra, GucSource source) return true; } +/* + * check_cluster_undo_gcs_coherence -- GUC check_hook for + * cluster.undo_gcs_coherence (spec-5.22b D2-2). + * + * Turning the shared-undo data plane ON physically migrates own-instance + * runtime + redo undo onto the shared cluster_fs root, so it REQUIRES + * cluster.shared_data_dir to name that root. Reject the ON transition when + * the root is unset: a clear config-time error beats a runtime read that has + * to fail closed with a generic SQLSTATE. (The runtime path additionally + * fails closed -- an unresolvable shared path yields the 53R97 visibility + * fail-close, never a bogus local fallback -- so this hook is operability, + * not the correctness backstop.) + */ +static bool +check_cluster_undo_gcs_coherence(bool *newval, void **extra, GucSource source) +{ + if (*newval && (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0')) { + GUC_check_errdetail( + "cluster.undo_gcs_coherence requires cluster.shared_data_dir to name the shared " + "cluster_fs mount (the same root used by shared_catalog and the recovery anchor)."); + return false; + } + return true; +} + static bool check_cluster_block_device_path(char **newval, void **extra, GucSource source) { @@ -1588,6 +1618,28 @@ cluster_init_guc(void) "(SQLSTATE 53R97)."), &cluster_crossnode_runtime_visibility, false, PGC_SUSET, 0, NULL, NULL, NULL); + /* + * cluster.undo_gcs_coherence -- spec-5.22b D2 master switch for the + * shared-undo block data plane. OFF (default): undo segments stay on the + * local DataDir and cross-instance undo reads take the 6.12i unmastered + * fetch path (fresh ref => 53R97) -- byte-identical to pre-D2, a safe + * rollback surface. ON: own-instance runtime AND redo undo writes migrate + * to the shared cluster_fs root (cluster.shared_data_dir), and undo blocks + * become owner-as-master GCS resources (grant / PI land in D2-3/D2-4). + * Gating physical migration on this switch (not merely peer-mode) keeps + * runtime and redo writes consistent -- both move only when it is on, so a + * default deployment never splits runtime-shared vs redo-local + * (Hardening v1.0.1 裁决 A). PGC_SIGHUP: flippable for the D6 end-to-end + * validation window; a default-ON flip is a separate decision after D3-D6. + */ + DefineCustomBoolVariable( + "cluster.undo_gcs_coherence", + gettext_noop("Enable the shared-undo block GCS data plane (spec-5.22b)."), + gettext_noop("Off keeps undo on the local data dir and cross-instance undo " + "fail-closed (SQLSTATE 53R97). Requires cluster.shared_data_dir when on."), + &cluster_undo_gcs_coherence, false, PGC_SIGHUP, 0, check_cluster_undo_gcs_coherence, NULL, + NULL); + /* * cluster.xid_striping -- spec-6.15 D1 (AD-012 exception 10 xid * space segmentation). When on, this node only issues 32-bit xids diff --git a/src/backend/cluster/cluster_tt_durable.c b/src/backend/cluster/cluster_tt_durable.c index de68784c52..0f96ab6c4e 100644 --- a/src/backend/cluster/cluster_tt_durable.c +++ b/src/backend/cluster/cluster_tt_durable.c @@ -183,7 +183,8 @@ tt_slot_write_committed(uint32 segment_id, uint8 owner, uint16 slot_offset, Tran cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&slot, sizeof(slot))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot read slot %u of undo segment %u", @@ -196,8 +197,8 @@ tt_slot_write_committed(uint32 segment_id, uint8 owner, uint16 slot_offset, Tran slot.commit_scn = commit_scn; slot.first_undo_block = InvalidUbaVal; /* spec-4.8 D7-A (P1#1): no stale head */ - if (!cluster_undo_smgr_write_header_bytes(segment_id, owner, off, (const char *)&slot, - sizeof(slot))) { + if (!cluster_undo_smgr_write_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (const char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot write slot %u of undo segment %u", @@ -283,7 +284,8 @@ cluster_tt_slot_durable_abort(uint32 segment_id, uint16 slot_offset, Transaction cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&slot, sizeof(slot))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot read slot %u of undo segment %u", @@ -296,8 +298,8 @@ cluster_tt_slot_durable_abort(uint32 segment_id, uint16 slot_offset, Transaction slot.commit_scn = InvalidScn; slot.first_undo_block = InvalidUbaVal; /* spec-4.8 D7-A (P1#1): cleared; 0x90 re-attaches */ - if (!cluster_undo_smgr_write_header_bytes(segment_id, owner, off, (const char *)&slot, - sizeof(slot))) { + if (!cluster_undo_smgr_write_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (const char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot write slot %u of undo segment %u", @@ -336,7 +338,8 @@ cluster_tt_slot_durable_set_head(uint32 segment_id, uint16 slot_offset, Transact cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&slot, sizeof(slot))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot read slot %u of undo segment %u", @@ -347,8 +350,8 @@ cluster_tt_slot_durable_set_head(uint32 segment_id, uint16 slot_offset, Transact * (xid, wrap); a recycled slot belongs to a newer owner -> leave untouched. */ if (slot.xid == xid && slot.wrap == wrap) { slot.first_undo_block = first_undo_block; - if (!cluster_undo_smgr_write_header_bytes(segment_id, owner, off, (const char *)&slot, - sizeof(slot))) { + if (!cluster_undo_smgr_write_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (const char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster durable TT: cannot write slot %u of undo segment %u", @@ -375,7 +378,8 @@ cluster_tt_slot_durable_lookup(uint32 segment_id, uint16 slot_offset, Transactio off = tt_slot_file_offset(slot_offset); cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&slot, sizeof(slot))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&slot, sizeof(slot))) { cluster_tt_durable_io_wait_end(); cluster_tt_durable_count_lookup(false); return false; /* segment absent / I/O -> miss (caller fail-closes) */ @@ -416,8 +420,8 @@ cluster_tt_slot_durable_lookup_committed_stable(uint32 segment_id, uint16 slot_o off = tt_slot_file_offset(slot_offset); cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&first, - sizeof(first))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&first, sizeof(first))) { cluster_tt_durable_io_wait_end(); cluster_tt_durable_count_lookup(false); return false; @@ -436,8 +440,8 @@ cluster_tt_slot_durable_lookup_committed_stable(uint32 segment_id, uint16 slot_o } cluster_tt_durable_io_wait_start(); - if (!cluster_undo_smgr_read_header_bytes(segment_id, owner, off, (char *)&second, - sizeof(second))) { + if (!cluster_undo_smgr_read_header_bytes(cluster_undo_intent_for_owner(owner), segment_id, + owner, off, (char *)&second, sizeof(second))) { cluster_tt_durable_io_wait_end(); cluster_tt_durable_count_lookup(false); return false; @@ -665,7 +669,8 @@ cluster_tt_slot_durable_resolve_by_xid_origin(int origin_node, TransactionId xid UndoSegmentHeaderData *hdr; uint16 i; - if (!cluster_undo_smgr_read_block(segment_id, owner, 0, blockbuf.data)) { + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner), segment_id, owner, + 0, blockbuf.data)) { /* absent segment -> sound skip; existing+unreadable -> incomplete. */ if (cluster_undo_segment_file_exists(owner, segment_id)) scan_complete = false; @@ -763,7 +768,8 @@ cluster_undo_segment_tt_header_scan_pass(uint32 segment_id, uint8 owner_instance /* Whole-block read mirrors the by-xid scan shape (one smgr surface). */ cluster_undo_cleaner_scan_wait_start(); - if (!cluster_undo_smgr_read_block(segment_id, owner_instance, 0, block.data)) { + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner_instance), segment_id, + owner_instance, 0, block.data)) { cluster_undo_cleaner_scan_wait_end(); return false; /* absent / I/O: caller counts and moves on */ } diff --git a/src/backend/cluster/cluster_tt_recovery.c b/src/backend/cluster/cluster_tt_recovery.c index 228198b42b..f38a108218 100644 --- a/src/backend/cluster/cluster_tt_recovery.c +++ b/src/backend/cluster/cluster_tt_recovery.c @@ -157,7 +157,8 @@ cluster_tt_recovery_resolve_active_slots(void) UndoSegmentHeaderData *hdr; uint16 i; - if (!cluster_undo_smgr_read_block(segment_id, owner, 0, blockbuf.data)) + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner), segment_id, owner, + 0, blockbuf.data)) continue; /* absent / unreadable -> skip (never false-abort) */ hdr = (UndoSegmentHeaderData *)blockbuf.data; @@ -235,7 +236,8 @@ cluster_tt_recovery_observe_scn_highwater(void) UndoSegmentHeaderData *hdr; uint16 i; - if (!cluster_undo_smgr_read_block(segment_id, owner, 0, blockbuf.data)) + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner), segment_id, owner, + 0, blockbuf.data)) continue; /* absent / unreadable -> skip */ hdr = (UndoSegmentHeaderData *)blockbuf.data; @@ -483,7 +485,8 @@ cluster_tt_recovery_physical_rollback(void) UndoSegmentHeaderData *hdr; uint16 i; - if (!cluster_undo_smgr_read_block(segment_id, owner, 0, blockbuf.data)) + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner), segment_id, owner, + 0, blockbuf.data)) continue; /* absent / unreadable -> skip */ hdr = (UndoSegmentHeaderData *)blockbuf.data; diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c index bd6d953116..85d991a1c7 100644 --- a/src/backend/cluster/cluster_undo_gcs.c +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -68,3 +68,36 @@ cluster_undo_block_master_is_self(const ClusterResId *undo_resid) return cluster_undo_resid_master(undo_resid) == cluster_node_id; } + +/* + * cluster_undo_path_uses_shared_root -- pure physical-root decision (D2-2). + * + * The single source of truth that both undo path builders consult: + * cluster_undo_path_resolve (runtime segment I/O) and the redo write + * surface in cluster_undo_xlog.c. Keeping the decision here (not + * duplicated in each builder) is what prevents own-instance undo + * split-brain -- runtime reading the shared copy while redo stamps the + * local copy (Hardening v1.0.1 裁决 A). + */ +bool +cluster_undo_path_uses_shared_root(ClusterUndoPathIntent intent, bool peer_mode, bool coherence_on) +{ + /* + * P1-3 hard contract: a dead-origin materialized copy (recovery rebuilt + * it in the local DataDir) NEVER migrates to the shared root, in any + * mode. Redirecting it would send dead-origin resolve / CR reads to an + * empty shared path (spec-5.22b §3.6, R1). + */ + if (intent == CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL) + return false; + + Assert(intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED); + + /* + * Own live runtime undo migrates to the shared cluster_fs root only when + * the whole physical migration is armed: a declared multi-node + * deployment (peer_mode) AND cluster.undo_gcs_coherence on. Either off + * => local DataDir, so D2 lands inert at the default (裁决 A). + */ + return peer_mode && coherence_on; +} diff --git a/src/backend/cluster/cluster_undo_record.c b/src/backend/cluster/cluster_undo_record.c index 5029caf77d..623d8cd70b 100644 --- a/src/backend/cluster/cluster_undo_record.c +++ b/src/backend/cluster/cluster_undo_record.c @@ -760,7 +760,8 @@ cluster_undo_try_mark_record_segment_committed(uint32 seg, uint8 owner_instance, if (UndoRecordShared == NULL || seg == 0) return; - if (!cluster_undo_smgr_read_block(seg, owner_instance, 0, blockbuf.data)) + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner_instance), seg, + owner_instance, 0, blockbuf.data)) return; /* read fail -> retain (best-effort) */ if (!cluster_undo_segment_header_identity_ok(blockbuf.data, seg, owner_instance)) return; /* L212: identity, not template bytes -> retain */ @@ -801,7 +802,8 @@ cluster_undo_try_mark_record_segment_committed(uint32 seg, uint8 owner_instance, } if (dirty) - (void)cluster_undo_smgr_write_block(seg, owner_instance, 0, blockbuf.data, true); + (void)cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(owner_instance), seg, + owner_instance, 0, blockbuf.data, true); } uint64 @@ -864,7 +866,8 @@ read_undo_block(uint32 segment_id, uint8 owner_instance, uint32 block_no, char * */ img = cluster_undo_buf_pin(segment_id, owner_instance, block_no, CLUSTER_UNDO_BUF_SHARED, &pin); if (img == NULL) - return cluster_undo_smgr_read_block(segment_id, owner_instance, block_no, buf); + return cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner_instance), + segment_id, owner_instance, block_no, buf); memcpy(buf, img, BLCKSZ); cluster_undo_buf_unpin(&pin); return true; @@ -893,7 +896,8 @@ write_undo_block_ext(uint32 segment_id, uint8 owner_instance, uint32 block_no, c * do_fsync request goes straight to the direct smgr write. */ if (do_fsync) - return cluster_undo_smgr_write_block(segment_id, owner_instance, block_no, buf, true); + return cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(owner_instance), + segment_id, owner_instance, block_no, buf, true); /* * Write-through the pool for DATA blocks: update the cached image and @@ -911,7 +915,8 @@ write_undo_block_ext(uint32 segment_id, uint8 owner_instance, uint32 block_no, c */ if (keep_clean) return false; - return cluster_undo_smgr_write_block(segment_id, owner_instance, block_no, buf, false); + return cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(owner_instance), + segment_id, owner_instance, block_no, buf, false); } /* diff --git a/src/backend/cluster/storage/cluster_shared_fs.c b/src/backend/cluster/storage/cluster_shared_fs.c index 354d047aca..951459e4dd 100644 --- a/src/backend/cluster/storage/cluster_shared_fs.c +++ b/src/backend/cluster/storage/cluster_shared_fs.c @@ -521,4 +521,61 @@ cluster_shared_fs_writeback(ClusterSharedFsHandle *handle, BlockNumber blocknum, cluster_shared_fs_active_ops->writeback(handle, blocknum, nblocks); } + +/* + * cluster_shared_fs_undo_path_resolve -- resolve an undo segment's path on + * the shared cluster_fs root (spec-5.22b D2-2 undo namespace). + * + * Undo is not a RelFileLocator, so it is addressed by resolving the SAME + * absolute path under cluster.shared_data_dir on every node -- exactly the + * pattern the shared-root sentinel, recovery anchor, and CF authority use. + * Owner-partitioned: the directory component is owner_instance-1 (= the + * owning node_id), a straight lift of the local $PGDATA/pg_undo layout so a + * segment's shared and local paths differ only in their root. + * + * Fail-closed (return -1) when the shared root is unset: this resolver is + * only reached on the RUNTIME_SHARED + coherence-on branch, so an unset root + * is a deployment error and must never silently fall back to a bogus path + * (mirrors build_anchor_path; the caller propagates -1 to a fail-closed + * read/write, and the GUC check-hook rejects coherence-on without a root). + */ +int +cluster_shared_fs_undo_path_resolve(uint8 owner_instance, uint32 segment_id, char *buf, + size_t buf_size) +{ + int ret; + + if (buf == NULL || buf_size == 0) + return -1; + Assert(owner_instance >= 1); + + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return -1; + + ret = snprintf(buf, buf_size, "%s/pg_undo/instance_%u/seg_%u.dat", cluster_shared_data_dir, + (unsigned)(owner_instance - 1), (unsigned)segment_id); + if (ret < 0 || (size_t)ret >= buf_size) + return -1; + return 0; +} + +int +cluster_shared_fs_undo_instance_dir_resolve(uint8 owner_instance, char *buf, size_t buf_size) +{ + int ret; + + if (buf == NULL || buf_size == 0) + return -1; + Assert(owner_instance >= 1); + + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return -1; + + ret = snprintf(buf, buf_size, "%s/pg_undo/instance_%u", cluster_shared_data_dir, + (unsigned)(owner_instance - 1)); + if (ret < 0 || (size_t)ret >= buf_size) + return -1; + return 0; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/storage/cluster_undo_alloc.c b/src/backend/cluster/storage/cluster_undo_alloc.c index 83c8694111..335a51e021 100644 --- a/src/backend/cluster/storage/cluster_undo_alloc.c +++ b/src/backend/cluster/storage/cluster_undo_alloc.c @@ -38,7 +38,11 @@ #include #include "access/xlog.h" -#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_guc.h" /* cluster_undo_gcs_coherence */ +#include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled */ +#include "common/file_perm.h" /* pg_mkdir_p / pg_dir_create_mode (shared subdir, D2-2) */ +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_undo_gcs.h" /* path intent + shared-root decision */ #include "cluster/cluster_undo_segment.h" #include "cluster/cluster_undo_segment_init.h" #include "cluster/cluster_undo_record_api.h" /* reuse counter note (3.13) */ @@ -48,7 +52,8 @@ #include "cluster/cluster_undo_segment_init.h" /* fresh header builder (D4 reuse) */ #include "access/xlog.h" /* XLogFlush (3.13 v0.3 (1)) */ #include "cluster/storage/cluster_undo_alloc.h" -#include "cluster/storage/cluster_undo_buf.h" /* invalidate_segment on reuse (3.18 D3.2) */ +#include "cluster/storage/cluster_shared_fs.h" /* undo namespace on shared root (D2-2) */ +#include "cluster/storage/cluster_undo_buf.h" /* invalidate_segment on reuse (3.18 D3.2) */ #include "cluster/storage/cluster_undo_xlog.h" #include "miscadmin.h" #include "storage/bufpage.h" @@ -79,7 +84,8 @@ extern int cluster_node_id; * the assert catches sentinel-0 misuse. */ int -cluster_undo_path_resolve(uint8 owner_instance, uint32 segment_id, char *buf, size_t buf_size) +cluster_undo_path_resolve(ClusterUndoPathIntent intent, uint8 owner_instance, uint32 segment_id, + char *buf, size_t buf_size) { int ret; @@ -87,7 +93,29 @@ cluster_undo_path_resolve(uint8 owner_instance, uint32 segment_id, char *buf, si return -1; Assert(owner_instance >= 1 && owner_instance <= UNDO_OWNER_INSTANCE_MAX); - /* directory uses cluster_node_id (= owner_instance - 1) */ + /* + * spec-5.22b D2-2 (threading strategy B): every caller derives intent via + * cluster_undo_intent_for_owner(owner), so in D2 RUNTIME_SHARED ⟺ own + * instance and MATERIALIZED_LOCAL ⟺ a foreign dead-origin owner. Assert + * that invariant here: a future spec (D4) that serves a foreign owner's + * live undo from shared storage passes RUNTIME_SHARED with owner!=self and + * MUST revisit this path -- the assert makes that divergence loud rather + * than a silent split. + */ + Assert((intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED) + == (owner_instance == (uint8)(cluster_node_id + 1))); + + /* + * RUNTIME_SHARED own undo migrates to the shared cluster_fs root only + * under peer-mode + cluster.undo_gcs_coherence; MATERIALIZED_LOCAL and + * every non-coherent mode stay on the local DataDir (P1-3 / 裁决 A). An + * unset shared root on the shared branch fails closed (resolver -1). + */ + if (cluster_undo_path_uses_shared_root(intent, cluster_peer_mode_enabled(), + cluster_undo_gcs_coherence)) + return cluster_shared_fs_undo_path_resolve(owner_instance, segment_id, buf, buf_size); + + /* local DataDir: directory uses cluster_node_id (= owner_instance - 1) */ ret = snprintf(buf, buf_size, "%s/pg_undo/instance_%u/seg_%u.dat", DataDir, (unsigned)(owner_instance - 1), (unsigned)segment_id); if (ret < 0 || (size_t)ret >= buf_size) @@ -146,7 +174,29 @@ ensure_instance_subdir(uint8 owner_instance) Assert(owner_instance >= 1 && owner_instance <= UNDO_OWNER_INSTANCE_MAX); - /* directory uses cluster_node_id (= owner_instance - 1); see + /* + * spec-5.22b D2-2: own-instance allocation under coherence lands the + * segment on the shared cluster_fs root, whose pg_undo/instance_ tree + * is not seeded by initdb -- create the whole path (idempotent). Foreign + * owners never reach the allocator (single-writer), so only the shared + * branch differs from the pre-D2 local mkdir. + */ + if (cluster_undo_path_uses_shared_root(cluster_undo_intent_for_owner(owner_instance), + cluster_peer_mode_enabled(), + cluster_undo_gcs_coherence)) { + if (cluster_shared_fs_undo_instance_dir_resolve(owner_instance, path, sizeof(path)) != 0) + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("undo shared instance subdir unresolved: owner_instance=%u " + "(cluster.shared_data_dir unset)", + (unsigned)owner_instance))); + if (pg_mkdir_p(path, pg_dir_create_mode) != 0 && errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create undo shared instance subdir \"%s\": %m", path))); + return; + } + + /* local DataDir: directory uses cluster_node_id (= owner_instance - 1); see * cluster_undo_path_resolve docstring. */ ret = snprintf(path, sizeof(path), "%s/pg_undo/instance_%u", DataDir, (unsigned)(owner_instance - 1)); @@ -270,7 +320,9 @@ cluster_undo_segment_allocate(uint32 segment_id, uint8 owner_instance) (unsigned)owner_instance, cluster_node_id, cluster_node_id + 1))); } - if (cluster_undo_path_resolve(owner_instance, segment_id, path, sizeof(path)) != 0) + if (cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), owner_instance, + segment_id, path, sizeof(path)) + != 0) ereport(ERROR, (errcode(ERRCODE_NAME_TOO_LONG), errmsg("undo segment path too long: instance=%u seg=%u", (unsigned)owner_instance, (unsigned)segment_id))); @@ -477,7 +529,8 @@ static bool read_segment_header_via_smgr(uint32 segment_id, uint8 owner_instance, char *blockbuf, UndoSegmentHeaderData **out_hdr) { - if (!cluster_undo_smgr_read_block(segment_id, owner_instance, 0, blockbuf)) + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner_instance), segment_id, + owner_instance, 0, blockbuf)) return false; *out_hdr = (UndoSegmentHeaderData *)blockbuf; return true; @@ -487,7 +540,8 @@ read_segment_header_via_smgr(uint32 segment_id, uint8 owner_instance, char *bloc static bool write_segment_header_via_smgr(uint32 segment_id, uint8 owner_instance, const char *blockbuf) { - return cluster_undo_smgr_write_block(segment_id, owner_instance, 0, blockbuf, true); + return cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(owner_instance), segment_id, + owner_instance, 0, blockbuf, true); } @@ -870,7 +924,13 @@ cluster_undo_segment_extend_or_create(uint8 owner_instance, bool *out_at_hard_ca * within that lock so current_pool_size is non-stale. */ { - extern int cluster_undo_segments_max_per_instance; + /* + * cluster_undo_segments_max_per_instance is declared in + * cluster/cluster_guc.h (included above), so no local extern is + * needed here. PGRAC: D2-2 added that include for + * cluster_undo_gcs_coherence; dropping the redundant local extern + * avoids a shadowed-declaration static-analysis finding. + */ int guc_val = cluster_undo_segments_max_per_instance; uint32 current_pool_size = 0; uint32 probe_slot; @@ -889,7 +949,8 @@ cluster_undo_segment_extend_or_create(uint8 owner_instance, bool *out_at_hard_ca = (uint32)(owner_instance - 1) * CLUSTER_UNDO_SEGS_PER_INSTANCE + probe_slot + 1; path_ret - = cluster_undo_path_resolve(owner_instance, probe_segment_id, path, sizeof(path)); + = cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), + owner_instance, probe_segment_id, path, sizeof(path)); if (path_ret != 0) break; @@ -927,7 +988,8 @@ cluster_undo_segment_extend_or_create(uint8 owner_instance, bool *out_at_hard_ca int fd; uint32 new_segment_id = base_segment_id + slot; - ret = cluster_undo_path_resolve(owner_instance, new_segment_id, path, sizeof(path)); + ret = cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), + owner_instance, new_segment_id, path, sizeof(path)); if (ret != 0) continue; @@ -1015,7 +1077,8 @@ cluster_undo_segment_scan_max_existing(uint8 owner_instance) int fd; uint32 probe_id = base_segment_id + slot; - ret = cluster_undo_path_resolve(owner_instance, probe_id, path, sizeof(path)); + ret = cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), + owner_instance, probe_id, path, sizeof(path)); if (ret != 0) break; @@ -1047,7 +1110,9 @@ cluster_undo_segment_file_exists(uint8 owner_instance, uint32 segment_id) if (owner_instance < 1 || owner_instance > UNDO_OWNER_INSTANCE_MAX) return false; - if (cluster_undo_path_resolve(owner_instance, segment_id, path, sizeof(path)) != 0) + if (cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), owner_instance, + segment_id, path, sizeof(path)) + != 0) return false; return access(path, F_OK) == 0; } diff --git a/src/backend/cluster/storage/cluster_undo_buf.c b/src/backend/cluster/storage/cluster_undo_buf.c index df8f7a927f..bc977cc76d 100644 --- a/src/backend/cluster/storage/cluster_undo_buf.c +++ b/src/backend/cluster/storage/cluster_undo_buf.c @@ -488,7 +488,8 @@ flush_dirty_slot(int slotno) * XLogFlush above is attributed to its own WAL wait event. */ pgstat_report_wait_start(WAIT_EVENT_CLUSTER_UNDO_BUF_FLUSH); - if (!cluster_undo_smgr_write_block(seg, owner, blk, localbuf.data, /* do_fsync = */ true)) { + if (!cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(owner), seg, owner, blk, + localbuf.data, /* do_fsync = */ true)) { pgstat_report_wait_end(); ereport(ERROR, (errcode(ERRCODE_CLUSTER_UNDO_RECORD_INVALID_UBA), errmsg("cluster undo buffer write-back flush failed " @@ -681,7 +682,8 @@ cluster_undo_buf_pin(uint32 segment_id, uint8 owner, uint32 block_no, ClusterUnd LWLockRelease(&UndoBufPool->map_lock); /* Disk read OUTSIDE the map_lock. */ - if (!cluster_undo_smgr_read_block(segment_id, owner, block_no, SLOT_DATA(slotno))) { + if (!cluster_undo_smgr_read_block(cluster_undo_intent_for_owner(owner), segment_id, + owner, block_no, SLOT_DATA(slotno))) { /* Fill failed — release reservation + report. */ LWLockRelease(&s->content_lock); LWLockAcquire(&UndoBufPool->map_lock, LW_EXCLUSIVE); @@ -732,8 +734,9 @@ cluster_undo_buf_mark_dirty(const ClusterUndoBufPin *pin, XLogRecPtr wal_lsn) * The block is NOT left buffered-dirty, so durability is identical to * today's per-block direct write. */ - if (!cluster_undo_smgr_write_block(s->segment_id, s->owner, s->block_no, - SLOT_DATA(pin->slot), /* do_fsync = */ false)) + if (!cluster_undo_smgr_write_block(cluster_undo_intent_for_owner(s->owner), s->segment_id, + s->owner, s->block_no, SLOT_DATA(pin->slot), + /* do_fsync = */ false)) ereport(ERROR, (errcode(ERRCODE_CLUSTER_UNDO_RECORD_INVALID_UBA), errmsg("cluster undo buffer write-through failed seg=%u owner=%u block=%u", diff --git a/src/backend/cluster/storage/cluster_undo_smgr.c b/src/backend/cluster/storage/cluster_undo_smgr.c index 676c976ce0..6dd82b008f 100644 --- a/src/backend/cluster/storage/cluster_undo_smgr.c +++ b/src/backend/cluster/storage/cluster_undo_smgr.c @@ -41,6 +41,7 @@ #include "storage/ipc.h" /* before_shmem_exit (fd cache cleanup) */ #include "utils/elog.h" +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_intent_for_owner (D2-2) */ #include "cluster/cluster_undo_record_api.h" /* smgr syscall counter bumps */ #include "cluster/cluster_undo_smgr.h" #include "cluster/cluster_undo_segment.h" @@ -61,6 +62,13 @@ */ static uint32 cached_fd_segment = 0; /* 0 = empty */ static uint8 cached_fd_owner = 0; +/* + * spec-5.22b D2-2: the intent is part of the cache key. A given (segment, + * owner) resolves to a DIFFERENT physical path for RUNTIME_SHARED (shared + * root) vs MATERIALIZED_LOCAL (local DataDir), so a cached fd opened under + * one intent must never be reused for the other. + */ +static ClusterUndoPathIntent cached_fd_intent = CLUSTER_UNDO_PATH_RUNTIME_SHARED; static int cached_fd = -1; static bool cached_fd_exit_registered = false; @@ -90,17 +98,18 @@ fd_cache_on_exit(int code, Datum arg) * NOT close the returned fd (the cache owns it). */ static int -get_segment_fd(uint32 segment_id, uint8 owner_instance) +get_segment_fd(ClusterUndoPathIntent intent, uint32 segment_id, uint8 owner_instance) { char path[MAXPGPATH]; int fd; - if (cached_fd >= 0 && cached_fd_segment == segment_id && cached_fd_owner == owner_instance) + if (cached_fd >= 0 && cached_fd_segment == segment_id && cached_fd_owner == owner_instance + && cached_fd_intent == intent) return cached_fd; /* hit */ fd_cache_close(); /* miss: drop the stale fd first */ - if (cluster_undo_path_resolve(owner_instance, segment_id, path, sizeof(path)) != 0) + if (cluster_undo_path_resolve(intent, owner_instance, segment_id, path, sizeof(path)) != 0) return -1; fd = BasicOpenFile(path, O_RDWR | PG_BINARY); if (fd < 0) @@ -110,6 +119,7 @@ get_segment_fd(uint32 segment_id, uint8 owner_instance) cached_fd = fd; cached_fd_segment = segment_id; cached_fd_owner = owner_instance; + cached_fd_intent = intent; if (!cached_fd_exit_registered) { before_shmem_exit(fd_cache_on_exit, (Datum)0); cached_fd_exit_registered = true; @@ -129,7 +139,8 @@ cluster_undo_smgr_fd_cache_reset(void) bool -cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance, uint32 block_no, char *buf) +cluster_undo_smgr_read_block(ClusterUndoPathIntent intent, uint32 segment_id, uint8 owner_instance, + uint32 block_no, char *buf) { int fd; off_t offset; @@ -139,7 +150,7 @@ cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance, uint32 blo if (buf == NULL || block_no >= UNDO_BLOCKS_PER_SEGMENT) return false; - fd = get_segment_fd(segment_id, owner_instance); + fd = get_segment_fd(intent, segment_id, owner_instance); if (fd < 0) return false; @@ -152,8 +163,8 @@ cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance, uint32 blo bool -cluster_undo_smgr_write_block(uint32 segment_id, uint8 owner_instance, uint32 block_no, - const char *buf, bool do_fsync) +cluster_undo_smgr_write_block(ClusterUndoPathIntent intent, uint32 segment_id, uint8 owner_instance, + uint32 block_no, const char *buf, bool do_fsync) { int fd; off_t offset; @@ -163,7 +174,7 @@ cluster_undo_smgr_write_block(uint32 segment_id, uint8 owner_instance, uint32 bl if (buf == NULL || block_no >= UNDO_BLOCKS_PER_SEGMENT) return false; - fd = get_segment_fd(segment_id, owner_instance); + fd = get_segment_fd(intent, segment_id, owner_instance); if (fd < 0) return false; @@ -195,8 +206,8 @@ cluster_undo_smgr_write_block(uint32 segment_id, uint8 owner_instance, uint32 bl * must stay inside block 0 (BLCKSZ). */ bool -cluster_undo_smgr_read_header_bytes(uint32 segment_id, uint8 owner_instance, uint32 offset, - char *buf, uint32 len) +cluster_undo_smgr_read_header_bytes(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 offset, char *buf, uint32 len) { int fd; ssize_t nread; @@ -204,7 +215,7 @@ cluster_undo_smgr_read_header_bytes(uint32 segment_id, uint8 owner_instance, uin if (buf == NULL || len == 0 || (uint64)offset + (uint64)len > (uint64)BLCKSZ) return false; - fd = get_segment_fd(segment_id, owner_instance); + fd = get_segment_fd(intent, segment_id, owner_instance); if (fd < 0) return false; @@ -214,8 +225,9 @@ cluster_undo_smgr_read_header_bytes(uint32 segment_id, uint8 owner_instance, uin } bool -cluster_undo_smgr_write_header_bytes(uint32 segment_id, uint8 owner_instance, uint32 offset, - const char *buf, uint32 len) +cluster_undo_smgr_write_header_bytes(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 offset, const char *buf, + uint32 len) { int fd; ssize_t nwritten; @@ -223,7 +235,7 @@ cluster_undo_smgr_write_header_bytes(uint32 segment_id, uint8 owner_instance, ui if (buf == NULL || len == 0 || (uint64)offset + (uint64)len > (uint64)BLCKSZ) return false; - fd = get_segment_fd(segment_id, owner_instance); + fd = get_segment_fd(intent, segment_id, owner_instance); if (fd < 0) return false; @@ -253,7 +265,8 @@ cluster_undo_smgr_create_segment_file(uint32 segment_id, uint8 owner_instance) int ret; int fd; - ret = cluster_undo_path_resolve(owner_instance, segment_id, path, sizeof(path)); + ret = cluster_undo_path_resolve(cluster_undo_intent_for_owner(owner_instance), + owner_instance, segment_id, path, sizeof(path)); if (ret != 0) return -2; @@ -272,7 +285,7 @@ cluster_undo_smgr_fsync_segment_file(uint32 segment_id, uint8 owner_instance) { int fd; - fd = get_segment_fd(segment_id, owner_instance); + fd = get_segment_fd(cluster_undo_intent_for_owner(owner_instance), segment_id, owner_instance); if (fd < 0) return false; diff --git a/src/backend/cluster/storage/cluster_undo_xlog.c b/src/backend/cluster/storage/cluster_undo_xlog.c index 6ceed05c7f..a18f0297d5 100644 --- a/src/backend/cluster/storage/cluster_undo_xlog.c +++ b/src/backend/cluster/storage/cluster_undo_xlog.c @@ -43,13 +43,18 @@ #include "access/xloginsert.h" #include "access/xlogreader.h" #include "access/xlogutils.h" +#include "cluster/cluster_guc.h" /* cluster_undo_gcs_coherence (D2-2) */ #include "cluster/cluster_hw.h" /* spec-5.7 D1 HW authority apply/encode */ +#include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled (D2-2) */ #include "cluster/cluster_tt_status.h" /* spec-3.16 D5 recovery counters */ #include "cluster/cluster_tt_durable.h" /* spec-3.11: redo decision predicate */ +#include "cluster/cluster_undo_gcs.h" /* shared-root decision (D2-2 裁决 A) */ #include "cluster/cluster_undo_segment.h" /* UNDO_SEGMENT_SIZE_BYTES */ -#include "cluster/storage/cluster_undo_alloc.h" /* header identity check (3.13 reuse redo) */ +#include "cluster/storage/cluster_shared_fs.h" /* undo shared-root dir resolve (D2-2) */ +#include "cluster/storage/cluster_undo_alloc.h" /* path resolve + intent (3.13 reuse redo) */ #include "cluster/storage/cluster_undo_buf.h" /* spec-3.18 D2b write-back gate */ #include "cluster/storage/cluster_undo_xlog.h" +#include "common/file_perm.h" /* pg_mkdir_p / pg_dir_create_mode (shared subdir, D2-2) */ #include "miscadmin.h" #include "storage/bufpage.h" #include "storage/fd.h" @@ -57,35 +62,27 @@ /* - * Build a path: $PGDATA/pg_undo/instance_/seg_.dat + * Build the undo segment path for the REDO write surface. * - * Both encoder (cluster_undo_emit_segment_init) and decoder - * (cluster_undo_redo) use this layout. pg_undo subdir is - * established by initdb (D4). instance_ subdir is created - * on demand by cluster_undo_redo_segment_init (Stage 1.22 ships - * only instance_0 from initdb; redo path may need other instance - * subdirs on standbys / cross-instance crash recovery). + * spec-5.22b D2-2 (Hardening v1.0.1 裁决 A): the redo write surface must + * resolve to the SAME physical root as the runtime write surface, or an + * own-instance segment would be read from the shared root (runtime) while + * redo re-stamps the local copy -- own-instance undo split-brain (block-0 + * TT is WAL-redo-only, so a lost redo write on the shared copy is a + * false-invisible 8.A hazard). So this delegates to the ONE decision + * source, cluster_undo_path_resolve: own redo (rec->instance == self) under + * coherence resolves to the shared cluster_fs root; a foreign rec->instance + * (dead-origin materialization rebuilt locally) resolves to the local + * DataDir. Callers pass cluster_undo_intent_for_owner(rec->instance). * - * Hardening v1.0.4 P1-1: directory naming uses cluster_node_id - * (= owner_instance - 1) so that single-node default - * (cluster_node_id = 0, owner_instance = 1) lands at instance_0/ - * matching the initdb seed. See cluster_undo_alloc.c - * cluster_undo_path_resolve docstring for full rationale. - * - * Returns 0 on success, -1 on path-too-long. Caller supplies - * buf with capacity >= MAXPGPATH. + * Returns 0 on success, -1 on path-too-long / unset shared root (fail- + * closed). Caller supplies buf with capacity >= MAXPGPATH. */ static int -build_undo_segment_path(uint8 owner_instance, uint32 segment_id, char *buf, size_t buf_size) +build_undo_segment_path(ClusterUndoPathIntent intent, uint8 owner_instance, uint32 segment_id, + char *buf, size_t buf_size) { - int ret; - - Assert(owner_instance >= 1 && owner_instance <= UNDO_OWNER_INSTANCE_MAX); - ret = snprintf(buf, buf_size, "%s/pg_undo/instance_%u/seg_%u.dat", DataDir, - (unsigned)(owner_instance - 1), (unsigned)segment_id); - if (ret < 0 || (size_t)ret >= buf_size) - return -1; - return 0; + return cluster_undo_path_resolve(intent, owner_instance, segment_id, buf, buf_size); } @@ -108,7 +105,28 @@ ensure_undo_instance_subdir(uint8 owner_instance) Assert(owner_instance >= 1 && owner_instance <= UNDO_OWNER_INSTANCE_MAX); - /* directory uses cluster_node_id (= owner_instance - 1) per Hardening v1.0.4 P1-1 */ + /* + * spec-5.22b D2-2: own-instance redo under coherence materializes the + * segment on the shared cluster_fs root, whose pg_undo/instance_ tree + * is not seeded by initdb -- create the whole path (idempotent). A + * foreign owner (dead-origin materialization) stays on the local DataDir + * (裁决 A), so only own redo under coherence takes the shared branch. + */ + if (cluster_undo_path_uses_shared_root(cluster_undo_intent_for_owner(owner_instance), + cluster_peer_mode_enabled(), + cluster_undo_gcs_coherence)) { + if (cluster_shared_fs_undo_instance_dir_resolve(owner_instance, path, sizeof(path)) != 0) + ereport(PANIC, (errmsg("undo shared instance subdir unresolved: owner_instance=%u " + "(cluster.shared_data_dir unset)", + (unsigned)owner_instance))); + if (pg_mkdir_p(path, pg_dir_create_mode) != 0 && errno != EEXIST) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not create undo shared instance subdir \"%s\": %m", path))); + return; + } + + /* local DataDir: pg_undo seeded by initdb; single-level mkdir (P1-1 naming). */ ret = snprintf(path, sizeof(path), "%s/pg_undo/instance_%u", DataDir, (unsigned)(owner_instance - 1)); if (ret < 0 || (size_t)ret >= sizeof(path)) @@ -633,7 +651,9 @@ cluster_undo_redo_segment_init(XLogReaderState *record) hdr = (xl_cluster_undo_segment_init *)payload; page_image = payload + sizeof(*hdr); - if (build_undo_segment_path(hdr->instance, hdr->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(hdr->instance), hdr->instance, + hdr->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", hdr->instance, hdr->segment_id))); @@ -760,7 +780,9 @@ cluster_tt_durable_redo_stamp_slot(uint8 instance, uint32 segment_id, uint16 slo ereport(PANIC, (errmsg("TT slot commit redo: slot_offset %u out of range (max %d)", slot_offset, TT_SLOTS_PER_SEGMENT - 1))); - if (build_undo_segment_path(instance, segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(instance), instance, segment_id, path, + sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", instance, segment_id))); @@ -889,7 +911,9 @@ cluster_undo_redo_tt_slot_abort(XLogReaderState *record) ereport(PANIC, (errmsg("XLOG_UNDO_TT_SLOT_ABORT slot_offset %u out of range (max %d)", rec->slot_offset, TT_SLOTS_PER_SEGMENT - 1))); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); @@ -998,7 +1022,9 @@ cluster_undo_redo_tt_slot_set_head(XLogReaderState *record) ereport(PANIC, (errmsg("XLOG_UNDO_TT_SLOT_SET_HEAD slot_offset %u out of range (max %d)", rec->slot_offset, TT_SLOTS_PER_SEGMENT - 1))); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); @@ -1094,7 +1120,9 @@ cluster_undo_redo_segment_recycle(XLogReaderState *record) XLogRecGetDataLen(record)))); rec = (xl_undo_segment_recycle *)XLogRecGetData(record); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); @@ -1202,7 +1230,9 @@ cluster_undo_redo_segment_reuse(XLogReaderState *record) rec = (xl_undo_segment_reuse *)XLogRecGetData(record); image = XLogRecGetData(record) + sizeof(*rec); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); @@ -1355,7 +1385,9 @@ cluster_undo_redo_block_write(XLogReaderState *record) if (rec->block_no == 0) ereport(PANIC, (errmsg("XLOG_UNDO_BLOCK_WRITE redo: block_no 0 is the segment header"))); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); @@ -1474,7 +1506,9 @@ cluster_undo_redo_block_write_multi(XLogReaderState *record) ereport(PANIC, (errmsg("XLOG_UNDO_BLOCK_WRITE_MULTI redo: block_no 0 is the segment header"))); - if (build_undo_segment_path(rec->instance, rec->segment_id, path, sizeof(path)) != 0) + if (build_undo_segment_path(cluster_undo_intent_for_owner(rec->instance), rec->instance, + rec->segment_id, path, sizeof(path)) + != 0) ereport(PANIC, (errmsg("undo segment path too long: instance=%u seg=%u", rec->instance, rec->segment_id))); diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..bd975372b7 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -199,6 +199,11 @@ extern bool cluster_past_image; * (undo-block CF fetch + live authority gate; default off = 53R97). */ extern bool cluster_crossnode_runtime_visibility; +/* spec-5.22b D2-2: shared-undo GCS coherence master switch (default off = + * undo stays on the local DataDir; on = own-instance runtime + redo undo + * migrate to the shared cluster_fs root under owner-as-master GCS). */ +extern bool cluster_undo_gcs_coherence; + /* spec-6.15 D1: xid space segmentation -- striped allocation (default * off = vanilla dense per-node xid allocation). */ extern bool cluster_xid_striping; diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h index f330bbc960..befe286e71 100644 --- a/src/include/cluster/cluster_undo_gcs.h +++ b/src/include/cluster/cluster_undo_gcs.h @@ -41,7 +41,8 @@ #ifndef CLUSTER_UNDO_GCS_H #define CLUSTER_UNDO_GCS_H -#include "cluster/cluster_grd.h" /* ClusterResId */ +#include "cluster/cluster_grd.h" /* ClusterResId */ +#include "cluster/storage/cluster_undo_alloc.h" /* ClusterUndoPathIntent */ /* * cluster_undo_block_lookup_master -- owner-as-master routing entry. @@ -66,4 +67,29 @@ extern int32 cluster_undo_block_lookup_master(const ClusterResId *undo_resid); */ extern bool cluster_undo_block_master_is_self(const ClusterResId *undo_resid); +/* + * cluster_undo_path_uses_shared_root -- pure physical-root decision (D2-2). + * + * Returns true iff an undo segment with the given path intent resolves to + * the shared cluster_fs root (cluster.shared_data_dir) rather than the + * local DataDir. Pure: takes the mode inputs explicitly (no globals), so + * cluster_unit drives every mode combination standalone. + * + * RUNTIME_SHARED -> shared iff (peer_mode && coherence_on) + * MATERIALIZED_LOCAL -> always local (P1-3 hard contract, spec-5.22b §3.6) + * + * cluster_undo_path_resolve (cluster_undo_alloc.c) and the redo write + * surface (cluster_undo_xlog.c) read the live peer_mode / coherence globals + * and delegate the decision here so the two path builders never diverge + * (own-instance undo split-brain, Hardening v1.0.1 裁决 A). + */ +extern bool cluster_undo_path_uses_shared_root(ClusterUndoPathIntent intent, bool peer_mode, + bool coherence_on); + +/* + * cluster_undo_intent_for_owner (the per-call intent derivation every undo + * smgr / path call site uses) is a static inline in cluster_undo_alloc.h so + * the ~30 call sites take no link dependency on this routing object. + */ + #endif /* CLUSTER_UNDO_GCS_H */ diff --git a/src/include/cluster/cluster_undo_smgr.h b/src/include/cluster/cluster_undo_smgr.h index 20dda91f1f..01311e64da 100644 --- a/src/include/cluster/cluster_undo_smgr.h +++ b/src/include/cluster/cluster_undo_smgr.h @@ -46,19 +46,30 @@ #include "postgres.h" #include "storage/block.h" +#include "cluster/storage/cluster_undo_alloc.h" /* ClusterUndoPathIntent (D2-2) */ + + +/* + * spec-5.22b D2-2: the block / header I/O entries take an explicit + * ClusterUndoPathIntent so the shared-vs-local physical root is chosen per + * call (RUNTIME_SHARED own undo -> shared cluster_fs root under coherence; + * MATERIALIZED_LOCAL foreign dead-origin copy -> local DataDir). Callers + * pass cluster_undo_intent_for_owner(owner_instance). + */ /* * cluster_undo_smgr_read_block -- read one 8KB block from undo segment file. * * Args: + * intent -- shared-vs-local root selector (D2-2) * segment_id, owner_instance -- per-instance segment identity * block_no -- 0..UNDO_BLOCKS_PER_SEGMENT-1 * buf -- caller-provided 8KB buffer (BLCKSZ) * * Returns: true on success, false on I/O error or short read. */ -extern bool cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance, uint32 block_no, - char *buf); +extern bool cluster_undo_smgr_read_block(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 block_no, char *buf); /* @@ -74,8 +85,9 @@ extern bool cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance * * NOT critical-section safe. */ -extern bool cluster_undo_smgr_write_block(uint32 segment_id, uint8 owner_instance, uint32 block_no, - const char *buf, bool do_fsync); +extern bool cluster_undo_smgr_write_block(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 block_no, const char *buf, + bool do_fsync); /* @@ -89,10 +101,12 @@ extern bool cluster_undo_smgr_write_block(uint32 segment_id, uint8 owner_instanc * Returns true on success, false on bad args / I/O error / short transfer. * NOT critical-section safe. */ -extern bool cluster_undo_smgr_read_header_bytes(uint32 segment_id, uint8 owner_instance, - uint32 offset, char *buf, uint32 len); -extern bool cluster_undo_smgr_write_header_bytes(uint32 segment_id, uint8 owner_instance, - uint32 offset, const char *buf, uint32 len); +extern bool cluster_undo_smgr_read_header_bytes(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 offset, char *buf, + uint32 len); +extern bool cluster_undo_smgr_write_header_bytes(ClusterUndoPathIntent intent, uint32 segment_id, + uint8 owner_instance, uint32 offset, + const char *buf, uint32 len); /* diff --git a/src/include/cluster/storage/cluster_shared_fs.h b/src/include/cluster/storage/cluster_shared_fs.h index 73e4f355f7..74f9fe7679 100644 --- a/src/include/cluster/storage/cluster_shared_fs.h +++ b/src/include/cluster/storage/cluster_shared_fs.h @@ -302,6 +302,38 @@ extern void cluster_shared_fs_writeback(ClusterSharedFsHandle *handle, BlockNumb BlockNumber nblocks); +/* ---------- + * Undo namespace on the shared root (spec-5.22b D2-2) + * + * Undo segments are NOT RelFileLocator relations, so the RelFileLocator + * vtable above cannot address them (see the shared-root sentinel note in + * cluster_shared_fs_sharedfs.c: non-RelFileLocator shared files are reached + * by resolving the SAME absolute path under cluster.shared_data_dir on every + * node, exactly as the recovery anchor and CF authority do). This resolver + * is that namespace for undo: it builds an owner-partitioned undo segment + * path under the shared root. cluster_undo_path_resolve routes here only for + * the CLUSTER_UNDO_PATH_RUNTIME_SHARED intent under peer-mode + + * cluster.undo_gcs_coherence. + * + * Returns 0 on success; -1 when the shared root is unset (fail-closed, so a + * misconfigured deployment never resolves a bogus path) or on buffer + * overflow. Caller supplies buf with capacity >= MAXPGPATH. + * ---------- + */ +extern int cluster_shared_fs_undo_path_resolve(uint8 owner_instance, uint32 segment_id, char *buf, + size_t buf_size); + +/* + * cluster_shared_fs_undo_instance_dir_resolve -- resolve the owner's undo + * instance DIRECTORY on the shared root (/pg_undo/ + * instance_). Unlike the local DataDir (where initdb pre-creates + * pg_undo), the shared root has no pre-seeded undo tree, so the caller + * pg_mkdir_p's this path before first use. Returns 0 / -1 (unset root). + */ +extern int cluster_shared_fs_undo_instance_dir_resolve(uint8 owner_instance, char *buf, + size_t buf_size); + + /* * Internal: built-in vtable instances exposed for cluster_shared_fs_init's * registration path and for cluster_unit linkage assertions. Backend diff --git a/src/include/cluster/storage/cluster_undo_alloc.h b/src/include/cluster/storage/cluster_undo_alloc.h index 38f316c2d8..2f65ac843a 100644 --- a/src/include/cluster/storage/cluster_undo_alloc.h +++ b/src/include/cluster/storage/cluster_undo_alloc.h @@ -34,6 +34,7 @@ #define CLUSTER_UNDO_ALLOC_H #include "c.h" +#include "cluster/cluster_scn.h" /* SCN (cluster_undo_segment_try_mark_recyclable) */ #include "storage/block.h" @@ -77,6 +78,54 @@ #define CLUSTER_UNDO_SEGS_PER_INSTANCE ((uint32)256) +/* + * ClusterUndoPathIntent -- spec-5.22b D2-2 (P1-3 hard contract). + * + * Declares, at every undo path-resolve call site, WHICH physical root a + * segment resolves to. The intent is stated EXPLICITLY by the caller and + * is never inferred from owner==self, because D4 will introduce a + * "foreign + runtime-shared serve" case (a survivor reading a dead owner's + * durable block from shared storage) where owner!=self no longer implies + * "local materialized copy". + * + * CLUSTER_UNDO_PATH_RUNTIME_SHARED + * The owner's own live runtime undo. Migrates to the shared + * cluster_fs root ONLY under peer-mode AND cluster.undo_gcs_coherence + * (both off => local DataDir, inert). + * + * CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL + * A dead-origin materialized copy that recovery rebuilt in the local + * DataDir (cluster_tt_durable.c by-xid resolve of a foreign origin, + * merged recovery, remote-xact outcome store). ALWAYS resolves to + * the local DataDir, in any mode -- D2 must never redirect these + * reads, else dead-origin recovery / CR regress (spec-5.22b §3.6, R1). + */ +typedef enum ClusterUndoPathIntent { + CLUSTER_UNDO_PATH_RUNTIME_SHARED, /* own live undo; shared under peer-mode+coherence */ + CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL /* dead-origin materialized copy; always local */ +} ClusterUndoPathIntent; + +/* cluster_node_id owns the +1 sentinel offset: owner_instance == node_id + 1. */ +extern int cluster_node_id; /* cluster_guc.c */ + +/* + * cluster_undo_intent_for_owner -- per-call intent derivation (D2-2, B). + * + * The single derivation every undo smgr / path call site consults: this + * node's own undo (owner_instance == cluster_node_id + 1) is + * RUNTIME_SHARED; a foreign owner -- in D2 always a dead-origin materialized + * copy that recovery rebuilt in the local DataDir -- is MATERIALIZED_LOCAL. + * Inline (mirrors cluster_mode.h's node-id gates) so the ~30 call sites take + * no link dependency on the GCS routing object. + */ +static inline ClusterUndoPathIntent +cluster_undo_intent_for_owner(uint8 owner_instance) +{ + return (owner_instance == (uint8)(cluster_node_id + 1)) ? CLUSTER_UNDO_PATH_RUNTIME_SHARED + : CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL; +} + + /* * cluster_undo_path_resolve * Build the segment file path: @@ -86,10 +135,19 @@ * in [0, 255] (uint8 max) * in [0, 4294967295] (uint32 max) * - * Returns 0 on success, -1 on buffer overflow. Caller supplies a - * buf with capacity >= MAXPGPATH. + * spec-5.22b D2-2: takes an explicit ClusterUndoPathIntent. A + * RUNTIME_SHARED own-instance segment resolves under the shared cluster_fs + * root when peer-mode + cluster.undo_gcs_coherence are on; MATERIALIZED_LOCAL + * (and every non-coherent mode) resolves under the local DataDir. Callers + * pass cluster_undo_intent_for_owner(owner) unless they have a stronger + * reason to name a literal intent. + * + * Returns 0 on success, -1 on buffer overflow (or an unset shared root on + * the shared branch -- fail-closed). Caller supplies a buf with capacity + * >= MAXPGPATH. */ -extern int cluster_undo_path_resolve(uint8 instance, uint32 segment_id, char *buf, size_t buf_size); +extern int cluster_undo_path_resolve(ClusterUndoPathIntent intent, uint8 instance, + uint32 segment_id, char *buf, size_t buf_size); /* diff --git a/src/test/cluster_unit/test_cluster_shared_fs.c b/src/test/cluster_unit/test_cluster_shared_fs.c index 71f7f05aa1..0d8724b75a 100644 --- a/src/test/cluster_unit/test_cluster_shared_fs.c +++ b/src/test/cluster_unit/test_cluster_shared_fs.c @@ -731,6 +731,32 @@ UT_TEST(test_sharedfs_sentinel_symbols_linkable) UT_ASSERT_NOT_NULL((void *)has_fn); } +/* ============================================================ + * spec-5.22b D2-2: undo namespace path resolve on the shared root. + * Non-RelFileLocator; mirrors build_anchor_path (fail-closed when the + * shared root is unset), owner-partitioned instance_. + * ============================================================ */ +UT_TEST(test_undo_shared_path_resolve) +{ + char buf[MAXPGPATH]; + + /* fail-closed when the shared root is unset: never resolve a bogus path */ + cluster_shared_data_dir = NULL; + UT_ASSERT_EQ(cluster_shared_fs_undo_path_resolve(1, 1, buf, sizeof(buf)), -1); + + /* owner-partitioned instance_ under the shared root; the directory uses + * owner_instance-1 (= node_id), matching the local $PGDATA/pg_undo layout */ + cluster_shared_data_dir = "/u01/pgrac/shared"; + UT_ASSERT_EQ(cluster_shared_fs_undo_path_resolve(1, 1, buf, sizeof(buf)), 0); + UT_ASSERT_STR_EQ(buf, "/u01/pgrac/shared/pg_undo/instance_0/seg_1.dat"); + + cluster_shared_data_dir = "/mnt/shared"; + UT_ASSERT_EQ(cluster_shared_fs_undo_path_resolve(3, 257, buf, sizeof(buf)), 0); + UT_ASSERT_STR_EQ(buf, "/mnt/shared/pg_undo/instance_2/seg_257.dat"); + + cluster_shared_data_dir = NULL; /* restore stub default */ +} + /* ============================================================ * Test runner @@ -739,7 +765,7 @@ UT_TEST(test_sharedfs_sentinel_symbols_linkable) int main(void) { - UT_PLAN(15); + UT_PLAN(16); UT_RUN(test_shared_fs_backend_max_constant); UT_RUN(test_shared_fs_backend_id_enum_frozen); UT_RUN(test_shared_fs_vtable_struct_nonempty); @@ -755,6 +781,7 @@ main(void) UT_RUN(test_sharedfs_vtable_callbacks_nonnull); UT_RUN(test_sharedfs_vtable_identity); UT_RUN(test_sharedfs_sentinel_symbols_linkable); + UT_RUN(test_undo_shared_path_resolve); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_tt_durable.c b/src/test/cluster_unit/test_cluster_tt_durable.c index bab49dde88..3b286c6a9d 100644 --- a/src/test/cluster_unit/test_cluster_tt_durable.c +++ b/src/test/cluster_unit/test_cluster_tt_durable.c @@ -193,7 +193,8 @@ cluster_undo_cleaner_scan_wait_end(void) {} bool -cluster_undo_smgr_read_header_bytes(uint32 segment_id pg_attribute_unused(), +cluster_undo_smgr_read_header_bytes(ClusterUndoPathIntent intent pg_attribute_unused(), + uint32 segment_id pg_attribute_unused(), uint8 owner_instance pg_attribute_unused(), uint32 offset, char *buf, uint32 len) { @@ -212,7 +213,8 @@ static int g_write_hdr_calls = 0; /* spec-3.13 D2-B scan-only invariant probe * static TTSlot g_last_written_slot; /* spec-3.15: capture write payload */ bool -cluster_undo_smgr_write_header_bytes(uint32 segment_id pg_attribute_unused(), +cluster_undo_smgr_write_header_bytes(ClusterUndoPathIntent intent pg_attribute_unused(), + uint32 segment_id pg_attribute_unused(), uint8 owner_instance pg_attribute_unused(), uint32 offset pg_attribute_unused(), const char *buf, uint32 len) @@ -224,8 +226,8 @@ cluster_undo_smgr_write_header_bytes(uint32 segment_id pg_attribute_unused(), } bool -cluster_undo_smgr_read_block(uint32 segment_id, uint8 owner_instance pg_attribute_unused(), - uint32 block_no, char *buf) +cluster_undo_smgr_read_block(ClusterUndoPathIntent intent pg_attribute_unused(), uint32 segment_id, + uint8 owner_instance pg_attribute_unused(), uint32 block_no, char *buf) { if (buf == NULL || block_no != 0 || segment_id != g_canned_block_segment) return false; /* other segments "don't exist" -> by-xid skips them */ diff --git a/src/test/cluster_unit/test_cluster_undo_buf.c b/src/test/cluster_unit/test_cluster_undo_buf.c index c959f0b3cf..63d09579e5 100644 --- a/src/test/cluster_unit/test_cluster_undo_buf.c +++ b/src/test/cluster_unit/test_cluster_undo_buf.c @@ -63,6 +63,14 @@ UT_DEFINE_GLOBALS(); +/* + * Self-node stub: cluster_undo_buf.o now derives the undo path intent via the + * cluster_undo_intent_for_owner inline (spec-5.22b D2-2), which reads + * cluster_node_id. The buffer-pool unit exercises own-instance segments, so + * node 0 keeps every smgr call on the RUNTIME_SHARED (own) branch. + */ +int cluster_node_id = 0; + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -217,7 +225,8 @@ static uint32 smgr_last_write_block = 0; static char smgr_last_write_byte0 = 0; bool -cluster_undo_smgr_read_block(uint32 segment_id pg_attribute_unused(), +cluster_undo_smgr_read_block(ClusterUndoPathIntent intent pg_attribute_unused(), + uint32 segment_id pg_attribute_unused(), uint8 owner_instance pg_attribute_unused(), uint32 block_no, char *buf) { smgr_read_calls++; @@ -227,7 +236,8 @@ cluster_undo_smgr_read_block(uint32 segment_id pg_attribute_unused(), } bool -cluster_undo_smgr_write_block(uint32 segment_id pg_attribute_unused(), +cluster_undo_smgr_write_block(ClusterUndoPathIntent intent pg_attribute_unused(), + uint32 segment_id pg_attribute_unused(), uint8 owner_instance pg_attribute_unused(), uint32 block_no, const char *buf, bool do_fsync) { diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 4f81d986a8..8270348702 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -35,6 +35,7 @@ #include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ #include "cluster/cluster_undo_gcs.h" #include "cluster/cluster_undo_resid.h" +#include "cluster/storage/cluster_undo_alloc.h" /* ClusterUndoPathIntent */ #undef printf #undef fprintf @@ -108,12 +109,81 @@ UT_TEST(test_undo_gcs_master_is_self) UT_ASSERT(cluster_undo_block_master_is_self(&r)); } +/* ====================================================================== + * U3 -- physical-path root decision (D2-2): a RUNTIME_SHARED (own-instance + * live) undo segment migrates to the shared cluster_fs root ONLY under + * peer-mode AND cluster.undo_gcs_coherence; every other mode combination + * stays on the local DataDir (inert, zero behaviour change) (spec-5.22b + * §2.3/§3.5, Hardening v1.0.1 裁决 A, Q8/Q9) + * ====================================================================== */ +UT_TEST(test_undo_gcs_path_runtime_shared_mode_branch) +{ + /* migrate: own runtime undo, peer-mode, coherence on */ + UT_ASSERT(cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED, true /*peer*/, + true /*coherence*/)); + + /* coherence off => inert, stay local (回归安全, 回 6.12i 无主权路径) */ + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED, true /*peer*/, + false /*coherence*/)); + + /* single-node / non-peer-mode => local regardless of coherence */ + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED, false /*peer*/, + true /*coherence*/)); + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED, false /*peer*/, + false /*coherence*/)); +} + +/* ====================================================================== + * U10 -- path-intent layering (P1-3 hard contract): a MATERIALIZED_LOCAL + * (dead-origin materialized copy that recovery rebuilt in the local + * DataDir) NEVER migrates to the shared root, in ANY mode. D2 must not + * redirect dead-origin materialized reads, else dead-origin recovery / CR + * regress (spec-5.22b §3.6, R1, Q9) + * ====================================================================== */ +UT_TEST(test_undo_gcs_path_materialized_never_migrates) +{ + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL, + true /*peer*/, true /*coherence*/)); + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL, + true /*peer*/, false /*coherence*/)); + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL, + false /*peer*/, true /*coherence*/)); + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL, + false /*peer*/, false /*coherence*/)); +} + +/* ====================================================================== + * U11 -- per-call intent derivation (spec-5.22b D2-2, threading strategy B): + * an undo segment whose owner IS this node maps to RUNTIME_SHARED (own live + * undo); any foreign owner maps to MATERIALIZED_LOCAL (dead-origin + * materialized copy). This is the single derivation every smgr call site + * uses, so a misclassification cannot hide at an individual site. + * owner_instance == cluster_node_id + 1 is "self" (the +1 sentinel offset). + * ====================================================================== */ +UT_TEST(test_undo_gcs_intent_for_owner) +{ + cluster_node_id = 0; /* self owner_instance = 1 */ + UT_ASSERT_EQ(cluster_undo_intent_for_owner(1), CLUSTER_UNDO_PATH_RUNTIME_SHARED); + UT_ASSERT_EQ(cluster_undo_intent_for_owner(2), CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL); + UT_ASSERT_EQ(cluster_undo_intent_for_owner(3), CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL); + + cluster_node_id = 2; /* self owner_instance = 3 */ + UT_ASSERT_EQ(cluster_undo_intent_for_owner(3), CLUSTER_UNDO_PATH_RUNTIME_SHARED); + UT_ASSERT_EQ(cluster_undo_intent_for_owner(1), CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL); + UT_ASSERT_EQ(cluster_undo_intent_for_owner(4), CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL); + + cluster_node_id = 0; /* restore stub default */ +} + int main(void) { - UT_PLAN(2); + UT_PLAN(5); UT_RUN(test_undo_gcs_lookup_master_is_owner); UT_RUN(test_undo_gcs_master_is_self); + UT_RUN(test_undo_gcs_path_runtime_shared_mode_branch); + UT_RUN(test_undo_gcs_path_materialized_never_migrates); + UT_RUN(test_undo_gcs_intent_for_owner); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 4addbd55bd52a467fa9e307a028af7527b65ecf3 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 22:02:05 +0800 Subject: [PATCH 04/38] feat(cluster): undo-block reader S-grant coherent read primitive (spec-5.22b D2-3) Add cluster_undo_block_acquire_shared: a peer acquires a coherent shared view of an owning instance's undo block through owner-as-master routing. The owner (authority and holder) ships the current image over the reused undo-TT fetch wire; the requesting peer consumes the shipped image and never opens the foreign undo file. Admission is fail-closed on two ANDed dimensions -- segment-generation anti-ABA and owner-incarnation epoch plus durable coverage -- so any miss/DENIED/doubt leaves the caller's existing fail-closed visibility boundary untouched. Structure follows the pure-policy split so cluster_unit links the decision core standalone: - cluster_undo_gcs.c gains the pure predicates cluster_undo_grant_armed (coherence x peer-mode gate), cluster_undo_grant_admissible (reuses cluster_vis_live_authority_covers_policy and the D1 generation check), and cluster_undo_grant_reader_pcm_mode/_transition (S via N->S). - cluster_undo_gcs_grant.c (new) holds the runtime wrapper with its heavy fetch/epoch/GUC dependencies, registered in the backend OBJS. Everything is gated behind cluster.undo_gcs_coherence (default off), so the default path is byte-for-byte unchanged. The master==self local fast path is fail-closed in this increment pending its owner-incarnation self-check; the live-owner peer read path is fully implemented. test_cluster_undo_gcs gains U4/U6/U7/U9 (reader lock contract, epoch and generation admissibility, coherence gate); its recipe links the pure live-authority policy object. Runtime wrapper is covered by the later TAP legs. Local gates (coherence off): cluster_unit 159/159 (undo_gcs 9/9), PG 219/219, cluster_regress 13/13, undo/CR/recovery TAP 6 files/160 tests, clang-format-18 clean, comment-headers clean. Spec: spec-5.22b-undo-block-gcs-integration.md --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_undo_gcs.c | 68 +++++++++ src/backend/cluster/cluster_undo_gcs_grant.c | 130 ++++++++++++++++++ src/include/cluster/cluster_undo_gcs.h | 87 ++++++++++++ src/test/cluster_unit/Makefile | 17 ++- src/test/cluster_unit/test_cluster_undo_gcs.c | 101 +++++++++++++- 6 files changed, 398 insertions(+), 6 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_gcs_grant.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 33f337ddeb..77e4a0d4e3 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -185,6 +185,7 @@ OBJS = \ cluster_undo_record.o \ cluster_undo_resid.o \ cluster_undo_gcs.o \ + cluster_undo_gcs_grant.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c index 85d991a1c7..09bfc0ad2f 100644 --- a/src/backend/cluster/cluster_undo_gcs.c +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -101,3 +101,71 @@ cluster_undo_path_uses_shared_root(ClusterUndoPathIntent intent, bool peer_mode, */ return peer_mode && coherence_on; } + +/* + * cluster_undo_grant_armed -- pure coherence gate (D2-3). + * + * See cluster_undo_gcs.h for the contract. Mirrors the physical-migration + * gate above (peer_mode && coherence_on): the grant/PI data plane and the + * physical migration arm together, so a peer never grants against undo bytes + * that were never migrated to the shared root. + */ +bool +cluster_undo_grant_armed(bool coherence_on, bool peer_mode) +{ + return coherence_on && peer_mode; +} + +/* + * cluster_undo_grant_admissible -- pure reader S-grant admissibility (D2-3). + * + * See cluster_undo_gcs.h for the two-dimension fail-closed contract. The two + * admit conditions are ANDed; any single failure returns false so the caller + * keeps the pre-existing 53R97 fail-closed boundary (Rule 8.A -- this only + * widens "admit when provable", never "admit when unprovable"). + */ +bool +cluster_undo_grant_admissible(const ClusterResId *undo_resid, uint32 expected_generation, + ClusterLiveAuthority auth, uint64 local_epoch, XLogRecPtr anchor_lsn) +{ + if (undo_resid == NULL) + return false; + + /* + * (1) Segment-generation anti-ABA (D1 §3.3 dim 1). A recycled segment + * reuses (undo_segment, block_no) for different content, so a generation + * mismatch means the reference is stale -- fail closed, never a match. + */ + if (!cluster_undo_resid_generation_matches(undo_resid, expected_generation)) + return false; + + /* + * (2) Owner-incarnation epoch + durable coverage (D-i2/D-i3). Reuse the + * pure live-authority window gate: authority sampled under a different + * membership epoch cannot be trusted, a reply with no live authority + * (invalid hwm) is never guessed, and the durable high-water must cover the + * version anchor (Invalid at the block level -> epoch + presence only). + */ + if (!cluster_vis_live_authority_covers_policy(anchor_lsn, auth, local_epoch)) + return false; + + return true; +} + +/* + * cluster_undo_grant_reader_pcm_mode / _transition -- reader lock contract + * (D2-3). The reader takes the undo block in PCM S mode via the read-first + * N->S transition (the legal read-first pair; cluster_pcm_lock owns and tests + * the transition-legality table). The writer/cleaner X path is D2-4. + */ +PcmState +cluster_undo_grant_reader_pcm_mode(void) +{ + return PCM_LOCK_MODE_S; +} + +PcmLockTransition +cluster_undo_grant_reader_pcm_transition(void) +{ + return PCM_TRANS_N_TO_S; +} diff --git a/src/backend/cluster/cluster_undo_gcs_grant.c b/src/backend/cluster/cluster_undo_gcs_grant.c new file mode 100644 index 0000000000..431e99ad7a --- /dev/null +++ b/src/backend/cluster/cluster_undo_gcs_grant.c @@ -0,0 +1,130 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_gcs_grant.c + * Shared-undo block GCS integration -- reader S-grant runtime primitive + * (spec-5.22b D2-3). + * + * This is the runtime wrapper that composes the pure decision core + * (cluster_undo_gcs.c: cluster_undo_grant_armed / _admissible) with the + * live wire: owner-as-master routing (D2-1), the reused 6.12i undo-TT + * fetch (owner ships its own image; the peer never opens the foreign undo + * file, invariant #8), and the fail-closed admission gate (Rule 8.A). + * + * It lives in its own object because it references the heavy backend + * surface (the GCS block fetch, the membership epoch, the coherence GUC) + * that the pure routing/decision object (cluster_undo_gcs.o) deliberately + * does NOT, so cluster_unit links that object standalone. This runtime + * wrapper is forward-covered by the D2-7 / D6 TAP legs (a peer really + * reading a foreign owner's undo block end-to-end), not by cluster_unit. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_gcs_grant.c + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22b-undo-block-gcs-integration.md (D2, §2.2/§3.1/§3.2) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_cr_server.h" /* cluster_gcs_block_undo_tt_fetch_and_wait */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER */ +#include "cluster/cluster_guc.h" /* cluster_undo_gcs_coherence */ +#include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled, cluster_node_id */ +#include "cluster/cluster_undo_gcs.h" +#include "cluster/cluster_undo_resid.h" + +/* + * cluster_undo_block_acquire_shared -- reader S-grant primitive (D2-3). + * + * See cluster_undo_gcs.h for the full contract. A peer acquires a coherent + * S view of owner N's undo block through owner-as-master routing; any + * miss / DENIED / doubt returns false so the caller keeps its 53R97 + * fail-closed boundary (Rule 8.A -- MVCC/visibility never forward-links a + * false-visible edge). + */ +bool +cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expected_generation, + char *dst_block, ClusterUndoGrantResult *res) +{ + int32 owner; + uint32 seg; + uint32 block; + uint32 gen; + uint64 local_epoch; + ClusterLiveAuthority auth; + + if (undo_resid == NULL || dst_block == NULL || res == NULL) + return false; /* fail-closed */ + if (!cluster_undo_resid_is_undo(undo_resid)) + return false; + + memset(res, 0, sizeof(*res)); + + /* + * Coherence gate. At the default (coherence off, or a single-node / non + * peer-mode deployment) the path is not armed: the caller keeps its + * unchanged authority-less fetch path, so D2 lands inert (回归安全). + */ + if (!cluster_undo_grant_armed(cluster_undo_gcs_coherence, cluster_peer_mode_enabled())) + return false; + + cluster_undo_resid_decode(undo_resid, &owner, &seg, &block, &gen); + local_epoch = cluster_epoch_get_current(); + + if (cluster_undo_block_master_is_self(undo_resid)) { + /* + * master==self: the owner reading its OWN undo needs no network grant + * (local PCM suffices, §3.1). The owner-incarnation epoch self-check + * (L364) that must gate serving from this local fast path -- proving + * THIS node is still the legitimate owner incarnation at local_epoch, + * not a fenced zombie -- has no standalone primitive yet, and this API + * has no D2-3 consumer (the owner reads its own undo through the + * existing local undo paths). Rather than serve a local read that + * cannot yet be proven safe (Rule 8.A), the self fast path is + * fail-closed here and lands with its incarnation check + consumer in + * D6. The peer-reads-foreign S-grant below is D2-3's delivered heart. + */ + return false; /* honest fail-closed forward (D6); never a stale local serve */ + } + + /* + * master!=self (live owner): the owner is the authority AND the holder. + * Reuse the 6.12i owner-ships-image wire verbatim (zero new wire; Q3★A + * maps a success to GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER at this layer). + * The owner reads its own undo and ships the current image + the authority + * co-sampled with it; this peer consumes the shipped image and NEVER opens + * the foreign undo file (invariant #8). A DENIED_RESOURCE_RECOVERING + * (owner dead / undo shard remastering) or any timeout / checksum / + * missing-trailer collapses the fetch to false -> fail-closed 53R97 + * (dead-owner serve is D4, not D2). + */ + if (!cluster_gcs_block_undo_tt_fetch_and_wait(owner, seg, block, dst_block, &auth)) + return false; + + /* + * Admit only a coherent view: segment-generation anti-ABA AND the owner's + * incarnation epoch equals ours AND the shipped durable high-water is + * present (Rule 8.A -- never admit on doubt). anchor_lsn is Invalid at + * the block level; the version-coverage (hwm >= page_lsn) gate is applied + * by the D3/D6 verdict consumer that owns the tuple's anchor. + */ + if (!cluster_undo_grant_admissible(undo_resid, expected_generation, auth, local_epoch, + InvalidXLogRecPtr)) + return false; + + res->origin_epoch = auth.origin_epoch; + res->live_hwm_lsn = auth.live_hwm_lsn; + res->tt_generation = auth.tt_generation; + res->status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; + return true; +} diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h index befe286e71..60f7ef1a5e 100644 --- a/src/include/cluster/cluster_undo_gcs.h +++ b/src/include/cluster/cluster_undo_gcs.h @@ -42,6 +42,8 @@ #define CLUSTER_UNDO_GCS_H #include "cluster/cluster_grd.h" /* ClusterResId */ +#include "cluster/cluster_pcm_lock.h" /* PcmState / PcmLockTransition */ +#include "cluster/cluster_runtime_visibility.h" /* ClusterLiveAuthority */ #include "cluster/storage/cluster_undo_alloc.h" /* ClusterUndoPathIntent */ /* @@ -92,4 +94,89 @@ extern bool cluster_undo_path_uses_shared_root(ClusterUndoPathIntent intent, boo * the ~30 call sites take no link dependency on this routing object. */ +/* + * ClusterUndoGrantResult -- reader S-grant outcome (spec-5.22b §2.2, D2-3). + * + * The authority triple co-sampled with the granted block image (never + * max-merged: it must stay the authority the bytes were shipped under), plus + * the semantic reply status. Under owner-as-master the owner is both master + * and holder, so a successful grant maps to GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER + * (Q3★A: 2-way holder ship; the 6.12i undo-TT-fetch wire is reused verbatim, + * no new reply enum). + */ +typedef struct ClusterUndoGrantResult { + uint64 origin_epoch; /* owner incarnation epoch co-sampled with bytes */ + XLogRecPtr live_hwm_lsn; /* owner durable AND TT-applied high-water */ + uint64 tt_generation; /* owner TT-slot generation (anti-alias; D3 uses) */ + uint8 status; /* GCS_BLOCK_REPLY_* semantic status (Q3★A) */ +} ClusterUndoGrantResult; + +/* + * cluster_undo_grant_armed -- pure coherence gate (D2-3, §3.5/Q8). + * + * The mastered reader S-grant path is armed iff cluster.undo_gcs_coherence is + * on AND the deployment is peer-mode. Off in either dimension => the caller + * keeps its pre-D2 authority-less fetch path (D2 lands inert at the default). + * Pure (mode inputs explicit, no globals) so cluster_unit drives it standalone. + */ +extern bool cluster_undo_grant_armed(bool coherence_on, bool peer_mode); + +/* + * cluster_undo_grant_admissible -- pure reader S-grant admissibility (D2-3, + * §3.2/§3.3). + * + * Decide whether an owner-shipped undo-block image + co-sampled authority may + * be admitted as a coherent S view of undo_resid. Two ANDed fail-closed + * dimensions (Rule 8.A -- any doubt returns false, never admits): + * (1) SEGMENT-generation anti-ABA: the ref's encoded generation (segment + * wrap_count) must still match the reader's expected generation (D1 + * cluster_undo_resid_generation_matches), else the segment was recycled + * and the ref is stale. + * (2) owner-incarnation epoch + durable coverage: reuses the pure D-i2 gate + * cluster_vis_live_authority_covers_policy (epoch match + hwm present + + * hwm >= anchor_lsn). anchor_lsn is InvalidXLogRecPtr for a block-level + * grant with no specific page version (the version-coverage gate is the + * D3/D6 verdict consumer's, which owns the tuple's anchor). + * + * The per-slot TT generation (auth.tt_generation) is a separate finer ABA + * dimension resolved at D3 (§3.3 dim 3), not here. + */ +extern bool cluster_undo_grant_admissible(const ClusterResId *undo_resid, + uint32 expected_generation, ClusterLiveAuthority auth, + uint64 local_epoch, XLogRecPtr anchor_lsn); + +/* + * cluster_undo_grant_reader_pcm_mode / _transition -- reader lock contract + * (D2-3, §2.2). The reader takes the undo block in PCM S mode via the + * read-first N->S transition; the writer/cleaner path (X via N->X) is D2-4. + * The transition-legality table itself is owned + tested by cluster_pcm_lock. + */ +extern PcmState cluster_undo_grant_reader_pcm_mode(void); +extern PcmLockTransition cluster_undo_grant_reader_pcm_transition(void); + +/* + * cluster_undo_block_acquire_shared -- reader S-grant primitive (D2-3, §2.2). + * + * A peer reads owner N's undo block (block0 TT header is D3's first consumer) + * through owner-as-master routing: + * - not armed (coherence off / non-peer) -> false: caller keeps its old + * authority-less path (fresh refs stay 53R97 fail-closed). + * - master==self -> the owner reads its own undo; the owner-incarnation + * self-check + local fast-path land in D6 (see cluster_undo_gcs_grant.c), + * so this increment fail-closes self rather than serve an unproven local + * read (Rule 8.A). + * - master!=self (live owner) -> the owner (authority + holder) ships the + * current image over the reused 6.12i wire; the requesting peer consumes + * the shipped image and NEVER opens the foreign undo file (invariant #8). + * Admitted only through cluster_undo_grant_admissible. + * + * Returns true with *res populated and *dst_block holding the coherent image; + * false (any miss / DENIED / doubt) -> the caller MUST keep 53R97 (Rule 8.A, + * never false-visible). Runtime object (heavy deps): not in the standalone + * cluster_unit link; forward-covered by the D2-7/D6 TAP legs. + */ +extern bool cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, + uint32 expected_generation, char *dst_block, + ClusterUndoGrantResult *res); + #endif /* CLUSTER_UNDO_GCS_H */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 93b4c74a8d..05e6d1e4ec 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -1844,13 +1844,20 @@ test_cluster_undo_resid: test_cluster_undo_resid.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_HW_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ -# spec-5.22b D2-1: test_cluster_undo_gcs — owner-as-master routing layer -# (cluster_undo_block_lookup_master / _master_is_self). Links the D1 undo -# resid pure object plus the D2 routing object; the test owns the -# cluster_node_id stub so the routing object links without the GUC layer. +# spec-5.22b D2-1/D2-3: test_cluster_undo_gcs — owner-as-master routing layer +# (D2-1: cluster_undo_block_lookup_master / _master_is_self) + the pure reader +# S-grant decision core (D2-3: cluster_undo_grant_armed / _admissible / +# _reader_pcm_*). Links the D1 undo resid pure object, the D2 routing object, +# and the pure live-authority policy object (cluster_undo_grant_admissible +# reuses cluster_vis_live_authority_covers_policy); the test owns the +# cluster_node_id stub so the routing object links without the GUC layer. The +# runtime acquire_shared wrapper (heavy fetch/epoch/smgr deps) lives in +# cluster_undo_gcs_grant.o, NOT here — it is forward-covered by the D2-7/D6 TAP. CLUSTER_UNDO_GCS_O = $(top_builddir)/src/backend/cluster/cluster_undo_gcs.o test_cluster_undo_gcs: test_cluster_undo_gcs.c unit_test.h \ - $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ + $(CLUSTER_RUNTIME_VIS_POLICY_O) $(CC) $(CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ + $(CLUSTER_RUNTIME_VIS_POLICY_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 8270348702..47e1d9d726 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -175,15 +175,114 @@ UT_TEST(test_undo_gcs_intent_for_owner) cluster_node_id = 0; /* restore stub default */ } +/* ====================================================================== + * U9 -- reader S-grant coherence gate (spec-5.22b §3.5, Q8): the mastered + * S-grant path is ARMED only when cluster.undo_gcs_coherence is on AND the + * deployment is peer-mode. Off in either dimension => not armed => the caller + * keeps its pre-D2 authority-less fetch path (回归安全, zero behaviour change at + * the default). Pure: branch-only integer logic, no globals. + * ====================================================================== */ +UT_TEST(test_undo_gcs_grant_armed_gate) +{ + UT_ASSERT(cluster_undo_grant_armed(true /*coherence*/, true /*peer*/)); + UT_ASSERT(!cluster_undo_grant_armed(false /*coherence*/, true /*peer*/)); + UT_ASSERT(!cluster_undo_grant_armed(true /*coherence*/, false /*peer*/)); + UT_ASSERT(!cluster_undo_grant_armed(false /*coherence*/, false /*peer*/)); +} + +/* ====================================================================== + * U4 -- reader S-grant lock contract (spec-5.22b §2.2/§4.1): the reader + * acquires the undo block in PCM S mode via the read-first N->S transition. + * The N->S transition-legality TABLE is owned + exhaustively tested by + * cluster_pcm_lock (test_cluster_pcm_lock, spec-2.30); D2-3 only pins that the + * reader selects that legal read-first pair, never a write/upgrade transition. + * Pure: compile-time constants. + * ====================================================================== */ +UT_TEST(test_undo_gcs_grant_reader_pcm_contract) +{ + UT_ASSERT_EQ(cluster_undo_grant_reader_pcm_mode(), PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_undo_grant_reader_pcm_transition(), PCM_TRANS_N_TO_S); + + /* a reader must never select a write-side mode / transition */ + UT_ASSERT(cluster_undo_grant_reader_pcm_mode() != PCM_LOCK_MODE_X); + UT_ASSERT(cluster_undo_grant_reader_pcm_transition() != PCM_TRANS_N_TO_X); +} + +/* ====================================================================== + * U7 -- reader S-grant admissibility: SEGMENT-generation anti-ABA (spec-5.22b + * §3.3 dim 1, D1). A reference whose encoded generation (segment wrap_count) + * no longer matches the reader's expected generation is STALE (the segment was + * recycled) -- admit MUST fail closed (Rule 8.A), never treat as a match. The + * per-slot TT generation (auth.tt_generation) is a SEPARATE finer dimension + * resolved at D3, not here. + * ====================================================================== */ +UT_TEST(test_undo_gcs_grant_admissible_generation) +{ + ClusterResId r; + ClusterLiveAuthority auth; + + /* authority that is otherwise admissible (epoch match, hwm present) */ + memset(&auth, 0, sizeof(auth)); + auth.origin_epoch = 42; + auth.live_hwm_lsn = 0x1000; + auth.tt_generation = 9; + + cluster_undo_resid_encode(3, 7, 129, 5 /*generation*/, &r); + + /* generation matches => admissible (epoch 42 == local 42, hwm >= anchor 0) */ + UT_ASSERT(cluster_undo_grant_admissible(&r, 5 /*expected_gen*/, auth, 42 /*local_epoch*/, + InvalidXLogRecPtr)); + + /* generation mismatch (recycled segment) => fail closed */ + UT_ASSERT( + !cluster_undo_grant_admissible(&r, 6 /*expected_gen != 5*/, auth, 42, InvalidXLogRecPtr)); +} + +/* ====================================================================== + * U6 -- reader S-grant admissibility: owner-incarnation epoch + durable + * coverage (spec-5.22b §3.2, D-i2/D-i3). Authority sampled under a different + * membership epoch than the reader's current epoch cannot be trusted (a + * reconfig may have remastered/fenced the owner between sample and use) => fail + * closed. A missing live authority (hwm invalid) also fails closed -- never + * guess. Reuses the pure cluster_vis_live_authority_covers_policy gate. + * ====================================================================== */ +UT_TEST(test_undo_gcs_grant_admissible_epoch) +{ + ClusterResId r; + ClusterLiveAuthority auth; + + cluster_undo_resid_encode(3, 7, 129, 5, &r); + + memset(&auth, 0, sizeof(auth)); + auth.origin_epoch = 100; + auth.live_hwm_lsn = 0x2000; + auth.tt_generation = 1; + + /* epoch match => admissible */ + UT_ASSERT(cluster_undo_grant_admissible(&r, 5, auth, 100 /*local_epoch*/, InvalidXLogRecPtr)); + + /* epoch mismatch (reconfig between sample and use) => fail closed */ + UT_ASSERT( + !cluster_undo_grant_admissible(&r, 5, auth, 101 /*local_epoch != 100*/, InvalidXLogRecPtr)); + + /* missing live authority (hwm invalid) => fail closed, never guess */ + auth.live_hwm_lsn = InvalidXLogRecPtr; + UT_ASSERT(!cluster_undo_grant_admissible(&r, 5, auth, 100, InvalidXLogRecPtr)); +} + int main(void) { - UT_PLAN(5); + UT_PLAN(9); UT_RUN(test_undo_gcs_lookup_master_is_owner); UT_RUN(test_undo_gcs_master_is_self); UT_RUN(test_undo_gcs_path_runtime_shared_mode_branch); UT_RUN(test_undo_gcs_path_materialized_never_migrates); UT_RUN(test_undo_gcs_intent_for_owner); + UT_RUN(test_undo_gcs_grant_armed_gate); + UT_RUN(test_undo_gcs_grant_reader_pcm_contract); + UT_RUN(test_undo_gcs_grant_admissible_generation); + UT_RUN(test_undo_gcs_grant_admissible_epoch); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From feaa1a942f2a60be7296ce0ec3a7e1c618a02e58 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 08:05:56 +0800 Subject: [PATCH 05/38] feat(cluster): undo-block writer/cleaner X-grant + PI-discard primitives (spec-5.22b D2-4) Owner-as-master write side, the dual of the D2-3 reader S-grant. - pure core (cluster_undo_gcs.c): writer PCM contract (X via N->X), SCN-only PI-discard coverage judge, and pure PI-holder targeting; all standalone unit-tested (test_cluster_undo_gcs U5, 9 -> 12 cases). - runtime (cluster_undo_gcs_grant.c): acquire_exclusive (the owner takes a local PCM X on its OWN undo block; a peer taking X on a foreign owner's undo fails closed -- single-writer invariant) and invalidate_peers (durable-write PI-discard broadcast keyed by the synthetic undo tag, reusing the shipped PI_DISCARD wire). - cluster_gcs_block.c: extract cluster_gcs_block_send_pi_discard_invalidate so the undo data plane reuses the wire + checksum instead of duplicating it (behaviour-preserving refactor; the gcs_block unit tests stay green). - cluster_undo_record.c: refresh the W3-wall contract comment + errhint (admit set unchanged; documents that a runtime-warm foreign undo read is served by the owner-shipped grant, never a local self-read). All coherence-gated (cluster.undo_gcs_coherence, default off) -> inert at the default. No live S/PI holder yet, so the runtime primitives are skeleton-ahead-of-consumer, forward-covered by the D2-7/D6 TAP. Spec: spec-5.22b-undo-block-gcs-integration.md (D2-4) --- src/backend/cluster/cluster_gcs_block.c | 37 ++++-- src/backend/cluster/cluster_undo_gcs.c | 55 +++++++++ src/backend/cluster/cluster_undo_gcs_grant.c | 112 ++++++++++++++++++ src/backend/cluster/cluster_undo_record.c | 23 +++- src/include/cluster/cluster_gcs_block.h | 3 + src/include/cluster/cluster_undo_gcs.h | 105 +++++++++++++++- src/test/cluster_unit/test_cluster_undo_gcs.c | 101 +++++++++++++++- 7 files changed, 419 insertions(+), 17 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..3e541e751d 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -5699,6 +5699,31 @@ gcs_block_pi_kept_note_send(BufferTag tag, int32 master_node) } } +/* + * Send one unsolicited PI_DISCARD INVALIDATE ride to `target_node` for `tag`: + * "drop your Past Image of this block". request_id 0, fire-and-forget, never + * ACKed (spec-6.12h D-h2). Public so the shared-undo data plane (spec-5.22b + * D2-4, owner-as-master) reuses the exact wire + checksum rather than + * duplicating the payload build. Emits tier-1 IC -> must run in LMON + * dispatch/tick context (L172 family); the caller owns the self/range guard. + */ +void +cluster_gcs_block_send_pi_discard_invalidate(BufferTag tag, int32 target_node) +{ + GcsBlockInvalidatePayload inv; + + memset(&inv, 0, sizeof(inv)); + inv.request_id = 0; /* unsolicited: no broadcast slot, no ACK */ + inv.epoch = cluster_epoch_get_current(); + inv.tag = tag; + inv.master_node = cluster_node_id; + inv.invalidating_for_x_node = 0; + inv.reserved_0[0] = GCS_BLOCK_INVALIDATE_KIND_PI_DISCARD; + inv.checksum = gcs_block_compute_invalidate_checksum(&inv); + (void)cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE, target_node, &inv, + sizeof(inv)); +} + /* * Master side: a durable-note for `tag` arrived (locally routed or via the * status-3 wire ride). If the written pd_block_scn covers the SCN watermark @@ -5728,17 +5753,7 @@ gcs_block_pi_discard_master_apply(BufferTag tag, SCN written_scn) if (n == cluster_node_id) { cluster_lever_h_note_discard_result(cluster_bufmgr_discard_pi_block(tag)); } else { - GcsBlockInvalidatePayload inv; - - memset(&inv, 0, sizeof(inv)); - inv.request_id = 0; /* unsolicited: no broadcast slot, no ACK */ - inv.epoch = cluster_epoch_get_current(); - inv.tag = tag; - inv.master_node = cluster_node_id; - inv.invalidating_for_x_node = 0; - inv.reserved_0[0] = GCS_BLOCK_INVALIDATE_KIND_PI_DISCARD; - inv.checksum = gcs_block_compute_invalidate_checksum(&inv); - (void)cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE, n, &inv, sizeof(inv)); + cluster_gcs_block_send_pi_discard_invalidate(tag, n); } } } diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c index 09bfc0ad2f..36804c4d5d 100644 --- a/src/backend/cluster/cluster_undo_gcs.c +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -169,3 +169,58 @@ cluster_undo_grant_reader_pcm_transition(void) { return PCM_TRANS_N_TO_S; } + +/* + * cluster_undo_grant_writer_pcm_mode / _transition -- writer/cleaner lock + * contract (D2-4). The owner takes its own undo block in PCM X via the + * write-first N->X transition before mutating/recycling it. See + * cluster_undo_gcs.h for the contract; cluster_pcm_lock owns and tests the + * transition-legality table. + */ +PcmState +cluster_undo_grant_writer_pcm_mode(void) +{ + return PCM_LOCK_MODE_X; +} + +PcmLockTransition +cluster_undo_grant_writer_pcm_transition(void) +{ + return PCM_TRANS_N_TO_X; +} + +/* + * cluster_undo_pi_discard_covered -- pure PI-discard coverage judge (D2-4). + * + * Delegates to the shipped cross-node coverage primitive + * cluster_pcm_pi_discard_covered (SCN-only, scn_local order). This undo + * domain wrapper exists to (a) name the owner-write "did my durable copy + * obsolete peer PIs" decision in the undo data plane and (b) pin the + * Q-D24-2 resolution at the type boundary: the judge is SCN-only, the LSN + * redo-coverage dimension is discharged by the collect (both-watermark + * clear) + retire_if_durable, never re-judged here. See cluster_undo_gcs.h. + */ +bool +cluster_undo_pi_discard_covered(SCN wm_scn, SCN written_scn) +{ + return cluster_pcm_pi_discard_covered(wm_scn, written_scn); +} + +/* + * cluster_undo_pi_discard_notify_target -- pure PI-discard targeting (D2-4). + * + * A peer `node` receives a PI_DISCARD iff its bit is set in `holders`, it is + * not `self_node` (the owner is the writer, never a PI holder of its own + * block), and its id is a valid 32-wide bitmap slot. Any out-of-range id + * fails closed (never index the bitmap out of bounds). See + * cluster_undo_gcs.h. + */ +bool +cluster_undo_pi_discard_notify_target(uint32 holders, int32 node, int32 self_node) +{ + if (node < 0 || node >= 32) + return false; /* invalid slot -> never a target (no OOB shift) */ + if (node == self_node) + return false; /* owner never notifies itself */ + return (holders & ((uint32)1u << (uint32)node)) != 0; +} diff --git a/src/backend/cluster/cluster_undo_gcs_grant.c b/src/backend/cluster/cluster_undo_gcs_grant.c index 431e99ad7a..f2f60bb4a6 100644 --- a/src/backend/cluster/cluster_undo_gcs_grant.c +++ b/src/backend/cluster/cluster_undo_gcs_grant.c @@ -128,3 +128,115 @@ cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expecte res->status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; return true; } + +/* + * cluster_undo_block_acquire_exclusive -- writer/cleaner X-grant primitive + * (D2-4). + * + * See cluster_undo_gcs.h for the full contract. The owner takes X on its OWN + * undo block before mutating/recycling it; a peer must never take X on a + * foreign owner's undo (single-writer invariant), so master != self fails + * closed (Rule 8.A). Runtime object; its live consumer + the D6 + * owner-incarnation self-check are skeleton-ahead (Q-D24-3). + */ +bool +cluster_undo_block_acquire_exclusive(const ClusterResId *undo_resid) +{ + if (undo_resid == NULL) + return false; /* fail-closed */ + if (!cluster_undo_resid_is_undo(undo_resid)) + return false; + + /* + * Coherence gate. At the default (coherence off / non peer-mode) the owner + * keeps its unchanged local undo write path, so D2 lands inert (回归安全). + */ + if (!cluster_undo_grant_armed(cluster_undo_gcs_coherence, cluster_peer_mode_enabled())) + return false; + + /* + * Single-writer invariant: only the owner ever writes/recycles its own undo + * (cluster_undo_record.c owner_instance == self). A peer taking X on a + * foreign owner's undo would be writing foreign undo -> fail closed + * (Rule 8.A), never grant. master == self => local PCM X, zero network (the + * owner is already both master and the sole writer). The owner-incarnation + * self-check (L364) that must gate actually serving from this local fast + * path lands with the write-path consumer in D6. + */ + if (!cluster_undo_block_master_is_self(undo_resid)) + return false; + + return true; +} + +/* + * cluster_undo_block_invalidate_peers -- owner-write PI-discard broadcast + * (D2-4). + * + * See cluster_undo_gcs.h for the full contract. After the owner durably + * writes its undo block's current copy, direct every peer holding an obsolete + * Past Image to drop it. SCN-only (Q-D24-2); LMON-context (L172); returns the + * number of peers notified, all misses fail-safe (0). Skeleton-ahead until + * D6 registers undo PI holders (Q-D24-3). + */ +int +cluster_undo_block_invalidate_peers(const ClusterResId *undo_resid, SCN write_scn) +{ + int32 owner; + uint32 seg; + uint32 block; + uint32 gen; + BufferTag tag; + uint32 holders = 0; + int notified = 0; + int n; + + if (undo_resid == NULL) + return 0; + if (!cluster_undo_resid_is_undo(undo_resid)) + return 0; + + /* Not armed (single-node / coherence off) -> no peers to invalidate. */ + if (!cluster_undo_grant_armed(cluster_undo_gcs_coherence, cluster_peer_mode_enabled())) + return 0; + + cluster_undo_resid_decode(undo_resid, &owner, &seg, &block, &gen); + (void)owner; /* synthetic undo tag deliberately omits owner (the origin only + * ever serves its own undo; fail-closed by construction) */ + (void)gen; /* segment generation is the READER's anti-ABA dimension + * (cluster_undo_grant_admissible), not the write-side notify */ + + /* + * The undo block's synthetic address tag (spec-6.12i D-i1): undo segments + * live outside shared buffers, so the PI protocol keys them by this magic + * tag exactly as the fetch wire does. + */ + tag = GcsBlockUndoFetchTagMake(seg, block); + + /* + * SCN coverage judge under the entry lock: if the just-durable write reaches + * the block's SCN watermark, collect + clear the pi_holders_bitmap (both + * watermarks) and hand back the pre-clear set to notify. Missing entry / + * not-covered / unarmed watermark -> false (fail-safe: the PI merely lingers + * until buffer pressure / anti-ABA reread, never a correctness loss). Until + * D6 registers undo PI holders this fail-safes to 0 (Q-D24-3). + */ + if (!cluster_pcm_lock_pi_discard_collect(tag, write_scn, &holders)) + return 0; + + /* + * Direct a PI_DISCARD to each PEER whose bit is set (the owner is the + * current writer, never a PI holder of its own block -- notify_target + * excludes self). Reuses the shipped single-target INVALIDATE send + * (LMON-context; L172 -- the D6 live caller runs in the owner-as-master + * LMON dispatch/tick path). + */ + for (n = 0; n < 32; n++) { + if (!cluster_undo_pi_discard_notify_target(holders, n, cluster_node_id)) + continue; + cluster_gcs_block_send_pi_discard_invalidate(tag, n); + notified++; + } + + return notified; +} diff --git a/src/backend/cluster/cluster_undo_record.c b/src/backend/cluster/cluster_undo_record.c index 623d8cd70b..401f1277a1 100644 --- a/src/backend/cluster/cluster_undo_record.c +++ b/src/backend/cluster/cluster_undo_record.c @@ -1765,14 +1765,29 @@ cluster_undo_get_record(UBA uba, void *out_buffer, size_t buffer_size) * blanket-converted to ERROR. The CR-construct caller no longer relies on * this branch: the spec-5.57 D2 pre-check in cluster_cr.c fail-closes a * runtime-warm cross-instance origin with 53R9G BEFORE this read, so for the - * CR path the boundary is consolidated to one errcode. The runtime - * cross-instance undo data plane lands in Stage 6 (#119). */ + * CR path the boundary is consolidated to one errcode. + * + * spec-5.22b D2 (#119) — this wall now formally ENFORCES invariant #8: a + * peer NEVER self-reads a foreign owner's undo bytes. The coherent runtime + * cross-instance undo read is cluster_undo_block_acquire_shared (D2-3): the + * owner is the master AND the holder, so it ships its own block image over + * owner-as-master routing and the requesting peer consumes THAT shipped + * image, never opening the foreign pg_undo/instance_ tree here. + * Hence this local-open path correctly STAYS fail-closed for a runtime-warm + * foreign owner -- the coherent bytes come from the grant, not from this + * read (admitting a local foreign read here would BREAK invariant #8). The + * admit set is unchanged (own-instance OR materialized-remote, both P1-3 + * local): only this contract text is updated (spec-5.22b Q-D24-1=A, zero + * behaviour change). Routing the recovery/SRF/CR callers onto the grant + * (so a runtime-warm foreign read reaches acquire_shared instead of this + * warning) is the D6 consumer wiring. */ if (owner_instance != (uint8)(cluster_node_id + 1) && !cluster_merged_instance_is_materialized((int)owner_instance - 1)) { ereport(WARNING, (errmsg("cluster_undo_get_record: runtime cross-instance undo read not supported"), - errhint("Cross-instance undo/CR data plane lands in Stage 6 (#119 undo-block " - "Cache Fusion); see Spec: spec-5.57."))); + errhint("The coherent cross-instance undo path is the owner's shipped block " + "image (spec-5.22b #119 undo-block Cache Fusion), not a local read; " + "see Spec: spec-5.57."))); return 0; } if (owner_instance != (uint8)(cluster_node_id + 1)) diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 06662971b9..3966218bc2 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1901,6 +1901,9 @@ extern void cluster_gcs_block_pi_write_note(BufferTag tag, SCN page_scn); extern uint64 cluster_gcs_block_pi_note_presync_snapshot(void); extern void cluster_gcs_block_pi_note_confirm(uint64 presync_seq); extern void cluster_gcs_block_pi_discard_drain(void); +/* spec-5.22b D2-4 — single-target PI_DISCARD send, reused by the shared-undo + * owner-as-master data plane (LMON-context; caller owns self/range guard). */ +extern void cluster_gcs_block_send_pi_discard_invalidate(BufferTag tag, int32 target_node); /* PGRAC: spec-4.7 D6 — 8 warm-recovery observability accessors. */ extern uint64 cluster_gcs_get_recovery_block_resources_recovering(void); diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h index 60f7ef1a5e..ffa390f6bb 100644 --- a/src/include/cluster/cluster_undo_gcs.h +++ b/src/include/cluster/cluster_undo_gcs.h @@ -42,8 +42,9 @@ #define CLUSTER_UNDO_GCS_H #include "cluster/cluster_grd.h" /* ClusterResId */ -#include "cluster/cluster_pcm_lock.h" /* PcmState / PcmLockTransition */ +#include "cluster/cluster_pcm_lock.h" /* PcmState / PcmLockTransition / pi_discard */ #include "cluster/cluster_runtime_visibility.h" /* ClusterLiveAuthority */ +#include "cluster/cluster_scn.h" /* SCN (D2-4 PI-discard coverage) */ #include "cluster/storage/cluster_undo_alloc.h" /* ClusterUndoPathIntent */ /* @@ -154,6 +155,57 @@ extern bool cluster_undo_grant_admissible(const ClusterResId *undo_resid, extern PcmState cluster_undo_grant_reader_pcm_mode(void); extern PcmLockTransition cluster_undo_grant_reader_pcm_transition(void); +/* + * cluster_undo_grant_writer_pcm_mode / _transition -- writer/cleaner lock + * contract (D2-4, §2.2). Before an owner mutates (or a cleaner recycles) its + * OWN undo block it takes the block in PCM X mode via the write-first N->X + * transition -- the pair symmetric to the reader's S / N->S (D2-3). The + * transition-legality table itself is owned + tested by cluster_pcm_lock + * (spec-2.30); this only names which pair the undo writer selects, so a + * write path can never accidentally take the read-first pair. + */ +extern PcmState cluster_undo_grant_writer_pcm_mode(void); +extern PcmLockTransition cluster_undo_grant_writer_pcm_transition(void); + +/* + * cluster_undo_pi_discard_covered -- pure PI-discard coverage judge (D2-4, + * §3.1; Q-D24-2). + * + * true iff the owner's just-durable write of an undo block's CURRENT copy + * (written_scn = its pd_block_scn) proves every peer Past Image of that block + * obsolete: the written version reaches the block's SCN watermark (the newest + * shipped version), so every PI is that version or older. Delegates to the + * shipped cross-node coverage primitive cluster_pcm_pi_discard_covered, which + * compares on scn_local (AD-008 time order), never the raw SCN (node-id + * dominated). Fail-safe: an unarmed (Invalid) watermark or an unknown + * (Invalid) written version returns false (keep the PI, never a false + * discard). + * + * SCN-only BY CONSTRUCTION (Q-D24-2 resolution): per-node WAL (spec-4.1) + * gives every node its own LSN space, so the LSN unit is incomparable + * cross-node. The LSN redo-coverage dimension is NOT judged here -- it is + * discharged by cluster_pcm_lock_pi_discard_collect (which clears BOTH + * watermarks once the SCN covers -- durable >= newest-shipped also + * discharges the redo-coverage claim) plus retire_if_durable (the single + * stream LSN fixture). So this D2-4 judge deliberately adds no LSN + * dimension; doing so would double-count or wrongly compare across streams. + */ +extern bool cluster_undo_pi_discard_covered(SCN wm_scn, SCN written_scn); + +/* + * cluster_undo_pi_discard_notify_target -- pure PI-discard targeting (D2-4, + * §1.2; Q4-A per-block). + * + * Once the coverage judge clears a block's pi_holders_bitmap, the owner + * directs a PI_DISCARD to each PEER whose bit is set. `node` is a legal + * target iff its bit is set in `holders` AND it is not `self_node` (the owner + * is the current writer of its own undo, never a Past-Image holder of it) AND + * its id is in range [0,32) (the 32-wide bitmap; ㉕ self-send precedent). + * Pure bitmap logic (no globals) so cluster_unit drives it standalone; the + * runtime invalidate_peers loops peers through this predicate. + */ +extern bool cluster_undo_pi_discard_notify_target(uint32 holders, int32 node, int32 self_node); + /* * cluster_undo_block_acquire_shared -- reader S-grant primitive (D2-3, §2.2). * @@ -179,4 +231,55 @@ extern bool cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expected_generation, char *dst_block, ClusterUndoGrantResult *res); +/* + * cluster_undo_block_acquire_exclusive -- writer/cleaner X-grant primitive + * (D2-4, §2.2/§3.1). + * + * The owner takes its OWN undo block in exclusive mode before mutating it (a + * record writer) or recycling its segment (the cleaner). Under owner-as + * master an undo block's master IS its owner, and the single-writer invariant + * means only the owner ever writes its own undo (cluster_undo_record.c + * owner_instance == self), so: + * - not armed (coherence off / non-peer) -> false: the caller keeps its + * unchanged local write path (D2 inert at the default, 回归安全). + * - master != self -> false, fail-closed: a peer must NEVER take X on a + * foreign owner's undo (that would be writing foreign undo, breaking the + * single-writer invariant, Rule 8.A). + * - master == self (armed) -> true: the owner grants itself the local PCM X + * (zero network -- it is already the master and the sole writer). + * + * X-grant here means "hold X so peer S copies are invalidated" (the paired + * invalidate_peers), NEVER "a peer may write foreign undo". Runtime object + * (heavy deps): not in the standalone cluster_unit link; its live consumer + * (the owner's undo write path taking X + the D6 owner-incarnation self-check) + * lands in D6, so this increment is skeleton-ahead-of-consumer (Q-D24-3), + * forward-covered by the D2-7/D6 TAP. + */ +extern bool cluster_undo_block_acquire_exclusive(const ClusterResId *undo_resid); + +/* + * cluster_undo_block_invalidate_peers -- owner-write PI-discard broadcast + * (D2-4, §1.2/§3.1; Q-D24-2/Q-D24-3). + * + * After the owner durably writes its undo block's CURRENT copy, it directs + * every peer holding a now-obsolete Past Image to drop it. Builds the block's + * synthetic undo address tag (GcsBlockUndoFetchTagMake) and reuses the shipped + * spec-6.12h PI-discard machinery: cluster_pcm_lock_pi_discard_collect judges + * SCN coverage (written_scn vs the block's SCN watermark) under the entry + * lock and, when covered, clears the watermarks + hands back the pre-clear + * pi_holders_bitmap; each set PEER then gets a PI_DISCARD ride on the + * INVALIDATE wire (unsolicited, never ACKed). Returns the number of peers + * notified (0 when not armed, no entry, or not covered -- all fail-safe: an + * un-discarded PI merely lingers until buffer pressure / anti-ABA reread, + * never a correctness loss). + * + * SCN-only (Q-D24-2): write_scn is the only cross-node comparable unit. + * LMON-context contract: it emits tier-1 IC (L172 family) so its live caller + * is the owner-as-master LMON dispatch/tick path (D6), NOT a writer backend; + * with no undo PI holder registered before D6 the collect fail-safes to 0, + * so this increment is skeleton-ahead-of-consumer (Q-D24-3), forward-covered + * by the D2-7/D6 TAP. + */ +extern int cluster_undo_block_invalidate_peers(const ClusterResId *undo_resid, SCN write_scn); + #endif /* CLUSTER_UNDO_GCS_H */ diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 47e1d9d726..03ef7a87b8 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -208,6 +208,102 @@ UT_TEST(test_undo_gcs_grant_reader_pcm_contract) UT_ASSERT(cluster_undo_grant_reader_pcm_transition() != PCM_TRANS_N_TO_X); } +/* ====================================================================== + * U5 -- writer/cleaner X-grant lock contract (spec-5.22b D2-4, §2.2): the + * owner takes its OWN undo block in PCM X mode via the write-first N->X + * transition before mutating it. The N->X transition-legality TABLE is owned + * + exhaustively tested by cluster_pcm_lock (spec-2.30); D2-4 only pins that + * the writer selects that legal write-first pair, never the reader S/N->S + * pair. Pure: compile-time constants. + * ====================================================================== */ +UT_TEST(test_undo_gcs_grant_writer_pcm_contract) +{ + UT_ASSERT_EQ(cluster_undo_grant_writer_pcm_mode(), PCM_LOCK_MODE_X); + UT_ASSERT_EQ(cluster_undo_grant_writer_pcm_transition(), PCM_TRANS_N_TO_X); + + /* a writer must never select the reader-side S mode / read-first pair */ + UT_ASSERT(cluster_undo_grant_writer_pcm_mode() != PCM_LOCK_MODE_S); + UT_ASSERT(cluster_undo_grant_writer_pcm_transition() != PCM_TRANS_N_TO_S); + + /* writer and reader select DIFFERENT modes/transitions (no aliasing) */ + UT_ASSERT(cluster_undo_grant_writer_pcm_mode() != cluster_undo_grant_reader_pcm_mode()); + UT_ASSERT(cluster_undo_grant_writer_pcm_transition() + != cluster_undo_grant_reader_pcm_transition()); +} + +/* ====================================================================== + * U5 -- PI-discard coverage judge (spec-5.22b D2-4, §3.1; Q-D24-2): the + * owner's durable write of the block's CURRENT copy obsoletes every peer Past + * Image IFF the written pd_block_scn reaches the SCN watermark (the newest + * shipped version). The comparison MUST be on scn_local (AD-008 time order), + * never the raw SCN (whose high bits are node-id dominated) -- a durable write + * from a high node-id must not "cover" a watermark from a low node-id merely + * because its raw value is larger. Fail-safe: an unarmed (Invalid) watermark + * or an unknown (Invalid) written version never discards (keep the PI). + * SCN-only by construction (Q-D24-2): per-node WAL makes the LSN unit + * incomparable cross-node, so the LSN redo-coverage dimension is discharged by + * cluster_pcm_lock_pi_discard_collect (clears BOTH watermarks on cover) + + * retire_if_durable (the single-stream LSN fixture), not here. + * ====================================================================== */ +UT_TEST(test_undo_gcs_pi_discard_covered) +{ + /* written local == watermark local (different nodes) => covered (equal is + * enough; proves the compare is on scn_local, node-independent) */ + UT_ASSERT(cluster_undo_pi_discard_covered(scn_encode(2, 500), scn_encode(1, 500))); + + /* written local strictly newer => covered */ + UT_ASSERT(cluster_undo_pi_discard_covered(scn_encode(1, 400), scn_encode(3, 900))); + + /* written local older => NOT covered (keep PI) */ + UT_ASSERT(!cluster_undo_pi_discard_covered(scn_encode(1, 900), scn_encode(3, 400))); + + /* node-domination trap: written raw value is LARGER (high node id) but its + * scn_local is SMALLER -> must NOT be treated as covered (raw compare would + * wrongly discard a still-live PI, an 8.A hazard) */ + UT_ASSERT(!cluster_undo_pi_discard_covered(scn_encode(1, 100), scn_encode(5, 50))); + + /* unarmed watermark (Invalid) => nothing provable cross-node => keep */ + UT_ASSERT(!cluster_undo_pi_discard_covered(InvalidScn, scn_encode(1, 500))); + + /* unknown written version (Invalid) => fail-safe keep */ + UT_ASSERT(!cluster_undo_pi_discard_covered(scn_encode(1, 500), InvalidScn)); +} + +/* ====================================================================== + * U5 -- PI-discard notify targeting (spec-5.22b D2-4, §1.2, Q4-A per-block): + * once the coverage judge clears a block's pi_holders_bitmap, the owner directs + * a PI_DISCARD to each PEER whose bit is set. A peer is a legal target IFF its + * bit is set AND it is not the owner itself (the owner is the current writer, + * never a Past-Image holder of its own block) AND its node id is in range + * [0,32). Pure bitmap logic, no globals. + * ====================================================================== */ +UT_TEST(test_undo_gcs_pi_discard_notify_target) +{ + uint32 holders = (1u << 1) | (1u << 3); /* peers 1 and 3 hold a PI */ + + /* set bits, not self => target */ + UT_ASSERT(cluster_undo_pi_discard_notify_target(holders, 1, 0 /*self*/)); + UT_ASSERT(cluster_undo_pi_discard_notify_target(holders, 3, 0 /*self*/)); + + /* unset bit => never a target */ + UT_ASSERT(!cluster_undo_pi_discard_notify_target(holders, 2, 0 /*self*/)); + UT_ASSERT(!cluster_undo_pi_discard_notify_target(holders, 0, 0 /*self*/)); + + /* self is never notified even if its bit is set (owner is the writer) */ + { + uint32 with_self = holders | (1u << 2); + + UT_ASSERT(!cluster_undo_pi_discard_notify_target(with_self, 2, 2 /*self==2*/)); + /* other peers still targeted when self==2 */ + UT_ASSERT(cluster_undo_pi_discard_notify_target(with_self, 1, 2)); + } + + /* out-of-range node ids fail closed (32-wide bitmap, ㉕ precedent) */ + UT_ASSERT(!cluster_undo_pi_discard_notify_target(0xffffffffu, -1, 0)); + UT_ASSERT(!cluster_undo_pi_discard_notify_target(0xffffffffu, 32, 0)); + UT_ASSERT(!cluster_undo_pi_discard_notify_target(0xffffffffu, 99, 0)); +} + /* ====================================================================== * U7 -- reader S-grant admissibility: SEGMENT-generation anti-ABA (spec-5.22b * §3.3 dim 1, D1). A reference whose encoded generation (segment wrap_count) @@ -273,7 +369,7 @@ UT_TEST(test_undo_gcs_grant_admissible_epoch) int main(void) { - UT_PLAN(9); + UT_PLAN(12); UT_RUN(test_undo_gcs_lookup_master_is_owner); UT_RUN(test_undo_gcs_master_is_self); UT_RUN(test_undo_gcs_path_runtime_shared_mode_branch); @@ -283,6 +379,9 @@ main(void) UT_RUN(test_undo_gcs_grant_reader_pcm_contract); UT_RUN(test_undo_gcs_grant_admissible_generation); UT_RUN(test_undo_gcs_grant_admissible_epoch); + UT_RUN(test_undo_gcs_grant_writer_pcm_contract); + UT_RUN(test_undo_gcs_pi_discard_covered); + UT_RUN(test_undo_gcs_pi_discard_notify_target); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 9cfe5daab57f2bc55a7ed8d9b227778de50e602a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:24:54 +0800 Subject: [PATCH 06/38] feat(cluster): undo-block remaster serve-gate (spec-5.22b D2-5) Owner-as-master serve-gate: an undo resource's master IS its owner, so the owner-death / remaster window is the undo shard's recovery phase. - pure core (cluster_undo_gcs.c): cluster_undo_serve_allowed(owner_alive, remaster_in_progress) = owner_alive && !remaster_in_progress. The owner-as-master mirror of cluster_gcs_block_phase_for_tag's CSSD-DEAD + recovery-in-progress fence and cluster_hw_serve_allowed's pure shape, keyed on the owner rather than a hash static master. Standalone unit-tested (test_cluster_undo_gcs U8, 12 -> 13 cases). - runtime (cluster_undo_gcs_grant.c): acquire_shared applies the gate on the peer-reads-foreign branch BEFORE the fetch -- a converged-DEAD owner or an in-flight remaster episode denies DENIED_RESOURCE_RECOVERING and fails closed, instead of stalling on a doomed fetch to a dead node. Dead-owner SERVE from shared storage is a later increment; this proves only the fail-closed deny (never a false-resolve). Undo folds into the reconfig episode by reading its state (cluster_cssd_get_peer_state + cluster_grd_recovery_in_progress), never by joining the hash-shard master map (undo never hash-routes). So cluster_grd.c and cluster_gcs_block.c are unchanged. All coherence-gated (cluster.undo_gcs_coherence, default off) -> inert at the default. Spec: spec-5.22b-undo-block-gcs-integration.md (D2-5) --- src/backend/cluster/cluster_undo_gcs.c | 17 +++++++++++ src/backend/cluster/cluster_undo_gcs_grant.c | 21 +++++++++++++ src/include/cluster/cluster_undo_gcs.h | 28 +++++++++++++++++ src/test/cluster_unit/test_cluster_undo_gcs.c | 30 ++++++++++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c index 36804c4d5d..19860d419c 100644 --- a/src/backend/cluster/cluster_undo_gcs.c +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -224,3 +224,20 @@ cluster_undo_pi_discard_notify_target(uint32 holders, int32 node, int32 self_nod return false; /* owner never notifies itself */ return (holders & ((uint32)1u << (uint32)node)) != 0; } + +/* + * cluster_undo_serve_allowed -- pure owner-as-master remaster serve-gate (D2-5). + * + * See cluster_undo_gcs.h for the full contract. A live-owner undo grant may + * proceed only when the owner is CSSD-alive AND no remaster episode is fencing + * its shard; either doubt fails closed so the caller denies with + * DENIED_RESOURCE_RECOVERING (dead-owner SERVE is D4, never a D2 false-resolve + * -- Rule 8.A/8.B). Undo folds into the reconfig episode by READING its state + * (this predicate's two inputs), never by joining the hash-shard master map + * (undo is owner-as-master, D1-5). + */ +bool +cluster_undo_serve_allowed(bool owner_alive, bool remaster_in_progress) +{ + return owner_alive && !remaster_in_progress; +} diff --git a/src/backend/cluster/cluster_undo_gcs_grant.c b/src/backend/cluster/cluster_undo_gcs_grant.c index f2f60bb4a6..6ae748e0fd 100644 --- a/src/backend/cluster/cluster_undo_gcs_grant.c +++ b/src/backend/cluster/cluster_undo_gcs_grant.c @@ -36,8 +36,10 @@ #include "postgres.h" #include "cluster/cluster_cr_server.h" /* cluster_gcs_block_undo_tt_fetch_and_wait */ +#include "cluster/cluster_cssd.h" /* cluster_cssd_get_peer_state (D2-5 serve-gate) */ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_gcs_block.h" /* GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER */ +#include "cluster/cluster_grd.h" /* cluster_grd_recovery_in_progress (D2-5 serve-gate) */ #include "cluster/cluster_guc.h" /* cluster_undo_gcs_coherence */ #include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled, cluster_node_id */ #include "cluster/cluster_undo_gcs.h" @@ -97,6 +99,25 @@ cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expecte return false; /* honest fail-closed forward (D6); never a stale local serve */ } + /* + * D2-5 remaster serve-gate (§3.4). Under owner-as-master the undo shard's + * master IS its owner, so the owner-death / remaster window IS the undo + * shard's recovery phase. Deny BEFORE the fetch when the owner is not a + * live serving master -- a converged-DEAD owner OR an in-flight remaster + * episode -- so the peer fail-closes as DENIED_RESOURCE_RECOVERING instead + * of stalling on a doomed fetch to a dead node (dead-owner SERVE from shared + * storage is D4, not D2; Rule 8.A never false-resolves). The owner-liveness + * read mirrors cluster_gcs_block_phase_for_tag's CSSD-DEAD + + * recovery-in-progress fence, keyed on the OWNER (owner-as-master) rather + * than a hash static master; undo folds into the reconfig episode by reading + * its state, never by joining the hash-shard master map (D1-5). + */ + if (!cluster_undo_serve_allowed(cluster_cssd_get_peer_state(owner) != CLUSTER_CSSD_PEER_DEAD, + cluster_grd_recovery_in_progress())) { + res->status = (uint8)GCS_BLOCK_REPLY_DENIED_RESOURCE_RECOVERING; + return false; /* fail-closed; serve = D4 */ + } + /* * master!=self (live owner): the owner is the authority AND the holder. * Reuse the 6.12i owner-ships-image wire verbatim (zero new wire; Q3★A diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h index ffa390f6bb..82fde7ef4c 100644 --- a/src/include/cluster/cluster_undo_gcs.h +++ b/src/include/cluster/cluster_undo_gcs.h @@ -206,6 +206,34 @@ extern bool cluster_undo_pi_discard_covered(SCN wm_scn, SCN written_scn); */ extern bool cluster_undo_pi_discard_notify_target(uint32 holders, int32 node, int32 self_node); +/* + * cluster_undo_serve_allowed -- pure owner-as-master remaster serve-gate (D2-5, + * §3.4; Rule 8.A/8.B). + * + * Under owner-as-master an undo resource's master IS its owner, so the undo + * shard folds into the SAME reconfig/remaster episode as the rest of that + * owner's resources -- not into the hash-shard master map (undo never + * hash-routes, D1-5). This is the owner-as-master mirror of + * cluster_gcs_block_phase_for_tag's (static-master DEAD + recovery-in-progress) + * fence for hash blocks, and of cluster_hw_serve_allowed's pure shape. + * + * true (may serve the live-owner grant) iff BOTH: + * - owner_alive: the owner is CSSD-alive (not converged DEAD), and + * - !remaster_in_progress: no death-driven remaster episode is fencing the + * owner's shard (cluster_grd_recovery_in_progress()). + * false (any doubt) => the caller denies with DENIED_RESOURCE_RECOVERING and + * fails closed (keeps 53R97); dead-owner SERVE from shared storage is D4, NOT + * D2 -- D2 only proves the fail-closed deny, never a false-resolve. + * + * remaster_in_progress is a per-node episode flag (any dead peer's episode), + * so this conservatively denies a LIVE owner's undo during an unrelated + * reconfig window too; that over-deny is a bounded, fail-closed availability + * cost (spec-5.22b §6 R8) -- the seed happy path (owner node0 live, no episode) + * is never gated. Pure (inputs explicit, no globals) so cluster_unit drives + * every owner-state combination standalone. + */ +extern bool cluster_undo_serve_allowed(bool owner_alive, bool remaster_in_progress); + /* * cluster_undo_block_acquire_shared -- reader S-grant primitive (D2-3, §2.2). * diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 03ef7a87b8..95ed7162e0 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -366,10 +366,37 @@ UT_TEST(test_undo_gcs_grant_admissible_epoch) UT_ASSERT(!cluster_undo_grant_admissible(&r, 5, auth, 100, InvalidXLogRecPtr)); } +/* ====================================================================== + * U8 -- remaster serve-gate (spec-5.22b D2-5, §3.4, Rule 8.A). Under + * owner-as-master an undo resource's master IS its owner, so the serve-gate + * keys on the OWNER's incarnation liveness -- the owner-as-master mirror of + * cluster_gcs_block_phase_for_tag's (CSSD-DEAD + recovery-in-progress) fence + * for hash blocks, and of cluster_hw_serve_allowed's pure shape. An undo + * grant may proceed ONLY when the owner is a live serving master AND no + * remaster episode is fencing its shard; a dead owner OR an in-flight remaster + * window fails closed (the requester keeps 53R97 and DENIED_RESOURCE_RECOVERING + * -- dead-owner SERVE is D4, never a D2 false-resolve). Pure: branch-only, no + * globals, so cluster_unit drives every owner-state combination standalone. + * ====================================================================== */ +UT_TEST(test_undo_gcs_serve_gate) +{ + /* live owner, no remaster => serve (the D2 seed happy path: node0 live) */ + UT_ASSERT(cluster_undo_serve_allowed(true /*owner_alive*/, false /*remaster*/)); + + /* dead owner => deny (serve = D4; never a false-resolve, Rule 8.A) */ + UT_ASSERT(!cluster_undo_serve_allowed(false /*owner_alive*/, false /*remaster*/)); + + /* remaster episode in flight (even a live owner) => deny (fail-closed window) */ + UT_ASSERT(!cluster_undo_serve_allowed(true /*owner_alive*/, true /*remaster*/)); + + /* dead owner AND remastering => deny */ + UT_ASSERT(!cluster_undo_serve_allowed(false /*owner_alive*/, true /*remaster*/)); +} + int main(void) { - UT_PLAN(12); + UT_PLAN(13); UT_RUN(test_undo_gcs_lookup_master_is_owner); UT_RUN(test_undo_gcs_master_is_self); UT_RUN(test_undo_gcs_path_runtime_shared_mode_branch); @@ -382,6 +409,7 @@ main(void) UT_RUN(test_undo_gcs_grant_writer_pcm_contract); UT_RUN(test_undo_gcs_pi_discard_covered); UT_RUN(test_undo_gcs_pi_discard_notify_target); + UT_RUN(test_undo_gcs_serve_gate); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 7b78f1d8b96948d89cd3a05181da54d3ef8863ef Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:51:22 +0800 Subject: [PATCH 07/38] feat(cluster): undo-block grant-plane wait events (spec-5.22b D2-6) Register three owner-as-master undo-block grant-plane wait events so an operator can see a backend blocked in the undo GCS data plane: UndoBlockGrantWait / UndoBlockInvalidateWait / UndoBlockRemasterWait. Full 4-place wait-event symmetry: the WaitEventCluster enum + name lookup (pg_stat_activity.wait_event), plus the pg_stat_cluster_wait_events roster (registry + count). The Undo class grows 8 -> 11 in the enum and the surfaced roster grows 4 -> 7 (the I/O-internal CR_CONSTRUCT / TT_DURABLE_IO / BUF_FLUSH / EXTENT_CLAIM stay off the roster, as before). The PCM convert/downgrade waits an undo grant blocks on reuse the PCM class (WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT/DOWNGRADE), so no new events for those. The three new events are register-ahead of their report sites, which land with the grant-path consumer (the static registration table convention, cluster_views.h). Update-required snapshot tripwires bumped in lockstep: the per-stage acceptance count snapshots (118 -> 121), the two roster count tests (Undo 4 -> 7, total 118 -> 121), and the enum-range count (Undo 8 -> 11). Spec: spec-5.22b-undo-block-gcs-integration.md (D2-6) --- src/backend/cluster/cluster_views.c | 7 ++++++- src/backend/utils/activity/wait_event.c | 9 +++++++++ src/include/cluster/cluster_views.h | 2 +- src/include/utils/wait_event.h | 17 +++++++++++------ src/test/cluster_tap/t/010_views.pl | 8 ++++---- src/test/cluster_tap/t/011_gviews.pl | 10 +++++----- .../test_cluster_gcs_block_retransmit.c | 8 ++++---- .../test_cluster_stage2_acceptance.c | 10 +++++----- .../test_cluster_stage3_acceptance.c | 10 +++++----- .../test_cluster_stage4_acceptance.c | 8 ++++---- .../test_cluster_stage5_5_cr_acceptance.c | 8 ++++---- .../test_cluster_stage5_beta_acceptance.c | 10 +++++----- .../test_cluster_stage5_integrated_acceptance.c | 4 ++-- src/test/cluster_unit/test_cluster_views.c | 12 +++++++----- .../cluster_unit/test_cluster_wait_events.c | 7 ++++--- 15 files changed, 76 insertions(+), 54 deletions(-) diff --git a/src/backend/cluster/cluster_views.c b/src/backend/cluster/cluster_views.c index a1554dbbb2..d517ae1af1 100644 --- a/src/backend/cluster/cluster_views.c +++ b/src/backend/cluster/cluster_views.c @@ -287,11 +287,16 @@ static const uint32 cluster_wait_event_infos[CLUSTER_WAIT_EVENTS_COUNT] = { WAIT_EVENT_INTERCONNECT_TIER_SWITCH, WAIT_EVENT_INTERCONNECT_CONNECT_RETRY, - /* Cluster: Undo (4) */ + /* Cluster: Undo (7 = 4 remote/retention + spec-5.22b D2-6 3 owner-as-master + * grant-plane waits; the I/O-internal CR_CONSTRUCT / TT_DURABLE_IO / BUF_FLUSH + * / EXTENT_CLAIM enum events stay off this roster, as before) */ WAIT_EVENT_UNDO_REMOTE_READ, WAIT_EVENT_UNDO_TT_LOOKUP_REMOTE, WAIT_EVENT_UNDO_SEGMENT_FETCH, WAIT_EVENT_UNDO_RETENTION_WAIT, + WAIT_EVENT_UNDO_BLOCK_GRANT_WAIT, + WAIT_EVENT_UNDO_BLOCK_INVALIDATE_WAIT, + WAIT_EVENT_UNDO_BLOCK_REMASTER_WAIT, /* Cluster: ADG (4) */ WAIT_EVENT_ADG_MRP_APPLY_WAIT, diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index a1844aac56..f23b2a71ad 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -1302,6 +1302,15 @@ pgstat_get_wait_cluster_undo(WaitEventCluster w) case WAIT_EVENT_CLUSTER_UNDO_EXTENT_CLAIM: event_name = "ClusterUndoExtentClaim"; /* spec-3.18 D7 */ break; + case WAIT_EVENT_UNDO_BLOCK_GRANT_WAIT: + event_name = "UndoBlockGrantWait"; /* spec-5.22b D2-6 */ + break; + case WAIT_EVENT_UNDO_BLOCK_INVALIDATE_WAIT: + event_name = "UndoBlockInvalidateWait"; /* spec-5.22b D2-6 */ + break; + case WAIT_EVENT_UNDO_BLOCK_REMASTER_WAIT: + event_name = "UndoBlockRemasterWait"; /* spec-5.22b D2-6 */ + break; default: break; } diff --git a/src/include/cluster/cluster_views.h b/src/include/cluster/cluster_views.h index 46b702f4df..bfbb6b5c04 100644 --- a/src/include/cluster/cluster_views.h +++ b/src/include/cluster/cluster_views.h @@ -51,7 +51,7 @@ * internal table in cluster_views.c stays in sync with the enum. */ #define CLUSTER_WAIT_EVENTS_COUNT \ - 118 /* spec-6.13 D8: RDMA busypoll + inline wait events included */ + 121 /* spec-6.13 D8 RDMA busypoll/inline; spec-5.22b D2-6 +3 undo-block grant-plane waits */ /* diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index c67502ae2c..c39b3aa5e0 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -469,16 +469,21 @@ typedef enum { WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT, WAIT_EVENT_CLUSTER_IC_RECONNECT, - /* Cluster: Undo (8 events) -- AD-010; spec-3.9 adds CR_CONSTRUCT; - * spec-3.11 adds TT_DURABLE_IO; spec-3.18 D7 adds BUF_FLUSH + EXTENT_CLAIM */ + /* Cluster: Undo (11 events) -- AD-010; spec-3.9 adds CR_CONSTRUCT; + * spec-3.11 adds TT_DURABLE_IO; spec-3.18 D7 adds BUF_FLUSH + EXTENT_CLAIM; + * spec-5.22b D2-6 adds BLOCK_GRANT/INVALIDATE/REMASTER (owner-as-master + * grant data plane; PCM convert/downgrade waits reuse the PCM class above) */ WAIT_EVENT_UNDO_REMOTE_READ = PG_WAIT_CLUSTER_UNDO, WAIT_EVENT_UNDO_TT_LOOKUP_REMOTE, WAIT_EVENT_UNDO_SEGMENT_FETCH, WAIT_EVENT_UNDO_RETENTION_WAIT, - WAIT_EVENT_CLUSTER_CR_CONSTRUCT, /* spec-3.9: own-instance CR block construction */ - WAIT_EVENT_UNDO_TT_DURABLE_IO, /* spec-3.11: durable TT slot header I/O */ - WAIT_EVENT_CLUSTER_UNDO_BUF_FLUSH, /* spec-3.18 D7: undo buffer write-back I/O */ - WAIT_EVENT_CLUSTER_UNDO_EXTENT_CLAIM, /* spec-3.18 D7: extent claim autoextend I/O */ + WAIT_EVENT_CLUSTER_CR_CONSTRUCT, /* spec-3.9: own-instance CR block construction */ + WAIT_EVENT_UNDO_TT_DURABLE_IO, /* spec-3.11: durable TT slot header I/O */ + WAIT_EVENT_CLUSTER_UNDO_BUF_FLUSH, /* spec-3.18 D7: undo buffer write-back I/O */ + WAIT_EVENT_CLUSTER_UNDO_EXTENT_CLAIM, /* spec-3.18 D7: extent claim autoextend I/O */ + WAIT_EVENT_UNDO_BLOCK_GRANT_WAIT, /* spec-5.22b D2-6: owner-as-master S/X grant */ + WAIT_EVENT_UNDO_BLOCK_INVALIDATE_WAIT, /* spec-5.22b D2-6: PI-discard invalidate broadcast */ + WAIT_EVENT_UNDO_BLOCK_REMASTER_WAIT, /* spec-5.22b D2-6: remaster serve-gate wait */ /* Cluster: ADG (4 events) -- #95 */ WAIT_EVENT_ADG_MRP_APPLY_WAIT = PG_WAIT_CLUSTER_ADG, diff --git a/src/test/cluster_tap/t/010_views.pl b/src/test/cluster_tap/t/010_views.pl index 524beb3e71..24438c0513 100644 --- a/src/test/cluster_tap/t/010_views.pl +++ b/src/test/cluster_tap/t/010_views.pl @@ -46,12 +46,12 @@ # ---------- -# Total row count: 118 (spec-6.13 adds RDMA busypoll + inline send waits). +# Total row count: 121 (spec-6.13 RDMA + spec-5.22b D2-6 undo-block grant-plane waits). # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', @@ -89,7 +89,7 @@ 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits - 'Cluster: Undo' => 4, + 'Cluster: Undo' => 7, # spec-5.22b D2-6: +UndoBlock Grant/Invalidate/Remaster waits 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_tap/t/011_gviews.pl b/src/test/cluster_tap/t/011_gviews.pl index 61dda90bdc..3f5dbb885a 100644 --- a/src/test/cluster_tap/t/011_gviews.pl +++ b/src/test/cluster_tap/t/011_gviews.pl @@ -15,7 +15,7 @@ # # What this test verifies: # - The global view exists and is queryable. -# - It returns exactly 118 rows (1 node x 118 cluster wait events). +# - It returns exactly 121 rows (1 node x 121 cluster wait events). # - It exposes exactly 1 distinct node_id at 0.17 (placeholder). # - The single node_id matches the cluster.node_id GUC. # - Per-class row counts match docs/wait-events-design.md §2.1. @@ -58,12 +58,12 @@ # ---------- -# Total row count: 1 node x 118 events (spec-6.13 adds RDMA busypoll + inline send waits). +# Total row count: 1 node x 121 events (spec-6.13 RDMA + spec-5.22b D2-6 undo-block waits). # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_gcluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', @@ -103,7 +103,7 @@ 'Cluster: Recovery' => 7, # spec-4.12 D6: +ClusterWriteFenceVerify 'Cluster: Sinval' => 6, 'Cluster: Interconnect' => 9, # spec-6.13: +busypoll + inline send waits - 'Cluster: Undo' => 4, + 'Cluster: Undo' => 7, # spec-5.22b D2-6: +UndoBlock Grant/Invalidate/Remaster waits 'Cluster: ADG' => 4, ); diff --git a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c index 095c16b2de..623912b7b4 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c @@ -196,7 +196,7 @@ UT_TEST(test_new_wait_events_distinct) } -UT_TEST(test_cluster_wait_events_count_118) +UT_TEST(test_cluster_wait_events_count_121) { /* spec-2.34 D7: 83 → 85 (+ 2 reliability wait events). * spec-2.36 D8: 85 → 88 (+ 3 CF 3-way wait events). @@ -206,8 +206,8 @@ UT_TEST(test_cluster_wait_events_count_118) * spec-4.6 D4: 97 → 98 (+ 1 GRD shard remaster short-wait). * spec-4.7 D1: 98 → 99 (+ 1 GCS block RECOVERING short-wait). * spec-4.11 D5: 99 → 100 (+ 1 online thread recovery short-wait). - * spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * spec-6.13 D8 -> 118; spec-5.22b D2-6 +3 undo-block waits -> 121. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -313,7 +313,7 @@ main(void) UT_RUN(test_retry_total_backoff_default_1500ms); UT_RUN(test_lwtranche_distinct); UT_RUN(test_new_wait_events_distinct); - UT_RUN(test_cluster_wait_events_count_118); + UT_RUN(test_cluster_wait_events_count_121); UT_RUN(test_dedup_full_status_distinct_from_master_not_holder); UT_RUN(test_block_data_size_equals_blcksz); UT_RUN(test_dedup_entry_collision_field_layout); diff --git a/src/test/cluster_unit/test_cluster_stage2_acceptance.c b/src/test/cluster_unit/test_cluster_stage2_acceptance.c index 8a6390e67f..d343bf67ae 100644 --- a/src/test/cluster_unit/test_cluster_stage2_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage2_acceptance.c @@ -208,16 +208,16 @@ UT_TEST(test_stage2_fault_inject_point_names) } -/* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ +/* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 121 ===== */ -UT_TEST(test_stage2_wait_events_count_snapshot_118) +UT_TEST(test_stage2_wait_events_count_snapshot_121) { /* spec-2.39 D13 ship value. Future spec adding wait events MUST * update this snapshot (update-required contract per spec v0.2 F5 * — current state, not "==93 forever"). spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering). spec-4.11 D5: 99 → 100 - * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * (+ ClusterThreadRecovery). spec-6.13 D8 -> 118; spec-5.22b D2-6 +3 undo-block waits -> 121. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -277,7 +277,7 @@ main(void) UT_RUN(test_stage2_msg_types_cumulative_registration); UT_RUN(test_stage2_capability_counter_symbols_linkable); UT_RUN(test_stage2_fault_inject_point_names); - UT_RUN(test_stage2_wait_events_count_snapshot_118); + UT_RUN(test_stage2_wait_events_count_snapshot_121); UT_RUN(test_stage2_sqlstate_53r60_through_95_encodable); UT_RUN(test_stage2_guc_enum_snapshot); UT_RUN(test_stage2_ic_msg_reserved_0_sentinel); diff --git a/src/test/cluster_unit/test_cluster_stage3_acceptance.c b/src/test/cluster_unit/test_cluster_stage3_acceptance.c index 448d2b6c59..285676ae8f 100644 --- a/src/test/cluster_unit/test_cluster_stage3_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage3_acceptance.c @@ -373,16 +373,16 @@ UT_TEST(test_stage3_sqlstate_mvcc_surface_encodable) } -/* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 118 ===== */ +/* ===== L5 — CLUSTER_WAIT_EVENTS_COUNT current snapshot 121 ===== */ -UT_TEST(test_stage3_wait_events_count_snapshot_118) +UT_TEST(test_stage3_wait_events_count_snapshot_121) { /* spec-4.2 D5 value (95 + 2 wal-state registry I/O). Update-required contract: a future spec * adding a wait event MUST bump this snapshot (it is current state, not * "==93 forever"). spec-4.6 D4: 97 → 98; spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering); spec-4.11 D5: 99 → 100 - * (+ ClusterThreadRecovery). spec-6.13 D8 current snapshot: 118. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + * (+ ClusterThreadRecovery). spec-6.13 D8 -> 118; spec-5.22b D2-6 +3 undo-block waits -> 121. * */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -475,7 +475,7 @@ main(void) UT_RUN(test_undo_4_8ab_redo_determinism_converges); UT_RUN(test_stage3_capability_dump_category_names); UT_RUN(test_stage3_sqlstate_mvcc_surface_encodable); - UT_RUN(test_stage3_wait_events_count_snapshot_118); + UT_RUN(test_stage3_wait_events_count_snapshot_121); UT_RUN(test_stage3_tt_enum_values_locked); UT_RUN(test_stage3_retention_active_retains_invariant); UT_RUN(test_stage3_bind_opcode_reserved); diff --git a/src/test/cluster_unit/test_cluster_stage4_acceptance.c b/src/test/cluster_unit/test_cluster_stage4_acceptance.c index 3e0617f786..b75cb485d9 100644 --- a/src/test/cluster_unit/test_cluster_stage4_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage4_acceptance.c @@ -25,7 +25,7 @@ * recovering / 53R9N undo-writeback-boundary / 53RA0 wal-thread- * routing-mismatch / 53RA3 merged-recovery-blocked / 53RA4 thread- * recovery-blocked. - * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 118 (spec-6.13 D8 + * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 121 (spec-6.13 D8 * value; update-required contract — any future spec adding wait * events MUST bump this snapshot). * L6 write-fence wire/ABI enums locked: ClusterFenceMarkerKind @@ -203,13 +203,13 @@ UT_TEST(test_stage4_sqlstate_recovery_fence_surface_encodable) /* ===== L5 — wait-events count snapshot ===== */ -UT_TEST(test_stage4_wait_events_count_snapshot_118) +UT_TEST(test_stage4_wait_events_count_snapshot_121) { /* Current Stage 4 surface value; spec-6.2 adds the latest 4 Smart Fusion * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -312,7 +312,7 @@ main(void) UT_RUN(test_stage4_undo_opcodes_preserved_and_info_mask_clear); UT_RUN(test_stage4_recovery_dump_category_names); UT_RUN(test_stage4_sqlstate_recovery_fence_surface_encodable); - UT_RUN(test_stage4_wait_events_count_snapshot_118); + UT_RUN(test_stage4_wait_events_count_snapshot_121); UT_RUN(test_stage4_write_fence_enums_locked); UT_RUN(test_stage4_thread_recovery_scope_enum_complete); UT_RUN(test_stage4_undo_writeback_boundary_enum_complete); diff --git a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c index 181c36b0f3..d33843c4fb 100644 --- a/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_5_cr_acceptance.c @@ -27,7 +27,7 @@ * OFF=0 / BOUNDARY=1(default) and the 4-counter ClusterCrCoordCounter * enum complete (CR_COORD_COUNTER__COUNT == 4) — the cr_coord * observability surface 5.58 HG#3 asserts. - * L6 CLUSTER_WAIT_EVENTS_COUNT snapshot = 118 — spec-6.13 adds the RDMA + * L6 CLUSTER_WAIT_EVENTS_COUNT snapshot = 121 — spec-6.13 adds the RDMA * wait-event pair after the block_device band; update- * required contract: a future spec adding cluster wait events MUST bump * this snapshot (and the dump/test baselines that count them). @@ -182,13 +182,13 @@ UT_TEST(test_stage5_5_cross_instance_coordinator_enums_locked) /* ===== L6 — wait-events count snapshot ===== */ -UT_TEST(test_stage5_5_wait_events_count_snapshot_118) +UT_TEST(test_stage5_5_wait_events_count_snapshot_121) { /* The whole CR read-path band (5.51-5.57) adds NO new wait events — it reuses * the spec-3.9 ClusterCRConstruct event. spec-6.0a adds 7 block_device * wait events after that band; spec-6.1 adds 2 RDMA wait events; spec-6.2 * adds 4 Smart Fusion authority waits; spec-6.13 adds 2 RDMA tier3 waits. */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -200,6 +200,6 @@ main(void) UT_RUN(test_stage5_5_cr_dump_category_names); UT_RUN(test_stage5_5_admission_policy_enum_locked); UT_RUN(test_stage5_5_cross_instance_coordinator_enums_locked); - UT_RUN(test_stage5_5_wait_events_count_snapshot_118); + UT_RUN(test_stage5_5_wait_events_count_snapshot_121); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c index 63750488ed..b5f1140186 100644 --- a/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_beta_acceptance.c @@ -27,7 +27,7 @@ * progress / 53R61 join-rejected-stale / 53R62 clean-leave-in-progress * / 53R64 node-removed-fenced / 53R70 ges-timeout / 55R01 * pcm-state-invalid. - * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 118 (update-required + * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 121 (update-required * contract) — a future spec adding cluster wait events MUST bump this * snapshot and the dump/test baselines that count them. * L6 7 RAC core presence (roadmap: only all seven make it "RAC core"): @@ -188,10 +188,10 @@ UT_TEST(test_beta_sqlstate_acceptance_surface_encodable) UT_TEST(test_beta_wait_events_count) { - /* Current Stage 5 beta surface value. update-required contract: a future - * spec adding cluster wait events MUST bump this snapshot (and the dump/test - * baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + /* Current surface value. update-required contract: a future spec adding + * cluster wait events MUST bump this snapshot (and the dump/test baselines + * that count them). spec-5.22b D2-6: +3 undo-block grant-plane waits -> 121. */ + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } diff --git a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c index c672b41494..7c0d766e4a 100644 --- a/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage5_integrated_acceptance.c @@ -27,7 +27,7 @@ * reconfig-in-progress / 53R61 join-rejected-stale / 53R62 clean- * leave-in-progress / 53R64 node-removed-fenced / 53R70 ges-timeout * / 55R01 pcm-state-invalid. - * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 118 (spec-6.13 D8 + * L5 CLUSTER_WAIT_EVENTS_COUNT current snapshot = 121 (spec-6.13 D8 * value; update-required contract) + the multi-node write-path * wait events present and pairwise distinct (GES_S4 / GES_REPLY / * CF_ENQUEUE / CR_CONSTRUCT / REL_EXTEND_WAIT — the MG-B M2 share). @@ -192,7 +192,7 @@ UT_TEST(test_stage5_wait_events_count_and_multinode_set) * authority waits and spec-6.13 adds 2 RDMA tier3 waits. update-required * contract: a future spec adding cluster wait events MUST bump this snapshot * (and the dump/test baselines that count them). */ - UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 118); + UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); /* The multi-node write-path wait events MG-B aggregates for the M2 share * must all be present and pairwise distinct (a reorder/removal would change diff --git a/src/test/cluster_unit/test_cluster_views.c b/src/test/cluster_unit/test_cluster_views.c index 00027881db..b7fa8cbdbe 100644 --- a/src/test/cluster_unit/test_cluster_views.c +++ b/src/test/cluster_unit/test_cluster_views.c @@ -207,7 +207,7 @@ cluster_shmem_iter_regions(int *idx pg_attribute_unused(), UT_DEFINE_GLOBALS(); -UT_TEST(test_cluster_wait_events_count_is_118) +UT_TEST(test_cluster_wait_events_count_is_121) { /* * Cumulative registration roster: 61 prior + 3 added by spec-2.6 D11 @@ -234,13 +234,15 @@ UT_TEST(test_cluster_wait_events_count_is_118) * (block_device production wait events) + spec-6.1 D8 RDMA * send/recv/poll/connect/fallback events + 4 added by spec-6.2 D10 * (Smart Fusion commit/DBWR/origin durable brakes + terminal resolve) + - * 2 added by spec-6.13 D8 (RDMA busypoll + inline send). + * 2 added by spec-6.13 D8 (RDMA busypoll + inline send) + 3 added by + * spec-5.22b D2-6 (UndoBlock Grant/Invalidate/Remaster grant-plane waits). * If a future subsystem spec adds new cluster wait events, both the * enum in wait_event.h and CLUSTER_WAIT_EVENTS_COUNT must move * together, and this test number must be bumped in lockstep. */ - /* spec-6.13 D8: RDMA tier3 wait events included -> 118. */ - UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 118); + /* spec-6.13 D8: RDMA tier3 wait events -> 118; + * spec-5.22b D2-6: +3 undo-block grant-plane waits -> 121. */ + UT_ASSERT_EQ(CLUSTER_WAIT_EVENTS_COUNT, 121); } @@ -285,7 +287,7 @@ int main(void) { UT_PLAN(5); - UT_RUN(test_cluster_wait_events_count_is_118); + UT_RUN(test_cluster_wait_events_count_is_121); UT_RUN(test_srf_symbol_linkable); UT_RUN(test_adg_srf_symbol_linkable); UT_RUN(test_first_event_is_ges_enqueue_acquire); diff --git a/src/test/cluster_unit/test_cluster_wait_events.c b/src/test/cluster_unit/test_cluster_wait_events.c index 1f3d56996d..55d919d410 100644 --- a/src/test/cluster_unit/test_cluster_wait_events.c +++ b/src/test/cluster_unit/test_cluster_wait_events.c @@ -237,10 +237,11 @@ UT_TEST(test_per_category_event_counts) - (uint32)WAIT_EVENT_INTERCONNECT_RDMA_SEND + 1, 9); /* Undo category: 4 (AD-010) + CR_CONSTRUCT (spec-3.9) + TT_DURABLE_IO - * (spec-3.11) + BUF_FLUSH + EXTENT_CLAIM (spec-3.18 D7) = 8. Count the - * full range to the last event so future additions are caught here (F12). */ + * (spec-3.11) + BUF_FLUSH + EXTENT_CLAIM (spec-3.18 D7) + BLOCK_GRANT_WAIT + + * BLOCK_INVALIDATE_WAIT + BLOCK_REMASTER_WAIT (spec-5.22b D2-6) = 11. Count + * the full range to the last event so future additions are caught here (F12). */ UT_ASSERT_EQ( - (uint32)WAIT_EVENT_CLUSTER_UNDO_EXTENT_CLAIM - (uint32)WAIT_EVENT_UNDO_REMOTE_READ + 1, 8); + (uint32)WAIT_EVENT_UNDO_BLOCK_REMASTER_WAIT - (uint32)WAIT_EVENT_UNDO_REMOTE_READ + 1, 11); UT_ASSERT_EQ((uint32)WAIT_EVENT_ADG_SCN_SYNC_WAIT - (uint32)WAIT_EVENT_ADG_MRP_APPLY_WAIT + 1, 4); UT_ASSERT_EQ((uint32)WAIT_EVENT_CLUSTER_BLOCK_DEVICE_PR_REGISTER From 612ea2f631947032b6642bac33b1385c3d3955a7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 10:23:37 +0800 Subject: [PATCH 08/38] feat(cluster): undo GCS grant-plane counters + shmem region (spec-5.22b D2-6) New ClusterUndoGcsShared shmem region (six atomic counters, no LWLock): grant_shared / grant_exclusive / ship_bytes / invalidate_notify / remaster_deny / local_fast_path. The owner-as-master grant primitives bump them at their existing sites; dump_undo (pg_cluster_state, category='undo') reads them lock-free -> the undo category grows 54 -> 60 rows. The counters are register-ahead of a live consumer (the grant primitives have no live caller until the grant-path consumer lands), so at rest every counter reads 0, but the surface is present + queryable now. Region roster: +"pgrac cluster undo gcs" (t/020 roster + region count). Also completes the wait-event count baselines the prior commit (undo-block grant-plane wait events) moved 118 -> 121 but did not update everywhere: the earlier commit's local check ran only the three view TAP files, so ~20 other TAP files and the cluster_smoke regress snapshot that pin the total wait-event count were left stale. Swept and bumped here (pg_stat_cluster_wait_events total 118 -> 121 across t/012-118, t/203, t/030, cluster_smoke.{sql,out}). Unit-test stubs added for the new dump accessors + shmem register (test_cluster_debug, test_cluster_shmem link cluster_debug.o / cluster_shmem.o standalone). Spec: spec-5.22b-undo-block-gcs-integration.md (D2-6) --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_debug.c | 24 ++- src/backend/cluster/cluster_shmem.c | 8 + src/backend/cluster/cluster_undo_gcs_grant.c | 8 +- src/backend/cluster/cluster_undo_gcs_stat.c | 189 ++++++++++++++++++ src/include/cluster/cluster_undo_gcs.h | 26 +++ .../expected/cluster_smoke.out | 6 +- .../cluster_regress/sql/cluster_smoke.sql | 2 +- src/test/cluster_tap/t/012_ic.pl | 8 +- src/test/cluster_tap/t/013_conf.pl | 8 +- src/test/cluster_tap/t/014_ic_mock.pl | 4 +- src/test/cluster_tap/t/015_inject.pl | 4 +- src/test/cluster_tap/t/016_perfmon.pl | 4 +- src/test/cluster_tap/t/017_debug.pl | 4 +- src/test/cluster_tap/t/020_shmem_registry.pl | 8 +- src/test/cluster_tap/t/021_block_format.pl | 6 +- src/test/cluster_tap/t/022_itl_slot.pl | 6 +- .../cluster_tap/t/023_buffer_descriptor.pl | 6 +- src/test/cluster_tap/t/030_acceptance.pl | 6 +- .../cluster_tap/t/108_pcm_state_machine.pl | 4 +- src/test/cluster_tap/t/110_gcs_loopback.pl | 8 +- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 6 +- .../t/112_gcs_block_retransmit_2node.pl | 4 +- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 4 +- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 6 +- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 4 +- .../t/116_gcs_block_lost_write_2node.pl | 4 +- .../t/117_sinval_broadcast_2node.pl | 4 +- .../t/118_sinval_ddl_propagation_2node.pl | 8 +- .../t/203_cluster_tt_status_foundation.pl | 4 +- .../t/214_cluster_3_8_undo_lifecycle.pl | 16 +- src/test/cluster_unit/test_cluster_debug.c | 31 +++ src/test/cluster_unit/test_cluster_shmem.c | 5 + .../test_cluster_stage3_acceptance.c | 3 +- 34 files changed, 369 insertions(+), 70 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_gcs_stat.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 77e4a0d4e3..8f3c5003c5 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -186,6 +186,7 @@ OBJS = \ cluster_undo_resid.o \ cluster_undo_gcs.o \ cluster_undo_gcs_grant.o \ + cluster_undo_gcs_stat.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..d26fa4b6b3 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -116,6 +116,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_cssd.h" /* cluster_cssd_status (spec-2.5 D12) */ #include "cluster/cluster_stats.h" /* cluster_stats_status (spec-1.14 D12) */ #include "cluster/cluster_undo_cleaner.h" /* dump_undo_cleaner (spec-3.13 D1) */ +#include "cluster/cluster_undo_gcs.h" /* undo GCS grant-plane counters (spec-5.22b D2-6) */ #include "cluster/cluster_lmon.h" /* cluster_lmon_status (spec-1.11 Sprint B D12) */ #include "cluster/cluster_guc.h" #include "catalog/pg_control.h" /* DBState (spec-4.3 plan dump) */ @@ -2078,7 +2079,7 @@ dump_gcs(ReturnSetInfo *rsinfo) /* * dump_undo -- spec-3.7 D10 + D6 真激活 counter observability. * - * Emits 53 rows under category='undo': 5 record-level allocator counters + * Emits 59 rows under category='undo': 5 record-level allocator counters * (spec-3.7) + 4 segment-lifecycle counters (spec-3.8) + 3 commit-fsync + * 4 smgr counters (the latter 7 added by the perf-merge undo * instrumentation) + 5 durable TT slot counters (spec-3.11 D8) + 5 retention @@ -2088,7 +2089,10 @@ dump_gcs(ReturnSetInfo *rsinfo) * 6 cleaner/reuse counters (spec-3.13 D6) + * 4 checkpoint-writeback boundary counters (spec-4.8ab D7: * undo_buf_held_wal / undo_buf_held_evidence / undo_buf_boundary_violations / - * undo_buf_remote_evidence_holds). Backs + * undo_buf_remote_evidence_holds) + + * 6 owner-as-master undo GCS grant-plane counters (spec-5.22b D2-6: + * grant_shared / grant_exclusive / ship_bytes / invalidate_notify / + * remaster_deny / local_fast_path). Backs * cluster_tap t/213 + t/214 + t/219 L2 + t/220 + t/270 verification + perf * class 7 baseline tracking. */ @@ -2483,6 +2487,22 @@ dump_undo(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_undo_buf_get_boundary_violation_count())); emit_row(rsinfo, "undo", "undo_buf_remote_evidence_holds", fmt_int64((int64)cluster_undo_buf_get_remote_evidence_hold_count())); + + /* spec-5.22b D2-6: owner-as-master undo GCS grant-plane counters (6). + * Register-ahead of the D6 consumer -> read 0 at rest, but the surface is + * present + queryable now. */ + emit_row(rsinfo, "undo", "undo_gcs_grant_shared_count", + fmt_int64((int64)cluster_undo_gcs_grant_shared_count())); + emit_row(rsinfo, "undo", "undo_gcs_grant_exclusive_count", + fmt_int64((int64)cluster_undo_gcs_grant_exclusive_count())); + emit_row(rsinfo, "undo", "undo_gcs_ship_bytes", + fmt_int64((int64)cluster_undo_gcs_ship_bytes())); + emit_row(rsinfo, "undo", "undo_gcs_invalidate_notify_count", + fmt_int64((int64)cluster_undo_gcs_invalidate_notify_count())); + emit_row(rsinfo, "undo", "undo_gcs_remaster_deny_count", + fmt_int64((int64)cluster_undo_gcs_remaster_deny_count())); + emit_row(rsinfo, "undo", "undo_gcs_local_fast_path_count", + fmt_int64((int64)cluster_undo_gcs_local_fast_path_count())); } /* diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..2dc81089cf 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -118,6 +118,7 @@ #include "cluster/cluster_cr_coordinator_stat.h" /* cluster_cr_coordinator_shmem_register (spec-5.57 D3) */ #include "cluster/cluster_xid_stripe.h" /* CLUSTER_XID_STRIDE boot validation (spec-6.15 D1) */ #include "cluster/cluster_tt_durable.h" /* cluster_tt_durable_shmem_register (spec-3.11 D7) */ +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_gcs_shmem_register (spec-5.22b D2-6) */ #include "cluster/cluster_sf_dep.h" /* cluster_sf_dep_shmem_register (spec-6.2 D6) */ #include "cluster/cluster_visibility_inject.h" /* cluster_visibility_inject_shmem_register (spec-3.2 D5b) */ #include "cluster/cluster_itl.h" /* cluster_lock_path_shmem_register (spec-3.4e D6) */ @@ -551,6 +552,13 @@ cluster_init_shmem_module(void) */ cluster_tt_durable_shmem_register(); + /* + * PGRAC spec-5.22b D2-6: register undo GCS data-plane counters shmem + * region (grant S/X, ship bytes, invalidate notify, remaster deny, + * local-fast-path; 6 atomic counters, 0 LWLock). + */ + cluster_undo_gcs_shmem_register(); + /* spec-6.2 D6: Smart Fusion dependency-vector store. Size is zero unless * cluster.smart_fusion=on, preserving default-off behavior. */ if (cluster_shmem_lookup_region("pgrac cluster smart fusion deps") == NULL) diff --git a/src/backend/cluster/cluster_undo_gcs_grant.c b/src/backend/cluster/cluster_undo_gcs_grant.c index 6ae748e0fd..c325be780e 100644 --- a/src/backend/cluster/cluster_undo_gcs_grant.c +++ b/src/backend/cluster/cluster_undo_gcs_grant.c @@ -96,6 +96,7 @@ cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expecte * fail-closed here and lands with its incarnation check + consumer in * D6. The peer-reads-foreign S-grant below is D2-3's delivered heart. */ + cluster_undo_gcs_count_local_fast_path(); /* D2-6: master==self routing taken */ return false; /* honest fail-closed forward (D6); never a stale local serve */ } @@ -115,7 +116,8 @@ cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expecte if (!cluster_undo_serve_allowed(cluster_cssd_get_peer_state(owner) != CLUSTER_CSSD_PEER_DEAD, cluster_grd_recovery_in_progress())) { res->status = (uint8)GCS_BLOCK_REPLY_DENIED_RESOURCE_RECOVERING; - return false; /* fail-closed; serve = D4 */ + cluster_undo_gcs_count_remaster_deny(); /* D2-6 */ + return false; /* fail-closed; serve = D4 */ } /* @@ -147,6 +149,7 @@ cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expecte res->live_hwm_lsn = auth.live_hwm_lsn; res->tt_generation = auth.tt_generation; res->status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; + cluster_undo_gcs_count_grant_shared(GCS_BLOCK_DATA_SIZE); /* D2-6: S-grant + shipped bytes */ return true; } @@ -187,6 +190,8 @@ cluster_undo_block_acquire_exclusive(const ClusterResId *undo_resid) if (!cluster_undo_block_master_is_self(undo_resid)) return false; + cluster_undo_gcs_count_grant_exclusive(); /* D2-6: local X grant */ + cluster_undo_gcs_count_local_fast_path(); /* D2-6: master==self, zero network */ return true; } @@ -259,5 +264,6 @@ cluster_undo_block_invalidate_peers(const ClusterResId *undo_resid, SCN write_sc notified++; } + cluster_undo_gcs_count_invalidate_notify(notified); /* D2-6 */ return notified; } diff --git a/src/backend/cluster/cluster_undo_gcs_stat.c b/src/backend/cluster/cluster_undo_gcs_stat.c new file mode 100644 index 0000000000..80ad90c7de --- /dev/null +++ b/src/backend/cluster/cluster_undo_gcs_stat.c @@ -0,0 +1,189 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_gcs_stat.c + * Shared-undo block GCS data-plane observability (spec-5.22b D2-6). + * + * Shmem counter region for the owner-as-master undo grant plane + * implemented in cluster_undo_gcs_grant.c. Kept in a separate translation + * unit so the pure decision core (cluster_undo_gcs.c) links standalone into + * cluster_unit without dragging in shmem symbols: the runtime primitives in + * cluster_undo_gcs_grant.c (already a backend-only object) call the extern + * bump hooks below, and dump_undo reads them lock-free via the accessors. + * + * Six atomic counters (no LWLock), surfaced under category='undo' by + * cluster_debug.c dump_undo: grant_shared / grant_exclusive / ship_bytes / + * invalidate_notify / remaster_deny / local_fast_path. + * + * The counters are register-ahead of a live consumer: the grant primitives + * they observe have no live caller until the D6 consumer wiring lands (the + * same skeleton-ahead posture as D2-3/D2-4), so at rest every counter reads + * 0 -- but the observability surface (shmem region + dump keys) is present + * and queryable now. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_gcs_stat.c + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22b-undo-block-gcs-integration.md (D2-6) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "port/atomics.h" +#include "storage/shmem.h" + +#include "cluster/cluster_shmem.h" +#include "cluster/cluster_undo_gcs.h" + +/* + * ClusterUndoGcsShared -- per-instance shmem counters (spec-5.22b D2-6). + * + * Six atomic counters, no LWLock (region lwlock_count = 0): bumped with + * pg_atomic_fetch_add_u64 from the granting / invalidating backend, read + * lock-free by dump_undo. + */ +typedef struct ClusterUndoGcsShared { + pg_atomic_uint64 grant_shared_count; /* reader S-grant admitted (D2-3) */ + pg_atomic_uint64 grant_exclusive_count; /* writer/cleaner X-grant taken (D2-4) */ + pg_atomic_uint64 ship_bytes; /* bytes admitted on a coherent S view */ + pg_atomic_uint64 invalidate_notify_count; /* peers sent a PI_DISCARD (D2-4) */ + pg_atomic_uint64 remaster_deny_count; /* serve-gate RESOURCE_RECOVERING deny (D2-5) */ + pg_atomic_uint64 local_fast_path_count; /* master==self routing taken (no network) */ +} ClusterUndoGcsShared; + +#ifdef USE_PGRAC_CLUSTER + +static ClusterUndoGcsShared *UndoGcsShared = NULL; + +/* ------------------------------------------------------------ */ +/* shmem region */ +/* ------------------------------------------------------------ */ + +Size +cluster_undo_gcs_shmem_size(void) +{ + return MAXALIGN(sizeof(ClusterUndoGcsShared)); +} + +void +cluster_undo_gcs_shmem_init(void) +{ + bool found; + + UndoGcsShared = ShmemInitStruct("ClusterUndoGcsShared", cluster_undo_gcs_shmem_size(), &found); + + if (!found) { + pg_atomic_init_u64(&UndoGcsShared->grant_shared_count, 0); + pg_atomic_init_u64(&UndoGcsShared->grant_exclusive_count, 0); + pg_atomic_init_u64(&UndoGcsShared->ship_bytes, 0); + pg_atomic_init_u64(&UndoGcsShared->invalidate_notify_count, 0); + pg_atomic_init_u64(&UndoGcsShared->remaster_deny_count, 0); + pg_atomic_init_u64(&UndoGcsShared->local_fast_path_count, 0); + } +} + +static const ClusterShmemRegion cluster_undo_gcs_region = { + .name = "pgrac cluster undo gcs", + .size_fn = cluster_undo_gcs_shmem_size, + .init_fn = cluster_undo_gcs_shmem_init, + .lwlock_count = 0, /* atomic counters only; no LWLock */ + .owner_subsys = "cluster_undo_gcs", + .reserved_flags = 0, +}; + +void +cluster_undo_gcs_shmem_register(void) +{ + cluster_shmem_register_region(&cluster_undo_gcs_region); +} + +/* ------------------------------------------------------------ */ +/* counter bump hooks (called from cluster_undo_gcs_grant.c) */ +/* ------------------------------------------------------------ */ + +void +cluster_undo_gcs_count_grant_shared(uint32 bytes) +{ + if (UndoGcsShared == NULL) + return; + pg_atomic_fetch_add_u64(&UndoGcsShared->grant_shared_count, 1); + pg_atomic_fetch_add_u64(&UndoGcsShared->ship_bytes, (uint64)bytes); +} + +void +cluster_undo_gcs_count_grant_exclusive(void) +{ + if (UndoGcsShared != NULL) + pg_atomic_fetch_add_u64(&UndoGcsShared->grant_exclusive_count, 1); +} + +void +cluster_undo_gcs_count_invalidate_notify(int peers) +{ + if (UndoGcsShared != NULL && peers > 0) + pg_atomic_fetch_add_u64(&UndoGcsShared->invalidate_notify_count, (uint64)peers); +} + +void +cluster_undo_gcs_count_remaster_deny(void) +{ + if (UndoGcsShared != NULL) + pg_atomic_fetch_add_u64(&UndoGcsShared->remaster_deny_count, 1); +} + +void +cluster_undo_gcs_count_local_fast_path(void) +{ + if (UndoGcsShared != NULL) + pg_atomic_fetch_add_u64(&UndoGcsShared->local_fast_path_count, 1); +} + +/* ------------------------------------------------------------ */ +/* lock-free read accessors (dump_undo) */ +/* ------------------------------------------------------------ */ + +uint64 +cluster_undo_gcs_grant_shared_count(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->grant_shared_count) : 0; +} + +uint64 +cluster_undo_gcs_grant_exclusive_count(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->grant_exclusive_count) : 0; +} + +uint64 +cluster_undo_gcs_ship_bytes(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->ship_bytes) : 0; +} + +uint64 +cluster_undo_gcs_invalidate_notify_count(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->invalidate_notify_count) : 0; +} + +uint64 +cluster_undo_gcs_remaster_deny_count(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->remaster_deny_count) : 0; +} + +uint64 +cluster_undo_gcs_local_fast_path_count(void) +{ + return UndoGcsShared != NULL ? pg_atomic_read_u64(&UndoGcsShared->local_fast_path_count) : 0; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_undo_gcs.h b/src/include/cluster/cluster_undo_gcs.h index 82fde7ef4c..00f0e1b6cd 100644 --- a/src/include/cluster/cluster_undo_gcs.h +++ b/src/include/cluster/cluster_undo_gcs.h @@ -310,4 +310,30 @@ extern bool cluster_undo_block_acquire_exclusive(const ClusterResId *undo_resid) */ extern int cluster_undo_block_invalidate_peers(const ClusterResId *undo_resid, SCN write_scn); +/* + * Undo GCS data-plane observability (D2-6, cluster_undo_gcs_stat.c). + * + * Shmem region (six atomic counters, no LWLock) + bump hooks + lock-free read + * accessors for the owner-as-master grant plane. The runtime primitives above + * call the bump hooks; dump_undo (cluster_debug.c) reads the accessors. The + * counters are register-ahead of a live consumer (skeleton-ahead, D2-3/D2-4): + * at rest every counter reads 0, but the surface is present + queryable now. + */ +extern Size cluster_undo_gcs_shmem_size(void); +extern void cluster_undo_gcs_shmem_init(void); +extern void cluster_undo_gcs_shmem_register(void); + +extern void cluster_undo_gcs_count_grant_shared(uint32 bytes); +extern void cluster_undo_gcs_count_grant_exclusive(void); +extern void cluster_undo_gcs_count_invalidate_notify(int peers); +extern void cluster_undo_gcs_count_remaster_deny(void); +extern void cluster_undo_gcs_count_local_fast_path(void); + +extern uint64 cluster_undo_gcs_grant_shared_count(void); +extern uint64 cluster_undo_gcs_grant_exclusive_count(void); +extern uint64 cluster_undo_gcs_ship_bytes(void); +extern uint64 cluster_undo_gcs_invalidate_notify_count(void); +extern uint64 cluster_undo_gcs_remaster_deny_count(void); +extern uint64 cluster_undo_gcs_local_fast_path_count(void); + #endif /* CLUSTER_UNDO_GCS_H */ diff --git a/src/test/cluster_regress/expected/cluster_smoke.out b/src/test/cluster_regress/expected/cluster_smoke.out index 062304fd1c..21699ba199 100644 --- a/src/test/cluster_regress/expected/cluster_smoke.out +++ b/src/test/cluster_regress/expected/cluster_smoke.out @@ -76,14 +76,14 @@ SELECT attname, format_type(atttypid, atttypmod) (7 rows) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 121 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- SELECT count(*) FROM pg_stat_cluster_wait_events; count ------- - 118 + 121 (1 row) -- ---------- @@ -107,7 +107,7 @@ SELECT count(DISTINCT type) FROM pg_stat_cluster_wait_events; SELECT count(*) FROM pg_stat_gcluster_wait_events; count ------- - 118 + 121 (1 row) -- ---------- diff --git a/src/test/cluster_regress/sql/cluster_smoke.sql b/src/test/cluster_regress/sql/cluster_smoke.sql index 04fd94fa1c..457eb7f9eb 100644 --- a/src/test/cluster_regress/sql/cluster_smoke.sql +++ b/src/test/cluster_regress/sql/cluster_smoke.sql @@ -44,7 +44,7 @@ SELECT attname, format_type(atttypid, atttypmod) -- ---------- --- 3. Cluster wait events: 118 rows (anchored by +-- 3. Cluster wait events: 121 rows (anchored by -- CLUSTER_WAIT_EVENTS_COUNT, spec-0.11 + StaticAssertDecl -- in cluster_views.c; spec-6.2 D10 +4 authority waits). -- ---------- diff --git a/src/test/cluster_tap/t/012_ic.pl b/src/test/cluster_tap/t/012_ic.pl index f894be28ad..914c2daf2d 100644 --- a/src/test/cluster_tap/t/012_ic.pl +++ b/src/test/cluster_tap/t/012_ic.pl @@ -102,13 +102,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_gcluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/013_conf.pl b/src/test/cluster_tap/t/013_conf.pl index a9200432d6..c992833379 100644 --- a/src/test/cluster_tap/t/013_conf.pl +++ b/src/test/cluster_tap/t/013_conf.pl @@ -113,13 +113,13 @@ # ---------- is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', - 'pg_stat_gcluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_gcluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql('postgres', q{SHOW "cluster.interconnect_tier"}), 'stub', diff --git a/src/test/cluster_tap/t/014_ic_mock.pl b/src/test/cluster_tap/t/014_ic_mock.pl index 08f3cfbdda..a64a8a5971 100644 --- a/src/test/cluster_tap/t/014_ic_mock.pl +++ b/src/test/cluster_tap/t/014_ic_mock.pl @@ -171,8 +171,8 @@ is( $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..9a62da52fa 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -189,8 +189,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ---------- # Test 11 (Hardening v1.0.1 / codex review P2-2): SQL SRF rejects diff --git a/src/test/cluster_tap/t/016_perfmon.pl b/src/test/cluster_tap/t/016_perfmon.pl index cf0c4bbfb1..57595f729e 100644 --- a/src/test/cluster_tap/t/016_perfmon.pl +++ b/src/test/cluster_tap/t/016_perfmon.pl @@ -154,8 +154,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..954074844e 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -157,8 +157,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..0ed660eb50 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -109,9 +109,9 @@ # between "pgrac cluster cr relgen" and "pgrac cluster cr tuple stats"). # spec-6.12h D-h3a: +1 "pgrac cluster pi shadow" (PI ship-SCN shadow table; # sorts between "pgrac cluster oid lease,pgrac cluster pcm grd" and "pgrac cluster qvotec"). -my $expected_region_count = $has_visibility_inject ? '79' : '78'; +my $expected_region_count = $has_visibility_inject ? '80' : '79'; my $expected_regions = - 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo record cursor'; + 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo gcs,pgrac cluster undo record cursor'; $expected_regions .= ',pgrac cluster visibility inject' if $has_visibility_inject; # spec-4.12 D7: cooperative write-fence region; always registered. Sorts after @@ -309,8 +309,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L17 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L17 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..1356d80d35 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -56,7 +56,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -199,8 +199,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L12 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); $node->stop; diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..5bc3f49d9a 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -71,7 +71,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -210,8 +210,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L12b pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L12b pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..30385ba290 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -61,7 +61,7 @@ # admission reason counters; +1 "pgrac cluster clean_leave" (spec-5.13); +1 # "pgrac cluster cr relgen" (spec-5.56 D4); +1 "pgrac cluster cr tuple stats" # (spec-5.54 D5); full list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '79' : '78'; # spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -137,8 +137,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L9 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..ab3aec7b44 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -146,8 +146,8 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'E1 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'E1 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); ok($node->safe_psql('postgres', q{SELECT count(*) > 0 FROM pg_stat_cluster_wait_events WHERE type='Cluster: GES'}) @@ -159,7 +159,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_gcluster_wait_events'), - '118', 'E4 pg_stat_gcluster_wait_events returns 118 rows (single-node, spec-6.13 RDMA wait surface)'); + '121', 'E4 pg_stat_gcluster_wait_events returns 121 rows (single-node, spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index cebe221c82..583e59229d 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -81,8 +81,8 @@ # L6 — wait event count baseline through spec-6.1. my $wait_event_count = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_stat_cluster_wait_events"); -is($wait_event_count, '118', - 'L6 wait event baseline 118 (spec-6.13 RDMA wait surface)'); +is($wait_event_count, '121', + 'L6 wait event baseline 121 (spec-6.13 RDMA wait surface)'); # L7 — no PCM wire opcode smoke (no SQL-visible PCM wire opcode enum surface) my $pcm_grd_init_event = $node_default->safe_psql( diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 84faab2ede..c8abcfca60 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -10,7 +10,7 @@ # L1 fresh cluster startup: pg_cluster_state.gcs has 67 keys # L2 api_state = "active" after postmaster phase 1 init # L3 WAIT_EVENT_GCS_REPLY_WAIT registered in pg_stat_cluster_wait_events -# L4 CLUSTER_WAIT_EVENTS_COUNT == 118 (cumulative through spec-6.13) +# L4 CLUSTER_WAIT_EVENTS_COUNT == 121 (cumulative through spec-6.13) # L5 msg_type registry surface visible: pg_cluster_ic_msg_types has # gcs_request + gcs_reply rows # L6 workload (SELECT/UPDATE/VACUUM) does NOT inc send_request_count @@ -87,11 +87,11 @@ sub gcs_value { 'L3 ClusterGcsReplyWait wait event registered (spec-2.32 D7)'); -# L4 — CLUSTER_WAIT_EVENTS_COUNT == 118 (spec-6.13). +# L4 — CLUSTER_WAIT_EVENTS_COUNT == 121 (spec-6.13). my $total_wait_events = $node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'); -is($total_wait_events, '118', - 'L4 wait_events count 118 (spec-6.13 RDMA wait surface)'); +is($total_wait_events, '121', + 'L4 wait_events count 121 (spec-6.13 RDMA wait surface)'); # L6 — Production workload does NOT trigger wire path (HC72 short-circuit). diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 9b5015e4e9..df637b12c1 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -147,13 +147,13 @@ sub gcs_int_value { # ============================================================ -# L5: total wait event count = 118. +# L5: total wait event count = 121. # ============================================================ is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L5 total cluster wait event count = 118 (spec-6.13 RDMA wait surface)'); + '121', + 'L5 total cluster wait event count = 121 (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 6b489f8c64..fc3685ba90 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -144,8 +144,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L5 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L5 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index 085bf47bd0..f99ca5305b 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -211,8 +211,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L9 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); done_testing(); diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 2a479ae004..930a843205 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -13,7 +13,7 @@ # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 # L3 pg_cluster_state.gcs has 67 keys (cumulative through spec-6.14a) -# L4 catversion lower-bound >= 202605430; wait event count == 118 +# L4 catversion lower-bound >= 202605430; wait event count == 121 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader # sees starvation_denied_pending_x_count tick @@ -132,8 +132,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L4 wait event count == 118 (spec-6.13 RDMA wait surface)'); + '121', + 'L4 wait event count == 121 (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 8f0e94d343..10131c9953 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -125,8 +125,8 @@ sub gcs_int is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - "L4 node$i wait event count == 118 (spec-6.13 RDMA wait surface)"); + '121', + "L4 node$i wait event count == 121 (spec-6.13 RDMA wait surface)"); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 5c3f5504ed..7023f3346a 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -101,8 +101,8 @@ sub gcs_int is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L2 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L2 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); is($pair->node0->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl index aea5668348..ae1fe02769 100644 --- a/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl +++ b/src/test/cluster_tap/t/117_sinval_broadcast_2node.pl @@ -93,8 +93,8 @@ sub sinval_int # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L3 wait event count == 118 (spec-6.13 RDMA wait surface)'); + '121', + 'L3 wait event count == 121 (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl index 89e2273a7f..9960b64f7b 100644 --- a/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl +++ b/src/test/cluster_tap/t/118_sinval_ddl_propagation_2node.pl @@ -13,7 +13,7 @@ # L1 catversion bump 202605450 → 202605460 # L2 pg_cluster_state sinval category has 16 keys (D8 +3 fanout + D9 +3 ack + D1 applied) # L3 3 NEW GUC defaults (sinval_ack_mode=peer_enqueued / timeout 5000 / slots 256) -# L4 pg_stat_cluster_wait_events count == 118 (cumulative through spec-6.13) +# L4 pg_stat_cluster_wait_events count == 121 (cumulative through spec-6.13) # L5 ClusterSinvalAckWait / ClusterSinvalAckSend / ClusterSinvalAckReceive visible # L6 53R95 ERRCODE_CLUSTER_SINVAL_ACK_TIMEOUT encodable # L7 node0 CREATE TABLE → node1 SELECT visible within 5s ack timeout @@ -77,12 +77,12 @@ 'L3c cluster.sinval_ack_wait_slots default 256'); # ============================================================ -# L4: pg_stat_cluster_wait_events count == 118. +# L4: pg_stat_cluster_wait_events count == 121. # ============================================================ is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L4 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L4 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ============================================================ # L5: 3 NEW ack wait events visible. diff --git a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl index 5b033f5f43..95ff94755e 100644 --- a/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl +++ b/src/test/cluster_tap/t/203_cluster_tt_status_foundation.pl @@ -215,8 +215,8 @@ sub tt_int # additions to sinval surface). is($pair->node0->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_wait_events'), - '118', - 'L9 pg_stat_cluster_wait_events returns 118 rows (spec-6.13 RDMA wait surface)'); + '121', + 'L9 pg_stat_cluster_wait_events returns 121 rows (spec-6.13 RDMA wait surface)'); # ============================================================ diff --git a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl index 75e4d98326..ab66119dc8 100644 --- a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl +++ b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl @@ -87,11 +87,23 @@ # ---------- my $undo_row_count = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='undo'}); -is($undo_row_count, '54', - "L2 undo category has 54 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4])" +is($undo_row_count, '60', + "L2 undo category has 60 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4] + 6 undo GCS grant-plane [spec-5.22b D2-6])" ); +# ---------- +# L2b (spec-5.22b D2-6): the 6 owner-as-master undo GCS grant-plane counter +# keys are present + queryable (register-ahead; read 0 at rest). +# ---------- +my $undo_gcs_keys = $node0->safe_psql('postgres', + q{SELECT string_agg(key, ',' ORDER BY key) FROM pg_cluster_state + WHERE category='undo' AND key LIKE 'undo_gcs_%'}); +is($undo_gcs_keys, + 'undo_gcs_grant_exclusive_count,undo_gcs_grant_shared_count,undo_gcs_invalidate_notify_count,undo_gcs_local_fast_path_count,undo_gcs_remaster_deny_count,undo_gcs_ship_bytes', + "L2b all 6 undo GCS grant-plane counter keys present (spec-5.22b D2-6)"); + + # ---------- # L3: 4 NEW lifecycle counter keys present # ---------- diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..67c7691666 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1570,6 +1570,37 @@ cluster_undo_reader_lookup_count(void) { return 0; } +/* spec-5.22b D2-6 stubs: 6 undo GCS grant-plane counters referenced by dump_undo. */ +uint64 +cluster_undo_gcs_grant_shared_count(void) +{ + return 0; +} +uint64 +cluster_undo_gcs_grant_exclusive_count(void) +{ + return 0; +} +uint64 +cluster_undo_gcs_ship_bytes(void) +{ + return 0; +} +uint64 +cluster_undo_gcs_invalidate_notify_count(void) +{ + return 0; +} +uint64 +cluster_undo_gcs_remaster_deny_count(void) +{ + return 0; +} +uint64 +cluster_undo_gcs_local_fast_path_count(void) +{ + return 0; +} /* spec-3.8 D11 stubs: 4 NEW lifecycle counters referenced by dump_undo. */ uint64 cluster_undo_autoextend_count(void) diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index d5bfd4ca76..7b98bdceac 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -424,6 +424,11 @@ cluster_tt_status_shmem_register(void) void cluster_tt_local_shmem_register(void) {} +/* spec-5.22b D2-6 stub: cluster_init_shmem_module also calls + * cluster_undo_gcs_shmem_register. */ +void +cluster_undo_gcs_shmem_register(void) +{} /* spec-3.2 D3 stub: cluster_init_shmem_module also calls * cluster_tt_status_hint_shmem_register. */ diff --git a/src/test/cluster_unit/test_cluster_stage3_acceptance.c b/src/test/cluster_unit/test_cluster_stage3_acceptance.c index 285676ae8f..1e1009a76d 100644 --- a/src/test/cluster_unit/test_cluster_stage3_acceptance.c +++ b/src/test/cluster_unit/test_cluster_stage3_acceptance.c @@ -381,7 +381,8 @@ UT_TEST(test_stage3_wait_events_count_snapshot_121) * adding a wait event MUST bump this snapshot (it is current state, not * "==93 forever"). spec-4.6 D4: 97 → 98; spec-4.7 D1: 98 → 99 * (+ ClusterGCSBlockRecovering); spec-4.11 D5: 99 → 100 - * (+ ClusterThreadRecovery). spec-6.13 D8 -> 118; spec-5.22b D2-6 +3 undo-block waits -> 121. * */ + * (+ ClusterThreadRecovery). spec-6.13 D8 -> 118; + * spec-5.22b D2-6 +3 undo-block waits -> 121. */ UT_ASSERT_EQ((int)CLUSTER_WAIT_EVENTS_COUNT, 121); } From 0afb5d03eb74bd4450be3487a43db43f8dca2e8c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 11:03:25 +0800 Subject: [PATCH 09/38] test(cluster): reconcile t/254 cr counter baseline (41 -> 57) The pi/cr recovery acceptance test pins per-category dump counter counts. The 'cr' category baseline (41) was stale: 16 more cr counters shipped after the last reconciliation -- spec-6.12i runtime-visibility rtvis_* / cr_server_undo_* / cr_server_verdict_*, spec-6.15 origin verdict -- all unconditional in dump_cr, so the deterministic count is now 57. Update the expected value + document the delta. This drift went undetected in CI because t/254 falls in the un-sharded t/249-272 nightly gap (its earlier stale value was "first caught by a local full run", per the existing comment). Test-only; no C change. --- .../cluster_tap/t/254_pi_cr_recovery_acceptance.pl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl index 1ee7bcc7a2..c8c44697f9 100644 --- a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl +++ b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl @@ -125,7 +125,7 @@ sub _new_single_node recovery => 39, # 3.16(4)+4.10 block(2)+4.11 thread(4)+4.3 plan(13)+4.4 worker(8)+4.5/4.7 merge(8) tt_recovery => 8, # 4.8 verdict counters gcs_recovery => 10, # 4.7 warm-recovery(8) + spec-2.41 D7 redo-coverage serve-gate(2) - cr => 41, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + cr => 57, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) # + 11 post-5.54 keys the baseline missed (stale on # main; first caught by a local full run): 6.12b # cr_server_{full,partial,denied} + @@ -134,6 +134,14 @@ sub _new_single_node # cr_global_epoch_fallback_bump, # cr_retention_horizon_advance_noted, # cr_reconfig_intra_survived + # + 16 more still stale on main (reconciled 2026-07-09, + # a full-band run under spec-5.22b): 6.12i runtime + # visibility rtvis_undo_fetch_{wire,cache_hit,failclosed}, + # cr_server_undo_{served,denied}, rtvis_resolve_{committed, + # aborted,failclosed}, rtvis_verdict_{wire,failclosed,exact, + # below_horizon,inadmissible}, cr_server_verdict_{served, + # denied}, rtvis_underivable_failclosed (all unconditional + # in dump_cr, so the count is deterministic = 57) pcm => 22, # 2.37 PI watermark + lock state + 6.14 D5 aux-deferred release # + spec-6.14a (b)-leg nonholder fail-closed counter ); From dd7334e416fc8eab33c7ebb46e74c0d9e69d779b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 11:03:43 +0800 Subject: [PATCH 10/38] test(cluster): spec-5.22b D2-7 undo-block GCS shared-storage data-plane TAP Add t/255: a 2-node ClusterPair + shared cluster_fs behavioral TAP for the owner-as-master undo data plane. cluster_unit (test_cluster_undo_gcs, U1-U10) already covers the pure decision cores; this pins the one thing it cannot -- the physical migration wired through the real allocator: - coherence off (default): own runtime undo stays on the local DataDir, the shared root holds none (inert / regression-safe negative control) - coherence on: after a forced fresh segment, own runtime undo migrates to the shared cluster_fs root (owner-as-master, observed end-to-end) - the migrated shared-root undo write path is functional (DML + rollback) - grant-plane counters stay 0 at rest -- the peer-really-reads-foreign -undo consumer is forward-D6; asserted, not faked Register t/255 in the nightly stage4-wal shard per the "every new t/ file lands in a shard the same commit" rule. The broader un-sharded t/249-272 gap is pre-existing and tracked separately. Test-only; no C change. Spec: spec-5.22b-undo-block-gcs-integration.md --- .github/workflows/nightly.yml | 8 +- .../t/255_cluster_5_22b_undo_gcs_shared.pl | 259 ++++++++++++++++++ 2 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 src/test/cluster_tap/t/255_cluster_5_22b_undo_gcs_shared.pl diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..a2206ca501 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -147,7 +147,13 @@ jobs: - { name: stage2-gcs, ranges: "100-118", unit: false, regress: false } - { name: stage3-core, ranges: "203-225", unit: false, regress: false } - { name: stage3-recent, ranges: "228-237", unit: false, regress: false } - - { name: stage4-wal, ranges: "242-248 273 275", unit: false, regress: false } + # 255 = spec-5.22b D2-7 undo-block GCS shared-storage data-plane TAP + # (2-node ClusterPair + shared cluster_fs; owner-as-master physical + # migration). Registered here (undo/recovery bucket) so the new file + # runs in nightly per the "every new t/ file lands in a shard the same + # commit" rule. NOTE: t/249-254 + t/256-272 remain an un-sharded + # pre-existing gap (tracked separately), not opened by this commit. + - { name: stage4-wal, ranges: "242-248 255 273 275", unit: false, regress: false } - { name: stage4-hardgate, ranges: "274-274", unit: false, regress: false } - { name: stage5-ges-locking, ranges: "276-326", unit: false, regress: false } # spec-5.19 D9: Stage 5 integrated acceptance (heavy 4-node ClusterQuad diff --git a/src/test/cluster_tap/t/255_cluster_5_22b_undo_gcs_shared.pl b/src/test/cluster_tap/t/255_cluster_5_22b_undo_gcs_shared.pl new file mode 100644 index 0000000000..5a037ec244 --- /dev/null +++ b/src/test/cluster_tap/t/255_cluster_5_22b_undo_gcs_shared.pl @@ -0,0 +1,259 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 255_cluster_5_22b_undo_gcs_shared.pl +# spec-5.22b D2-7 — Shared-undo block GCS integration data-plane TAP +# on a 2-node ClusterPair + shared cluster_fs root. +# +# This is the behavioral (end-to-end) half of D2-7; the pure decision +# cores (owner-as-master routing, S/X grant contracts, admissibility, +# serve-gate, PI-discard) are exhaustively covered standalone by +# cluster_unit (test_cluster_undo_gcs, U1-U10). This TAP pins the one +# thing cluster_unit cannot: the D2-2 physical migration wired through +# the real undo allocator on a running peer-mode pair. +# +# L1 ClusterPair startup: both nodes alive under peer-mode + +# shared cluster_fs, coherence OFF at boot (default, inert). +# L2 the 6 owner-as-master undo GCS grant-plane counter keys are +# present + queryable (register-ahead). +# L3 coherence OFF (default): node0's own runtime undo lands on the +# LOCAL DataDir, and the shared root holds NO node0 undo segment +# -- the inert / regression-safe path (negative control). +# L4 coherence ON (ALTER SYSTEM + reload): after a forced fresh +# segment allocation, node0's own runtime undo now lands on the +# SHARED cluster_fs root -- owner-as-master physical migration, +# the D2-2 heart, observed end-to-end at the filesystem. +# L5 the migrated (shared-root) undo write path is functional: +# INSERT / UPDATE / DELETE round-trip correctly with no error, +# so the segment relocation did not break the undo data path. +# L6 grant-plane counters stay 0 at rest: acquire_shared / +# acquire_exclusive / invalidate_peers have no live consumer at +# D2 (the W3 wall still fail-closes a foreign-undo self-read, and +# master==self fails closed), so the peer-really-reads-foreign- +# undo leg is forward-D6. This is asserted, not faked. +# +# NOT covered here (honestly forward-linked, never a faked pass): +# - a peer really reading a foreign owner's undo end-to-end +# ("committed seed visible") -> D6 wires the seed consumer. +# - X-transfer invalidation of a live peer S/PI holder -> D6. +# - dead-owner SERVE from shared storage (the positive leg) -> D4. +# +# IDENTIFICATION +# src/test/cluster_tap/t/255_cluster_5_22b_undo_gcs_shared.pl +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# NOTES +# Spec: spec-5.22b-undo-block-gcs-integration.md (D2-7, §4.2) +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + + +# Own-instance undo for node0 (cluster_node_id = 0) lives under the +# owner-partitioned instance_0 subtree; the shared and local paths differ +# only in their root (cluster_shared_fs_undo_path_resolve). +my $INSTANCE_SUBDIR = 'pg_undo/instance_0'; + +# Count node0 undo segment files under a given root's instance_0 subtree. +sub undo_seg_count +{ + my ($root) = @_; + my @segs = glob("$root/$INSTANCE_SUBDIR/seg_*.dat"); + return scalar(@segs); +} + + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'spec_5_22b_undo_gcs', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.pcm_grd_max_entries = 0', + 'cluster.undo_segments_max_per_instance = 256', + 'cluster.undo_segment_create_timeout_ms = 5000', + 'max_prepared_transactions = 4', + # coherence stays OFF at boot -- L3 is the inert negative control; + # L4 arms it via ALTER SYSTEM so the off->on transition is proven, + # not just the on state. + ]); +$pair->start_pair; +usleep(2_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; +my $data0 = $node0->data_dir; +my $shared_root = $pair->shared_data_root; + + +# ---------- +# L1: both nodes alive under peer-mode + shared cluster_fs (coherence off). +# ---------- +{ + my $r0 = $node0->safe_psql('postgres', 'SELECT 1'); + is($r0, '1', "L1a node0 alive under peer-mode + shared cluster_fs"); + + my $r1 = $node1->safe_psql('postgres', 'SELECT 1'); + is($r1, '1', "L1b node1 alive under peer-mode + shared cluster_fs"); + + # The pair really is peer-mode (declared 2-node topology), the precondition + # for the D2-2 migration gate. + my $ncount = $node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_cluster_nodes}); + ok($ncount >= 2, "L1c node0 sees a >=2-node topology (peer-mode, ncount=$ncount)"); + + # coherence is OFF at boot (the inert default). + my $coh = $node0->safe_psql('postgres', + q{SELECT setting FROM pg_settings WHERE name = 'cluster.undo_gcs_coherence'}); + is($coh, 'off', "L1d cluster.undo_gcs_coherence default = off (inert)"); +} + + +# ---------- +# L2: the 6 owner-as-master undo GCS grant-plane counter keys are present. +# ---------- +{ + my $keys = $node0->safe_psql('postgres', + q{SELECT string_agg(key, ',' ORDER BY key) FROM pg_cluster_state + WHERE category='undo' AND key LIKE 'undo_gcs_%'}); + is($keys, + 'undo_gcs_grant_exclusive_count,undo_gcs_grant_shared_count,undo_gcs_invalidate_notify_count,undo_gcs_local_fast_path_count,undo_gcs_remaster_deny_count,undo_gcs_ship_bytes', + "L2 all 6 undo GCS grant-plane counter keys present"); +} + + +# ---------- +# L3: coherence OFF -> node0 own undo lands LOCAL, shared root has none. +# (the inert / regression-safe negative control for the migration gate) +# ---------- +{ + $node0->safe_psql('postgres', q{ + CREATE TABLE t_undo_gcs (id int, v text); + INSERT INTO t_undo_gcs SELECT g, 'boot-local' || g FROM generate_series(1, 20) g; + }); + + # Force a fresh segment allocation so the placement decision is exercised + # by the allocator under the current (off) coherence value. + my $force0 = $node0->safe_psql('postgres', + q{SELECT cluster_undo_test_force_segment_end()}); + is($force0, 't', "L3a force_segment_end true after active segment claimed"); + $node0->safe_psql('postgres', + q{INSERT INTO t_undo_gcs VALUES (1001, 'local-after-force')}); + + my $local_off = undo_seg_count($data0); + my $shared_off = undo_seg_count($shared_root); + + ok($local_off >= 1, + "L3b coherence off: node0 undo on LOCAL DataDir ($local_off seg file(s))"); + is($shared_off, 0, + "L3c coherence off: shared cluster_fs root holds NO node0 undo (inert)"); +} + + +# ---------- +# L4: coherence ON -> node0 own undo migrates to the SHARED cluster_fs root. +# (owner-as-master physical migration -- the D2-2 heart, observed e2e) +# ---------- +{ + my $shared_before = undo_seg_count($shared_root); + is($shared_before, 0, "L4a shared root empty of node0 undo before arming coherence"); + + $node0->safe_psql('postgres', + 'ALTER SYSTEM SET cluster.undo_gcs_coherence = on'); + $node0->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(500_000); + + # A fresh backend reads the armed value (postgresql.auto.conf); force a + # fresh segment so the next allocation resolves under coherence = on. + my $coh_on = $node0->safe_psql('postgres', + q{SELECT setting FROM pg_settings WHERE name = 'cluster.undo_gcs_coherence'}); + is($coh_on, 'on', "L4b cluster.undo_gcs_coherence armed (off -> on)"); + + my $force1 = $node0->safe_psql('postgres', + q{SELECT cluster_undo_test_force_segment_end()}); + is($force1, 't', "L4c force_segment_end true under coherence on"); + $node0->safe_psql('postgres', + q{INSERT INTO t_undo_gcs VALUES (2001, 'shared-after-arm')}); + + my $shared_after = undo_seg_count($shared_root); + ok($shared_after >= 1, + "L4d coherence on: node0 own undo migrated to SHARED root ($shared_after seg file(s))"); +} + + +# ---------- +# L5: the migrated (shared-root) undo write path is functional. +# INSERT / UPDATE / DELETE round-trip with no error, so relocating the +# segment onto the shared root did not break the undo data path. +# ---------- +{ + $node0->safe_psql('postgres', q{ + INSERT INTO t_undo_gcs SELECT g, 'shared-dml' || g FROM generate_series(3000, 3040) g; + UPDATE t_undo_gcs SET v = v || '-upd' WHERE id BETWEEN 3000 AND 3040; + DELETE FROM t_undo_gcs WHERE id BETWEEN 3000 AND 3010; + }); + + my $updated = $node0->safe_psql('postgres', + q{SELECT count(*) FROM t_undo_gcs WHERE v LIKE 'shared-dml%-upd'}); + is($updated, '30', + "L5a shared-root undo write path functional: 30 rows survived update+delete"); + + # A rolled-back update must leave the row unchanged (undo applied from the + # shared-root segment). + $node0->safe_psql('postgres', q{ + BEGIN; + UPDATE t_undo_gcs SET v = 'ROLLED-BACK' WHERE id = 2001; + ROLLBACK; + }); + my $rb = $node0->safe_psql('postgres', + q{SELECT v FROM t_undo_gcs WHERE id = 2001}); + is($rb, 'shared-after-arm', + "L5b rollback restored the pre-image from shared-root undo"); +} + + +# ---------- +# L6: grant-plane counters stay 0 at rest -- the peer-really-reads-foreign- +# undo consumer is forward-D6, asserted (not faked). +# +# acquire_shared / acquire_exclusive / invalidate_peers are skeleton-ahead +# at D2: the W3 wall (cluster_undo_record.c) still fail-closes a foreign +# undo self-read and master==self fails closed, so no live caller drives +# these counters. A non-zero here would mean a consumer landed early, +# contradicting the skeleton-ahead state -- so 0 is a real assertion. +# ---------- +{ + my $counters = $node0->safe_psql('postgres', + q{SELECT string_agg(key || '=' || value, ',' ORDER BY key) + FROM pg_cluster_state + WHERE category='undo' AND key LIKE 'undo_gcs_%'}); + is($counters, + 'undo_gcs_grant_exclusive_count=0,undo_gcs_grant_shared_count=0,' + . 'undo_gcs_invalidate_notify_count=0,undo_gcs_local_fast_path_count=0,' + . 'undo_gcs_remaster_deny_count=0,undo_gcs_ship_bytes=0', + "L6 grant-plane counters all 0 at rest (peer-read consumer is forward-D6)"); + + note('D2-7 forward-D6 (not asserted here, never faked): a peer really ' + . 'reading a foreign owner\'s undo end-to-end ("committed seed visible"), ' + . 'X-transfer invalidation of a live peer holder, and dead-owner SERVE ' + . 'from shared storage. The D2 delivery is the owner-as-master routing + ' + . 'physical migration proven in L1-L5.'); +} + + +$pair->stop_pair; +done_testing(); From c68511f039e602e3e2532e6c239bd16cb5f18334 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 14:47:54 +0800 Subject: [PATCH 11/38] feat(cluster): spec-5.22c D3-1 undo verdict taxonomy + pure mappers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the five-value cross-node xid->commit_scn verdict taxonomy and its two pure mapping helpers, the structural layer D3 builds on. new src/include/cluster/cluster_undo_verdict.h ClusterUndoVerdictKind {UNKNOWN_FAIL_CLOSED=0, COMMITTED_EXACT=1, COMMITTED_BOUND=2, ABORTED=3, IN_PROGRESS=4} + ClusterUndoVerdictResult {kind, commit_scn, wrap}. kind alone distinguishes EXACT vs BOUND (no is_bound side axis): a switch handling only EXACT lets a BOUND fall to a fail-closed default, so a horizon bound is never stamped or cached as an exact commit SCN. The zero value is the fail-closed sentinel. mod cluster_runtime_visibility_policy.c cluster_undo_verdict_from_wire_page(): CP5 verdict page -> taxonomy (unusable page -> UNKNOWN; EXACT -> EXACT; BELOW_HORIZON -> BOUND; ABORTED -> ABORTED). The read_scn admissibility of a BOUND needs scn_time_cmp and is deferred to the consumer, keeping this policy object dependency-free (as cluster_vis_undo_verdict_page_usable does). cluster_undo_verdict_from_block_proof(): CP3 proof -> taxonomy (COMMITTED -> EXACT{scn,wrap}; ABORTED -> ABORTED; NONE -> UNKNOWN). Both pure: no I/O, shmem, or elog. new test_cluster_undo_verdict.c (U1-U3, standalone link against the policy object) + Makefile registration. Pure taxonomy layer only; not yet wired into the resolve path (D3-2+), so this increment is zero behaviour change. Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3-1, §2.1/§4.1) --- .../cluster_runtime_visibility_policy.c | 80 ++++++++ src/include/cluster/cluster_undo_verdict.h | 99 ++++++++++ src/test/cluster_unit/Makefile | 20 +- .../cluster_unit/test_cluster_undo_verdict.c | 172 ++++++++++++++++++ 4 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 src/include/cluster/cluster_undo_verdict.h create mode 100644 src/test/cluster_unit/test_cluster_undo_verdict.c diff --git a/src/backend/cluster/cluster_runtime_visibility_policy.c b/src/backend/cluster/cluster_runtime_visibility_policy.c index 711b5068cd..609b4965e3 100644 --- a/src/backend/cluster/cluster_runtime_visibility_policy.c +++ b/src/backend/cluster/cluster_runtime_visibility_policy.c @@ -32,6 +32,7 @@ #include "cluster/cluster_gcs_block.h" /* ClusterGcsUndoVerdictPage (CP5) */ #include "cluster/cluster_runtime_visibility.h" #include "cluster/cluster_undo_segment.h" /* UndoSegmentHeaderData, TTSlot */ +#include "cluster/cluster_undo_verdict.h" /* D3 verdict taxonomy + mappers */ /* * cluster_vis_live_authority_covers_policy @@ -210,3 +211,82 @@ cluster_vis_undo_verdict_page_usable(const struct ClusterGcsUndoVerdictPage *v, return false; /* unknown kind: refuse, never guess */ } } + +/* + * cluster_undo_verdict_from_wire_page + * + * D3-1 pure mapper: a CP5 origin verdict page -> the five-value taxonomy. + * Fail-closed by construction: a page that does not structurally answer + * asked_xid (cluster_vis_undo_verdict_page_usable) yields UNKNOWN_FAIL_CLOSED, + * so the caller keeps the 53R97 boundary (Rule 8.A). A BELOW_HORIZON page + * maps to COMMITTED_BOUND unconditionally -- the read_scn admissibility of + * that bound (requester leg (e)) needs scn_time_cmp and is applied by the + * D3-3 consumer, deliberately outside this dependency-free policy object (see + * cluster_vis_undo_verdict_page_usable NOTES). Pure: no I/O, no elog. + */ +ClusterUndoVerdictResult +cluster_undo_verdict_from_wire_page(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid) +{ + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + + /* Structural gate first: an unusable page is not evidence about the xid. */ + if (!cluster_vis_undo_verdict_page_usable(v, asked_xid)) + return r; + + switch (v->verdict) { + case (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT: + r.kind = CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = (SCN)v->commit_scn; + r.wrap = v->wrap; + return r; + + case (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_BELOW_HORIZON: + /* Bound-only commit: kind == BOUND is itself the "never stamp the + * scn as exact" marker. The read_scn gate is the consumer's. */ + r.kind = CLUSTER_UNDO_VERDICT_COMMITTED_BOUND; + r.commit_scn = (SCN)v->horizon_scn; + return r; + + case (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED: + r.kind = CLUSTER_UNDO_VERDICT_ABORTED; + return r; + + default: + /* page_usable() already fenced the kind range; defense in depth. */ + return r; + } +} + +/* + * cluster_undo_verdict_from_block_proof + * + * D3-1 pure mapper: a CP3 single-block positive proof (exact xid+wrap slot + * match on the shipped block0 bytes) -> the taxonomy. COMMITTED carries the + * true commit SCN (EXACT); ABORTED is terminal; NONE is UNKNOWN_FAIL_CLOSED, + * which the orchestrator reads as "fall to the CP5 origin verdict", never as + * a terminal answer to the caller (Rule 8.A). Pure. + */ +ClusterUndoVerdictResult +cluster_undo_verdict_from_block_proof(ClusterVisTtProof proof, SCN commit_scn, uint16 wrap) +{ + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + + switch (proof) { + case CLUSTER_VIS_TT_PROOF_COMMITTED: + r.kind = CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = commit_scn; + r.wrap = wrap; + return r; + + case CLUSTER_VIS_TT_PROOF_ABORTED: + r.kind = CLUSTER_UNDO_VERDICT_ABORTED; + return r; + + case CLUSTER_VIS_TT_PROOF_NONE: + default: + return r; + } +} diff --git a/src/include/cluster/cluster_undo_verdict.h b/src/include/cluster/cluster_undo_verdict.h new file mode 100644 index 0000000000..6663e4aa5f --- /dev/null +++ b/src/include/cluster/cluster_undo_verdict.h @@ -0,0 +1,99 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_verdict.h + * Cross-node xid -> transaction-outcome verdict taxonomy over shared + * undo (spec-5.22c D3, shared undo physical data plane under GCS, + * Layer-2). + * + * D2 (spec-5.22b) shipped the owner-as-master coherent block-read + * primitive (cluster_undo_block_acquire_shared). D3 consumes it to + * answer "did xid commit, and at what SCN?" across nodes, producing the + * five-value verdict below. Amendment-2 splits the brief's single + * COMMITTED_EXACT into EXACT (a true commit SCN, may be stamped/cached) + * and BOUND (a retention horizon bound valid only against the asking + * read_scn, never stamped/cached) so a consumer switch that handles only + * EXACT falls through to a fail-closed default on a BOUND -- a horizon + * bound can never be mistaken for an exact commit SCN. + * + * The two mapping helpers here are PURE (no scn_time_cmp, no shmem, no + * elog), so the taxonomy layer links standalone against the + * dependency-free runtime-visibility policy object. The read_scn + * admissibility gate of a COMMITTED_BOUND (requester leg (e)) is applied + * by the D3-3 consumer, deliberately outside these mappers -- exactly as + * cluster_vis_undo_verdict_page_usable defers it. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_undo_verdict.h + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3, §2.1) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_UNDO_VERDICT_H +#define CLUSTER_UNDO_VERDICT_H + +#include "cluster/cluster_runtime_visibility.h" /* ClusterVisTtProof */ +#include "cluster/cluster_scn.h" /* SCN, InvalidScn */ + +struct ClusterGcsUndoVerdictPage; /* cluster_gcs_block.h */ + +/* + * Cross-node xid -> terminal-state verdict (brief line 33 taxonomy; + * Amendment-2 refines COMMITTED_EXACT into EXACT / BOUND -> five values). + * + * 0 == UNKNOWN_FAIL_CLOSED: any path that cannot PROVE a terminal outcome + * lands here so the caller keeps the 53R97 fail-closed boundary (Rule 8.A / + * L10: the zero value is fail-closed, never defaults to visible). + */ +typedef enum ClusterUndoVerdictKind { + CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED = 0, + CLUSTER_UNDO_VERDICT_COMMITTED_EXACT = 1, /* true commit SCN (may stamp/cache) */ + CLUSTER_UNDO_VERDICT_COMMITTED_BOUND = 2, /* horizon bound only (never stamp/cache) */ + CLUSTER_UNDO_VERDICT_ABORTED = 3, /* proven rolled back (CLOG abort) */ + CLUSTER_UNDO_VERDICT_IN_PROGRESS = 4 /* proven live (Q6: D3 folds to UNKNOWN) */ +} ClusterUndoVerdictKind; + +/* + * kind alone distinguishes EXACT vs BOUND (there is deliberately no + * commit_scn_is_bound side axis, Amendment-2): a switch that only handles + * COMMITTED_EXACT lets a BOUND fall to default -> fail-closed, so a horizon + * bound is never stamped/cached as an exact commit_scn. + */ +typedef struct ClusterUndoVerdictResult { + uint8 kind; /* ClusterUndoVerdictKind */ + SCN commit_scn; /* EXACT=true commit SCN / BOUND=horizon bound; else InvalidScn */ + uint16 wrap; /* COMMITTED_EXACT slot wrap evidence */ +} ClusterUndoVerdictResult; + +/* + * cluster_undo_verdict_from_wire_page -- map a CP5 origin verdict page to the + * taxonomy. Refuses (UNKNOWN_FAIL_CLOSED) any page that does not structurally + * answer asked_xid (cluster_vis_undo_verdict_page_usable). A BELOW_HORIZON + * page maps to COMMITTED_BOUND unconditionally; the read_scn admissibility of + * that bound is the consumer's gate, not this pure mapper's. Pure: no I/O, no + * shmem, no elog. + */ +extern ClusterUndoVerdictResult +cluster_undo_verdict_from_wire_page(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid); + +/* + * cluster_undo_verdict_from_block_proof -- map a CP3 single-block positive + * proof to the taxonomy: COMMITTED -> COMMITTED_EXACT{commit_scn,wrap} (the + * shipped block0 bytes carry the true commit SCN), ABORTED -> ABORTED, NONE -> + * UNKNOWN_FAIL_CLOSED (the orchestrator reads a CP3 UNKNOWN as "fall to the + * CP5 origin verdict", it is not the caller's terminal answer). Pure. + */ +extern ClusterUndoVerdictResult cluster_undo_verdict_from_block_proof(ClusterVisTtProof proof, + SCN commit_scn, uint16 wrap); + +#endif /* CLUSTER_UNDO_VERDICT_H */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 05e6d1e4ec..b3854803aa 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -88,7 +88,8 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_hang_acceptance \ test_cluster_xid_stripe \ test_cluster_undo_resid \ - test_cluster_undo_gcs + test_cluster_undo_gcs \ + test_cluster_undo_verdict # Path to the cluster_version object (no PG deps, safe to link standalone). CLUSTER_VERSION_O = $(top_builddir)/src/backend/cluster/cluster_version.o @@ -180,7 +181,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs test_cluster_undo_verdict,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1861,3 +1862,18 @@ test_cluster_undo_gcs: test_cluster_undo_gcs.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ $(CLUSTER_RUNTIME_VIS_POLICY_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# spec-5.22c D3-1: test_cluster_undo_verdict — cross-node xid->commit_scn +# verdict taxonomy (five-value ClusterUndoVerdictKind) + the two pure mapping +# helpers (wire ClusterGcsUndoVerdictPage -> taxonomy, block-proof +# ClusterVisTtProof -> taxonomy). The mappers live in the dependency-free +# runtime-visibility policy object (no scn_time_cmp / shmem / elog), so the +# taxonomy layer links standalone against it plus libpgport. The read_scn +# admissibility gate of a COMMITTED_BOUND is the D3-3 consumer's, not these +# pure mappers'; the S-grant / verdict orchestration wiring is backend-heavy +# and forward-covered by the D3-7 / D6 TAP. +test_cluster_undo_verdict: test_cluster_undo_verdict.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_RUNTIME_VIS_POLICY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_RUNTIME_VIS_POLICY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_undo_verdict.c b/src/test/cluster_unit/test_cluster_undo_verdict.c new file mode 100644 index 0000000000..e62d664532 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_undo_verdict.c @@ -0,0 +1,172 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_undo_verdict.c + * Unit tests for the D3 cross-node xid -> commit_scn verdict taxonomy + * (spec-5.22c, shared undo physical data plane under GCS, Layer-2 D3). + * + * D3-1 (this increment) pins the five-value verdict taxonomy and the two + * pure mapping helpers: wire ClusterGcsUndoVerdictPage -> taxonomy and + * block-proof ClusterVisTtProof -> taxonomy. The mappers are pure + * (dependency-free: no scn_time_cmp, no shmem, no elog) so the taxonomy + * layer links standalone against the runtime-visibility policy object. + * The read_scn admissibility gate of a COMMITTED_BOUND is the consumer's + * (D3-3), deliberately outside these dependency-free mappers -- exactly + * as cluster_vis_undo_verdict_page_usable defers it (policy.c NOTES). + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_undo_verdict.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3, §4.1) + * + * The mapping helpers live in cluster_runtime_visibility_policy.o (the + * dependency-free vis policy object); the test links that object plus + * libpgport and needs no self-node or SCN-clock globals. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_gcs_block.h" /* ClusterGcsUndoVerdictPage + wire kinds */ +#include "cluster/cluster_runtime_visibility.h" /* ClusterVisTtProof */ +#include "cluster/cluster_scn.h" /* SCN, InvalidScn */ +#include "cluster/cluster_undo_verdict.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} + +/* + * Build a structurally valid verdict page for the asked xid, so the mapper's + * internal cluster_vis_undo_verdict_page_usable() gate accepts it and the + * test exercises the taxonomy mapping (not the structural refusal). + */ +static void +make_verdict_page(ClusterGcsUndoVerdictPage *v, uint8 kind, TransactionId xid, SCN commit_scn, + SCN horizon_scn, uint16 wrap) +{ + memset(v, 0, sizeof(*v)); + v->magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; + v->version = CLUSTER_GCS_UNDO_VERDICT_VERSION; + v->xid_echo = (uint64)xid; + v->verdict = kind; + v->commit_scn = commit_scn; + v->horizon_scn = horizon_scn; + v->wrap = wrap; +} + +/* ====================================================================== + * U1 -- wire page -> five-value taxonomy mapping (D3-1, Amendment-2): + * COMMITTED_EXACT -> COMMITTED_EXACT{commit_scn,wrap}; + * COMMITTED_BELOW_HORIZON -> COMMITTED_BOUND{commit_scn=horizon}; + * ABORTED -> ABORTED; structurally-unusable page -> UNKNOWN_FAIL_CLOSED. + * BOUND is a distinct kind from EXACT so a switch handling only EXACT falls + * through to a fail-closed default (never treats a horizon bound as exact). + * ====================================================================== */ +UT_TEST(test_undo_verdict_from_wire_page_mapping) +{ + ClusterGcsUndoVerdictPage v; + ClusterUndoVerdictResult r; + TransactionId xid = 791; + + /* EXACT -> COMMITTED_EXACT{commit_scn,wrap} */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT, xid, 5000, InvalidScn, 7); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + UT_ASSERT_EQ(r.commit_scn, 5000); + UT_ASSERT_EQ(r.wrap, 7); + + /* BELOW_HORIZON -> COMMITTED_BOUND{commit_scn=horizon}; the read_scn + * admissibility gate is the consumer's, not this pure mapper's. */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_BELOW_HORIZON, xid, InvalidScn, 6000, + 0); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_BOUND); + UT_ASSERT_EQ(r.commit_scn, 6000); + + /* Amendment-2: BOUND is never EXACT (distinct kind values), and the + * BOUND result's kind must never equal EXACT. */ + UT_ASSERT_NE(CLUSTER_UNDO_VERDICT_COMMITTED_BOUND, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + UT_ASSERT_NE(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + + /* ABORTED -> ABORTED */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_ABORTED, xid, InvalidScn, InvalidScn, 0); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_ABORTED); + + /* structurally unusable (echo != asked xid) -> UNKNOWN_FAIL_CLOSED */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT, xid, 5000, InvalidScn, 7); + r = cluster_undo_verdict_from_wire_page(&v, xid + 1); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); +} + +/* ====================================================================== + * U2 -- taxonomy zero-value contract (D3-1, L10 / Rule 8.A): a + * zero-initialised result is UNKNOWN_FAIL_CLOSED with InvalidScn, so any + * struct that is not explicitly proven terminal defaults to fail-closed. + * ====================================================================== */ +UT_TEST(test_undo_verdict_zero_value_is_fail_closed) +{ + ClusterUndoVerdictResult r = { 0 }; + + /* the enum's zero value MUST be the fail-closed sentinel */ + UT_ASSERT_EQ((int)CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, 0); + UT_ASSERT_EQ((uint64)InvalidScn, 0); + + /* a {0} result therefore reads as UNKNOWN with no scn */ + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + UT_ASSERT_EQ((uint64)r.commit_scn, (uint64)InvalidScn); + UT_ASSERT_EQ(r.wrap, 0); +} + +/* ====================================================================== + * U3 -- block-proof -> taxonomy mapping (D3-1, CP3 leg): COMMITTED -> + * COMMITTED_EXACT{scn,wrap}; ABORTED -> ABORTED; NONE -> UNKNOWN_FAIL_CLOSED + * (the orchestrator reads a CP3 NONE/UNKNOWN as "fall to CP5", never as the + * caller's terminal answer). + * ====================================================================== */ +UT_TEST(test_undo_verdict_from_block_proof_mapping) +{ + ClusterUndoVerdictResult r; + + r = cluster_undo_verdict_from_block_proof(CLUSTER_VIS_TT_PROOF_COMMITTED, 4242, 9); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + UT_ASSERT_EQ((uint64)r.commit_scn, 4242); + UT_ASSERT_EQ(r.wrap, 9); + + r = cluster_undo_verdict_from_block_proof(CLUSTER_VIS_TT_PROOF_ABORTED, InvalidScn, 0); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_ABORTED); + + r = cluster_undo_verdict_from_block_proof(CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); +} + +int +main(void) +{ + UT_PLAN(3); + UT_RUN(test_undo_verdict_from_wire_page_mapping); + UT_RUN(test_undo_verdict_zero_value_is_fail_closed); + UT_RUN(test_undo_verdict_from_block_proof_mapping); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From 6546bd8645030aec1a16b45278c2270f4df46042 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 14:57:54 +0800 Subject: [PATCH 12/38] feat(cluster): spec-5.22c D3-2 CP3 block0 read via owner-as-master S-grant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the CP3 single-block TT read in the runtime-visibility resolver from the 6.12i best-effort authority-less fetch to the D2 owner-as-master coherent S-grant, gated on cluster.undo_gcs_coherence + peer-mode. cluster_runtime_visibility_try_resolve_remote (CP3 leg): coherence on + peer -> cluster_undo_resid_encode(origin, seg, block0, gen) + cluster_undo_block_acquire_shared() (owner-as-master routing + D2-5 serve-gate + epoch/generation admissibility; the owner ships the image, this peer never opens the foreign undo file). The grant result's authority triple is reconstructed so the version-coverage gate D2 deferred (admissibility used anchor=Invalid) is applied here: hwm >= this tuple's anchor_lsn, else fall to the CP5 verdict leg. coherence off keeps the best-effort fetch byte-for-byte (zero regression); every miss still falls closed to CP5 and ultimately 53R97. Amendment-1: the requester has no independent source for the owner's current segment generation, so it encodes rid.field3 == gen (its own known value). D2's generation_matches then compares two equal caller values and is structurally true -- NOT the anti-ABA gate on this path. The real anti-ABA is the slot-level xid+wrap positive proof (content-based; a recycled segment changes the slot wrap -> proof NONE -> fail-closed). test_cluster_undo_verdict.c U4/U6/U11: pin the pure contracts the CP3 wiring composes (D1 resid encode/master, the version-coverage policy, the D2 armed gate). The orchestration wiring is backend-heavy and forward-covered by the D3-7 / D6 TAP; links the D1 + D2 objects with a cluster_node_id stub. Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3-2, §2.3/§3.5/§4.1) --- .../cluster/cluster_runtime_visibility.c | 50 +++++++++- src/test/cluster_unit/Makefile | 23 +++-- .../cluster_unit/test_cluster_undo_verdict.c | 95 ++++++++++++++++++- 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index 289a063130..0b04315b31 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -49,8 +49,11 @@ #include "cluster/cluster_elog.h" /* cluster_node_id */ #include "cluster/cluster_epoch.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled (D3-2) */ #include "cluster/cluster_runtime_visibility.h" #include "cluster/cluster_uba.h" +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ /* * cluster_vis_live_authority_covers @@ -320,6 +323,7 @@ cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segme ClusterLiveAuthority auth; SCN scn = InvalidScn; uint16 wrap = TT_WRAP_INVALID; + bool got_block = false; if (out_committed != NULL) *out_committed = false; @@ -349,10 +353,50 @@ cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segme * are asking — a fetch miss there is a routing artifact, not evidence. * A successful exact xid+wrap proof still short-circuits the wire; a * fetch miss or a proof NONE falls to the origin-verdict leg. + * + * Block0 acquisition (D3-2): under cluster.undo_gcs_coherence + peer-mode + * the read goes through the D2 owner-as-master coherent S-grant + * (cluster_undo_block_acquire_shared -- owner-as-master routing + D2-5 + * serve-gate + epoch/generation admissibility; the owner ships the image + * and this peer never opens the foreign undo file, invariant #8). Off + * that gate it keeps the 6.12i best-effort authority-less fetch verbatim, + * so coherence off is byte-for-byte the old path (回归安全). + * + * Amendment-1 (generation): the requester has no independent source for + * the owner's current segment generation, so it encodes rid.field3 == gen + * (its own known value: a carried ITL-ref generation if any, else a + * neutral 0). D2's generation_matches(rid, gen) then compares two equal + * caller values and is structurally true -- it is NOT the anti-ABA gate on + * this path. The real anti-ABA is the slot-level xid+wrap positive proof + * below (content-based on the shipped bytes; a recycled segment changes + * the slot wrap, so the proof fails NONE -> fail-closed). */ - if (cluster_undo_block_fetch_for_visibility(origin_node, uba_encode(undo_segment_id, 0, 0, 0), - page.data, &auth) - && cluster_vis_live_authority_covers(anchor_lsn, auth)) { + if (cluster_undo_gcs_coherence && cluster_peer_mode_enabled()) { + ClusterResId rid; + ClusterUndoGrantResult res; + uint32 gen = 0; /* Amendment-1: rid.field3 == gen, self-consistent */ + + cluster_undo_resid_encode((int32)origin_node, undo_segment_id, 0 /* block0 */, gen, &rid); + if (cluster_undo_block_acquire_shared(&rid, gen, page.data, &res)) { + /* Reconstruct the co-sampled authority the D3 version-coverage + * gate below needs (D2 kept only the triple, not the struct). */ + auth.origin_epoch = res.origin_epoch; + auth.live_hwm_lsn = res.live_hwm_lsn; + auth.tt_generation = res.tt_generation; + got_block = true; + } + } else + got_block = cluster_undo_block_fetch_for_visibility( + origin_node, uba_encode(undo_segment_id, 0, 0, 0), page.data, &auth); + + /* + * Version-coverage gate. For the S-grant path this is the gate D2 + * deferred (its admissibility used anchor=Invalid): the owner's shipped + * hwm must cover THIS tuple's anchor_lsn (§3.5). For the best-effort path + * it is the unchanged 6.12i D-i2 window gate. hwm < anchor -> the origin + * durable TT does not yet cover this version -> fall to the verdict leg. + */ + if (got_block && cluster_vis_live_authority_covers(anchor_lsn, auth)) { switch (cluster_vis_tt_block_positive_proof( page.data, undo_segment_id, (uint8)(origin_node + 1), raw_xid, &scn, &wrap)) { case CLUSTER_VIS_TT_PROOF_COMMITTED: diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index b3854803aa..d9cec7e71e 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -1863,17 +1863,22 @@ test_cluster_undo_gcs: test_cluster_undo_gcs.c unit_test.h \ $(CLUSTER_RUNTIME_VIS_POLICY_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ -# spec-5.22c D3-1: test_cluster_undo_verdict — cross-node xid->commit_scn +# spec-5.22c D3-1/D3-2: test_cluster_undo_verdict — cross-node xid->commit_scn # verdict taxonomy (five-value ClusterUndoVerdictKind) + the two pure mapping # helpers (wire ClusterGcsUndoVerdictPage -> taxonomy, block-proof -# ClusterVisTtProof -> taxonomy). The mappers live in the dependency-free -# runtime-visibility policy object (no scn_time_cmp / shmem / elog), so the -# taxonomy layer links standalone against it plus libpgport. The read_scn -# admissibility gate of a COMMITTED_BOUND is the D3-3 consumer's, not these -# pure mappers'; the S-grant / verdict orchestration wiring is backend-heavy -# and forward-covered by the D3-7 / D6 TAP. +# ClusterVisTtProof -> taxonomy), plus the pure contract pins the D3-2 CP3 +# wiring composes (D1 resid encode/master, D2 armed gate, the version-coverage +# policy). All pure decision objects link standalone: the taxonomy mappers + +# covers policy live in the dependency-free runtime-visibility policy object, +# resid encode/master in the D1 object, the armed gate in the D2 routing +# object; the test owns the cluster_node_id stub the routing object needs. +# The read_scn admissibility of a COMMITTED_BOUND is the D3-3 consumer's, and +# the S-grant / verdict orchestration wiring is backend-heavy and +# forward-covered by the D3-7 / D6 TAP. test_cluster_undo_verdict: test_cluster_undo_verdict.c unit_test.h \ - $(CLUSTER_VERSION_O) $(CLUSTER_RUNTIME_VIS_POLICY_O) + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ + $(CLUSTER_RUNTIME_VIS_POLICY_O) $(CC) $(CFLAGS) $(CPPFLAGS) $< \ - $(CLUSTER_VERSION_O) $(CLUSTER_RUNTIME_VIS_POLICY_O) \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_UNDO_GCS_O) \ + $(CLUSTER_RUNTIME_VIS_POLICY_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_undo_verdict.c b/src/test/cluster_unit/test_cluster_undo_verdict.c index e62d664532..d69084543a 100644 --- a/src/test/cluster_unit/test_cluster_undo_verdict.c +++ b/src/test/cluster_unit/test_cluster_undo_verdict.c @@ -36,8 +36,10 @@ #include "postgres.h" #include "cluster/cluster_gcs_block.h" /* ClusterGcsUndoVerdictPage + wire kinds */ -#include "cluster/cluster_runtime_visibility.h" /* ClusterVisTtProof */ +#include "cluster/cluster_runtime_visibility.h" /* ClusterVisTtProof + covers policy */ #include "cluster/cluster_scn.h" /* SCN, InvalidScn */ +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_grant_armed (D3-2 pin) */ +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2 pin) */ #include "cluster/cluster_undo_verdict.h" #undef printf @@ -48,6 +50,13 @@ UT_DEFINE_GLOBALS(); +/* + * Stub self-node global. cluster_undo_gcs.o (the D2 routing object linked for + * the D3-2 armed-gate pin) resolves cluster_node_id from cluster_guc.c in a + * real backend; the test owns it so the routing object links standalone. + */ +int cluster_node_id = 0; + void ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) { @@ -160,13 +169,95 @@ UT_TEST(test_undo_verdict_from_block_proof_mapping) UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); } +/* ====================================================================== + * U4 -- resid encoding contract the D3-2 CP3 path composes (spec-5.22c + * §2.3): owner->field4, seg->field1, block0->field2, gen->field3; the master + * of an undo resid IS the encoded owner (owner-as-master, never a hash). A + * carried ITL-ref generation is preserved in field3 (Amendment-1). + * ====================================================================== */ +UT_TEST(test_undo_verdict_resid_encode_contract) +{ + ClusterResId rid; + int32 owner = -1; + uint32 seg = 0; + uint32 block = 99; + uint32 gen = 99; + + /* D3-2 encodes (origin, segment, block0, gen) */ + cluster_undo_resid_encode(2, 7, 0 /* block0 */, 0 /* gen, Amendment-1 neutral */, &rid); + cluster_undo_resid_decode(&rid, &owner, &seg, &block, &gen); + UT_ASSERT_EQ(owner, 2); + UT_ASSERT_EQ(seg, 7); + UT_ASSERT_EQ(block, 0); + UT_ASSERT_EQ(gen, 0); + + /* owner-as-master: the master IS the owner, never a hash of the tag */ + UT_ASSERT_EQ(cluster_undo_resid_master(&rid), 2); + + /* a carried ITL-ref generation is preserved in field3 */ + cluster_undo_resid_encode(3, 9, 0, 5, &rid); + cluster_undo_resid_decode(&rid, &owner, &seg, &block, &gen); + UT_ASSERT_EQ(gen, 5); + UT_ASSERT_EQ(cluster_undo_resid_master(&rid), 3); +} + +/* ====================================================================== + * U6 -- version-coverage gate the D3-2 CP3 path applies on the S-grant + * authority (D2-deferred, §3.5): hwm >= anchor + matching epoch -> covers; + * hwm < anchor -> refuse (origin durable TT does not yet cover this tuple + * version); epoch mismatch -> refuse (authority from a different reconfig + * generation). Fail-closed on every doubt (Rule 8.A). + * ====================================================================== */ +UT_TEST(test_undo_verdict_version_coverage_gate) +{ + ClusterLiveAuthority auth; + uint64 epoch = 7; + + auth.origin_epoch = epoch; + auth.tt_generation = 1; + + /* hwm >= anchor, epoch matches -> covers */ + auth.live_hwm_lsn = 2000; + UT_ASSERT(cluster_vis_live_authority_covers_policy(1000, auth, epoch)); + + /* hwm < anchor -> does not cover (fall-closed) */ + auth.live_hwm_lsn = 500; + UT_ASSERT(!cluster_vis_live_authority_covers_policy(1000, auth, epoch)); + + /* epoch mismatch -> refuse even with a covering hwm */ + auth.live_hwm_lsn = 2000; + UT_ASSERT(!cluster_vis_live_authority_covers_policy(1000, auth, epoch + 1)); +} + +/* ====================================================================== + * U11 -- coherence-off fallback gate (spec-5.22c §3.4, Q2): the D3-2 CP3 + * path takes the D2 owner-as-master S-grant ONLY when armed (coherence on + * AND peer-mode); every other combination keeps the 6.12i best-effort fetch + * verbatim (回归安全 -- coherence off is byte-for-byte the old path). + * ====================================================================== */ +UT_TEST(test_undo_verdict_coherence_off_fallback) +{ + /* armed only when BOTH coherence and peer-mode hold */ + UT_ASSERT(cluster_undo_grant_armed(true /*coherence*/, true /*peer*/)); + + /* coherence off -> not armed -> best-effort fallback (no regression) */ + UT_ASSERT(!cluster_undo_grant_armed(false /*coherence*/, true /*peer*/)); + + /* single-node / non-peer -> not armed regardless of coherence */ + UT_ASSERT(!cluster_undo_grant_armed(true /*coherence*/, false /*peer*/)); + UT_ASSERT(!cluster_undo_grant_armed(false, false)); +} + int main(void) { - UT_PLAN(3); + UT_PLAN(6); UT_RUN(test_undo_verdict_from_wire_page_mapping); UT_RUN(test_undo_verdict_zero_value_is_fail_closed); UT_RUN(test_undo_verdict_from_block_proof_mapping); + UT_RUN(test_undo_verdict_resid_encode_contract); + UT_RUN(test_undo_verdict_version_coverage_gate); + UT_RUN(test_undo_verdict_coherence_off_fallback); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From d232fe6ff719b7a5f1e777a5a9c6415f7f257a26 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 15:13:15 +0800 Subject: [PATCH 13/38] feat(cluster): spec-5.22c D3-3/D3-4 verdict_resolve entry + master==self local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the D3 cross-node verdict entry the D6 consumer will call, with both routing legs, the CP5 serve-gate precheck, and the shared origin/self resolve. cluster_undo_verdict_resolve (D3-3, cluster_runtime_visibility.c): the entry point. master==self routes to the local durable resolve; master!=self to the CP3 owner-as-master S-grant + CP5 origin verdict, whose (ok, committed, scn, is_bound) outcome is folded into the taxonomy by the new pure cluster_undo_verdict_from_resolve() (Amendment-2: is_bound collapses into the COMMITTED_BOUND kind, so a consumer never sees the side axis). Every unproven path returns UNKNOWN_FAIL_CLOSED (caller keeps 53R97, Rule 8.A). rtvis_resolve_own_xid (D3-4): the owner resolving its OWN xid uses its own durable TT + own CLOG, never acquire_shared (which is fail-closed-forward-D6 for master==self). It reuses cluster_lms_undo_verdict_fill_page -- extracted from lms_undo_verdict_serve (behavior-preserving; the origin serve still calls it after zeroing the full BLCKSZ wire page and co-sampling authority) -- so a node's self answer and its served answer over one xid can never diverge. A COMMITTED_BOUND then applies the same read_scn admissibility (observe horizon, admit iff read_scn at/after it) the foreign path applies. CP5 serve-gate precheck (Q9, rtvis_try_origin_verdict): under the coherent regime a dead owner / in-flight remaster fails closed early with no doomed verdict request on the wire, symmetric with the D2-5 acquire_shared gate; serve = D4. Off the coherent gate the 6.12i best-effort path is unchanged. test_cluster_undo_verdict.c U13: from_resolve folding (ok/committed/is_bound -> kind; BOUND never EXACT). cr_server_policy 7/7 confirms the extraction is behavior-preserving; the origin/self verdict e2e is D3-7 / D6 TAP. Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3-3/D3-4, §2.2/§2.4/§3.1) --- src/backend/cluster/cluster_cr_server.c | 39 ++++++- .../cluster/cluster_runtime_visibility.c | 104 +++++++++++++++++- .../cluster_runtime_visibility_policy.c | 27 +++++ src/include/cluster/cluster_cr_server.h | 8 ++ src/include/cluster/cluster_undo_verdict.h | 28 +++++ .../cluster_unit/test_cluster_undo_verdict.c | 35 +++++- 6 files changed, 235 insertions(+), 6 deletions(-) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 034a76eec1..ab8b1df04d 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -392,9 +392,6 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) { ClusterGcsUndoVerdictPage *v = (ClusterGcsUndoVerdictPage *)slot->result_page; TransactionId xid = slot->undo_xid; - SCN scn = InvalidScn; - SCN horizon = InvalidScn; - uint16 wrap = 0; if (!cluster_crossnode_runtime_visibility) return false; @@ -410,7 +407,43 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) slot->undo_auth.live_hwm_lsn = GetFlushRecPtr(NULL); slot->undo_auth.tt_generation = cluster_undo_tt_retention_rollover_count(); + /* + * Zero the full BLCKSZ wire page (the reply checksum covers it all), then + * fill the verdict fields from the OWN durable TT + CLOG. fill_page is + * shared with the master==self local verdict resolve (D3-4) so the served + * and self answers over one xid can never diverge (Rule 8.A). + */ memset(slot->result_page, 0, BLCKSZ); + return cluster_lms_undo_verdict_fill_page(xid, v); +} + +/* + * cluster_lms_undo_verdict_fill_page -- resolve an OWN xid into a verdict page + * over the COMPLETE own durable TT (cluster_tt_slot_durable_resolve_by_xid) + * cross-checked against the OWN CLOG (AD-006: CLOG is the committed-ness + * authority; the TT carries commit_scn). The caller has already zeroed the + * page buffer (origin: the full BLCKSZ wire page; D3-4 self: sizeof the + * struct) and owns any authority co-sampling; this fills only + * magic/version/xid_echo and the verdict taxonomy fields. true = *v holds + * the verdict; false = refuse (in-doubt / ambiguous / not-own / unresolvable + * -> caller keeps the 53R97 fail-closed boundary, Rule 8.A). Shared so the + * origin-served verdict and the master==self local verdict are byte-for-byte + * the same decision. + */ +bool +cluster_lms_undo_verdict_fill_page(TransactionId xid, ClusterGcsUndoVerdictPage *v) +{ + SCN scn = InvalidScn; + SCN horizon = InvalidScn; + uint16 wrap = 0; + + if (!TransactionIdIsNormal(xid)) + return false; + + /* spec-6.15 D4: only answer for provably-own xids (see banner). */ + if (!cluster_xid_is_mine(xid)) + return false; + v->magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; v->version = CLUSTER_GCS_UNDO_VERDICT_VERSION; v->xid_echo = (uint64)xid; diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index 0b04315b31..dc1bb39b84 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -46,14 +46,16 @@ #include "cluster/cluster_cr_cache.h" #include "cluster/cluster_cr_pool.h" #include "cluster/cluster_cr_server.h" +#include "cluster/cluster_cssd.h" /* cluster_cssd_get_peer_state (D3-3 Q9 serve-gate) */ #include "cluster/cluster_elog.h" /* cluster_node_id */ #include "cluster/cluster_epoch.h" #include "cluster/cluster_guc.h" #include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled (D3-2) */ #include "cluster/cluster_runtime_visibility.h" #include "cluster/cluster_uba.h" -#include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ -#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ +#include "cluster/cluster_undo_verdict.h" /* verdict taxonomy + entry (D3-3/D3-4) */ /* * cluster_vis_live_authority_covers @@ -228,6 +230,23 @@ rtvis_try_origin_verdict(int origin_node, uint32 undo_segment_id, TransactionId ClusterGcsUndoVerdictPage verdict; ClusterLiveAuthority auth; + /* + * Q9: owner-as-master serve-gate precheck (coherent regime only). A dead + * owner or an in-flight remaster fails closed early — with no doomed + * verdict request on the wire — symmetric with the D2-5 acquire_shared + * serve-gate (keyed on the OWNER under owner-as-master, not a hash static + * master). Dead-owner SERVE from shared storage is D4, never a D2/D3 + * false-resolve (Rule 8.A). Off the coherent gate the 6.12i best-effort + * verdict path is unchanged (回归安全). + */ + if (cluster_undo_gcs_coherence && cluster_peer_mode_enabled() + && !cluster_undo_serve_allowed(cluster_cssd_get_peer_state((int32)origin_node) + != CLUSTER_CSSD_PEER_DEAD, + cluster_grd_recovery_in_progress())) { + cluster_rtvis_verdict_note_failclosed(); + return false; + } + cluster_rtvis_verdict_note_wire(); if (!cluster_gcs_block_undo_verdict_fetch_and_wait((int32)origin_node, undo_segment_id, raw_xid, &verdict, &auth)) { @@ -431,4 +450,85 @@ cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segme return false; } +/* + * rtvis_resolve_own_xid — D3-4 master==self local verdict resolve (Q5). + * + * The owner resolving its OWN xid needs no network grant: its own durable TT + * + own CLOG are the authority. It must NOT take acquire_shared, which is + * fail-closed-forward-D6 for master==self (cluster_undo_gcs_grant.c:97). The + * resolution reuses cluster_lms_undo_verdict_fill_page — the very function + * the origin serves foreign requesters with — so a node's self answer and + * its served answer over one xid can never diverge (Rule 8.A). A + * COMMITTED_BOUND then applies the same read_scn admissibility (requester leg + * (e)) the foreign path applies: observe the horizon (Lamport self-heal), + * admit only when read_scn is at/after it, else fail-closed for THIS + * snapshot. + */ +static ClusterUndoVerdictResult +rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) +{ + ClusterGcsUndoVerdictPage v; + ClusterUndoVerdictResult r; + ClusterUndoVerdictResult unknown + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + + memset(&v, 0, sizeof(v)); + if (!cluster_lms_undo_verdict_fill_page(raw_xid, &v)) + return unknown; /* in-doubt / ambiguous / not-own -> fail-closed */ + + r = cluster_undo_verdict_from_wire_page(&v, raw_xid); + + if (r.kind == CLUSTER_UNDO_VERDICT_COMMITTED_BOUND) { + cluster_scn_observe(r.commit_scn); + if (!SCN_VALID(read_scn) || scn_time_cmp(r.commit_scn, read_scn) > 0) + return unknown; /* bound inadmissible for this snapshot; heals next */ + } + return r; +} + +/* + * cluster_undo_verdict_resolve — D3 cross-node xid -> commit_scn verdict entry + * (D3-3). The D6 consumer calls this to decide a seed / fresh-ref tuple. + * + * master==self (owner reads its own xid) routes to the local durable resolve + * (rtvis_resolve_own_xid); master!=self (a peer reading a foreign owner — the + * seed/joiner main scene) routes to the CP3 owner-as-master S-grant + CP5 + * origin verdict (cluster_runtime_visibility_try_resolve_remote), whose + * outcome is folded into the taxonomy (Amendment-2 collapses is_bound into + * the kind, so a consumer never sees the side axis). Every unproven path + * returns UNKNOWN_FAIL_CLOSED so the caller keeps its 53R97 boundary + * (Rule 8.A — never a false-visible edge). + * + * The foreign-path result carries wrap == 0: the wrap is the anti-ABA + * evidence already consumed inside try_resolve_remote (proof + wrap-suspect + * gate), not a consumer output; the self path carries it verbatim from the + * verdict page. + */ +ClusterUndoVerdictResult +cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, + XLogRecPtr anchor_lsn, SCN read_scn) +{ + ClusterUndoVerdictResult unknown + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + bool committed = false; + bool is_bound = false; + SCN commit_scn = InvalidScn; + + if (!cluster_crossnode_runtime_visibility) + return unknown; + if (!TransactionIdIsNormal(raw_xid) || origin_node < 0) + return unknown; + + /* master==self: own CLOG + own durable TT authority (Q5/D3-4). */ + if (origin_node == cluster_node_id) + return rtvis_resolve_own_xid(raw_xid, read_scn); + + /* master!=self: CP3 owner-as-master S-grant + CP5 origin verdict. */ + if (cluster_runtime_visibility_try_resolve_remote(origin_node, undo_segment_id, raw_xid, + anchor_lsn, read_scn, &committed, &commit_scn, + &is_bound)) + return cluster_undo_verdict_from_resolve(true, committed, commit_scn, is_bound); + return unknown; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_runtime_visibility_policy.c b/src/backend/cluster/cluster_runtime_visibility_policy.c index 609b4965e3..c99696cb09 100644 --- a/src/backend/cluster/cluster_runtime_visibility_policy.c +++ b/src/backend/cluster/cluster_runtime_visibility_policy.c @@ -290,3 +290,30 @@ cluster_undo_verdict_from_block_proof(ClusterVisTtProof proof, SCN commit_scn, u return r; } } + +/* + * cluster_undo_verdict_from_resolve + * + * D3-3 Amendment-2 folding point: collapse the runtime resolver's + * (ok, committed, commit_scn, is_bound) outcome into the taxonomy so the + * legacy is_bound boolean never escapes to a consumer. is_bound maps to + * COMMITTED_BOUND (never COMMITTED_EXACT), so a switch that only handles + * EXACT falls to fail-closed on a bound -- a horizon bound is never stamped + * or cached as an exact commit SCN. Pure. + */ +ClusterUndoVerdictResult +cluster_undo_verdict_from_resolve(bool ok, bool committed, SCN commit_scn, bool is_bound) +{ + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + + if (!ok) + return r; /* fetch / covers / deny miss -> fail-closed */ + if (!committed) { + r.kind = CLUSTER_UNDO_VERDICT_ABORTED; + return r; + } + r.kind = is_bound ? CLUSTER_UNDO_VERDICT_COMMITTED_BOUND : CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = commit_scn; + return r; +} diff --git a/src/include/cluster/cluster_cr_server.h b/src/include/cluster/cluster_cr_server.h index 6ac15e08c7..711ee950f3 100644 --- a/src/include/cluster/cluster_cr_server.h +++ b/src/include/cluster/cluster_cr_server.h @@ -181,6 +181,14 @@ extern bool cluster_lms_undo_fetch_submit(const GcsBlockForwardPayload *fwd); * requester keeps its unchanged 53R97). */ extern bool cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd); +/* spec-5.22c D3-4: resolve an OWN xid into a verdict page over the complete + * own durable TT + own CLOG. Shared by the origin-served verdict + * (lms_undo_verdict_serve) and the master==self local verdict resolve so the + * two answers over one xid can never diverge (Rule 8.A). The caller zeroes + * the page buffer and owns any authority co-sampling; true = *v holds the + * verdict, false = refuse (caller keeps 53R97 fail-closed). */ +extern bool cluster_lms_undo_verdict_fill_page(TransactionId xid, ClusterGcsUndoVerdictPage *v); + /* LMS main-loop side: construct every PENDING slot (errors become DENIED * results — fail-closed to the requester, never an LMS restart). */ extern void cluster_lms_cr_drain(void); diff --git a/src/include/cluster/cluster_undo_verdict.h b/src/include/cluster/cluster_undo_verdict.h index 6663e4aa5f..b157c0829a 100644 --- a/src/include/cluster/cluster_undo_verdict.h +++ b/src/include/cluster/cluster_undo_verdict.h @@ -96,4 +96,32 @@ cluster_undo_verdict_from_wire_page(const struct ClusterGcsUndoVerdictPage *v, extern ClusterUndoVerdictResult cluster_undo_verdict_from_block_proof(ClusterVisTtProof proof, SCN commit_scn, uint16 wrap); +/* + * cluster_undo_verdict_from_resolve -- fold the runtime resolver's outcome + * (ok, committed, commit_scn, is_bound) into the taxonomy. This is the + * Amendment-2 folding point: the legacy is_bound boolean becomes the + * COMMITTED_BOUND kind (never COMMITTED_EXACT), so a consumer switch that + * handles only EXACT falls to fail-closed on a bound. !ok -> UNKNOWN; + * committed && is_bound -> BOUND; committed && !is_bound -> EXACT; !committed + * -> ABORTED. Pure. + */ +extern ClusterUndoVerdictResult cluster_undo_verdict_from_resolve(bool ok, bool committed, + SCN commit_scn, bool is_bound); + +/* + * cluster_undo_verdict_resolve — D3 cross-node xid -> commit_scn verdict entry + * (D3-3), the primitive the D6 consumer calls to decide a seed / fresh-ref + * tuple. origin_node = the xid's owner; undo_segment_id = the ITL-ref segment + * (CP3 block0 locator); anchor_lsn = this tuple's page LSN (version-coverage + * gate); read_scn = the snapshot SCN (COMMITTED_BOUND admissibility), or + * InvalidScn for callers without snapshot semantics. master==self routes to + * the local durable resolve; master!=self to the CP3 S-grant + CP5 verdict. + * kind == UNKNOWN_FAIL_CLOSED => the caller keeps 53R97 (never false-visible). + * Compiled only in --enable-cluster builds. + */ +extern ClusterUndoVerdictResult cluster_undo_verdict_resolve(int origin_node, + uint32 undo_segment_id, + TransactionId raw_xid, + XLogRecPtr anchor_lsn, SCN read_scn); + #endif /* CLUSTER_UNDO_VERDICT_H */ diff --git a/src/test/cluster_unit/test_cluster_undo_verdict.c b/src/test/cluster_unit/test_cluster_undo_verdict.c index d69084543a..ef183a91ef 100644 --- a/src/test/cluster_unit/test_cluster_undo_verdict.c +++ b/src/test/cluster_unit/test_cluster_undo_verdict.c @@ -248,16 +248,49 @@ UT_TEST(test_undo_verdict_coherence_off_fallback) UT_ASSERT(!cluster_undo_grant_armed(false, false)); } +/* ====================================================================== + * U13 -- verdict_resolve outcome folding (spec-5.22c D3-3, Amendment-2): + * the entry folds the runtime resolver's (ok, committed, scn, is_bound) into + * the taxonomy. !ok -> UNKNOWN_FAIL_CLOSED; committed & !is_bound -> EXACT; + * committed & is_bound -> BOUND (never EXACT); !committed -> ABORTED. This + * is the point the legacy is_bound boolean is collapsed into the kind so a + * consumer never sees the side axis. + * ====================================================================== */ +UT_TEST(test_undo_verdict_from_resolve_folding) +{ + ClusterUndoVerdictResult r; + + /* resolver failed (fetch/covers/deny miss) -> fail-closed */ + r = cluster_undo_verdict_from_resolve(false, false, InvalidScn, false); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + + /* committed, exact scn -> COMMITTED_EXACT{scn} */ + r = cluster_undo_verdict_from_resolve(true, true, 8000, false); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + UT_ASSERT_EQ((uint64)r.commit_scn, 8000); + + /* committed, bound only -> COMMITTED_BOUND{scn}, NEVER EXACT (Amendment-2) */ + r = cluster_undo_verdict_from_resolve(true, true, 9000, true); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_BOUND); + UT_ASSERT_EQ((uint64)r.commit_scn, 9000); + UT_ASSERT_NE(r.kind, CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + + /* not committed -> ABORTED */ + r = cluster_undo_verdict_from_resolve(true, false, InvalidScn, false); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_ABORTED); +} + int main(void) { - UT_PLAN(6); + UT_PLAN(7); UT_RUN(test_undo_verdict_from_wire_page_mapping); UT_RUN(test_undo_verdict_zero_value_is_fail_closed); UT_RUN(test_undo_verdict_from_block_proof_mapping); UT_RUN(test_undo_verdict_resid_encode_contract); UT_RUN(test_undo_verdict_version_coverage_gate); UT_RUN(test_undo_verdict_coherence_off_fallback); + UT_RUN(test_undo_verdict_from_resolve_folding); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From a81b0bf4742f7056758027b27972603b472decfa Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 15:19:44 +0800 Subject: [PATCH 14/38] feat(cluster): spec-5.22c D3-5/D3-6 8.A negative tests + self-path notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D3-5 (8.A gate closure): pin the fail-closed boundaries of the verdict layer. Every unproven / malformed / ambiguous outcome must land UNKNOWN_FAIL_CLOSED, never a guessed commit. test_cluster_undo_verdict.c U7/U8/U10/U12: U7 malformed verdict page (EXACT w/o scn, BELOW_HORIZON w/ stray scn, unknown kind, wrong magic) -> UNKNOWN. U8 segment-generation gate is structural, not anti-ABA (Amendment-1): rid.field3==gen makes generation_matches structurally true; it is a real comparison (a genuine mismatch fails closed) that D3 makes always-true by construction, so it is no independent proof. U10 IN_PROGRESS is reserved (value 4) but never produced -- every mapper folds an unproven-terminal outcome to UNKNOWN, never IN_PROGRESS. U12 the slot-wrap positive proof is the real anti-ABA even under a structurally-true generation gate: a proof NONE -> UNKNOWN. The proof truth table itself is covered by test_cluster_runtime_visibility (the pure helper D3-2's CP3 reuses verbatim), not duplicated here. No new production code: the fail-closed behaviour is by construction in D3-1..D3-4; these pin it. D3-6 (observability): the master==self local resolve now counts its terminal outcome on the same rtvis resolve counters the foreign path uses (committed / aborted / fail-closed), so both entry legs are observable via the existing dump. No new shmem region or wait event (reuses D2-6). cluster_unit test_cluster_undo_verdict 11/11. Spec: spec-5.22c-ttslot-verdict-shared-undo.md (D3-5/D3-6, §3.2/§4.1/§1.2) --- .../cluster/cluster_runtime_visibility.c | 20 ++- .../cluster_unit/test_cluster_undo_verdict.c | 120 +++++++++++++++++- 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index dc1bb39b84..ea2911e4b8 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -473,16 +473,30 @@ rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; memset(&v, 0, sizeof(v)); - if (!cluster_lms_undo_verdict_fill_page(raw_xid, &v)) - return unknown; /* in-doubt / ambiguous / not-own -> fail-closed */ + if (!cluster_lms_undo_verdict_fill_page(raw_xid, &v)) { + cluster_rtvis_resolve_note_failclosed(); /* D3-6: self-path observability */ + return unknown; /* in-doubt / ambiguous / not-own -> fail-closed */ + } r = cluster_undo_verdict_from_wire_page(&v, raw_xid); if (r.kind == CLUSTER_UNDO_VERDICT_COMMITTED_BOUND) { cluster_scn_observe(r.commit_scn); - if (!SCN_VALID(read_scn) || scn_time_cmp(r.commit_scn, read_scn) > 0) + if (!SCN_VALID(read_scn) || scn_time_cmp(r.commit_scn, read_scn) > 0) { + cluster_rtvis_resolve_note_failclosed(); return unknown; /* bound inadmissible for this snapshot; heals next */ + } } + + /* D3-6: count the self-path terminal outcome on the same rtvis resolve + * counters the foreign path uses (total verdict resolutions by outcome). */ + if (r.kind == CLUSTER_UNDO_VERDICT_COMMITTED_EXACT + || r.kind == CLUSTER_UNDO_VERDICT_COMMITTED_BOUND) + cluster_rtvis_resolve_note_committed(); + else if (r.kind == CLUSTER_UNDO_VERDICT_ABORTED) + cluster_rtvis_resolve_note_aborted(); + else + cluster_rtvis_resolve_note_failclosed(); return r; } diff --git a/src/test/cluster_unit/test_cluster_undo_verdict.c b/src/test/cluster_unit/test_cluster_undo_verdict.c index ef183a91ef..e0fab794b5 100644 --- a/src/test/cluster_unit/test_cluster_undo_verdict.c +++ b/src/test/cluster_unit/test_cluster_undo_verdict.c @@ -280,10 +280,124 @@ UT_TEST(test_undo_verdict_from_resolve_folding) UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_ABORTED); } +/* ====================================================================== + * U7 -- 8.A gate: a structurally malformed verdict page is never evidence + * (spec-5.22c §3.2, Q7). An EXACT kind with no commit_scn, a BELOW_HORIZON + * carrying a stray exact scn, an unknown kind, or a wrong magic all fail + * closed to UNKNOWN_FAIL_CLOSED -- never guessed committed. + * ====================================================================== */ +UT_TEST(test_undo_verdict_wire_page_malformed_fail_closed) +{ + ClusterGcsUndoVerdictPage v; + ClusterUndoVerdictResult r; + TransactionId xid = 555; + + /* EXACT kind but no commit_scn (unstamped) -> UNKNOWN */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT, xid, InvalidScn, InvalidScn, 0); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + + /* BELOW_HORIZON but carries a stray exact commit_scn -> UNKNOWN */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_BELOW_HORIZON, xid, 5000, 6000, 0); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + + /* unknown verdict kind (99) -> UNKNOWN (never guess) */ + make_verdict_page(&v, 99, xid, InvalidScn, InvalidScn, 0); + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + + /* wrong magic (corrupt / foreign carrier) -> UNKNOWN */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT, xid, 5000, InvalidScn, 3); + v.magic = 0xDEADBEEFu; + r = cluster_undo_verdict_from_wire_page(&v, xid); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); +} + +/* ====================================================================== + * U8 -- segment-generation gate is structural, not the anti-ABA gate + * (spec-5.22c Amendment-1, Q4). D3 encodes rid.field3 == gen (its own known + * value), so D2's generation_matches compares two equal caller values and is + * structurally TRUE -- it is a real comparison (a genuine mismatch fails + * closed), but D3 makes it always-true by construction, so it provides NO + * independent anti-ABA proof on the requester path. + * ====================================================================== */ +UT_TEST(test_undo_verdict_generation_structural_not_anti_aba) +{ + ClusterResId rid; + + cluster_undo_resid_encode(2, 7, 0, 4, &rid); + + /* field3 == gen -> structurally passes (D3's construction) */ + UT_ASSERT(cluster_undo_resid_generation_matches(&rid, 4)); + + /* it IS a real comparison: a genuine mismatch fails closed. D3 chooses + * gen == field3 to make it pass, it does not disable the check. */ + UT_ASSERT(!cluster_undo_resid_generation_matches(&rid, 5)); + + /* The real anti-ABA on the requester path is the slot-level xid+wrap + * positive proof (cluster_vis_tt_block_positive_proof), whose recycle / + * ambiguity truth table is covered by test_cluster_runtime_visibility and + * consumed verbatim by the D3-2 CP3 leg. */ +} + +/* ====================================================================== + * U10 -- IN_PROGRESS is reserved but never produced by D3 (spec-5.22c Q6, + * §3.3). The taxonomy keeps the value位 for a future write-path consumer, + * but every D3 mapper folds an unproven-terminal outcome to UNKNOWN, never + * IN_PROGRESS (cross-node has no origin-live ProcArray proof; conflating + * "running" with "crash-lost" would breach 8.A). + * ====================================================================== */ +UT_TEST(test_undo_verdict_in_progress_never_produced) +{ + ClusterGcsUndoVerdictPage v; + TransactionId xid = 321; + + UT_ASSERT_EQ((int)CLUSTER_UNDO_VERDICT_IN_PROGRESS, 4); /* value位 reserved */ + + /* no wire kind maps to IN_PROGRESS (wire is EXACT/BELOW_HORIZON/ABORTED) */ + make_verdict_page(&v, CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT, xid, 5000, InvalidScn, 1); + UT_ASSERT_NE(cluster_undo_verdict_from_wire_page(&v, xid).kind, + CLUSTER_UNDO_VERDICT_IN_PROGRESS); + + /* from_resolve never yields IN_PROGRESS across its whole domain */ + UT_ASSERT_NE(cluster_undo_verdict_from_resolve(false, false, InvalidScn, false).kind, + CLUSTER_UNDO_VERDICT_IN_PROGRESS); + UT_ASSERT_NE(cluster_undo_verdict_from_resolve(true, true, 1, false).kind, + CLUSTER_UNDO_VERDICT_IN_PROGRESS); + UT_ASSERT_NE(cluster_undo_verdict_from_resolve(true, true, 1, true).kind, + CLUSTER_UNDO_VERDICT_IN_PROGRESS); + UT_ASSERT_NE(cluster_undo_verdict_from_resolve(true, false, InvalidScn, false).kind, + CLUSTER_UNDO_VERDICT_IN_PROGRESS); +} + +/* ====================================================================== + * U12 -- the slot-wrap positive proof is the real anti-ABA, even when the + * segment-generation gate is structurally true (spec-5.22c Amendment-1). A + * recycled segment changes the slot bytes so the xid+wrap proof comes back + * NONE; D3 maps a NONE proof to UNKNOWN_FAIL_CLOSED (never committed). The + * proof truth table itself lives in test_cluster_runtime_visibility; here we + * pin the D3 layer's fail-closed handling of a proof miss under a + * structurally-true generation gate. + * ====================================================================== */ +UT_TEST(test_undo_verdict_slot_wrap_is_the_real_anti_aba) +{ + ClusterResId rid; + ClusterUndoVerdictResult r; + + /* generation gate structurally true (does not weaken safety) ... */ + cluster_undo_resid_encode(2, 7, 0, 4, &rid); + UT_ASSERT(cluster_undo_resid_generation_matches(&rid, 4)); + + /* ... yet a recycled segment -> proof NONE -> D3 fail-closes to UNKNOWN */ + r = cluster_undo_verdict_from_block_proof(CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + UT_ASSERT_EQ(r.kind, CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); +} + int main(void) { - UT_PLAN(7); + UT_PLAN(11); UT_RUN(test_undo_verdict_from_wire_page_mapping); UT_RUN(test_undo_verdict_zero_value_is_fail_closed); UT_RUN(test_undo_verdict_from_block_proof_mapping); @@ -291,6 +405,10 @@ main(void) UT_RUN(test_undo_verdict_version_coverage_gate); UT_RUN(test_undo_verdict_coherence_off_fallback); UT_RUN(test_undo_verdict_from_resolve_folding); + UT_RUN(test_undo_verdict_wire_page_malformed_fail_closed); + UT_RUN(test_undo_verdict_generation_structural_not_anti_aba); + UT_RUN(test_undo_verdict_in_progress_never_produced); + UT_RUN(test_undo_verdict_slot_wrap_is_the_real_anti_aba); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 36098ecaf60abf2d0a35fa64ece3211aad1a52f5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 21:16:51 +0800 Subject: [PATCH 15/38] feat(cluster): resolve fresh remote-ITL-ref visibility via cross-node undo verdict A fresh joiner reading a catalog tuple that a peer committed before the joiner came up finds an empty local TT overlay, so the remote-ITL-ref classification left {REMOTE, UNKNOWN} on the miss and the visibility consumer failed closed (53R97 -> false-invisible). Route that branch through the shared-undo cross-node verdict (cluster_undo_verdict_resolve) so the owner-as-master resolves the tuple's commit outcome. - cluster_visibility_resolve.c: widen the fresh-remote-ITL-ref branch to ask the origin verdict when cluster.crossnode_runtime_visibility is on and the origin is a peer; every unproven leg keeps the unchanged 53R97 fail-closed boundary. - cluster_visibility_verdict.c: pure helpers cluster_vis_from_undo_verdict() and cluster_vis_freshref_origin_decision() (derivable-time integrity check; an underivable origin still asks the verdict rather than failing closed). - KIND_UNDO_VERDICT wire: carry an authoritative (physical-binding) flag. A fresh-ref request whose origin is the page's physical ITL binding lets the origin serve from its own durable TT + CLOG, skipping the stripe self-check precheck; derived (recycled) requests keep cluster_xid_is_mine unchanged so the overlapping-value guard is not relaxed for them. The authoritative path does not weaken the positive proof (RESOLVED_SCN still requires TransactionIdDidCommit + wrap-suspect). - cluster_cr.c / cluster_debug.c: two cr counters vis_freshref_verdict_{resolved,failclosed}_count (cr category 57 -> 59). - tests: unit test_cluster_vis_undo_verdict_map (U1-U10); 2-node e2e t/359_cluster_5_22f_seed_visibility.pl; sync the six cr category-count assertions 57 -> 59. Local gates: cluster_unit 161/161, PG 219/219 (cassert), cluster_regress 13/13, cr/verdict/recovery TAP band green. t/347 fails a pre-existing same-epoch-restart recovery-ownership path on this host (reproduced on the base commit without these changes, node restart FATAL "crash recovery of a cluster-coherent page requires a recovery-ownership window"); merge-window fast-gate/nightly gate it. Spec: spec-5.22f-shared-catalog-seed-visibility-consumer.md --- src/backend/cluster/cluster_cr.c | 33 ++ src/backend/cluster/cluster_cr_server.c | 24 +- src/backend/cluster/cluster_debug.c | 5 + src/backend/cluster/cluster_gcs_block.c | 4 +- .../cluster/cluster_runtime_visibility.c | 19 +- .../cluster/cluster_visibility_resolve.c | 63 ++++ .../cluster/cluster_visibility_verdict.c | 83 +++++ src/include/cluster/cluster_cr.h | 6 + src/include/cluster/cluster_cr_server.h | 11 +- src/include/cluster/cluster_gcs_block.h | 23 +- .../cluster/cluster_runtime_visibility.h | 1 + src/include/cluster/cluster_undo_verdict.h | 11 +- .../cluster/cluster_visibility_resolve.h | 52 +++ .../t/215_cluster_3_9_cr_construction.pl | 4 +- .../t/216_cluster_3_10_cr_cache.pl | 4 +- .../t/254_pi_cr_recovery_acceptance.pl | 3 +- .../t/300_cluster_5_50_cr_profile.pl | 4 +- .../t/306_cluster_5_53_cr_key_reuse.pl | 2 +- .../t/311_cluster_5_56_cr_lifecycle.pl | 2 +- .../t/359_cluster_5_22f_seed_visibility.pl | 301 ++++++++++++++++++ src/test/cluster_unit/Makefile | 16 +- src/test/cluster_unit/test_cluster_debug.c | 11 + .../test_cluster_vis_undo_verdict_map.c | 249 +++++++++++++++ 23 files changed, 897 insertions(+), 34 deletions(-) create mode 100644 src/test/cluster_tap/t/359_cluster_5_22f_seed_visibility.pl create mode 100644 src/test/cluster_unit/test_cluster_vis_undo_verdict_map.c diff --git a/src/backend/cluster/cluster_cr.c b/src/backend/cluster/cluster_cr.c index 1f2b8ed1af..7817d7fbf9 100644 --- a/src/backend/cluster/cluster_cr.c +++ b/src/backend/cluster/cluster_cr.c @@ -169,6 +169,17 @@ typedef struct ClusterCRShared { * striped cluster). */ pg_atomic_uint64 rtvis_underivable_failclosed_count; + /* + * spec-5.22f D6-3: fresh-remote-ITL-ref widening outcome counters. A + * fresh ref (local_xid == raw_xid, origin != self) that missed the overlay + * and was resolved by the D3 cross-node verdict (classify_ref_guts D6 + * branch): resolved = a terminal COMMITTED/ABORTED verdict consumed + * (the L1 root-cause-#1 hard assert lands here, proving the fresh-ref path + * ran rather than an overlay / ordinary-propagation hit); failclosed = a + * STALE integrity mismatch or an UNKNOWN verdict kept 53R97. + */ + pg_atomic_uint64 vis_freshref_verdict_resolved_count; + pg_atomic_uint64 vis_freshref_verdict_failclosed_count; } ClusterCRShared; static ClusterCRShared *CRShared = NULL; @@ -251,6 +262,8 @@ cluster_cr_shmem_init(void) pg_atomic_init_u64(&CRShared->cr_server_verdict_served_count, 0); pg_atomic_init_u64(&CRShared->cr_server_verdict_denied_count, 0); pg_atomic_init_u64(&CRShared->rtvis_underivable_failclosed_count, 0); + pg_atomic_init_u64(&CRShared->vis_freshref_verdict_resolved_count, 0); + pg_atomic_init_u64(&CRShared->vis_freshref_verdict_failclosed_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_resolved_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_recycled_invisible_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_invalid_or_ambiguous_count, 0); @@ -411,6 +424,22 @@ cluster_rtvis_note_underivable_failclosed(void) if (CRShared != NULL) pg_atomic_fetch_add_u64(&CRShared->rtvis_underivable_failclosed_count, 1); } + +/* PGRAC: spec-5.22f D6-3 — fresh-remote-ITL-ref widening outcome bumps + * (classify_ref_guts, backend context). */ +void +cluster_vis_freshref_verdict_note_resolved(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->vis_freshref_verdict_resolved_count, 1); +} + +void +cluster_vis_freshref_verdict_note_failclosed(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->vis_freshref_verdict_failclosed_count, 1); +} CR_COUNTER_ACCESSOR(cluster_cr_corruption_count, cr_corruption_count) CR_COUNTER_ACCESSOR(cluster_cr_chain_walk_steps_sum, cr_chain_walk_steps_sum) CR_COUNTER_ACCESSOR(cluster_cr_inverse_insert_count, cr_inverse_insert_count) @@ -448,6 +477,10 @@ CR_COUNTER_ACCESSOR(cluster_rtvis_verdict_inadmissible_count, rtvis_verdict_inad CR_COUNTER_ACCESSOR(cluster_cr_server_verdict_served_count, cr_server_verdict_served_count) CR_COUNTER_ACCESSOR(cluster_cr_server_verdict_denied_count, cr_server_verdict_denied_count) CR_COUNTER_ACCESSOR(cluster_rtvis_underivable_failclosed_count, rtvis_underivable_failclosed_count) +/* spec-5.22f D6-3: fresh-remote-ITL-ref widening outcome counters. */ +CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_resolved_count, vis_freshref_verdict_resolved_count) +CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_failclosed_count, + vis_freshref_verdict_failclosed_count) /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ CR_COUNTER_ACCESSOR(cluster_cr_xmax_resolved_count, cr_xmax_resolved_count) CR_COUNTER_ACCESSOR(cluster_cr_xmax_recycled_invisible_count, cr_xmax_recycled_invisible_count) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index ab8b1df04d..d290b159ab 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -283,6 +283,7 @@ cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd) slot->undo_segment_id = segment_id; slot->undo_block_no = block_no; slot->undo_xid = (TransactionId)carrier; + slot->undo_authoritative = GcsBlockForwardPayloadIsUndoVerdictAuthoritative(fwd); memset(&slot->undo_auth, 0, sizeof(slot->undo_auth)); /* Publish the request fields before LMS can observe PENDING. */ @@ -398,8 +399,14 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) if (!TransactionIdIsNormal(xid)) return false; - /* spec-6.15 D4: only answer for provably-own xids (see banner). */ - if (!cluster_xid_is_mine(xid)) + /* + * spec-6.15 D4: only answer for provably-own xids (see banner) -- UNLESS + * the request is spec-5.22f D6-7 AUTHORITATIVE (origin chosen from the fresh + * ref's physical ITL binding, so the requester already proved this is the + * correct owner; the underivable own seed xid is answered over the durable-TT + * + CLOG authority below, the positive proof unchanged). + */ + if (!slot->undo_authoritative && !cluster_xid_is_mine(xid)) return false; /* Co-sample the authority triple BEFORE the scan (see banner). */ @@ -414,7 +421,7 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) * and self answers over one xid can never diverge (Rule 8.A). */ memset(slot->result_page, 0, BLCKSZ); - return cluster_lms_undo_verdict_fill_page(xid, v); + return cluster_lms_undo_verdict_fill_page(xid, slot->undo_authoritative, v); } /* @@ -431,7 +438,8 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) * the same decision. */ bool -cluster_lms_undo_verdict_fill_page(TransactionId xid, ClusterGcsUndoVerdictPage *v) +cluster_lms_undo_verdict_fill_page(TransactionId xid, bool authoritative, + ClusterGcsUndoVerdictPage *v) { SCN scn = InvalidScn; SCN horizon = InvalidScn; @@ -440,8 +448,12 @@ cluster_lms_undo_verdict_fill_page(TransactionId xid, ClusterGcsUndoVerdictPage if (!TransactionIdIsNormal(xid)) return false; - /* spec-6.15 D4: only answer for provably-own xids (see banner). */ - if (!cluster_xid_is_mine(xid)) + /* spec-6.15 D4: only answer for provably-own xids (see banner) -- UNLESS the + * caller is spec-5.22f D6-7 AUTHORITATIVE (physical-binding fresh ref), in + * which case the durable-TT + CLOG scan below is the authority for an + * underivable own xid. The derived (recycled) and master==self callers pass + * false and keep the self-check. */ + if (!authoritative && !cluster_xid_is_mine(xid)) return false; v->magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index d26fa4b6b3..6819ba3529 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2588,6 +2588,11 @@ dump_cr(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_cr_server_verdict_denied_count())); emit_row(rsinfo, "cr", "rtvis_underivable_failclosed_count", fmt_int64((int64)cluster_rtvis_underivable_failclosed_count())); + /* spec-5.22f D6-3: fresh-remote-ITL-ref widening outcomes. */ + emit_row(rsinfo, "cr", "vis_freshref_verdict_resolved_count", + fmt_int64((int64)cluster_vis_freshref_verdict_resolved_count())); + emit_row(rsinfo, "cr", "vis_freshref_verdict_failclosed_count", + fmt_int64((int64)cluster_vis_freshref_verdict_failclosed_count())); /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ emit_row(rsinfo, "cr", "cr_xmax_resolved_count", fmt_int64((int64)cluster_cr_xmax_resolved_count())); diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 3e541e751d..bb85e19fb4 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -2510,7 +2510,7 @@ cluster_gcs_block_undo_tt_fetch_and_wait(int32 origin_node, uint32 segment_id, u */ bool cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_id, - TransactionId xid, + TransactionId xid, bool authoritative, ClusterGcsUndoVerdictPage *verdict_out, ClusterLiveAuthority *auth_out) { @@ -2559,7 +2559,7 @@ cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_ fwd.requester_backend_id = (int32)MyBackendId; fwd.master_node = cluster_node_id; fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, true); + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, authoritative); /* The widened xid rides the watermark carrier (upper 32 bits zero). */ GcsBlockForwardPayloadSetExpectedPiWatermarkScn(&fwd, (SCN)(uint64)xid); diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index ea2911e4b8..039fec8416 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -224,7 +224,7 @@ cluster_undo_block_fetch_for_visibility(int origin_node, UBA uba, char *out_page */ static bool rtvis_try_origin_verdict(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, - XLogRecPtr anchor_lsn, SCN read_scn, bool *out_committed, + XLogRecPtr anchor_lsn, SCN read_scn, bool authoritative, bool *out_committed, SCN *out_commit_scn, bool *out_commit_scn_is_bound) { ClusterGcsUndoVerdictPage verdict; @@ -249,7 +249,7 @@ rtvis_try_origin_verdict(int origin_node, uint32 undo_segment_id, TransactionId cluster_rtvis_verdict_note_wire(); if (!cluster_gcs_block_undo_verdict_fetch_and_wait((int32)origin_node, undo_segment_id, raw_xid, - &verdict, &auth)) { + authoritative, &verdict, &auth)) { cluster_rtvis_verdict_note_failclosed(); return false; } @@ -335,7 +335,7 @@ rtvis_try_origin_verdict(int origin_node, uint32 undo_segment_id, TransactionId bool cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, XLogRecPtr anchor_lsn, - SCN read_scn, bool *out_committed, + SCN read_scn, bool authoritative, bool *out_committed, SCN *out_commit_scn, bool *out_commit_scn_is_bound) { PGAlignedBlock page; @@ -439,7 +439,8 @@ cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segme * complete own-TT verdict instead of failing closed outright. */ if (rtvis_try_origin_verdict(origin_node, undo_segment_id, raw_xid, anchor_lsn, read_scn, - out_committed, out_commit_scn, out_commit_scn_is_bound)) { + authoritative, out_committed, out_commit_scn, + out_commit_scn_is_bound)) { if (*out_committed) cluster_rtvis_resolve_note_committed(); else @@ -473,7 +474,9 @@ rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; memset(&v, 0, sizeof(v)); - if (!cluster_lms_undo_verdict_fill_page(raw_xid, &v)) { + /* master==self keeps the stripe self-check (authoritative=false): D6-7's + * physical-binding relaxation is for the FOREIGN fresh-ref serve only. */ + if (!cluster_lms_undo_verdict_fill_page(raw_xid, false, &v)) { cluster_rtvis_resolve_note_failclosed(); /* D3-6: self-path observability */ return unknown; /* in-doubt / ambiguous / not-own -> fail-closed */ } @@ -520,7 +523,7 @@ rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) */ ClusterUndoVerdictResult cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, - XLogRecPtr anchor_lsn, SCN read_scn) + XLogRecPtr anchor_lsn, SCN read_scn, bool authoritative) { ClusterUndoVerdictResult unknown = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; @@ -539,8 +542,8 @@ cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, Transactio /* master!=self: CP3 owner-as-master S-grant + CP5 origin verdict. */ if (cluster_runtime_visibility_try_resolve_remote(origin_node, undo_segment_id, raw_xid, - anchor_lsn, read_scn, &committed, &commit_scn, - &is_bound)) + anchor_lsn, read_scn, authoritative, &committed, + &commit_scn, &is_bound)) return cluster_undo_verdict_from_resolve(true, committed, commit_scn, is_bound); return unknown; } diff --git a/src/backend/cluster/cluster_visibility_resolve.c b/src/backend/cluster/cluster_visibility_resolve.c index 7c1db01671..049285536f 100644 --- a/src/backend/cluster/cluster_visibility_resolve.c +++ b/src/backend/cluster/cluster_visibility_resolve.c @@ -446,6 +446,7 @@ classify_ref_guts(TransactionId raw_xid, const ClusterUndoTTSlotRef *ref, XLogRe if (cluster_runtime_visibility_try_resolve_remote( derived_origin, (uint32)ref->undo_segment_id, raw_xid, anchor_lsn, read_scn, + false /* derived origin: keep the stripe self-check (6.12i P0) */, &committed, &scn, &is_bound)) { out->evidence = CLUSTER_VIS_EVIDENCE_REMOTE; out->status @@ -465,6 +466,68 @@ classify_ref_guts(TransactionId raw_xid, const ClusterUndoTTSlotRef *ref, XLogRe } resolve_from_remote_ref(raw_xid, ref, out); + + /* + * PGRAC: spec-5.22f D6-2 — fresh-remote-ITL-ref widening (root-cause #1 + * seed/joiner visibility consumer). + * + * A fresh remote ITL ref reaches here (origin != self, local_xid == + * raw_xid: the slot still binds raw_xid, so ref->origin_node_id + + * undo_segment_id are the tuple page's PHYSICAL ITL binding = the xid's + * true owner + CP3 block0 locator). A fresh joiner reading a seed-committed + * tuple has an EMPTY TT overlay, so resolve_from_remote_ref left + * {REMOTE, UNKNOWN} on the miss -> 53R97 -> false-invisible (命门 2). Ask + * the live owner via the D3 cross-node verdict instead of fail-closing. + * + * Unlike the recycled branch (:436), which MUST derive the origin from the + * xid value (a recycled ref names the slot's NEW owner, unrelated to + * raw_xid — the 6.12i alias P0), the fresh ref's ref->origin_node_id is + * authoritative. cluster_xid_origin_slot is only a derivable-time integrity + * cross-check (Q2 / Option B, P1-a): underivable (below-floor pre-striping + * seed / striping off) STILL asks the verdict — Rule 8.A safety is anchored + * INSIDE D3 (wrap-suspect anti-ABA / covers / serve gates fail-close every + * unproven leg), never on a "no alias" assumption. crossnode off / a + * derivable mismatch / an UNKNOWN verdict all keep STALE_OR_AMBIGUOUS -> + * 53R97 (Rule 8.A: this branch only widens "resolve when provable"). + */ + if (out->status == CLUSTER_TT_STATUS_UNKNOWN && cluster_crossnode_runtime_visibility + && (int32)ref->origin_node_id != cluster_node_id) { + int derived = cluster_xid_origin_slot(raw_xid); /* derivable-time check only */ + + switch (cluster_vis_freshref_origin_decision(derived, (int32)ref->origin_node_id)) { + case CLUSTER_VIS_FRESHREF_ORIGIN_STALE: + /* derivable mismatch: striping bug / page corruption / alias. No + * verdict is asked, so only the dedicated fresh-ref counter moves + * -- mirroring the recycled underivable leg (:458), which likewise + * does not touch the shared rtvis_resolve totals. */ + out->evidence = CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS; + cluster_vis_freshref_verdict_note_failclosed(); + break; + + case CLUSTER_VIS_FRESHREF_ORIGIN_ASK: + default: { + /* origin = ref->origin_node_id (physical binding, authoritative); + * every unproven leg inside D3 returns UNKNOWN_FAIL_CLOSED. + * cluster_undo_verdict_resolve -> try_resolve_remote already bumps + * the shared rtvis_resolve_{committed,aborted,failclosed} totals + * internally (cluster_runtime_visibility.c:424/427/444/446/449), so + * the dedicated fresh-ref counter is the ONLY extra bump here -- an + * explicit rtvis_resolve_note here would double-count. */ + ClusterUndoVerdictResult v + = cluster_undo_verdict_resolve((int)ref->origin_node_id, + (uint32)ref->undo_segment_id, raw_xid, anchor_lsn, + read_scn, true /* fresh ref: physical-binding authority */); + + if (cluster_vis_from_undo_verdict(v, out)) { + cluster_vis_freshref_verdict_note_resolved(); + return; + } + /* verdict UNKNOWN: cluster_vis_from_undo_verdict already set STALE. */ + cluster_vis_freshref_verdict_note_failclosed(); + break; + } + } + } } /* diff --git a/src/backend/cluster/cluster_visibility_verdict.c b/src/backend/cluster/cluster_visibility_verdict.c index 8b1c146ea3..5f4c7d0dc5 100644 --- a/src/backend/cluster/cluster_visibility_verdict.c +++ b/src/backend/cluster/cluster_visibility_verdict.c @@ -190,4 +190,87 @@ cluster_vis_dirty_verdict(ClusterTTStatus status, bool is_xmax, bool is_delete) } } +/* ============================================================ + * spec-5.22f D6-1: fresh-remote-ITL-ref visibility consumer helpers. + * + * Two PURE helpers the classify_ref_guts fresh-ref widening (D6-2) calls + * to consume a D3 cross-node verdict. No I/O, no shmem, no elog, so they + * unit-test by full enumeration (test_cluster_vis_undo_verdict_map, + * U1-U10) alongside the OBS-2~5 tables above. + * ============================================================ */ + +/* + * cluster_vis_from_undo_verdict -- map a D3 five-value verdict onto the local + * visibility out-params. See the header for the truth table. EVERY branch + * overwrites commit_scn + commit_scn_is_bound so a non-terminal verdict never + * leaks a residual scn (U7). Returns true iff a terminal (COMMITTED/ABORTED) + * outcome was produced; false keeps the caller on the 53R97 fail-closed path. + */ +bool +cluster_vis_from_undo_verdict(ClusterUndoVerdictResult v, ClusterVisResolve *out) +{ + switch (v.kind) { + case CLUSTER_UNDO_VERDICT_COMMITTED_EXACT: + out->evidence = CLUSTER_VIS_EVIDENCE_REMOTE; + out->status = CLUSTER_TT_STATUS_COMMITTED; + out->commit_scn = v.commit_scn; + out->commit_scn_is_bound = false; + return true; + + case CLUSTER_UNDO_VERDICT_COMMITTED_BOUND: + /* + * A below-horizon bound decides correctly ONLY against the read_scn + * this resolve ran under; commit_scn_is_bound forbids stamping/caching + * it as an exact commit_scn (spec-6.12i CP5 / Rule 8.A). + */ + out->evidence = CLUSTER_VIS_EVIDENCE_REMOTE; + out->status = CLUSTER_TT_STATUS_COMMITTED; + out->commit_scn = v.commit_scn; + out->commit_scn_is_bound = true; + return true; + + case CLUSTER_UNDO_VERDICT_ABORTED: + out->evidence = CLUSTER_VIS_EVIDENCE_REMOTE; + out->status = CLUSTER_TT_STATUS_ABORTED; + out->commit_scn = InvalidScn; + out->commit_scn_is_bound = false; + return true; + + case CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED: + case CLUSTER_UNDO_VERDICT_IN_PROGRESS: + default: + /* + * Not proven terminal -> STALE_OR_AMBIGUOUS so the caller keeps 53R97 + * (Rule 8.A / L10: a zero / unknown verdict is never visible). D3 + * folds a proven-live xid to UNKNOWN already; IN_PROGRESS is handled + * here only defensively. + */ + out->evidence = CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS; + out->status = CLUSTER_TT_STATUS_UNKNOWN; + out->commit_scn = InvalidScn; + out->commit_scn_is_bound = false; + return false; + } +} + +/* + * cluster_vis_freshref_origin_decision -- the fresh-ref origin decision + * (spec-5.22f Q2 / Option B, P1-a). A fresh ref's ref_origin is the tuple + * page's physical ITL binding (the true owner); the value-derived slot is only + * a derivable-time integrity cross-check. Underivable (-1) is deliberately + * ASK, not fail-closed: the physical binding is authoritative and D3's + * wrap-suspect / covers / serve gates fail-close every unproven leg inside the + * verdict. A derivable mismatch is a striping bug / page corruption / alias + * -> STALE (Rule 8.A integrity guard). Pure. + */ +ClusterVisFreshRefOriginDecision +cluster_vis_freshref_origin_decision(int derived_slot, int32 ref_origin) +{ + if (derived_slot < 0) + return CLUSTER_VIS_FRESHREF_ORIGIN_ASK; /* underivable (P1-a) */ + if (derived_slot == (int) ref_origin) + return CLUSTER_VIS_FRESHREF_ORIGIN_ASK; /* corroborated */ + return CLUSTER_VIS_FRESHREF_ORIGIN_STALE; /* derivable mismatch (8.A) */ +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_cr.h b/src/include/cluster/cluster_cr.h index 0f4f7b9293..d13b56cef9 100644 --- a/src/include/cluster/cluster_cr.h +++ b/src/include/cluster/cluster_cr.h @@ -266,6 +266,12 @@ extern void cluster_rtvis_verdict_note_inadmissible(void); extern uint64 cluster_rtvis_underivable_failclosed_count(void); extern void cluster_rtvis_note_underivable_failclosed(void); +/* spec-5.22f D6-3: fresh-remote-ITL-ref widening outcome counters. */ +extern uint64 cluster_vis_freshref_verdict_resolved_count(void); +extern uint64 cluster_vis_freshref_verdict_failclosed_count(void); +extern void cluster_vis_freshref_verdict_note_resolved(void); +extern void cluster_vis_freshref_verdict_note_failclosed(void); + /* * spec-6.12i CP5 (D-i4): origin-side pieces of the cross-instance verdict * serve (cluster_cr_server.c). diff --git a/src/include/cluster/cluster_cr_server.h b/src/include/cluster/cluster_cr_server.h index 711ee950f3..37bc1fef10 100644 --- a/src/include/cluster/cluster_cr_server.h +++ b/src/include/cluster/cluster_cr_server.h @@ -129,6 +129,12 @@ typedef struct ClusterLmsCrSlot { uint32 undo_segment_id; uint32 undo_block_no; TransactionId undo_xid; + /* spec-5.22f D6-7: the KIND_UNDO_VERDICT request was AUTHORITATIVE (origin + * chosen from the fresh ref's physical ITL binding, wire value 3) -> the + * serve skips the cluster_xid_is_mine stripe self-check and answers an + * underivable own xid over its durable-TT + CLOG authority. false for the + * derived (recycled) path, which keeps the self-check (6.12i P0 guard). */ + bool undo_authoritative; ClusterLiveAuthority undo_auth; char result_page[BLCKSZ]; /* the constructed CR page (FULL/PARTIAL), the * fetched undo header block (UNDO_FETCH), or @@ -187,7 +193,8 @@ extern bool cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd); * two answers over one xid can never diverge (Rule 8.A). The caller zeroes * the page buffer and owns any authority co-sampling; true = *v holds the * verdict, false = refuse (caller keeps 53R97 fail-closed). */ -extern bool cluster_lms_undo_verdict_fill_page(TransactionId xid, ClusterGcsUndoVerdictPage *v); +extern bool cluster_lms_undo_verdict_fill_page(TransactionId xid, bool authoritative, + ClusterGcsUndoVerdictPage *v); /* LMS main-loop side: construct every PENDING slot (errors become DENIED * results — fail-closed to the requester, never an LMS restart). */ @@ -225,7 +232,7 @@ extern bool cluster_gcs_block_undo_tt_fetch_and_wait(int32 origin_node, uint32 s * The caller MUST Lamport-observe verdict_out->horizon_scn (and any * commit_scn) it consumes — SCNs that crossed the wire (AD-008). */ extern bool cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_id, - TransactionId xid, + TransactionId xid, bool authoritative, ClusterGcsUndoVerdictPage *verdict_out, ClusterLiveAuthority *auth_out); diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 3966218bc2..c7f799d3e9 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1255,16 +1255,33 @@ GcsBlockForwardPayloadSetDirectLandFromRequest(GcsBlockForwardPayload *fwd, * complete durable-TT scan + CLOG cross-check + retention origin legs and * ships UNDO_VERDICT_RESULT, or a DENIED status which the requester maps * to the unchanged 53R97 fail-closed (Rule 8.A). */ +/* + * PGRAC: spec-5.22f D6-7 — reserved_0[6] value-multiplexes the verdict request + * into two sub-kinds: VALUE 2 = a DERIVED verdict (the spec-6.15 D4 recycled + * path, whose origin was derived from the xid value; the serve keeps the + * cluster_xid_is_mine self-check that guards the 6.12i P0 wrong-origin match), + * VALUE 3 = an AUTHORITATIVE verdict (the spec-5.22f fresh-ref path, whose + * origin is the tuple page's PHYSICAL ITL binding — the requester already + * proved this is the correct owner, so the serve skips the stripe pre-filter + * and answers underivable own xids over its own durable-TT + CLOG authority; + * the positive-proof gates are unchanged, Rule 8.A). + */ static inline void -GcsBlockForwardPayloadSetUndoVerdictRequest(GcsBlockForwardPayload *p, bool undo_verdict) +GcsBlockForwardPayloadSetUndoVerdictRequest(GcsBlockForwardPayload *p, bool authoritative) { - p->reserved_0[6] = undo_verdict ? (uint8)2 : (uint8)0; + p->reserved_0[6] = authoritative ? (uint8)3 : (uint8)2; } static inline bool GcsBlockForwardPayloadIsUndoVerdictRequest(const GcsBlockForwardPayload *p) { - return p->reserved_0[6] == (uint8)2; + return p->reserved_0[6] == (uint8)2 || p->reserved_0[6] == (uint8)3; +} + +static inline bool +GcsBlockForwardPayloadIsUndoVerdictAuthoritative(const GcsBlockForwardPayload *p) +{ + return p->reserved_0[6] == (uint8)3; } /* PGRAC: spec-6.12i D-i1 — synthetic undo-address tag for the fetch wire. diff --git a/src/include/cluster/cluster_runtime_visibility.h b/src/include/cluster/cluster_runtime_visibility.h index 15c325be5e..3c51c896b7 100644 --- a/src/include/cluster/cluster_runtime_visibility.h +++ b/src/include/cluster/cluster_runtime_visibility.h @@ -179,6 +179,7 @@ extern bool cluster_vis_undo_verdict_page_usable(const struct ClusterGcsUndoVerd extern bool cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, XLogRecPtr anchor_lsn, SCN read_scn, + bool authoritative, bool *out_committed, SCN *out_commit_scn, bool *out_commit_scn_is_bound); diff --git a/src/include/cluster/cluster_undo_verdict.h b/src/include/cluster/cluster_undo_verdict.h index b157c0829a..94cfd3ceb2 100644 --- a/src/include/cluster/cluster_undo_verdict.h +++ b/src/include/cluster/cluster_undo_verdict.h @@ -117,11 +117,18 @@ extern ClusterUndoVerdictResult cluster_undo_verdict_from_resolve(bool ok, bool * InvalidScn for callers without snapshot semantics. master==self routes to * the local durable resolve; master!=self to the CP3 S-grant + CP5 verdict. * kind == UNKNOWN_FAIL_CLOSED => the caller keeps 53R97 (never false-visible). - * Compiled only in --enable-cluster builds. + * + * authoritative (spec-5.22f D6-7) = the origin was chosen from the tuple page's + * PHYSICAL ITL binding (a fresh-ref consumer), not derived from the xid value. + * When true the origin serves underivable own xids over its own durable-TT + + * CLOG authority (skipping the stripe self-check that guards the derived-path + * 6.12i P0); the positive-proof gates are unchanged. Derived (recycled) callers + * pass false to keep cluster_xid_is_mine. Compiled only in --enable-cluster builds. */ extern ClusterUndoVerdictResult cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, - XLogRecPtr anchor_lsn, SCN read_scn); + XLogRecPtr anchor_lsn, SCN read_scn, + bool authoritative); #endif /* CLUSTER_UNDO_VERDICT_H */ diff --git a/src/include/cluster/cluster_visibility_resolve.h b/src/include/cluster/cluster_visibility_resolve.h index e063596bf0..a0804cc5e4 100644 --- a/src/include/cluster/cluster_visibility_resolve.h +++ b/src/include/cluster/cluster_visibility_resolve.h @@ -57,6 +57,7 @@ #include "cluster/cluster_scn.h" /* SCN */ #include "cluster/cluster_tt_slot.h" /* ClusterUndoTTSlotRef */ #include "cluster/cluster_tt_status.h" /* ClusterTTStatus */ +#include "cluster/cluster_undo_verdict.h" /* ClusterUndoVerdictResult (spec-5.22f D6) */ /* @@ -215,6 +216,57 @@ extern ClusterVisVerdict cluster_vis_dirty_verdict(ClusterTTStatus status, bool bool is_delete); +/* + * ============================================================ + * spec-5.22f D6: shared-catalog seed / fresh-ref visibility consumer. + * + * Two PURE helpers (no I/O / no shmem / no elog) that let the + * classify_ref_guts fresh-remote-ITL-ref branch consume the D3 + * cross-node verdict (cluster_undo_verdict_resolve). Kept pure so the + * mapping + origin-decision truth tables are a fully enumerable unit + * test (test_cluster_vis_undo_verdict_map, U1-U10) exactly as the OBS-2~5 + * tables above. Implementation lives with them in + * cluster_visibility_verdict.c (no cluster_visibility_verdict.h churn). + * ============================================================ + */ + +/* + * cluster_vis_from_undo_verdict -- map a D3 cross-node verdict onto the local + * visibility out-params. COMMITTED_EXACT/BOUND -> REMOTE/COMMITTED (BOUND + * sets commit_scn_is_bound so a below-horizon bound is never stamped/cached); + * ABORTED -> REMOTE/ABORTED; UNKNOWN_FAIL_CLOSED / IN_PROGRESS (D3 never + * produces IN_PROGRESS) / any other -> STALE_OR_AMBIGUOUS so the caller keeps + * the 53R97 fail-closed boundary (Rule 8.A / L10). ALL branches overwrite + * commit_scn + commit_scn_is_bound so a non-terminal verdict never leaks a + * residual scn. Returns true iff a terminal (COMMITTED/ABORTED) was produced. + */ +extern bool cluster_vis_from_undo_verdict(ClusterUndoVerdictResult v, ClusterVisResolve *out); + +/* + * cluster_vis_freshref_origin_decision -- the fresh-remote-ITL-ref origin + * decision (spec-5.22f Q2 / Option B, P1-a). A fresh ref (local_xid == + * raw_xid) carries the tuple page's PHYSICAL ITL binding, so ref_origin is the + * true owner; the value-derived cluster_xid_origin_slot(raw_xid) is only a + * derivable-time integrity cross-check: + * derived_slot < 0 (underivable: below-floor pre-striping seed / striping + * off) -> ASK (P1-a: + * NOT fail-closed -- the fresh ref's physical binding is + * authoritative and D3's wrap-suspect / covers / serve + * gates fail-close every unproven leg internally) + * derived_slot == ref_origin (corroborated) -> ASK + * derived_slot >= 0 && derived_slot != ref_origin (striping bug / page + * corruption / alias) -> STALE + * Pure. + */ +typedef enum ClusterVisFreshRefOriginDecision { + CLUSTER_VIS_FRESHREF_ORIGIN_ASK = 0, + CLUSTER_VIS_FRESHREF_ORIGIN_STALE = 1 +} ClusterVisFreshRefOriginDecision; + +extern ClusterVisFreshRefOriginDecision cluster_vis_freshref_origin_decision(int derived_slot, + int32 ref_origin); + + /* * spec-3.14 D5: cheap "does this tuple have any REMOTE writer evidence" * test for the prune / vacuum / surely-dead guards. Looks only at ITL diff --git a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl index 719c919676..81c6b46281 100644 --- a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl +++ b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl @@ -94,8 +94,8 @@ # ---------- my $cr_rows = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}); -is($cr_rows, '57', - 'L2 cr category has 57 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict)'); +is($cr_rows, '59', + 'L2 cr category has 59 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); # ---------- diff --git a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl index d912b07ede..3b849e1ea3 100644 --- a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl +++ b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl @@ -78,8 +78,8 @@ is( $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '57', - 'L1d cr category has 57 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict)'); + '59', + 'L1d cr category has 59 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); } diff --git a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl index c8c44697f9..c29e0d2d29 100644 --- a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl +++ b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl @@ -125,7 +125,8 @@ sub _new_single_node recovery => 39, # 3.16(4)+4.10 block(2)+4.11 thread(4)+4.3 plan(13)+4.4 worker(8)+4.5/4.7 merge(8) tt_recovery => 8, # 4.8 verdict counters gcs_recovery => 10, # 4.7 warm-recovery(8) + spec-2.41 D7 redo-coverage serve-gate(2) - cr => 57, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + cr => 59, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + # + ... + 6.12b(6) + 6.12i/6.15(16) + 2 spec-5.22f D6 fresh-ref verdict # + 11 post-5.54 keys the baseline missed (stale on # main; first caught by a local full run): 6.12b # cr_server_{full,partial,denied} + diff --git a/src/test/cluster_tap/t/300_cluster_5_50_cr_profile.pl b/src/test/cluster_tap/t/300_cluster_5_50_cr_profile.pl index 12ff7c2ac2..5258363021 100644 --- a/src/test/cluster_tap/t/300_cluster_5_50_cr_profile.pl +++ b/src/test/cluster_tap/t/300_cluster_5_50_cr_profile.pl @@ -126,8 +126,8 @@ is( $node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '57', - 'L1b cr category has 57 counters (17 + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict)'); + '59', + 'L1b cr category has 59 counters (17 + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); $node->safe_psql('postgres', 'CREATE TABLE t_l1 (id int, v int); INSERT INTO t_l1 VALUES (1, 100);'); diff --git a/src/test/cluster_tap/t/306_cluster_5_53_cr_key_reuse.pl b/src/test/cluster_tap/t/306_cluster_5_53_cr_key_reuse.pl index 4fd4be4074..84eb52ee52 100644 --- a/src/test/cluster_tap/t/306_cluster_5_53_cr_key_reuse.pl +++ b/src/test/cluster_tap/t/306_cluster_5_53_cr_key_reuse.pl @@ -106,7 +106,7 @@ # spec-6.12b six CR-server counters. is( $node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '57', 'L1b cr category has 57 rows (17 + 5 spec-5.53 + 8 spec-5.54 + 5 spec-5.56 + 6 spec-6.12b + 16 spec-6.12i/6.15)'); + '59', 'L1b cr category has 59 rows (17 + 5 spec-5.53 + 8 spec-5.54 + 5 spec-5.56 + 6 spec-6.12b + 16 spec-6.12i/6.15 + 2 spec-5.22f D6 fresh-ref verdict)'); for my $k (qw(cr_key_mismatch_count cr_epoch_mismatch_count cr_generation_mismatch_count cr_base_lsn_mismatch_count diff --git a/src/test/cluster_tap/t/311_cluster_5_56_cr_lifecycle.pl b/src/test/cluster_tap/t/311_cluster_5_56_cr_lifecycle.pl index 69b97ec460..a81455b268 100644 --- a/src/test/cluster_tap/t/311_cluster_5_56_cr_lifecycle.pl +++ b/src/test/cluster_tap/t/311_cluster_5_56_cr_lifecycle.pl @@ -106,7 +106,7 @@ 'postmaster', 'L7b rel_generation_slots is PGC_POSTMASTER'); is( $node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '57', 'L7c cr category has 57 rows (22 + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict)'); + '59', 'L7c cr category has 59 rows (22 + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); for my $k ( qw(cr_global_epoch_fallback_bump_count cr_rel_gen_bump_count cr_rel_gen_table_overflow_count cr_retention_horizon_advance_noted_count diff --git a/src/test/cluster_tap/t/359_cluster_5_22f_seed_visibility.pl b/src/test/cluster_tap/t/359_cluster_5_22f_seed_visibility.pl new file mode 100644 index 0000000000..a3bb4dac9d --- /dev/null +++ b/src/test/cluster_tap/t/359_cluster_5_22f_seed_visibility.pl @@ -0,0 +1,301 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 359_cluster_5_22f_seed_visibility.pl +# spec-5.22f D6 — shared-catalog seed / fresh-ref visibility consumer +# end-to-end on a 2-node ClusterPair + shared cluster_fs root. This is +# the ROOT-CAUSE #1 real fix: the "committed seed visible" leg that +# D2-7 (t/255 L6) explicitly forward-linked to D6. +# +# Substrate: node0 (cluster-enabled -> storage-mode, peer-independent) +# commits a NORMAL-xid INSERT while undo GCS coherence is ON, so the +# seed xact's undo segment (block0 TT + commit_scn) migrates to the +# SHARED cluster_fs root (the D2-2 heart). The heap page is +# phantom-shared, so node1 reads the SAME bytes node0 wrote, carrying +# node0's ITL binding. The D6 mechanism is tuple-agnostic (heap or +# catalog): a fresh remote ITL ref (local_xid == raw_xid, origin != self) +# that misses node1's empty TT overlay is exactly the seed/joiner case. +# +# Striping stays OFF, so cluster_xid_origin_slot(seed_xid) is +# underivable (-1) on node1 -- the true root-cause #1 timeline (a seed +# DDL committed pre-striping). Option B (Q2 / P1-a): underivable STILL +# asks the D3 verdict (ref->origin is the physical binding, authoritative; +# Rule 8.A safety is anchored inside D3's wrap-suspect / covers / serve +# gates), so the seed resolves visible instead of self-deadlocking on a +# derived>=0 guard. +# +# L1 pair boots + shared cluster_fs; seed table + coherence armed; the +# seed xact's undo lands on the SHARED root. +# L2 命门 2 baseline: crossnode_runtime_visibility OFF -> node1's fresh +# read of the seed tuple fails closed 53R97 (overlay miss, no +# widening) -- the RED state D6 fixes. +# L3 ROOT-CAUSE #1 FIX (spec L1): crossnode ON -> node1's fresh read +# SUCCEEDS via the D6 fresh-ref widening -> D3 verdict. The +# vis_freshref_verdict_resolved_count counter increment is a HARD +# assertion (P1-b): it proves the read went through the fresh-ref +# widening path, not an overlay hit or ordinary propagation -- else +# the test would silently degrade to a formed-cluster DDL test that +# does NOT prove root-cause #1. +# L4 aborted seed (spec L2): a rolled-back seed row resolves ABORTED via +# the verdict -> NOT visible (never false-visible). +# L5 off regression (spec L4): crossnode OFF again -> a fresh backend +# fails closed 53R97; the off path is byte-for-byte the pre-D6 +# resolve_from_remote_ref (no widening, zero regression). +# L6 dead-owner (spec L3): node0 fail-stops; node1's fresh read of the +# seed tuple hits the verdict serve-gate deny -> UNKNOWN -> 53R97, +# never a stale false-visible. The crash-rejoin positive leg +# (survivor SERVES the dead owner's verdict) is honestly forward-D4 +# (Rule 8.B): D6 tests only the deny side. +# +# Spec: spec-5.22f-shared-catalog-seed-visibility-consumer.md (D6, §4.2) +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/359_cluster_5_22f_seed_visibility.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +sub state_val +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub write_retry +{ + my ($node, $sql) = @_; + for my $i (1 .. 10) + { + my $ok = eval { $node->safe_psql('postgres', $sql); 1 }; + return 1 if $ok; + usleep(500_000); + } + return 0; +} + +sub poll_until +{ + my ($node, $sql, $want, $secs) = @_; + my $deadline = time() + $secs; + while (time() < $deadline) + { + my $v = eval { $node->safe_psql('postgres', $sql); }; + return 1 if defined $v && $v eq $want; + usleep(250_000); + } + return 0; +} + +sub arm_guc_both +{ + my ($pair, $guc, $val) = @_; + for my $n ($pair->node0, $pair->node1) + { + $n->safe_psql('postgres', "ALTER SYSTEM SET $guc = $val"); + $n->safe_psql('postgres', 'SELECT pg_reload_conf()'); + } + usleep(1_000_000); +} + +# ============================================================ +# L1: boot + shared cluster_fs; seed table + coherence armed; the seed +# xact's undo lands on the SHARED root (readable by node1 as foreign). +# ============================================================ +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'spec_5_22f_seed_vis', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 5', + 'cluster.undo_segments_max_per_instance = 256', + 'cluster.undo_segment_create_timeout_ms = 5000', + # crossnode_runtime_visibility stays OFF at boot (L2 is the 命门 2 + # baseline; L3 arms it so the off->on fix transition is proven). + # xid_striping stays OFF -> the seed xid is underivable on node1 + # (root-cause #1 pre-striping timeline; Option B still resolves it). + ]); +$pair->start_pair; +usleep(2_000_000); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); + +my ($node0, $node1) = ($pair->node0, $pair->node1); + +is($node0->safe_psql('postgres', 'SHOW cluster.crossnode_runtime_visibility'), + 'off', 'L1 crossnode_runtime_visibility default off (命门 2 baseline)'); + +# Phantom-shared seed heap table (coincident relfilepath, like t/346/t/255). +$node0->safe_psql('postgres', 'CREATE TABLE s_t (id int, v int)'); +$node1->safe_psql('postgres', 'CREATE TABLE s_t (id int, v int)'); +my $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}); +my $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}); +is($p0, $p1, 'L1 s_t relfilepath coincidence holds (phantom-shared)'); + +# Arm undo GCS coherence so node0's seed undo migrates to the shared root. +arm_guc_both($pair, 'cluster.undo_gcs_coherence', 'on'); +is($node0->safe_psql('postgres', 'SHOW cluster.undo_gcs_coherence'), + 'on', 'L1 undo_gcs_coherence armed on'); + +# Force a fresh segment under coherence=on so the seed xact's undo lands on +# the shared cluster_fs root, then commit the NORMAL-xid seed INSERT (ONE xact, +# NOT recycled -> the tuple's ITL slot still binds the seed xid: a FRESH ref). +ok(write_retry($node0, q{SELECT cluster_undo_test_force_segment_end()}), + 'L1 forced fresh undo segment under coherence=on'); +ok(write_retry($node0, 'INSERT INTO s_t SELECT g, g * 10 FROM generate_series(1, 8) g'), + 'L1 seed xact committed 8 NORMAL-xid rows (fresh ITL binding, undo on shared root)'); +ok(write_retry($node0, 'CHECKPOINT'), 'L1 checkpoint'); + +# ============================================================ +# L2: 命门 2 baseline -- crossnode OFF -> node1's fresh read of the seed +# tuple fails closed 53R97 (fresh remote ITL ref, overlay miss, no +# widening). This is the RED state D6 fixes. +# ============================================================ +{ + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + isnt($rc, 0, 'L2 crossnode-off: node1 fresh read of seed errors (fail-closed)'); + like($err, qr/cluster TT status unknown/, + 'L2 命门 2: overlay-miss on the fresh remote ITL ref keeps 53R97 (no widening)'); +} + +# ============================================================ +# L3: ROOT-CAUSE #1 FIX (spec L1) -- crossnode ON -> node1's fresh read +# SUCCEEDS via the D6 fresh-ref widening -> D3 verdict. The +# vis_freshref_verdict_resolved_count increment is a HARD assertion (P1-b). +# ============================================================ +arm_guc_both($pair, 'cluster.crossnode_runtime_visibility', 'on'); +is($node1->safe_psql('postgres', 'SHOW cluster.crossnode_runtime_visibility'), + 'on', 'L3 crossnode_runtime_visibility armed on both nodes'); + +{ + my $resolved0 = state_val($node1, 'cr', 'vis_freshref_verdict_resolved_count'); + + # A first read may fail 53R97 while the shared undo / verdict wire warms, + # or be refused by leg (e) clock skew (the Lamport observe heals the next). + # Poll a fresh backend until the seed resolves visible -- but the SUCCESS + # itself (8 rows) and the counter increment are real, never faked. + my $ok_rows = 0; + my $last_err = ''; + for my $try (1 .. 20) + { + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + if (defined $out && $out =~ /^8$/m) { $ok_rows = 1; last; } + $last_err = $err // ''; + usleep(500_000); + } + + # Diagnostic snapshot of the D2->D3->D6 chain (localizes a 0-resolve stall). + for my $k (qw(vis_freshref_verdict_resolved_count vis_freshref_verdict_failclosed_count + rtvis_undo_fetch_wire_count rtvis_undo_fetch_failclosed_count + rtvis_resolve_committed_count rtvis_resolve_failclosed_count + rtvis_verdict_wire_count rtvis_verdict_failclosed_count + rtvis_underivable_failclosed_count)) + { + diag("L3 node1 cr.$k = " . state_val($node1, 'cr', $k)); + } + for my $k (qw(cr_server_undo_served_count cr_server_undo_denied_count + cr_server_verdict_served_count cr_server_verdict_denied_count)) + { + diag("L3 node0 cr.$k = " . state_val($node0, 'cr', $k)); + } + diag("L3 last node1 read error: $last_err") if $last_err; + + ok($ok_rows, + 'L3 ROOT-CAUSE #1 FIX: node1 fresh read sees the 8 committed seed rows (D6 widening -> verdict)'); + + cmp_ok(state_val($node1, 'cr', 'vis_freshref_verdict_resolved_count'), '>', $resolved0, + 'L3 P1-b HARD ASSERT: vis_freshref_verdict_resolved_count moved (the fresh-ref widening path ran, not overlay/propagation)'); +} + +# ============================================================ +# L4: aborted seed (spec L2) -- a rolled-back seed row resolves ABORTED via +# the verdict -> NOT visible (never false-visible). +# ============================================================ +{ + # A distinguishable row inserted then rolled back on node0. Its tuple + # stays physically on the shared page (no vacuum), so node1 sees it and + # must resolve its xmin ABORTED -> invisible. + $node0->safe_psql('postgres', q{ + BEGIN; + INSERT INTO s_t VALUES (999, -1); + ROLLBACK; + }); + ok(write_retry($node0, 'CHECKPOINT'), 'L4 checkpoint after aborted seed row'); + + my $seen_aborted = 0; + for my $try (1 .. 20) + { + my ($rc, $out, $err) = $node1->psql('postgres', + 'SELECT count(*) FROM s_t WHERE id = 999'); + if (defined $out && $out =~ /^0$/m) { $seen_aborted = 1; last; } + usleep(500_000); + } + ok($seen_aborted, + 'L4 aborted seed row is NOT visible on node1 (verdict ABORTED, never false-visible)'); +} + +# ============================================================ +# L5: off regression (spec L4) -- crossnode OFF again -> a fresh backend +# fails closed 53R97; the off path is the pre-D6 resolve_from_remote_ref +# (no widening, zero regression to the single-node / off surface). +# ============================================================ +{ + arm_guc_both($pair, 'cluster.crossnode_runtime_visibility', 'off'); + + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + isnt($rc, 0, 'L5 crossnode-off regression: fresh backend read fails closed again'); + like($err, qr/cluster TT status unknown/, + 'L5 off path unchanged (widening gated off -> byte-for-byte pre-D6 overlay-miss 53R97)'); + + # Re-arm for the dead-owner leg. + arm_guc_both($pair, 'cluster.crossnode_runtime_visibility', 'on'); +} + +# ============================================================ +# L6: dead-owner (spec L3) -- node0 fail-stops; node1's fresh read of the +# seed hits the verdict serve-gate deny -> UNKNOWN -> 53R97, never a +# stale false-visible. The crash-rejoin positive leg (survivor SERVES +# the dead owner's verdict) is honestly forward-D4 (Rule 8.B). +# ============================================================ +{ + $pair->kill_node9(0); + ok( poll_until($node1, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 40), + 'L6 node1 CSSD marks node0 suspected/dead'); + + # A fresh backend read: the origin is dead, the verdict serve-gate denies, + # and the read MUST error -- above all it must NOT return the 8 seed rows + # from a stale path. + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + isnt($rc, 0, 'L6 dead-owner: node1 fresh read errors (fail-closed, not stale false-visible)'); + unlike(($out // ''), qr/^\s*8\s*$/m, + 'L6 dead-owner read did NOT serve the 8 seed rows from a stale path'); + like($err, + qr/cluster TT status unknown|failed to enqueue|fail-stop|could not obtain|connection/i, + 'L6 the failure is a cluster fail-closed error (serve-gate deny -> 53R97; positive serve = forward-D4)'); +} + +$pair->stop_pair if $pair->can('stop_pair'); +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index d9cec7e71e..55664241b6 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -89,7 +89,8 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_xid_stripe \ test_cluster_undo_resid \ test_cluster_undo_gcs \ - test_cluster_undo_verdict + test_cluster_undo_verdict \ + test_cluster_vis_undo_verdict_map # Path to the cluster_version object (no PG deps, safe to link standalone). CLUSTER_VERSION_O = $(top_builddir)/src/backend/cluster/cluster_version.o @@ -181,7 +182,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs test_cluster_undo_verdict,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -532,6 +533,17 @@ test_cluster_visibility_variants: test_cluster_visibility_variants.c unit_test.h $(CC) $(CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_VIS_VERDICT_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# spec-5.22f D6-1: test_cluster_vis_undo_verdict_map links the same pure +# verdict object (cluster_visibility_verdict.o) for the two D6 fresh-ref +# consumer helpers (cluster_vis_from_undo_verdict + cluster_vis_freshref_ +# origin_decision). Both are dependency-free (no PG backend), so nothing is +# stubbed -- identical link recipe to test_cluster_visibility_variants. +test_cluster_vis_undo_verdict_map: test_cluster_vis_undo_verdict_map.c unit_test.h \ + $(CLUSTER_VIS_VERDICT_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VIS_VERDICT_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ test_cluster_retention: test_cluster_retention.c unit_test.h \ $(CLUSTER_UNDO_RETENTION_O) $(CC) $(CFLAGS) $(CPPFLAGS) $< \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 67c7691666..0b848f9193 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1843,6 +1843,17 @@ cluster_rtvis_underivable_failclosed_count(void) { return 0; } +/* spec-5.22f D6-3: fresh-remote-ITL-ref widening outcome counters. */ +uint64 +cluster_vis_freshref_verdict_resolved_count(void) +{ + return 0; +} +uint64 +cluster_vis_freshref_verdict_failclosed_count(void) +{ + return 0; +} /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ uint64 cluster_cr_xmax_resolved_count(void) diff --git a/src/test/cluster_unit/test_cluster_vis_undo_verdict_map.c b/src/test/cluster_unit/test_cluster_vis_undo_verdict_map.c new file mode 100644 index 0000000000..1a7daeaef9 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_vis_undo_verdict_map.c @@ -0,0 +1,249 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_vis_undo_verdict_map.c + * Unit tests for the spec-5.22f D6 fresh-ref visibility consumer pure + * helpers (shared-catalog seed/joiner visibility, Layer-2 D6). + * + * D6-1 adds two PURE helpers to cluster_visibility_verdict.c: + * cluster_vis_from_undo_verdict -- map a D3 five-value verdict + * onto the ClusterVisResolve + * out-params (U1-U7); + * cluster_vis_freshref_origin_decision -- the fresh-ref origin + * decision ASK/STALE (U8-U10). + * Both are dependency-free (no scn_time_cmp, no shmem, no elog) so the + * mapping + origin-decision truth tables are a fully enumerable unit + * test, linking cluster_visibility_verdict.o standalone -- exactly as + * test_cluster_visibility_variants does for the OBS-2~5 tables. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_vis_undo_verdict_map.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22f-shared-catalog-seed-visibility-consumer.md (D6, §4.1). + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_scn.h" /* SCN, InvalidScn */ +#include "cluster/cluster_tt_status.h" /* ClusterTTStatus */ +#include "cluster/cluster_undo_verdict.h" /* ClusterUndoVerdictResult + kinds */ +#include "cluster/cluster_visibility_resolve.h" + +/* Drop PG's port.h printf -> pg_printf override; unit_test.h uses stdlib + * printf and we don't link libpgport into this binary. */ +#undef printf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + + +/* Seed a ClusterVisResolve with obvious garbage so a helper that fails to + * overwrite a field leaks it (U7 residue guard). */ +static void +poison_out(ClusterVisResolve *out) +{ + memset(out, 0, sizeof(*out)); + out->evidence = CLUSTER_VIS_EVIDENCE_LOCAL; + out->status = CLUSTER_TT_STATUS_IN_PROGRESS; + out->commit_scn = (SCN) 987654321; + out->commit_scn_is_bound = true; +} + +static ClusterUndoVerdictResult +make_verdict(ClusterUndoVerdictKind kind, SCN commit_scn) +{ + ClusterUndoVerdictResult v = { 0 }; + + v.kind = (uint8) kind; + v.commit_scn = commit_scn; + return v; +} + + +/* ====================================================================== + * U1 -- COMMITTED_EXACT{scn} -> REMOTE/COMMITTED/scn, is_bound=false, true. + * ====================================================================== */ +UT_TEST(test_map_committed_exact) +{ + ClusterVisResolve out; + bool terminal; + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_COMMITTED_EXACT, (SCN) 5000), &out); + + UT_ASSERT_EQ((int) terminal, 1); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_REMOTE); + UT_ASSERT_EQ((int) out.status, (int) CLUSTER_TT_STATUS_COMMITTED); + UT_ASSERT_EQ((uint64) out.commit_scn, (uint64) 5000); + UT_ASSERT_EQ((int) out.commit_scn_is_bound, 0); +} + +/* ====================================================================== + * U2 -- COMMITTED_BOUND{scn} -> REMOTE/COMMITTED/scn, is_bound=TRUE, true. + * The bound must never be mistaken for an exact commit_scn (Amendment-2 + * traced through: consumers never stamp / cache a bound). + * ====================================================================== */ +UT_TEST(test_map_committed_bound_sets_is_bound) +{ + ClusterVisResolve out; + bool terminal; + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_COMMITTED_BOUND, (SCN) 6000), &out); + + UT_ASSERT_EQ((int) terminal, 1); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_REMOTE); + UT_ASSERT_EQ((int) out.status, (int) CLUSTER_TT_STATUS_COMMITTED); + UT_ASSERT_EQ((uint64) out.commit_scn, (uint64) 6000); + UT_ASSERT_EQ((int) out.commit_scn_is_bound, 1); +} + +/* ====================================================================== + * U3 -- ABORTED -> REMOTE/ABORTED/InvalidScn, is_bound=false, true. + * ====================================================================== */ +UT_TEST(test_map_aborted) +{ + ClusterVisResolve out; + bool terminal; + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_ABORTED, InvalidScn), &out); + + UT_ASSERT_EQ((int) terminal, 1); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_REMOTE); + UT_ASSERT_EQ((int) out.status, (int) CLUSTER_TT_STATUS_ABORTED); + UT_ASSERT_EQ((uint64) out.commit_scn, (uint64) InvalidScn); + UT_ASSERT_EQ((int) out.commit_scn_is_bound, 0); +} + +/* ====================================================================== + * U4 -- UNKNOWN_FAIL_CLOSED -> STALE_OR_AMBIGUOUS/UNKNOWN/InvalidScn, false + * (Rule 8.A: caller keeps 53R97; never treated committed/visible). + * ====================================================================== */ +UT_TEST(test_map_unknown_is_stale_fail_closed) +{ + ClusterVisResolve out; + bool terminal; + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, InvalidScn), &out); + + UT_ASSERT_EQ((int) terminal, 0); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS); + UT_ASSERT_EQ((int) out.status, (int) CLUSTER_TT_STATUS_UNKNOWN); + UT_ASSERT_EQ((uint64) out.commit_scn, (uint64) InvalidScn); + UT_ASSERT_EQ((int) out.commit_scn_is_bound, 0); +} + +/* ====================================================================== + * U5 -- IN_PROGRESS (D3 folds it to UNKNOWN, defensive) -> STALE, false. + * ====================================================================== */ +UT_TEST(test_map_in_progress_is_stale_fail_closed) +{ + ClusterVisResolve out; + bool terminal; + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_IN_PROGRESS, InvalidScn), &out); + + UT_ASSERT_EQ((int) terminal, 0); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS); + UT_ASSERT_EQ((int) out.status, (int) CLUSTER_TT_STATUS_UNKNOWN); +} + +/* ====================================================================== + * U6 -- a zero-initialised verdict {0} (== UNKNOWN_FAIL_CLOSED, L10) maps to + * STALE / false, so an un-proven verdict struct is never accidentally visible. + * ====================================================================== */ +UT_TEST(test_map_zero_init_verdict_is_stale) +{ + ClusterUndoVerdictResult v = { 0 }; + ClusterVisResolve out; + bool terminal; + + UT_ASSERT_EQ((int) CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, 0); + + poison_out(&out); + terminal = cluster_vis_from_undo_verdict(v, &out); + + UT_ASSERT_EQ((int) terminal, 0); + UT_ASSERT_EQ((int) out.evidence, (int) CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS); +} + +/* ====================================================================== + * U7 -- residue guard: a STALE map must overwrite commit_scn + is_bound so a + * caller's earlier terminal residue never leaks (poison_out seeds 987654321 / + * is_bound=true; the STALE map must clear both). + * ====================================================================== */ +UT_TEST(test_map_stale_clears_scn_residue) +{ + ClusterVisResolve out; + + poison_out(&out); + (void) cluster_vis_from_undo_verdict( + make_verdict(CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, InvalidScn), &out); + + UT_ASSERT_EQ((uint64) out.commit_scn, (uint64) InvalidScn); + UT_ASSERT_EQ((int) out.commit_scn_is_bound, 0); +} + +/* ====================================================================== + * U8 -- origin decision: derived == ref_origin (corroborated) -> ASK. + * ====================================================================== */ +UT_TEST(test_origin_corroborated_is_ask) +{ + UT_ASSERT_EQ((int) cluster_vis_freshref_origin_decision(2, 2), + (int) CLUSTER_VIS_FRESHREF_ORIGIN_ASK); +} + +/* ====================================================================== + * U9 -- origin decision: derived == -1 (underivable: below-floor pre-striping + * seed / striping off) -> ASK (P1-a: NOT fail-closed; = root-cause #1 path). + * ====================================================================== */ +UT_TEST(test_origin_underivable_is_ask_not_failclosed) +{ + UT_ASSERT_EQ((int) cluster_vis_freshref_origin_decision(-1, 2), + (int) CLUSTER_VIS_FRESHREF_ORIGIN_ASK); +} + +/* ====================================================================== + * U10 -- origin decision: derived >= 0 && derived != ref_origin (striping bug + * / page corruption / alias) -> STALE (Rule 8.A integrity guard). + * ====================================================================== */ +UT_TEST(test_origin_derivable_mismatch_is_stale) +{ + UT_ASSERT_EQ((int) cluster_vis_freshref_origin_decision(3, 2), + (int) CLUSTER_VIS_FRESHREF_ORIGIN_STALE); +} + +int +main(void) +{ + UT_PLAN(10); + UT_RUN(test_map_committed_exact); + UT_RUN(test_map_committed_bound_sets_is_bound); + UT_RUN(test_map_aborted); + UT_RUN(test_map_unknown_is_stale_fail_closed); + UT_RUN(test_map_in_progress_is_stale_fail_closed); + UT_RUN(test_map_zero_init_verdict_is_stale); + UT_RUN(test_map_stale_clears_scn_residue); + UT_RUN(test_origin_corroborated_is_ask); + UT_RUN(test_origin_underivable_is_ask_not_failclosed); + UT_RUN(test_origin_derivable_mismatch_is_stale); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From 599bf8d1544c5c124071ecde053c0a7d5e844ac1 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 23:27:17 +0800 Subject: [PATCH 16/38] feat(cluster): D4-1 dead-owner undo serve-authority election (spec-5.22d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure lowest-fresh-survivor election + live-snapshot wrapper for electing a deterministic temporary serve authority when an undo resource's owner is dead/absent (spec-5.22d D4-1, Route B dead-owner block0 verdict serve). - cluster_undo_authority.{c,h}: pure cluster_undo_authority_decide() — no shmem/GRD/GUC deps; mirrors cluster_grd_master_map_remaster survivor rule (declared minus non-fresh). Fail-closed edges: epoch mismatch -> RECOVERING; owner not dead-decided (生死未定) -> UNKNOWN; no fresh survivor -> RECOVERING; malformed owner -> UNKNOWN. Never elects a non-(declared AND fresh) node (约束 #2 at derivation layer). - cluster_undo_authority_snapshot.c: wrapper projects membership SSOT (cluster_membership_get_state) + L420 heartbeat-freshness (observed_fresh_alive, NOT record existence) + current reconfig epoch into the pure decide(). TAP-covered (D4-8). - test_cluster_undo_authority: 8 unit cases RED->GREEN (U1 lowest-survivor, U2 owner-live, U7 undecided, epoch-mismatch, no-survivor, skips-lower, requires-declared, invalid-owner). Base = D6 tip 36098ecaf6 (spec-5.22b-undo-gcs, unpushed F1 window). --- src/backend/cluster/Makefile | 2 + src/backend/cluster/cluster_undo_authority.c | 91 +++++++ .../cluster/cluster_undo_authority_snapshot.c | 105 ++++++++ src/include/cluster/cluster_undo_authority.h | 125 ++++++++++ src/test/cluster_unit/Makefile | 17 +- .../test_cluster_undo_authority.c | 234 ++++++++++++++++++ 6 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 src/backend/cluster/cluster_undo_authority.c create mode 100644 src/backend/cluster/cluster_undo_authority_snapshot.c create mode 100644 src/include/cluster/cluster_undo_authority.h create mode 100644 src/test/cluster_unit/test_cluster_undo_authority.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 8f3c5003c5..933c777a57 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -184,6 +184,8 @@ OBJS = \ cluster_visibility_verdict.o \ cluster_undo_record.o \ cluster_undo_resid.o \ + cluster_undo_authority.o \ + cluster_undo_authority_snapshot.o \ cluster_undo_gcs.o \ cluster_undo_gcs_grant.o \ cluster_undo_gcs_stat.o \ diff --git a/src/backend/cluster/cluster_undo_authority.c b/src/backend/cluster/cluster_undo_authority.c new file mode 100644 index 0000000000..dbb073357c --- /dev/null +++ b/src/backend/cluster/cluster_undo_authority.c @@ -0,0 +1,91 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_authority.c + * Dead/absent-owner undo serve-authority election (spec-5.22d D4-1). + * + * See cluster_undo_authority.h for the model. This file holds the PURE + * decision core; the live-snapshot wrapper (reading real GRD / membership + * / epoch state) lands in D4-1's second half. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-1, §2.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_undo_authority.h" + +#ifdef USE_PGRAC_CLUSTER + +/* Test bit n in a 128-bit little-endian membership bitmap. */ +static inline bool +au_bit_test(const uint8 *bm, int32 node) +{ + return (bm[node >> 3] >> (node & 7)) & 1u; +} + +/* + * cluster_undo_authority_decide -- pure serve-authority election. + * + * Precedence (all fail-closed except OK / OWNER_LIVE): + * 1. malformed owner (outside [0, SCN_MAX_VALID_NODE_ID]) -> UNKNOWN + * 2. caller epoch != accepted snapshot epoch -> RECOVERING + * 3. owner is fresh-alive -> OWNER_LIVE + * 4. owner not dead-decided (生死未定; never guess) -> UNKNOWN + * 5. elect lowest (declared AND fresh-alive) survivor; + * none exists -> RECOVERING + * else -> OK + * + * Mirrors cluster_grd_master_map_remaster: survivors are the declared + * list minus non-fresh nodes; the election is a deterministic function + * of the accepted snapshot so every reader agrees on the authority. + */ +ClusterUndoAuthorityStatus +cluster_undo_authority_decide(const ClusterUndoAuthorityInput *in, + ClusterUndoAuthorityDecision *out) +{ + int32 node; + + out->authority_node = -1; + + /* 1. malformed owner */ + if (in->owner_node < 0 || in->owner_node > SCN_MAX_VALID_NODE_ID) + return (out->status = CLUSTER_UNDO_AUTHORITY_UNKNOWN); + + /* 2. caller scoped to a stale epoch */ + if (in->snapshot_epoch != in->request_epoch) + return (out->status = CLUSTER_UNDO_AUTHORITY_RECOVERING); + + /* 3. owner still fresh-alive: stay on the live-owner (D6) path */ + if (au_bit_test(in->alive_fresh, in->owner_node)) + return (out->status = CLUSTER_UNDO_AUTHORITY_OWNER_LIVE); + + /* 4. owner neither fresh-alive nor dead-decided: 生死未定, never guess */ + if (!au_bit_test(in->dead_decided, in->owner_node)) + return (out->status = CLUSTER_UNDO_AUTHORITY_UNKNOWN); + + /* 5. elect the lowest declared, fresh-alive survivor */ + for (node = 0; node <= SCN_MAX_VALID_NODE_ID; node++) { + if (au_bit_test(in->declared, node) && au_bit_test(in->alive_fresh, node)) { + out->authority_node = node; + return (out->status = CLUSTER_UNDO_AUTHORITY_OK); + } + } + + /* no fresh survivor -> transient; fail closed */ + return (out->status = CLUSTER_UNDO_AUTHORITY_RECOVERING); +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c new file mode 100644 index 0000000000..1cf18d21c4 --- /dev/null +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -0,0 +1,105 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_authority_snapshot.c + * Live membership-snapshot wrapper for dead/absent-owner undo + * serve-authority election (spec-5.22d D4-1, wrapper half). + * + * Reads the real membership-state SSOT (cluster_membership_get_state), + * heartbeat freshness (cluster_reconfig_get_observed_fresh_alive, the + * L420 freshness gate -- NOT record existence), and the current accepted + * reconfig epoch (cluster_epoch_get_current), projects them into a + * ClusterUndoAuthorityInput, and defers the actual election to the pure + * cluster_undo_authority_decide(). Kept out of cluster_undo_authority.c + * so the pure decision object links standalone in the D4-1 unit test; + * this wrapper is covered by the D4-8 TAP. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_authority_snapshot.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-1, §2.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_membership.h" /* cluster_membership_get_state */ +#include "cluster/cluster_reconfig.h" /* cluster_reconfig_get_observed_fresh_alive */ +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_undo_authority.h" + +#ifdef USE_PGRAC_CLUSTER + +static inline void +bm_set(uint8 *bm, int32 node) +{ + bm[node >> 3] |= (uint8)(1u << (node & 7)); +} + +/* + * cluster_undo_serve_authority_lookup -- live-snapshot wrapper. + * + * Projects the membership SSOT into the pure decide()'s bitmaps: + * alive_fresh <- state==MEMBER AND observed_fresh_alive (L420) + * dead_decided <- state==DEAD or state==REMOVED (durable fail-stop / + * decommission decision; NOT a transient live read, L419) + * declared <- every non-absent, non-rejected known node + * ABSENT / JOINING / REJECTED owners are neither fresh nor dead-decided, + * so decide() fails closed (UNKNOWN) -- we never guess a node dead. + */ +ClusterUndoAuthorityStatus +cluster_undo_serve_authority_lookup(int32 owner_node, uint64 reconfig_epoch, int32 *out_authority) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + int32 node; + + *out_authority = -1; + + memset(&in, 0, sizeof(in)); + in.owner_node = owner_node; + in.request_epoch = reconfig_epoch; + in.snapshot_epoch = cluster_epoch_get_current(); + + for (node = 0; node <= SCN_MAX_VALID_NODE_ID && node < CLUSTER_MAX_NODES; node++) { + ClusterMembershipState st = cluster_membership_get_state(node); + + switch (st) { + case CLUSTER_MEMBER_MEMBER: + bm_set(in.declared, node); + if (cluster_reconfig_get_observed_fresh_alive(node)) + bm_set(in.alive_fresh, node); + break; + case CLUSTER_MEMBER_DEAD: + case CLUSTER_MEMBER_REMOVED: + bm_set(in.declared, node); + bm_set(in.dead_decided, node); + break; + case CLUSTER_MEMBER_JOINING: + /* known/declared but not eligible either way (may become live) */ + bm_set(in.declared, node); + break; + case CLUSTER_MEMBER_ABSENT: + case CLUSTER_MEMBER_REJECTED: + default: + /* not a usable authority; owner here => decide() UNKNOWN */ + break; + } + } + + (void)cluster_undo_authority_decide(&in, &out); + if (out.status == CLUSTER_UNDO_AUTHORITY_OK) + *out_authority = out.authority_node; + return out.status; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h new file mode 100644 index 0000000000..17f62f666d --- /dev/null +++ b/src/include/cluster/cluster_undo_authority.h @@ -0,0 +1,125 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_authority.h + * Dead/absent-owner undo serve-authority derivation (spec-5.22d D4-1). + * + * When an undo resource's owner instance is dead/absent, the block0 + * TTSlot verdict for that owner cannot be served by the owner itself. + * A single deterministic SURVIVOR is elected as the temporary serve + * authority for that owner's undo resource, scoped to the current + * reconfig (membership) epoch. The authority reads the owner's block0 + * from shared storage under GCS and serves the verdict; every reader + * agrees on WHO the authority is because the election is a pure + * deterministic function of the accepted membership snapshot -- never + * an ad-hoc local peer_state read (mirrors cluster_grd_master_map_ + * remaster's survivor rule). + * + * This header exposes the PURE decision core (cluster_undo_authority_ + * decide): it takes the membership snapshot as explicit bitmaps and + * returns the elected authority + status, with no shmem / GRD / GUC + * dependencies, so it is exhaustively unit-testable (spec-5.22d §4.1 + * U1-U8). The live-snapshot wrapper (cluster_undo_serve_authority_ + * lookup) that reads the real GRD / membership / epoch state lives in + * a separate object and is covered by the D4-8 TAP. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_undo_authority.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-1, §2.1) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_UNDO_AUTHORITY_H +#define CLUSTER_UNDO_AUTHORITY_H + +#include "c.h" +#include "cluster/cluster_reconfig.h" /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ + +#ifdef USE_PGRAC_CLUSTER + +/* + * Outcome of a serve-authority derivation for one dead/absent owner. + * + * OK -- out_authority_node holds the elected survivor authority; + * the caller routes the verdict request to it (or self- + * serves if authority == self). + * OWNER_LIVE -- the owner is a fresh-alive member; do NOT elect an + * authority, the caller stays on the live-owner path (D6). + * RECOVERING -- membership not yet settled at the requested epoch, or no + * fresh survivor exists. Transient; fail closed (retry). + * UNKNOWN -- owner is neither fresh-alive nor dead-decided (生死未定), + * or the input is malformed. Fail closed; NEVER guess. + * + * RECOVERING / UNKNOWN both map to a fail-closed verdict (RESOURCE_RECOVERING + * / 53R97) at the consumer -- the distinction is observability only. + */ +typedef enum ClusterUndoAuthorityStatus { + CLUSTER_UNDO_AUTHORITY_OK = 0, + CLUSTER_UNDO_AUTHORITY_OWNER_LIVE, + CLUSTER_UNDO_AUTHORITY_RECOVERING, + CLUSTER_UNDO_AUTHORITY_UNKNOWN +} ClusterUndoAuthorityStatus; + +/* + * Membership snapshot for one derivation, as explicit bitmaps (128-bit, + * CLUSTER_RECONFIG_DEAD_BITMAP_BYTES = 16, bit n = byte[n>>3] & (1<<(n&7))). + * The wrapper fills these from the accepted reconfig snapshot; the pure + * decide() reads ONLY this struct. + * + * request_epoch -- the reconfig epoch the caller is scoped to. + * snapshot_epoch -- the current accepted reconfig epoch. Mismatch => + * RECOVERING (the caller's epoch is stale; re-derive). + * declared -- declared topology (pgrac.conf), ascending node ids. + * alive_fresh -- declared members that are alive AND heartbeat-fresh + * (L420: cluster_reconfig_get_observed_fresh_alive), + * NOT mere record/slot existence. + * dead_decided -- the accepted reconfig dead set (durable quorum + * decision), NOT a transient live-signal read (L419). + */ +typedef struct ClusterUndoAuthorityInput { + int32 owner_node; + uint64 request_epoch; + uint64 snapshot_epoch; + uint8 declared[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint8 alive_fresh[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint8 dead_decided[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; +} ClusterUndoAuthorityInput; + +typedef struct ClusterUndoAuthorityDecision { + ClusterUndoAuthorityStatus status; + int32 authority_node; /* valid iff status == OK, else -1 */ +} ClusterUndoAuthorityDecision; + +/* + * Pure serve-authority election. No shmem / GRD / GUC dependencies. + * + * Rule (mirrors cluster_grd_master_map_remaster survivor selection): + * authority = lowest node id that is (declared AND alive_fresh), + * elected ONLY when the owner is dead-decided at the caller's epoch. + * Returns via *out; the return value echoes out->status for convenience. + */ +extern ClusterUndoAuthorityStatus cluster_undo_authority_decide(const ClusterUndoAuthorityInput *in, + ClusterUndoAuthorityDecision *out); + +/* + * Live-snapshot wrapper (D4-1 second half, implemented in a heavy object, + * TAP-covered). Reads the real GRD / membership / freshness / epoch state, + * fills a ClusterUndoAuthorityInput, and calls cluster_undo_authority_decide. + * owner_node -- undo resource owner (canonical, from D1 resid). + * reconfig_epoch -- caller's scoped epoch (cluster_epoch_get_current()). + * *out_authority -- elected authority node when the return is OK, else -1. + */ +extern ClusterUndoAuthorityStatus +cluster_undo_serve_authority_lookup(int32 owner_node, uint64 reconfig_epoch, int32 *out_authority); + +#endif /* USE_PGRAC_CLUSTER */ + +#endif /* CLUSTER_UNDO_AUTHORITY_H */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 55664241b6..729bf4af39 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -88,6 +88,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_hang_acceptance \ test_cluster_xid_stripe \ test_cluster_undo_resid \ + test_cluster_undo_authority \ test_cluster_undo_gcs \ test_cluster_undo_verdict \ test_cluster_vis_undo_verdict_map @@ -182,7 +183,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_authority test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1857,6 +1858,20 @@ test_cluster_undo_resid: test_cluster_undo_resid.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_RESID_O) $(CLUSTER_HW_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-5.22d D4-1: test_cluster_undo_authority — dead/absent-owner undo +# serve-authority election pure layer (deterministic lowest-fresh-survivor + +# fail-closed edges). The pure decide() has no PG-backend / GRD / GUC +# dependencies, so it links standalone (nothing stubbed beyond the Assert +# ExceptionalCondition hook). The live-snapshot wrapper (real GRD / +# membership / epoch reads) lives in cluster_undo_authority's heavy path and +# is forward-covered by the D4-8 TAP. +CLUSTER_UNDO_AUTHORITY_O = $(top_builddir)/src/backend/cluster/cluster_undo_authority.o +test_cluster_undo_authority: test_cluster_undo_authority.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_AUTHORITY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_AUTHORITY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + # spec-5.22b D2-1/D2-3: test_cluster_undo_gcs — owner-as-master routing layer # (D2-1: cluster_undo_block_lookup_master / _master_is_self) + the pure reader # S-grant decision core (D2-3: cluster_undo_grant_armed / _admissible / diff --git a/src/test/cluster_unit/test_cluster_undo_authority.c b/src/test/cluster_unit/test_cluster_undo_authority.c new file mode 100644 index 0000000000..1d96c39939 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_undo_authority.c @@ -0,0 +1,234 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_undo_authority.c + * Unit tests for the dead/absent-owner undo serve-authority election + * pure layer (spec-5.22d D4-1). + * + * Covers the pure decision core cluster_undo_authority_decide(): the + * deterministic lowest-fresh-survivor election, and every fail-closed + * edge (epoch mismatch, owner-live short-circuit, owner-undecided, + * no-survivor, malformed owner). Election NEVER returns a node that + * is not both declared and fresh-alive (the "no ad-hoc / no hash + * fallback" property, spec-5.22d §3.3 约束 #2 at the derivation layer). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_undo_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-1, §4.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_undo_authority.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} + +/* ---- bitmap helpers (mirror the on-wire 128-bit membership shape) ---- */ +static void +bm_set(uint8 *bm, int node) +{ + bm[node >> 3] |= (uint8)(1u << (node & 7)); +} + +/* + * Build a canonical input: declared = {0..n_declared-1}; alive_fresh and + * dead_decided start empty; epochs both = epoch. Callers then mark the + * fresh / dead members they want. + */ +static void +input_init(ClusterUndoAuthorityInput *in, int32 owner, uint64 epoch, int n_declared) +{ + int i; + + memset(in, 0, sizeof(*in)); + in->owner_node = owner; + in->request_epoch = epoch; + in->snapshot_epoch = epoch; + for (i = 0; i < n_declared; i++) + bm_set(in->declared, i); +} + +/* ====================================================================== + * U1 -- dead owner elects the lowest fresh-alive survivor + * ====================================================================== */ +UT_TEST(test_authority_dead_owner_elects_lowest_survivor) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + /* declared {0,1,2,3}; owner 0 dead-decided; 1,2,3 fresh-alive */ + input_init(&in, 0 /*owner*/, 42 /*epoch*/, 4); + bm_set(in.dead_decided, 0); + bm_set(in.alive_fresh, 1); + bm_set(in.alive_fresh, 2); + bm_set(in.alive_fresh, 3); + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_OK); + UT_ASSERT_EQ(out.status, CLUSTER_UNDO_AUTHORITY_OK); + UT_ASSERT_EQ(out.authority_node, 1); /* lowest fresh survivor */ +} + +/* ====================================================================== + * U2 -- owner is fresh-alive: no election, stay on live-owner (D6) path + * ====================================================================== */ +UT_TEST(test_authority_owner_live_no_election) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + input_init(&in, 2 /*owner*/, 42, 4); + bm_set(in.alive_fresh, 1); + bm_set(in.alive_fresh, 2); /* owner itself fresh */ + bm_set(in.alive_fresh, 3); + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_OWNER_LIVE); + UT_ASSERT_EQ(out.authority_node, -1); +} + +/* ====================================================================== + * U7 -- owner neither fresh-alive nor dead-decided: 生死未定 => UNKNOWN + * (fail closed; NEVER guess dead and elect an authority) + * ====================================================================== */ +UT_TEST(test_authority_owner_undecided_unknown) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + input_init(&in, 0 /*owner*/, 42, 4); + /* owner 0 NOT in alive_fresh and NOT in dead_decided */ + bm_set(in.alive_fresh, 1); + bm_set(in.alive_fresh, 2); + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_UNKNOWN); + UT_ASSERT_EQ(out.authority_node, -1); +} + +/* ====================================================================== + * U1b -- epoch mismatch: caller's scope is stale => RECOVERING + * ====================================================================== */ +UT_TEST(test_authority_epoch_mismatch_recovering) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + input_init(&in, 0, 42, 4); + bm_set(in.dead_decided, 0); + bm_set(in.alive_fresh, 1); + in.snapshot_epoch = 43; /* moved on */ + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_RECOVERING); + UT_ASSERT_EQ(out.authority_node, -1); +} + +/* ====================================================================== + * U1c -- dead owner but no fresh survivor exists => RECOVERING + * (never elect a non-fresh node; no hash / ad-hoc fallback) + * ====================================================================== */ +UT_TEST(test_authority_no_fresh_survivor_recovering) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + input_init(&in, 0, 42, 4); + bm_set(in.dead_decided, 0); + /* declared {0,1,2,3} but NONE fresh-alive */ + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_RECOVERING); + UT_ASSERT_EQ(out.authority_node, -1); +} + +/* ====================================================================== + * U1d -- lowest survivor skips a lower non-fresh (dead) node; the dead + * owner is itself excluded from survivors even if it would be lowest + * ====================================================================== */ +UT_TEST(test_authority_skips_lower_nonfresh) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + /* declared {0,1,2,3}; owner 0 dead; node 1 also NOT fresh (down); + * node 2,3 fresh -> authority must be 2, not 0 or 1 */ + input_init(&in, 0, 42, 4); + bm_set(in.dead_decided, 0); + bm_set(in.alive_fresh, 2); + bm_set(in.alive_fresh, 3); + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_OK); + UT_ASSERT_EQ(out.authority_node, 2); +} + +/* ====================================================================== + * U1e -- authority must be a DECLARED node: a fresh bit outside the + * declared set is never elected + * ====================================================================== */ +UT_TEST(test_authority_requires_declared) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + /* declared {0,1}; owner 0 dead; node 1 NOT fresh; node 5 fresh but + * NOT declared -> no eligible survivor -> RECOVERING */ + input_init(&in, 0, 42, 2); + bm_set(in.dead_decided, 0); + bm_set(in.alive_fresh, 5); /* undeclared */ + + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_RECOVERING); + UT_ASSERT_EQ(out.authority_node, -1); +} + +/* ====================================================================== + * U1f -- malformed owner (out of [0, SCN_MAX_VALID_NODE_ID]) => UNKNOWN + * ====================================================================== */ +UT_TEST(test_authority_invalid_owner_unknown) +{ + ClusterUndoAuthorityInput in; + ClusterUndoAuthorityDecision out; + + input_init(&in, -1 /*owner*/, 42, 4); + bm_set(in.alive_fresh, 1); + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_UNKNOWN); + UT_ASSERT_EQ(out.authority_node, -1); + + input_init(&in, 128 /*owner > 127*/, 42, 4); + bm_set(in.alive_fresh, 1); + UT_ASSERT_EQ(cluster_undo_authority_decide(&in, &out), CLUSTER_UNDO_AUTHORITY_UNKNOWN); + UT_ASSERT_EQ(out.authority_node, -1); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_authority_dead_owner_elects_lowest_survivor); + UT_RUN(test_authority_owner_live_no_election); + UT_RUN(test_authority_owner_undecided_unknown); + UT_RUN(test_authority_epoch_mismatch_recovering); + UT_RUN(test_authority_no_fresh_survivor_recovering); + UT_RUN(test_authority_skips_lower_nonfresh); + UT_RUN(test_authority_requires_declared); + UT_RUN(test_authority_invalid_owner_unknown); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From c9ef43533ea9ead97544b12ac06a51f8cbca15c8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 23:31:21 +0800 Subject: [PATCH 17/38] feat(cluster): D4-2 undo serve-authority routing layer (spec-5.22d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identity (canonical owner, D1 cluster_undo_resid_master) kept separate from serve DESTINATION (owner-live -> owner; dead -> elected authority). Pure cluster_undo_route_decide() maps (owner, epoch, status, authority) -> route with NO branch to a GRD hash-master: a lookup miss fails closed to destination -1, never falls back to hash routing (约束 #2). Heavy glue cluster_undo_serve_authority(resid, epoch) composes D1 owner + D4-1 wrapper + the mapper. 4 route unit cases (U3a-d) RED->GREEN; 12/12 total. --- src/backend/cluster/cluster_undo_authority.c | 35 +++++++++++++ .../cluster/cluster_undo_authority_snapshot.c | 22 ++++++++ src/include/cluster/cluster_undo_authority.h | 36 +++++++++++++ .../test_cluster_undo_authority.c | 51 ++++++++++++++++++- 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_undo_authority.c b/src/backend/cluster/cluster_undo_authority.c index dbb073357c..6b4e89b657 100644 --- a/src/backend/cluster/cluster_undo_authority.c +++ b/src/backend/cluster/cluster_undo_authority.c @@ -88,4 +88,39 @@ cluster_undo_authority_decide(const ClusterUndoAuthorityInput *in, return (out->status = CLUSTER_UNDO_AUTHORITY_RECOVERING); } +/* + * cluster_undo_route_decide -- pure serve routing. + * + * OWNER_LIVE -> serve from the (live) owner (D6 path). + * OK -> serve from the elected authority. + * RECOVERING/UNKNOWN -> destination -1 (fail closed). There is NO branch + * to a GRD hash-master: undo is owner-as-master, so a + * miss fails closed, never falls back to hash routing + * (spec-5.22d 约束 #2). + */ +ClusterUndoServeRoute +cluster_undo_route_decide(int32 owner_node, uint64 reconfig_epoch, + ClusterUndoAuthorityStatus status, int32 authority_node) +{ + ClusterUndoServeRoute r; + + r.reconfig_epoch = reconfig_epoch; + r.status = status; + + switch (status) { + case CLUSTER_UNDO_AUTHORITY_OWNER_LIVE: + r.destination_node = owner_node; + break; + case CLUSTER_UNDO_AUTHORITY_OK: + r.destination_node = authority_node; + break; + case CLUSTER_UNDO_AUTHORITY_RECOVERING: + case CLUSTER_UNDO_AUTHORITY_UNKNOWN: + default: + r.destination_node = -1; /* fail closed; never a hash master */ + break; + } + return r; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c index 1cf18d21c4..f187347314 100644 --- a/src/backend/cluster/cluster_undo_authority_snapshot.c +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -36,6 +36,7 @@ #include "cluster/cluster_reconfig.h" /* cluster_reconfig_get_observed_fresh_alive */ #include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ #include "cluster/cluster_undo_authority.h" +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_master (D1) */ #ifdef USE_PGRAC_CLUSTER @@ -102,4 +103,25 @@ cluster_undo_serve_authority_lookup(int32 owner_node, uint64 reconfig_epoch, int return out.status; } +/* + * cluster_undo_serve_authority -- resolve the serve route for one undo + * resource (D4-2 heavy glue). + * + * Canonical owner comes from the D1 pure identity (cluster_undo_resid_ + * master, never a hash route); the live serve authority comes from the + * D4-1 wrapper; the pure D4-2 mapper turns the two into a route. A lookup + * miss yields a fail-closed route (destination -1), NEVER the GRD hash + * master (spec-5.22d 约束 #2 / §2.2). + */ +ClusterUndoServeRoute +cluster_undo_serve_authority(const ClusterResId *undo_resid, uint64 reconfig_epoch) +{ + int32 owner_node = cluster_undo_resid_master(undo_resid); + int32 authority_node = -1; + ClusterUndoAuthorityStatus st; + + st = cluster_undo_serve_authority_lookup(owner_node, reconfig_epoch, &authority_node); + return cluster_undo_route_decide(owner_node, reconfig_epoch, st, authority_node); +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h index 17f62f666d..65901a34b1 100644 --- a/src/include/cluster/cluster_undo_authority.h +++ b/src/include/cluster/cluster_undo_authority.h @@ -41,6 +41,7 @@ #define CLUSTER_UNDO_AUTHORITY_H #include "c.h" +#include "cluster/cluster_grd.h" /* ClusterResId */ #include "cluster/cluster_reconfig.h" /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ #ifdef USE_PGRAC_CLUSTER @@ -120,6 +121,41 @@ extern ClusterUndoAuthorityStatus cluster_undo_authority_decide(const ClusterUnd extern ClusterUndoAuthorityStatus cluster_undo_serve_authority_lookup(int32 owner_node, uint64 reconfig_epoch, int32 *out_authority); +/* + * D4-2 -- serve routing. Identity (canonical owner, D1 cluster_undo_resid_ + * master) is deliberately kept separate from the serve DESTINATION: the + * request's owner never changes, only WHERE the verdict is served from. + * + * destination_node -- owner (OWNER_LIVE) or elected authority (OK); + * -1 for RECOVERING / UNKNOWN (fail-closed). It is + * NEVER a GRD shard hash-master -- undo is owner-as- + * master, so a lookup miss fails closed, it does NOT + * fall back to the hash route (spec-5.22d 约束 #2). + */ +typedef struct ClusterUndoServeRoute { + int32 destination_node; + uint64 reconfig_epoch; + ClusterUndoAuthorityStatus status; +} ClusterUndoServeRoute; + +/* + * Pure route mapper: (owner, epoch, lookup status, elected authority) -> route. + * No shmem deps; unit-testable. The only nodes it can ever emit are the + * passed owner (live) or authority (elected) -- structurally there is no path + * to a hash-master, so a fail-closed status yields destination -1. + */ +extern ClusterUndoServeRoute cluster_undo_route_decide(int32 owner_node, uint64 reconfig_epoch, + ClusterUndoAuthorityStatus status, + int32 authority_node); + +/* + * Heavy glue (snapshot object, TAP-covered): resolve the canonical owner from + * the undo resid (D1), look up the live serve authority (D4-1 wrapper), and + * map to a route. Never routes undo through the GRD hash master. + */ +extern ClusterUndoServeRoute cluster_undo_serve_authority(const ClusterResId *undo_resid, + uint64 reconfig_epoch); + #endif /* USE_PGRAC_CLUSTER */ #endif /* CLUSTER_UNDO_AUTHORITY_H */ diff --git a/src/test/cluster_unit/test_cluster_undo_authority.c b/src/test/cluster_unit/test_cluster_undo_authority.c index 1d96c39939..b278eefdf5 100644 --- a/src/test/cluster_unit/test_cluster_undo_authority.c +++ b/src/test/cluster_unit/test_cluster_undo_authority.c @@ -217,10 +217,55 @@ UT_TEST(test_authority_invalid_owner_unknown) UT_ASSERT_EQ(out.authority_node, -1); } +/* ====================================================================== + * D4-2 route mapper (U3 family) -- identity(owner) vs destination(authority) + * separation; a fail-closed status NEVER yields a node (no hash fallback). + * ====================================================================== */ + +/* U3a -- OWNER_LIVE routes to the owner itself (live-owner / D6 path) */ +UT_TEST(test_route_owner_live_routes_owner) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(2 /*owner*/, 42, CLUSTER_UNDO_AUTHORITY_OWNER_LIVE, -1); + + UT_ASSERT_EQ(r.status, CLUSTER_UNDO_AUTHORITY_OWNER_LIVE); + UT_ASSERT_EQ(r.destination_node, 2); + UT_ASSERT_EQ((int)r.reconfig_epoch, 42); +} + +/* U3b -- OK routes to the elected authority, NOT the (dead) owner */ +UT_TEST(test_route_ok_routes_authority) +{ + ClusterUndoServeRoute r = cluster_undo_route_decide(0 /*dead owner*/, 42, + CLUSTER_UNDO_AUTHORITY_OK, 3 /*authority*/); + + UT_ASSERT_EQ(r.status, CLUSTER_UNDO_AUTHORITY_OK); + UT_ASSERT_EQ(r.destination_node, 3); +} + +/* U3c -- RECOVERING fails closed to -1; NEVER a node / hash master (约束 #2) */ +UT_TEST(test_route_recovering_failclosed_not_node) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(0, 42, CLUSTER_UNDO_AUTHORITY_RECOVERING, -1); + + UT_ASSERT_EQ(r.status, CLUSTER_UNDO_AUTHORITY_RECOVERING); + UT_ASSERT_EQ(r.destination_node, -1); +} + +/* U3d -- UNKNOWN fails closed to -1 (no ad-hoc / hash fallback) */ +UT_TEST(test_route_unknown_failclosed) +{ + ClusterUndoServeRoute r = cluster_undo_route_decide(0, 42, CLUSTER_UNDO_AUTHORITY_UNKNOWN, -1); + + UT_ASSERT_EQ(r.status, CLUSTER_UNDO_AUTHORITY_UNKNOWN); + UT_ASSERT_EQ(r.destination_node, -1); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(12); UT_RUN(test_authority_dead_owner_elects_lowest_survivor); UT_RUN(test_authority_owner_live_no_election); UT_RUN(test_authority_owner_undecided_unknown); @@ -229,6 +274,10 @@ main(void) UT_RUN(test_authority_skips_lower_nonfresh); UT_RUN(test_authority_requires_declared); UT_RUN(test_authority_invalid_owner_unknown); + UT_RUN(test_route_owner_live_routes_owner); + UT_RUN(test_route_ok_routes_authority); + UT_RUN(test_route_recovering_failclosed_not_node); + UT_RUN(test_route_unknown_failclosed); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 086916d2b9b374826be03634c02b04f8fe3423d8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 07:35:01 +0800 Subject: [PATCH 18/38] feat(cluster): D4-3 undo path-intent AUTHORITY_BLOCK0 (spec-5.22d) Add a third ClusterUndoPathIntent value, CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, so a survivor node can resolve a foreign (dead) owner's durable block0 from the shared cluster_fs root, read-only. cluster_undo_path_uses_shared_root() now admits the new intent under the same peer-mode + undo_gcs_coherence arm gate as own RUNTIME_SHARED (both off => inert local DataDir). cluster_undo_path_resolve()'s owner/self assert is relaxed to permit AUTHORITY_BLOCK0 with owner != self only; a plain RUNTIME_SHARED foreign owner and AUTHORITY_BLOCK0 with owner == self are still rejected. The read stays block0-only by call-site contract. Adds U6 (test_undo_gcs_path_authority_block0_mode_branch) to test_cluster_undo_gcs.c: the AUTHORITY_BLOCK0 arm truth table (14/14). Spec: spec-5.22d-undo-dead-owner-verdict-serve.md --- src/backend/cluster/cluster_undo_gcs.c | 17 +++++++--- .../cluster/storage/cluster_undo_alloc.c | 24 ++++++++++---- .../cluster/storage/cluster_undo_alloc.h | 16 +++++++-- src/test/cluster_unit/test_cluster_undo_gcs.c | 33 ++++++++++++++++++- 4 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/backend/cluster/cluster_undo_gcs.c b/src/backend/cluster/cluster_undo_gcs.c index 19860d419c..8abc2441ef 100644 --- a/src/backend/cluster/cluster_undo_gcs.c +++ b/src/backend/cluster/cluster_undo_gcs.c @@ -91,13 +91,22 @@ cluster_undo_path_uses_shared_root(ClusterUndoPathIntent intent, bool peer_mode, if (intent == CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL) return false; - Assert(intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED); + /* + * spec-5.22d D4-3: RUNTIME_SHARED (own live undo) and + * RUNTIME_SHARED_AUTHORITY_BLOCK0 (a survivor authority reading a dead + * owner's durable block0 from shared storage) both resolve to the shared + * cluster_fs root under the SAME arm gate. MATERIALIZED_LOCAL already + * returned above; any other value is a caller bug. + */ + Assert(intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED + || intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0); /* - * Own live runtime undo migrates to the shared cluster_fs root only when - * the whole physical migration is armed: a declared multi-node + * The shared migration is armed only under a declared multi-node * deployment (peer_mode) AND cluster.undo_gcs_coherence on. Either off - * => local DataDir, so D2 lands inert at the default (裁决 A). + * => local DataDir, so both the own-instance runtime path (D2, 裁决 A) + * and the dead-owner authority block0 serve (D4) land inert at the + * default. */ return peer_mode && coherence_on; } diff --git a/src/backend/cluster/storage/cluster_undo_alloc.c b/src/backend/cluster/storage/cluster_undo_alloc.c index 335a51e021..73f8c3c808 100644 --- a/src/backend/cluster/storage/cluster_undo_alloc.c +++ b/src/backend/cluster/storage/cluster_undo_alloc.c @@ -96,14 +96,24 @@ cluster_undo_path_resolve(ClusterUndoPathIntent intent, uint8 owner_instance, ui /* * spec-5.22b D2-2 (threading strategy B): every caller derives intent via * cluster_undo_intent_for_owner(owner), so in D2 RUNTIME_SHARED ⟺ own - * instance and MATERIALIZED_LOCAL ⟺ a foreign dead-origin owner. Assert - * that invariant here: a future spec (D4) that serves a foreign owner's - * live undo from shared storage passes RUNTIME_SHARED with owner!=self and - * MUST revisit this path -- the assert makes that divergence loud rather - * than a silent split. + * instance and MATERIALIZED_LOCAL ⟺ a foreign dead-origin owner. + * + * spec-5.22d D4-3: the dead-owner authority serve introduces exactly one + * sanctioned foreign-owner shared read -- RUNTIME_SHARED_AUTHORITY_BLOCK0, + * which by construction targets a foreign owner (owner != self). Relax + * the invariant to admit that one case ONLY: + * - a plain RUNTIME_SHARED with a foreign owner is still a loud + * split-brain bug (it would silently read the shared copy of a + * segment it does not own); + * - AUTHORITY_BLOCK0 with owner == self is likewise rejected (a live + * owner serves via the D6 path, never via the authority block0 route). + * The read stays block0-only by the call-site contract (§2.3/§3.6); this + * assert only governs the owner-vs-self / intent pairing. */ - Assert((intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED) - == (owner_instance == (uint8)(cluster_node_id + 1))); + Assert(((intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED) + == (owner_instance == (uint8)(cluster_node_id + 1))) + || (intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0 + && owner_instance != (uint8)(cluster_node_id + 1))); /* * RUNTIME_SHARED own undo migrates to the shared cluster_fs root only diff --git a/src/include/cluster/storage/cluster_undo_alloc.h b/src/include/cluster/storage/cluster_undo_alloc.h index 2f65ac843a..4212a3eff1 100644 --- a/src/include/cluster/storage/cluster_undo_alloc.h +++ b/src/include/cluster/storage/cluster_undo_alloc.h @@ -99,10 +99,22 @@ * merged recovery, remote-xact outcome store). ALWAYS resolves to * the local DataDir, in any mode -- D2 must never redirect these * reads, else dead-origin recovery / CR regress (spec-5.22b §3.6, R1). + * + * CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0 (spec-5.22d D4-3) + * A survivor authority serving a DEAD owner's durable block0 (TTSlot + * verdict) from shared storage. Here owner != self, yet the read + * DOES migrate to the shared cluster_fs root -- the ONLY intent that + * resolves a FOREIGN owner's shared block0. Armed by the same gate + * as own RUNTIME_SHARED (peer-mode + cluster.undo_gcs_coherence). + * Read-only and block0-only by call-site contract (§2.3/§3.6); it + * never grants DATA-block access, and a plain RUNTIME_SHARED with + * owner != self is still rejected by cluster_undo_path_resolve. */ typedef enum ClusterUndoPathIntent { - CLUSTER_UNDO_PATH_RUNTIME_SHARED, /* own live undo; shared under peer-mode+coherence */ - CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL /* dead-origin materialized copy; always local */ + CLUSTER_UNDO_PATH_RUNTIME_SHARED, /* own live undo; shared under peer-mode+coherence */ + CLUSTER_UNDO_PATH_MATERIALIZED_LOCAL, /* dead-origin materialized copy; always local */ + CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0 /* D4: authority reads a dead owner's + * shared block0 (read-only, foreign) */ } ClusterUndoPathIntent; /* cluster_node_id owns the +1 sentinel offset: owner_instance == node_id + 1. */ diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 95ed7162e0..0f76ab8c4c 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -152,6 +152,36 @@ UT_TEST(test_undo_gcs_path_materialized_never_migrates) false /*peer*/, false /*coherence*/)); } +/* ====================================================================== + * U6 -- path-intent AUTHORITY_BLOCK0 root decision (spec-5.22d D4-3, + * 约束 #4): a survivor authority serving a DEAD owner's durable block0 from + * shared storage (owner != self) migrates to the shared cluster_fs root + * under the SAME arm gate as own-instance RUNTIME_SHARED -- peer-mode AND + * cluster.undo_gcs_coherence. Either off => inert (local DataDir), so D4 + * lands zero behaviour change at the default (回归安全). This is the ONLY + * intent that legitimately resolves a FOREIGN owner's shared block0 + * (read-only); a plain RUNTIME_SHARED foreign owner is rejected by the + * ownership assert in cluster_undo_path_resolve, so AUTHORITY_BLOCK0 does + * not generalise to DATA-block reads (spec-5.22d §2.3/§3.6). + * ====================================================================== */ +UT_TEST(test_undo_gcs_path_authority_block0_mode_branch) +{ + /* serve: survivor authority reads a dead owner's shared block0, + * peer-mode, coherence on -> shared root */ + UT_ASSERT(cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + true /*peer*/, true /*coherence*/)); + + /* coherence off => inert, stay local (回归安全) */ + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + true /*peer*/, false /*coherence*/)); + + /* non-peer-mode => local regardless of coherence (declared topology gate) */ + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + false /*peer*/, true /*coherence*/)); + UT_ASSERT(!cluster_undo_path_uses_shared_root(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + false /*peer*/, false /*coherence*/)); +} + /* ====================================================================== * U11 -- per-call intent derivation (spec-5.22b D2-2, threading strategy B): * an undo segment whose owner IS this node maps to RUNTIME_SHARED (own live @@ -396,11 +426,12 @@ UT_TEST(test_undo_gcs_serve_gate) int main(void) { - UT_PLAN(13); + UT_PLAN(14); UT_RUN(test_undo_gcs_lookup_master_is_owner); UT_RUN(test_undo_gcs_master_is_self); UT_RUN(test_undo_gcs_path_runtime_shared_mode_branch); UT_RUN(test_undo_gcs_path_materialized_never_migrates); + UT_RUN(test_undo_gcs_path_authority_block0_mode_branch); UT_RUN(test_undo_gcs_intent_for_owner); UT_RUN(test_undo_gcs_grant_armed_gate); UT_RUN(test_undo_gcs_grant_reader_pcm_contract); From 7476cb7ab5bd28e40e4794d73e07e553030b3fbc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 08:09:29 +0800 Subject: [PATCH 19/38] feat(cluster): D4-4 dead-owner verdict precision chain + block0 authority serve (spec-5.22d) Wire the dead/absent-owner serve authority into the cross-node undo verdict entry. Under the armed data plane (peer-mode + cluster.undo_gcs_coherence), cluster_undo_verdict_resolve now routes a foreign origin by owner liveness BEFORE asking the owner: - owner live: the CP3 S-grant + CP5 origin-verdict path, byte-for-byte unchanged; - owner dead, self is the elected authority: serve the verdict off the owner's durable shared block0 (AUTHORITY_BLOCK0 read-only intent, not an S-grant), admissible only under the three-way coverage AND -- claimed-at-current-epoch, block0 fully read, slot xid+wrap positive proof; any term false keeps the 53R97 fail-closed boundary; - owner dead with a peer/no authority, or liveness unproven: fail closed. Peer-authority wire serve lands with D4-5/D4-6. Never the native CLOG/hint path. New pure helpers cluster_undo_authority_serve_decide (route -> consumer action) and cluster_undo_authority_coverage_ok (three-term shape pinned by the U5 truth table); 6 new unit tests (18/18). New cr counters undo_authority_serve_hit / undo_authority_fail_closed (dump-exposed; cr category 59 -> 61, baselines reconciled in t/215, t/216, t/254). Gate off => chain inert; t/359 + t/346 stay green (the live-owner path is unchanged under the armed chain). Spec: spec-5.22d-undo-dead-owner-verdict-serve.md --- src/backend/cluster/cluster_cr.c | 31 ++++ src/backend/cluster/cluster_debug.c | 5 + .../cluster/cluster_runtime_visibility.c | 135 +++++++++++++++++- src/backend/cluster/cluster_undo_authority.c | 39 +++++ src/include/cluster/cluster_cr.h | 6 + src/include/cluster/cluster_undo_authority.h | 42 ++++++ .../t/215_cluster_3_9_cr_construction.pl | 4 +- .../t/216_cluster_3_10_cr_cache.pl | 4 +- .../t/254_pi_cr_recovery_acceptance.pl | 4 +- src/test/cluster_unit/test_cluster_debug.c | 11 ++ .../test_cluster_undo_authority.c | 99 ++++++++++++- 11 files changed, 370 insertions(+), 10 deletions(-) diff --git a/src/backend/cluster/cluster_cr.c b/src/backend/cluster/cluster_cr.c index 7817d7fbf9..7e9c617f61 100644 --- a/src/backend/cluster/cluster_cr.c +++ b/src/backend/cluster/cluster_cr.c @@ -180,6 +180,16 @@ typedef struct ClusterCRShared { */ pg_atomic_uint64 vis_freshref_verdict_resolved_count; pg_atomic_uint64 vis_freshref_verdict_failclosed_count; + /* + * spec-5.22d D4-4: dead/absent-owner authority block0 serve outcomes. + * serve_hit = self-as-authority served a terminal verdict off the dead + * owner's shared block0 (the D4-8 L1 hard assert lands here, proving the + * Route B serve ran rather than a materialized-copy read); fail_closed = + * the precision chain refused (peer/no authority, epoch moved, block0 + * unreadable, or no slot proof) and the caller kept 53R97. + */ + pg_atomic_uint64 undo_authority_serve_hit_count; + pg_atomic_uint64 undo_authority_fail_closed_count; } ClusterCRShared; static ClusterCRShared *CRShared = NULL; @@ -264,6 +274,8 @@ cluster_cr_shmem_init(void) pg_atomic_init_u64(&CRShared->rtvis_underivable_failclosed_count, 0); pg_atomic_init_u64(&CRShared->vis_freshref_verdict_resolved_count, 0); pg_atomic_init_u64(&CRShared->vis_freshref_verdict_failclosed_count, 0); + pg_atomic_init_u64(&CRShared->undo_authority_serve_hit_count, 0); + pg_atomic_init_u64(&CRShared->undo_authority_fail_closed_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_resolved_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_recycled_invisible_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_invalid_or_ambiguous_count, 0); @@ -440,6 +452,22 @@ cluster_vis_freshref_verdict_note_failclosed(void) if (CRShared != NULL) pg_atomic_fetch_add_u64(&CRShared->vis_freshref_verdict_failclosed_count, 1); } + +/* PGRAC: spec-5.22d D4-4 — dead-owner authority block0 serve bumps + * (cluster_runtime_visibility.c precision chain, backend context). */ +void +cluster_undo_authority_note_serve_hit(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->undo_authority_serve_hit_count, 1); +} + +void +cluster_undo_authority_note_failclosed(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->undo_authority_fail_closed_count, 1); +} CR_COUNTER_ACCESSOR(cluster_cr_corruption_count, cr_corruption_count) CR_COUNTER_ACCESSOR(cluster_cr_chain_walk_steps_sum, cr_chain_walk_steps_sum) CR_COUNTER_ACCESSOR(cluster_cr_inverse_insert_count, cr_inverse_insert_count) @@ -481,6 +509,9 @@ CR_COUNTER_ACCESSOR(cluster_rtvis_underivable_failclosed_count, rtvis_underivabl CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_resolved_count, vis_freshref_verdict_resolved_count) CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_failclosed_count, vis_freshref_verdict_failclosed_count) +/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +CR_COUNTER_ACCESSOR(cluster_undo_authority_serve_hit_count, undo_authority_serve_hit_count) +CR_COUNTER_ACCESSOR(cluster_undo_authority_fail_closed_count, undo_authority_fail_closed_count) /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ CR_COUNTER_ACCESSOR(cluster_cr_xmax_resolved_count, cr_xmax_resolved_count) CR_COUNTER_ACCESSOR(cluster_cr_xmax_recycled_invisible_count, cr_xmax_recycled_invisible_count) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 6819ba3529..41bfebefb5 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2593,6 +2593,11 @@ dump_cr(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_vis_freshref_verdict_resolved_count())); emit_row(rsinfo, "cr", "vis_freshref_verdict_failclosed_count", fmt_int64((int64)cluster_vis_freshref_verdict_failclosed_count())); + /* spec-5.22d D4-4: dead-owner authority block0 serve outcomes. */ + emit_row(rsinfo, "cr", "undo_authority_serve_hit_count", + fmt_int64((int64)cluster_undo_authority_serve_hit_count())); + emit_row(rsinfo, "cr", "undo_authority_fail_closed_count", + fmt_int64((int64)cluster_undo_authority_fail_closed_count())); /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ emit_row(rsinfo, "cr", "cr_xmax_resolved_count", fmt_int64((int64)cluster_cr_xmax_resolved_count())); diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index 039fec8416..a1dd1309e5 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -53,9 +53,11 @@ #include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled (D3-2) */ #include "cluster/cluster_runtime_visibility.h" #include "cluster/cluster_uba.h" -#include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ -#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ -#include "cluster/cluster_undo_verdict.h" /* verdict taxonomy + entry (D3-3/D3-4) */ +#include "cluster/cluster_undo_authority.h" /* dead-owner serve authority (D4-4) */ +#include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ +#include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block (D4-4) */ +#include "cluster/cluster_undo_verdict.h" /* verdict taxonomy + entry (D3-3/D3-4) */ /* * cluster_vis_live_authority_covers @@ -503,6 +505,97 @@ rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) return r; } +/* + * rtvis_authority_serve_block0 — D4-4 self-authority dead-owner block0 serve + * (spec-5.22d §2.4, Route B). + * + * Self IS the elected serve authority for a dead/absent owner's undo + * resource: read the owner's durable block0 straight from shared storage + * (AUTHORITY_BLOCK0 intent, read-only) and prove the slot verdict on the + * read bytes. Deliberately NOT an S-grant: the owner is dead and can grant + * nothing; block0 is synchronously durable (D2 Q3), so a successful full + * read is crash-consistent and no LSN-cover window applies (that gate + * belongs to the WAL-replay materialized path, which Route B does not use). + * + * Admissibility = the coverage predicate (约束 #3, three-way AND): + * (i) claimed_at_epoch -- the route derivation (OK) is still scoped to + * the CURRENT accepted epoch (an epoch move invalidates the claim); + * (ii) block0_readable -- the shared block0 bytes were fully read via + * the AUTHORITY_BLOCK0 foreign-owner intent (D4-3); + * (iii) wrap_match -- the slot-level xid+wrap positive proof + * matched (content-based anti-ABA -- the same proof the live CP3 + * leg trusts; a recycled segment changes the slot wrap, so the + * proof fails NONE). + * Any term false -> UNKNOWN_FAIL_CLOSED (the consumer keeps 53R97), never + * a native CLOG/hint fallback (Rule 8.A). A proof NONE has no CP5 + * fallback here: the origin is dead, there is nobody to ask -- the slot + * may live in another segment, so NONE proves nothing => fail closed. + */ +static ClusterUndoVerdictResult +rtvis_authority_serve_block0(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, + const ClusterUndoServeRoute *route) +{ + ClusterUndoVerdictResult unknown + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + PGAlignedBlock page; + SCN scn = InvalidScn; + uint16 wrap = TT_WRAP_INVALID; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; + bool claimed; + bool readable = false; + + /* + * Segment 0 is bootstrap-only (Assert-fenced in uba_encode): a ref + * carrying it is not resolvable evidence (same pre-validation as the + * live-owner remote leg). + */ + if (undo_segment_id == 0 || undo_segment_id > UINT16_MAX) { + cluster_undo_authority_note_failclosed(); + cluster_rtvis_resolve_note_failclosed(); + return unknown; + } + + /* (i) the claim must still be scoped to the CURRENT epoch */ + claimed = (route->status == CLUSTER_UNDO_AUTHORITY_OK + && cluster_epoch_get_current() == route->reconfig_epoch); + + /* (ii) read the dead owner's shared block0 (read-only foreign intent) */ + if (claimed) + readable = cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + undo_segment_id, (uint8)(origin_node + 1), + 0 /* block0 */, page.data); + + /* (iii) slot-level xid+wrap positive proof on the read bytes */ + if (readable) + proof = cluster_vis_tt_block_positive_proof(page.data, undo_segment_id, + (uint8)(origin_node + 1), raw_xid, &scn, &wrap); + + if (!cluster_undo_authority_coverage_ok(claimed, readable, + proof != CLUSTER_VIS_TT_PROOF_NONE)) { + cluster_undo_authority_note_failclosed(); + cluster_rtvis_resolve_note_failclosed(); + return unknown; + } + + cluster_undo_authority_note_serve_hit(); + if (proof == CLUSTER_VIS_TT_PROOF_COMMITTED) { + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_COMMITTED_EXACT, .commit_scn = scn, .wrap = wrap }; + + cluster_rtvis_resolve_note_committed(); + return r; + } + + /* CLUSTER_VIS_TT_PROOF_ABORTED: terminal, safe to consume */ + { + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_ABORTED, .commit_scn = InvalidScn, .wrap = wrap }; + + cluster_rtvis_resolve_note_aborted(); + return r; + } +} + /* * cluster_undo_verdict_resolve — D3 cross-node xid -> commit_scn verdict entry * (D3-3). The D6 consumer calls this to decide a seed / fresh-ref tuple. @@ -540,7 +633,41 @@ cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, Transactio if (origin_node == cluster_node_id) return rtvis_resolve_own_xid(raw_xid, read_scn); - /* master!=self: CP3 owner-as-master S-grant + CP5 origin verdict. */ + /* + * D4-4 precision chain (spec-5.22d §2.4): under the armed data plane, + * route by owner liveness BEFORE asking the owner. A dead/absent owner + * cannot serve (its S-grant / verdict wire has nobody behind it); the + * elected survivor authority serves its block0 verdict instead. Off the + * arm gate the chain is inert and the pre-D4 path below runs verbatim + * (回归安全, §2.6). + */ + if (cluster_undo_gcs_coherence && cluster_peer_mode_enabled()) { + ClusterResId rid; + ClusterUndoServeRoute route; + + cluster_undo_resid_encode((int32)origin_node, undo_segment_id, 0 /* block0 */, 0, &rid); + route = cluster_undo_serve_authority(&rid, cluster_epoch_get_current()); + switch (cluster_undo_authority_serve_decide(&route, cluster_node_id)) { + case CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0: + /* self IS the elected authority for the dead owner */ + return rtvis_authority_serve_block0(origin_node, undo_segment_id, raw_xid, &route); + case CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE: + break; /* live owner: unchanged CP3 + CP5 path below */ + case CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED: + default: + /* + * Dead owner with a peer/no authority (peer wire serve lands + * with D4-5/D4-6), or owner liveness unproven: fail closed, + * NEVER the native CLOG/hint path (Rule 8.A). + */ + cluster_undo_authority_note_failclosed(); + cluster_rtvis_resolve_note_failclosed(); + return unknown; + } + } + + /* master!=self, owner live (or data plane unarmed): CP3 S-grant + CP5 + * origin verdict, byte-for-byte the pre-D4 path. */ if (cluster_runtime_visibility_try_resolve_remote(origin_node, undo_segment_id, raw_xid, anchor_lsn, read_scn, authoritative, &committed, &commit_scn, &is_bound)) diff --git a/src/backend/cluster/cluster_undo_authority.c b/src/backend/cluster/cluster_undo_authority.c index 6b4e89b657..d59f7f3ed3 100644 --- a/src/backend/cluster/cluster_undo_authority.c +++ b/src/backend/cluster/cluster_undo_authority.c @@ -123,4 +123,43 @@ cluster_undo_route_decide(int32 owner_node, uint64 reconfig_epoch, return r; } +/* + * cluster_undo_authority_serve_decide -- pure consumer serve decision (D4-4). + * + * See cluster_undo_authority.h for the contract. The ONLY two provable + * actions are the live-owner path (D6, unchanged) and the self-authority + * block0 serve; every other route -- RECOVERING, UNKNOWN, a malformed + * destination, and an elected PEER authority (wire serve = D4-5/D4-6) -- + * fails closed. + */ +ClusterUndoAuthorityServeDecision +cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 self_node) +{ + switch (route->status) { + case CLUSTER_UNDO_AUTHORITY_OWNER_LIVE: + return CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE; + case CLUSTER_UNDO_AUTHORITY_OK: + if (self_node >= 0 && route->destination_node == self_node) + return CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0; + /* peer authority: wire serve lands with D4-5/D4-6; fail closed */ + return CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED; + case CLUSTER_UNDO_AUTHORITY_RECOVERING: + case CLUSTER_UNDO_AUTHORITY_UNKNOWN: + default: + return CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED; + } +} + +/* + * cluster_undo_authority_coverage_ok -- pure coverage predicate (D4-4). + * + * See cluster_undo_authority.h: three-way AND, each term independently + * required (约束 #3; U5 pins the shape). + */ +bool +cluster_undo_authority_coverage_ok(bool claimed_at_epoch, bool block0_readable, bool wrap_match) +{ + return claimed_at_epoch && block0_readable && wrap_match; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_cr.h b/src/include/cluster/cluster_cr.h index d13b56cef9..28b2c67381 100644 --- a/src/include/cluster/cluster_cr.h +++ b/src/include/cluster/cluster_cr.h @@ -272,6 +272,12 @@ extern uint64 cluster_vis_freshref_verdict_failclosed_count(void); extern void cluster_vis_freshref_verdict_note_resolved(void); extern void cluster_vis_freshref_verdict_note_failclosed(void); +/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +extern uint64 cluster_undo_authority_serve_hit_count(void); +extern uint64 cluster_undo_authority_fail_closed_count(void); +extern void cluster_undo_authority_note_serve_hit(void); +extern void cluster_undo_authority_note_failclosed(void); + /* * spec-6.12i CP5 (D-i4): origin-side pieces of the cross-instance verdict * serve (cluster_cr_server.c). diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h index 65901a34b1..d341944c3e 100644 --- a/src/include/cluster/cluster_undo_authority.h +++ b/src/include/cluster/cluster_undo_authority.h @@ -156,6 +156,48 @@ extern ClusterUndoServeRoute cluster_undo_route_decide(int32 owner_node, uint64 extern ClusterUndoServeRoute cluster_undo_serve_authority(const ClusterResId *undo_resid, uint64 reconfig_epoch); +/* + * D4-4 -- consumer serve decision (pure). Maps a route to the ONE action the + * verdict consumer takes: + * + * SERVE_FAIL_CLOSED -- everything unproven: RECOVERING / UNKNOWN, a + * malformed route, AND an elected PEER authority. The + * peer wire serve lands with D4-5/D4-6 (LMS authority + * serve + capability gate); until then routing a + * request there would be an unproven path, so it fails + * closed (never native, Rule 8.A). Value 0 so a + * zeroed decision is the safe one. + * SERVE_OWNER_LIVE -- owner is fresh-alive: stay on the D6 live-owner + * path, byte-for-byte unchanged. + * SERVE_SELF_BLOCK0 -- self IS the elected authority: read the dead + * owner's shared block0 (AUTHORITY_BLOCK0 intent) and + * serve the verdict locally. + */ +typedef enum ClusterUndoAuthorityServeDecision { + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED = 0, + CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE, + CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0 +} ClusterUndoAuthorityServeDecision; + +extern ClusterUndoAuthorityServeDecision +cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 self_node); + +/* + * D4-4 -- coverage predicate (spec-5.22d §2.4, 约束 #3). The authority may + * serve a dead owner's block0 verdict ONLY under the three-way AND: + * (i) claimed_at_epoch -- the authority derivation held at the CURRENT + * reconfig epoch (route OK and the epoch has not moved since); + * (ii) block0_readable -- the shared block0 bytes were actually read + * (AUTHORITY_BLOCK0 path resolve + full-block pread); + * (iii) wrap_match -- the slot-level xid+wrap positive proof matched + * (content-based anti-ABA; a recycled segment changes the wrap). + * Any term false => not covered => UNKNOWN_FAIL_CLOSED (53R97), never a + * native CLOG/hint fallback. Named (rather than an inline &&) so the + * three-term shape is pinned by U5's truth table. + */ +extern bool cluster_undo_authority_coverage_ok(bool claimed_at_epoch, bool block0_readable, + bool wrap_match); + #endif /* USE_PGRAC_CLUSTER */ #endif /* CLUSTER_UNDO_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl index 81c6b46281..23e08967e6 100644 --- a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl +++ b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl @@ -94,8 +94,8 @@ # ---------- my $cr_rows = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}); -is($cr_rows, '59', - 'L2 cr category has 59 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); +is($cr_rows, '61', + 'L2 cr category has 61 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 2 spec-5.22d D4 authority serve)'); # ---------- diff --git a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl index 3b849e1ea3..7251abab6e 100644 --- a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl +++ b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl @@ -78,8 +78,8 @@ is( $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '59', - 'L1d cr category has 59 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict)'); + '61', + 'L1d cr category has 61 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 2 spec-5.22d D4 authority serve)'); } diff --git a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl index c29e0d2d29..fe43dee397 100644 --- a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl +++ b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl @@ -125,8 +125,10 @@ sub _new_single_node recovery => 39, # 3.16(4)+4.10 block(2)+4.11 thread(4)+4.3 plan(13)+4.4 worker(8)+4.5/4.7 merge(8) tt_recovery => 8, # 4.8 verdict counters gcs_recovery => 10, # 4.7 warm-recovery(8) + spec-2.41 D7 redo-coverage serve-gate(2) - cr => 59, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + cr => 61, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) # + ... + 6.12b(6) + 6.12i/6.15(16) + 2 spec-5.22f D6 fresh-ref verdict + # + 2 spec-5.22d D4 authority serve (undo_authority_ + # {serve_hit,fail_closed}, unconditional in dump_cr) # + 11 post-5.54 keys the baseline missed (stale on # main; first caught by a local full run): 6.12b # cr_server_{full,partial,denied} + diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 0b848f9193..02329ac978 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1854,6 +1854,17 @@ cluster_vis_freshref_verdict_failclosed_count(void) { return 0; } +/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +uint64 +cluster_undo_authority_serve_hit_count(void) +{ + return 0; +} +uint64 +cluster_undo_authority_fail_closed_count(void) +{ + return 0; +} /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ uint64 cluster_cr_xmax_resolved_count(void) diff --git a/src/test/cluster_unit/test_cluster_undo_authority.c b/src/test/cluster_unit/test_cluster_undo_authority.c index b278eefdf5..b788baefc1 100644 --- a/src/test/cluster_unit/test_cluster_undo_authority.c +++ b/src/test/cluster_unit/test_cluster_undo_authority.c @@ -262,10 +262,101 @@ UT_TEST(test_route_unknown_failclosed) UT_ASSERT_EQ(r.destination_node, -1); } +/* ====================================================================== + * U5 -- coverage predicate (spec-5.22d D4-4, §2.4 约束 #3): the authority + * block0 serve is admissible ONLY under the three-way AND + * claimed-at-epoch ∧ block0-readable ∧ wrap-match. + * Each term singly false must fail the whole predicate (three negative + * sub-cases) -- a future edit cannot silently drop one term. + * ====================================================================== */ +UT_TEST(test_coverage_three_way_and) +{ + /* all three proven -> covered */ + UT_ASSERT(cluster_undo_authority_coverage_ok(true, true, true)); + + /* (i) authority not claimed at the current epoch -> not covered */ + UT_ASSERT(!cluster_undo_authority_coverage_ok(false, true, true)); + + /* (ii) shared block0 not readable -> not covered */ + UT_ASSERT(!cluster_undo_authority_coverage_ok(true, false, true)); + + /* (iii) generation/wrap mismatch (ABA suspect) -> not covered */ + UT_ASSERT(!cluster_undo_authority_coverage_ok(true, true, false)); +} + +/* ====================================================================== + * D4-4 serve-decision mapper -- route -> consumer action. OWNER_LIVE keeps + * the D6 live-owner path; OK with destination == self serves the dead + * owner's shared block0 locally; EVERYTHING else fails closed, including + * an elected PEER authority (its wire serve lands with D4-5/D4-6 -- until + * then routing a request there would be an unproven path). + * ====================================================================== */ + +/* owner live -> stay on the live-owner (D6) path, never block0-serve */ +UT_TEST(test_serve_decide_owner_live) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(2 /*owner*/, 42, CLUSTER_UNDO_AUTHORITY_OWNER_LIVE, -1); + + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1 /*self*/), + CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE); +} + +/* elected authority == self -> serve the dead owner's shared block0 */ +UT_TEST(test_serve_decide_self_block0) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(0 /*dead owner*/, 42, CLUSTER_UNDO_AUTHORITY_OK, 1); + + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1 /*self*/), + CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0); +} + +/* elected authority == a PEER -> fail closed (peer wire serve = D4-5/D4-6) */ +UT_TEST(test_serve_decide_peer_failclosed) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(0 /*dead owner*/, 42, CLUSTER_UNDO_AUTHORITY_OK, 3); + + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1 /*self*/), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); +} + +/* RECOVERING / UNKNOWN -> fail closed (never native, never a guess) */ +UT_TEST(test_serve_decide_recovering_unknown_failclosed) +{ + ClusterUndoServeRoute r + = cluster_undo_route_decide(0, 42, CLUSTER_UNDO_AUTHORITY_RECOVERING, -1); + + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); + + r = cluster_undo_route_decide(0, 42, CLUSTER_UNDO_AUTHORITY_UNKNOWN, -1); + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); +} + +/* defense in depth: an OK route carrying an invalid destination, or an + * invalid self node, can never yield a serve decision */ +UT_TEST(test_serve_decide_invalid_dest_failclosed) +{ + ClusterUndoServeRoute r; + + /* malformed: OK status but destination -1 (cannot happen through + * route_decide; guard against a hand-rolled route) */ + r.status = CLUSTER_UNDO_AUTHORITY_OK; + r.destination_node = -1; + r.reconfig_epoch = 42; + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, -1), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); +} + int main(void) { - UT_PLAN(12); + UT_PLAN(18); UT_RUN(test_authority_dead_owner_elects_lowest_survivor); UT_RUN(test_authority_owner_live_no_election); UT_RUN(test_authority_owner_undecided_unknown); @@ -278,6 +369,12 @@ main(void) UT_RUN(test_route_ok_routes_authority); UT_RUN(test_route_recovering_failclosed_not_node); UT_RUN(test_route_unknown_failclosed); + UT_RUN(test_coverage_three_way_and); + UT_RUN(test_serve_decide_owner_live); + UT_RUN(test_serve_decide_self_block0); + UT_RUN(test_serve_decide_peer_failclosed); + UT_RUN(test_serve_decide_recovering_unknown_failclosed); + UT_RUN(test_serve_decide_invalid_dest_failclosed); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From e64d542e932e70e68e5d6debe8ab76c8d9c9f174 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 08:22:56 +0800 Subject: [PATCH 20/38] feat(cluster): D4-5 shared authority block0-prove core + epoch-stale counter (spec-5.22d) Extract the dead-owner block0 serve into ONE prove core, cluster_undo_authority_block0_prove (heavy snapshot object): input validation, the claimed-at-current-epoch check, the AUTHORITY_BLOCK0 shared read, the slot xid+wrap positive proof, and the coverage three-way AND all live in a single implementation, so the requester self-serve leg (D4-4) and the wire-served LMS authority leg (D4-6) can never diverge over one (owner, segment, xid) -- the same discipline as cluster_lms_undo_verdict_fill_page on the live-owner path. The undo_authority_* counters are owned by the core, so no consumer can forget the observability contract. rtvis_authority_serve_block0 becomes a thin wrapper (route-OK assert + rtvis resolve counter fold); behavior unchanged, t/359 + t/346 stay green. Add the third serve counter, undo_authority_epoch_stale_reject: the epoch axis of a refusal, bumped in addition to fail_closed (which stays the total-refusal count). cr category 61 -> 62; baselines reconciled in t/215, t/216, t/254. Spec: spec-5.22d-undo-dead-owner-verdict-serve.md --- src/backend/cluster/cluster_cr.c | 19 +++- src/backend/cluster/cluster_debug.c | 4 +- .../cluster/cluster_runtime_visibility.c | 95 ++++--------------- .../cluster/cluster_undo_authority_snapshot.c | 85 ++++++++++++++++- src/include/cluster/cluster_cr.h | 4 +- src/include/cluster/cluster_undo_authority.h | 30 +++++- .../t/215_cluster_3_9_cr_construction.pl | 4 +- .../t/216_cluster_3_10_cr_cache.pl | 4 +- .../t/254_pi_cr_recovery_acceptance.pl | 7 +- src/test/cluster_unit/test_cluster_debug.c | 7 +- 10 files changed, 163 insertions(+), 96 deletions(-) diff --git a/src/backend/cluster/cluster_cr.c b/src/backend/cluster/cluster_cr.c index 7e9c617f61..19b80bfaac 100644 --- a/src/backend/cluster/cluster_cr.c +++ b/src/backend/cluster/cluster_cr.c @@ -186,10 +186,15 @@ typedef struct ClusterCRShared { * owner's shared block0 (the D4-8 L1 hard assert lands here, proving the * Route B serve ran rather than a materialized-copy read); fail_closed = * the precision chain refused (peer/no authority, epoch moved, block0 - * unreadable, or no slot proof) and the caller kept 53R97. + * unreadable, or no slot proof) and the caller kept 53R97; + * epoch_stale_reject (D4-5) = the specific epoch axis of a refusal (the + * claim's reconfig epoch no longer is the current one) -- bumped IN + * ADDITION to fail_closed, so fail_closed stays the total-refusal count + * (D4-8 L3 asserts the axis, not just the total). */ pg_atomic_uint64 undo_authority_serve_hit_count; pg_atomic_uint64 undo_authority_fail_closed_count; + pg_atomic_uint64 undo_authority_epoch_stale_reject_count; } ClusterCRShared; static ClusterCRShared *CRShared = NULL; @@ -276,6 +281,7 @@ cluster_cr_shmem_init(void) pg_atomic_init_u64(&CRShared->vis_freshref_verdict_failclosed_count, 0); pg_atomic_init_u64(&CRShared->undo_authority_serve_hit_count, 0); pg_atomic_init_u64(&CRShared->undo_authority_fail_closed_count, 0); + pg_atomic_init_u64(&CRShared->undo_authority_epoch_stale_reject_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_resolved_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_recycled_invisible_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_invalid_or_ambiguous_count, 0); @@ -468,6 +474,13 @@ cluster_undo_authority_note_failclosed(void) if (CRShared != NULL) pg_atomic_fetch_add_u64(&CRShared->undo_authority_fail_closed_count, 1); } + +void +cluster_undo_authority_note_epoch_stale_reject(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->undo_authority_epoch_stale_reject_count, 1); +} CR_COUNTER_ACCESSOR(cluster_cr_corruption_count, cr_corruption_count) CR_COUNTER_ACCESSOR(cluster_cr_chain_walk_steps_sum, cr_chain_walk_steps_sum) CR_COUNTER_ACCESSOR(cluster_cr_inverse_insert_count, cr_inverse_insert_count) @@ -509,9 +522,11 @@ CR_COUNTER_ACCESSOR(cluster_rtvis_underivable_failclosed_count, rtvis_underivabl CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_resolved_count, vis_freshref_verdict_resolved_count) CR_COUNTER_ACCESSOR(cluster_vis_freshref_verdict_failclosed_count, vis_freshref_verdict_failclosed_count) -/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +/* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve counters. */ CR_COUNTER_ACCESSOR(cluster_undo_authority_serve_hit_count, undo_authority_serve_hit_count) CR_COUNTER_ACCESSOR(cluster_undo_authority_fail_closed_count, undo_authority_fail_closed_count) +CR_COUNTER_ACCESSOR(cluster_undo_authority_epoch_stale_reject_count, + undo_authority_epoch_stale_reject_count) /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ CR_COUNTER_ACCESSOR(cluster_cr_xmax_resolved_count, cr_xmax_resolved_count) CR_COUNTER_ACCESSOR(cluster_cr_xmax_recycled_invisible_count, cr_xmax_recycled_invisible_count) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 41bfebefb5..af81eb2e15 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2593,11 +2593,13 @@ dump_cr(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_vis_freshref_verdict_resolved_count())); emit_row(rsinfo, "cr", "vis_freshref_verdict_failclosed_count", fmt_int64((int64)cluster_vis_freshref_verdict_failclosed_count())); - /* spec-5.22d D4-4: dead-owner authority block0 serve outcomes. */ + /* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve outcomes. */ emit_row(rsinfo, "cr", "undo_authority_serve_hit_count", fmt_int64((int64)cluster_undo_authority_serve_hit_count())); emit_row(rsinfo, "cr", "undo_authority_fail_closed_count", fmt_int64((int64)cluster_undo_authority_fail_closed_count())); + emit_row(rsinfo, "cr", "undo_authority_epoch_stale_reject_count", + fmt_int64((int64)cluster_undo_authority_epoch_stale_reject_count())); /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ emit_row(rsinfo, "cr", "cr_xmax_resolved_count", fmt_int64((int64)cluster_cr_xmax_resolved_count())); diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index a1dd1309e5..9e5721afae 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -56,7 +56,6 @@ #include "cluster/cluster_undo_authority.h" /* dead-owner serve authority (D4-4) */ #include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ #include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ -#include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block (D4-4) */ #include "cluster/cluster_undo_verdict.h" /* verdict taxonomy + entry (D3-3/D3-4) */ /* @@ -507,93 +506,35 @@ rtvis_resolve_own_xid(TransactionId raw_xid, SCN read_scn) /* * rtvis_authority_serve_block0 — D4-4 self-authority dead-owner block0 serve - * (spec-5.22d §2.4, Route B). + * (spec-5.22d §2.4, Route B), requester-leg wrapper. * * Self IS the elected serve authority for a dead/absent owner's undo - * resource: read the owner's durable block0 straight from shared storage - * (AUTHORITY_BLOCK0 intent, read-only) and prove the slot verdict on the - * read bytes. Deliberately NOT an S-grant: the owner is dead and can grant - * nothing; block0 is synchronously durable (D2 Q3), so a successful full - * read is crash-consistent and no LSN-cover window applies (that gate - * belongs to the WAL-replay materialized path, which Route B does not use). - * - * Admissibility = the coverage predicate (约束 #3, three-way AND): - * (i) claimed_at_epoch -- the route derivation (OK) is still scoped to - * the CURRENT accepted epoch (an epoch move invalidates the claim); - * (ii) block0_readable -- the shared block0 bytes were fully read via - * the AUTHORITY_BLOCK0 foreign-owner intent (D4-3); - * (iii) wrap_match -- the slot-level xid+wrap positive proof - * matched (content-based anti-ABA -- the same proof the live CP3 - * leg trusts; a recycled segment changes the slot wrap, so the - * proof fails NONE). - * Any term false -> UNKNOWN_FAIL_CLOSED (the consumer keeps 53R97), never - * a native CLOG/hint fallback (Rule 8.A). A proof NONE has no CP5 - * fallback here: the origin is dead, there is nobody to ask -- the slot - * may live in another segment, so NONE proves nothing => fail closed. + * resource. The read + coverage + proof runs in the SHARED prove core + * (cluster_undo_authority_block0_prove, D4-5) — the same core the + * wire-served LMS authority leg runs, so the self answer and the served + * answer over one (owner, segment, xid) can never diverge (Rule 8.A, the + * same discipline as cluster_lms_undo_verdict_fill_page on the live-owner + * path). This wrapper only folds the outcome into the rtvis resolve + * counters; the prove core owns the undo_authority_* counters. */ static ClusterUndoVerdictResult rtvis_authority_serve_block0(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, const ClusterUndoServeRoute *route) { - ClusterUndoVerdictResult unknown - = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; - PGAlignedBlock page; - SCN scn = InvalidScn; - uint16 wrap = TT_WRAP_INVALID; - ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; - bool claimed; - bool readable = false; - - /* - * Segment 0 is bootstrap-only (Assert-fenced in uba_encode): a ref - * carrying it is not resolvable evidence (same pre-validation as the - * live-owner remote leg). - */ - if (undo_segment_id == 0 || undo_segment_id > UINT16_MAX) { - cluster_undo_authority_note_failclosed(); - cluster_rtvis_resolve_note_failclosed(); - return unknown; - } - - /* (i) the claim must still be scoped to the CURRENT epoch */ - claimed = (route->status == CLUSTER_UNDO_AUTHORITY_OK - && cluster_epoch_get_current() == route->reconfig_epoch); - - /* (ii) read the dead owner's shared block0 (read-only foreign intent) */ - if (claimed) - readable = cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, - undo_segment_id, (uint8)(origin_node + 1), - 0 /* block0 */, page.data); - - /* (iii) slot-level xid+wrap positive proof on the read bytes */ - if (readable) - proof = cluster_vis_tt_block_positive_proof(page.data, undo_segment_id, - (uint8)(origin_node + 1), raw_xid, &scn, &wrap); - - if (!cluster_undo_authority_coverage_ok(claimed, readable, - proof != CLUSTER_VIS_TT_PROOF_NONE)) { - cluster_undo_authority_note_failclosed(); - cluster_rtvis_resolve_note_failclosed(); - return unknown; - } + ClusterUndoVerdictResult r; - cluster_undo_authority_note_serve_hit(); - if (proof == CLUSTER_VIS_TT_PROOF_COMMITTED) { - ClusterUndoVerdictResult r - = { .kind = CLUSTER_UNDO_VERDICT_COMMITTED_EXACT, .commit_scn = scn, .wrap = wrap }; + /* serve_decide only emits SELF_BLOCK0 on an OK route */ + Assert(route->status == CLUSTER_UNDO_AUTHORITY_OK); + r = cluster_undo_authority_block0_prove((int32)origin_node, undo_segment_id, raw_xid, + route->reconfig_epoch); + if (r.kind == CLUSTER_UNDO_VERDICT_COMMITTED_EXACT) cluster_rtvis_resolve_note_committed(); - return r; - } - - /* CLUSTER_VIS_TT_PROOF_ABORTED: terminal, safe to consume */ - { - ClusterUndoVerdictResult r - = { .kind = CLUSTER_UNDO_VERDICT_ABORTED, .commit_scn = InvalidScn, .wrap = wrap }; - + else if (r.kind == CLUSTER_UNDO_VERDICT_ABORTED) cluster_rtvis_resolve_note_aborted(); - return r; - } + else + cluster_rtvis_resolve_note_failclosed(); + return r; } /* diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c index f187347314..a22bc9225a 100644 --- a/src/backend/cluster/cluster_undo_authority_snapshot.c +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -30,13 +30,17 @@ */ #include "postgres.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ -#include "cluster/cluster_membership.h" /* cluster_membership_get_state */ -#include "cluster/cluster_reconfig.h" /* cluster_reconfig_get_observed_fresh_alive */ -#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ +#include "cluster/cluster_cr.h" /* undo_authority_* counters (D4-5) */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_membership.h" /* cluster_membership_get_state */ +#include "cluster/cluster_reconfig.h" /* cluster_reconfig_get_observed_fresh_alive */ +#include "cluster/cluster_runtime_visibility.h" /* cluster_vis_tt_block_positive_proof */ +#include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ +#include "cluster/cluster_tt_slot.h" /* TT_WRAP_INVALID */ #include "cluster/cluster_undo_authority.h" #include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_master (D1) */ +#include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block (D4-5) */ #ifdef USE_PGRAC_CLUSTER @@ -124,4 +128,75 @@ cluster_undo_serve_authority(const ClusterResId *undo_resid, uint64 reconfig_epo return cluster_undo_route_decide(owner_node, reconfig_epoch, st, authority_node); } +/* + * cluster_undo_authority_block0_prove -- shared authority block0 prove core + * (D4-5). See cluster_undo_authority.h for the contract. + * + * The coverage three-way AND (§2.4), with the counters owned HERE so the + * requester self-serve leg (D4-4) and the wire-served LMS leg (D4-5/D4-6) + * cannot drift on the observability contract: + * (i) claimed_at_epoch: claim_epoch still IS the current accepted + * epoch; a stale claim bumps epoch_stale_reject + fail_closed. + * (ii) block0_readable: the AUTHORITY_BLOCK0 foreign-owner intent + * (D4-3) resolved + fully read the shared block0 bytes. + * (iii) wrap_match: the slot-level xid+wrap positive proof matched + * (content-based anti-ABA; the same proof the live CP3 leg + * trusts). block0 is synchronously durable (D2 Q3): a full read + * is crash-consistent, no LSN-cover window applies. + * Proof NONE has no CP5 fallback: the owner is dead, there is nobody to + * ask, so NONE proves nothing => fail closed (never a native fallback). + */ +ClusterUndoVerdictResult +cluster_undo_authority_block0_prove(int32 owner_node, uint32 segment_id, TransactionId raw_xid, + uint64 claim_epoch) +{ + ClusterUndoVerdictResult unknown + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + PGAlignedBlock page; + SCN scn = InvalidScn; + uint16 wrap = TT_WRAP_INVALID; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; + bool claimed; + bool readable = false; + + /* malformed owner: never provable (defense in depth; the route layers + * upstream already refuse it) */ + if (owner_node < 0 || owner_node > SCN_MAX_VALID_NODE_ID || !TransactionIdIsNormal(raw_xid)) { + cluster_undo_authority_note_failclosed(); + return unknown; + } + + /* segment 0 is bootstrap-only (Assert-fenced in uba_encode): a ref + * carrying it is not resolvable evidence */ + if (segment_id == 0 || segment_id > UINT16_MAX) { + cluster_undo_authority_note_failclosed(); + return unknown; + } + + /* (i) the claim must still be scoped to the CURRENT epoch */ + claimed = (cluster_epoch_get_current() == claim_epoch); + if (!claimed) + cluster_undo_authority_note_epoch_stale_reject(); + + /* (ii) read the dead owner's shared block0 (read-only foreign intent) */ + if (claimed) + readable = cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + segment_id, (uint8)(owner_node + 1), 0 /* block0 */, + page.data); + + /* (iii) slot-level xid+wrap positive proof on the read bytes */ + if (readable) + proof = cluster_vis_tt_block_positive_proof(page.data, segment_id, (uint8)(owner_node + 1), + raw_xid, &scn, &wrap); + + if (!cluster_undo_authority_coverage_ok(claimed, readable, + proof != CLUSTER_VIS_TT_PROOF_NONE)) { + cluster_undo_authority_note_failclosed(); + return unknown; + } + + cluster_undo_authority_note_serve_hit(); + return cluster_undo_verdict_from_block_proof(proof, scn, wrap); +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_cr.h b/src/include/cluster/cluster_cr.h index 28b2c67381..66bcdd3e63 100644 --- a/src/include/cluster/cluster_cr.h +++ b/src/include/cluster/cluster_cr.h @@ -272,11 +272,13 @@ extern uint64 cluster_vis_freshref_verdict_failclosed_count(void); extern void cluster_vis_freshref_verdict_note_resolved(void); extern void cluster_vis_freshref_verdict_note_failclosed(void); -/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +/* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve counters. */ extern uint64 cluster_undo_authority_serve_hit_count(void); extern uint64 cluster_undo_authority_fail_closed_count(void); +extern uint64 cluster_undo_authority_epoch_stale_reject_count(void); extern void cluster_undo_authority_note_serve_hit(void); extern void cluster_undo_authority_note_failclosed(void); +extern void cluster_undo_authority_note_epoch_stale_reject(void); /* * spec-6.12i CP5 (D-i4): origin-side pieces of the cross-instance verdict diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h index d341944c3e..664cbdf53a 100644 --- a/src/include/cluster/cluster_undo_authority.h +++ b/src/include/cluster/cluster_undo_authority.h @@ -41,8 +41,9 @@ #define CLUSTER_UNDO_AUTHORITY_H #include "c.h" -#include "cluster/cluster_grd.h" /* ClusterResId */ -#include "cluster/cluster_reconfig.h" /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ +#include "cluster/cluster_grd.h" /* ClusterResId */ +#include "cluster/cluster_reconfig.h" /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ +#include "cluster/cluster_undo_verdict.h" /* ClusterUndoVerdictResult (D4-5) */ #ifdef USE_PGRAC_CLUSTER @@ -198,6 +199,31 @@ cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 se extern bool cluster_undo_authority_coverage_ok(bool claimed_at_epoch, bool block0_readable, bool wrap_match); +/* + * D4-5 -- shared authority block0 prove core (heavy, snapshot object). The + * ONE implementation both consumers run so the self-authority answer (D4-4 + * requester leg, cluster_runtime_visibility.c) and the wire-served authority + * answer (D4-5/D4-6 LMS leg) over one (owner, segment, xid) can never + * diverge (Rule 8.A, the same discipline as cluster_lms_undo_verdict_fill_ + * page for the live-owner path). + * + * Reads the dead owner's durable shared block0 (AUTHORITY_BLOCK0 read-only + * intent) and proves the slot verdict on the read bytes, admissible only + * under the coverage three-way AND (§2.4). The epoch axis re-checks + * claim_epoch against the CURRENT accepted epoch; a stale claim bumps + * undo_authority_epoch_stale_reject in addition to fail_closed. + * + * Returns COMMITTED_EXACT{commit_scn, wrap} / ABORTED{wrap}, or + * UNKNOWN_FAIL_CLOSED for every refusal (malformed input, bootstrap + * segment 0, stale epoch, unreadable block0, no slot proof). Bumps the + * undo_authority_{serve_hit,fail_closed,epoch_stale_reject} counters + * itself, so no consumer can forget the observability contract. + */ +extern ClusterUndoVerdictResult cluster_undo_authority_block0_prove(int32 owner_node, + uint32 segment_id, + TransactionId raw_xid, + uint64 claim_epoch); + #endif /* USE_PGRAC_CLUSTER */ #endif /* CLUSTER_UNDO_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl index 23e08967e6..8c2aa8acee 100644 --- a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl +++ b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl @@ -94,8 +94,8 @@ # ---------- my $cr_rows = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}); -is($cr_rows, '61', - 'L2 cr category has 61 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 2 spec-5.22d D4 authority serve)'); +is($cr_rows, '62', + 'L2 cr category has 62 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 3 spec-5.22d D4 authority serve)'); # ---------- diff --git a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl index 7251abab6e..0d226be5c0 100644 --- a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl +++ b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl @@ -78,8 +78,8 @@ is( $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '61', - 'L1d cr category has 61 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 2 spec-5.22d D4 authority serve)'); + '62', + 'L1d cr category has 62 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 3 spec-5.22d D4 authority serve)'); } diff --git a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl index fe43dee397..ddbecc30fc 100644 --- a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl +++ b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl @@ -125,10 +125,11 @@ sub _new_single_node recovery => 39, # 3.16(4)+4.10 block(2)+4.11 thread(4)+4.3 plan(13)+4.4 worker(8)+4.5/4.7 merge(8) tt_recovery => 8, # 4.8 verdict counters gcs_recovery => 10, # 4.7 warm-recovery(8) + spec-2.41 D7 redo-coverage serve-gate(2) - cr => 61, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + cr => 62, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) # + ... + 6.12b(6) + 6.12i/6.15(16) + 2 spec-5.22f D6 fresh-ref verdict - # + 2 spec-5.22d D4 authority serve (undo_authority_ - # {serve_hit,fail_closed}, unconditional in dump_cr) + # + 3 spec-5.22d D4 authority serve (undo_authority_ + # {serve_hit,fail_closed,epoch_stale_reject}, + # unconditional in dump_cr) # + 11 post-5.54 keys the baseline missed (stale on # main; first caught by a local full run): 6.12b # cr_server_{full,partial,denied} + diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 02329ac978..b43ad7e52e 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1854,7 +1854,7 @@ cluster_vis_freshref_verdict_failclosed_count(void) { return 0; } -/* spec-5.22d D4-4: dead-owner authority block0 serve counters. */ +/* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve counters. */ uint64 cluster_undo_authority_serve_hit_count(void) { @@ -1865,6 +1865,11 @@ cluster_undo_authority_fail_closed_count(void) { return 0; } +uint64 +cluster_undo_authority_epoch_stale_reject_count(void) +{ + return 0; +} /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ uint64 cluster_cr_xmax_resolved_count(void) From 39e196559b31abdee7cb97413f87b6c48601acde Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 13:01:23 +0800 Subject: [PATCH 21/38] feat(cluster): D4-6 dead-owner authority verdict wire serve (spec-5.22d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dead undo owner's xid verdict can now be served over the wire by the deterministically elected survivor authority (Route B), activating the D4-4 requester peer leg that until now failed closed. Wire (request zero-ABI, reserved/empty fields only): - reserved_0[6] VALUE 4 = authority verdict request (kind-4); the dead owner rides in the previously-empty tag.relNumber as owner+1 (0 stays absent; kinds 1/2/3 now strictly require an empty carrier). - Reply provenance: ClusterGcsUndoVerdictPage version 2 marks authority-served; the v1 strict ==1 gate refuses it (old requesters fail closed) and the authority leg accepts ONLY version 2, refusing the BELOW_HORIZON kind outright (the block0 prove has no horizon leg). - HELLO capability bit 0x2 (UNDO_AUTHORITY_SERVE_V1), advertised unconditionally as a protocol capability; requesters never route kind 4 to a peer without it and do NOT re-run the election. Serve side (LMS): triple check before a byte is answered -- request epoch == current epoch, re-derived authority for (owner, current) is self, then the shared block0 prove core (the same implementation the D4-4 self leg runs, so wire-served and self answers can never diverge). Requester side: full reply binding -- sender == elected authority AND reply epoch == stamped epoch EXACTLY (transport HC100 >= is only a pre-filter) -- plus the version-2 structural gate/mapper. The two verdict fetch wrappers now share one transport core (single-stamped epoch); acceptance policies stay separate per leg. Tests: serve_decide truth table gains the PEER_BLOCK0 arm; new unit truth tables for the kind-4 flag, owner tag roundtrip, version-2 usable/fill/mapper closure, and the reply binding (wrong-sender and epoch±1 negatives). cluster_unit 162/162; TAP regression band t/359 t/346 t/255 t/215 t/216 t/254 all green (live-owner path byte-identical: verdict_served=8/denied=0 unchanged). Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-5 rest + D4-6) --- src/backend/cluster/cluster_cr_server.c | 119 ++++++++- src/backend/cluster/cluster_gcs_block.c | 233 ++++++++++++++---- src/backend/cluster/cluster_ic.c | 4 + .../cluster/cluster_runtime_visibility.c | 35 ++- .../cluster_runtime_visibility_policy.c | 144 +++++++++++ src/backend/cluster/cluster_sf_dep.c | 22 ++ src/backend/cluster/cluster_undo_authority.c | 21 +- src/include/cluster/cluster_cr_server.h | 24 +- src/include/cluster/cluster_gcs_block.h | 85 ++++++- src/include/cluster/cluster_ic.h | 9 + .../cluster/cluster_runtime_visibility.h | 15 ++ src/include/cluster/cluster_sf_dep.h | 5 + src/include/cluster/cluster_undo_authority.h | 21 +- src/include/cluster/cluster_undo_verdict.h | 14 ++ .../cluster_unit/test_cluster_gcs_block.c | 94 ++++++- src/test/cluster_unit/test_cluster_ic.c | 30 ++- .../test_cluster_runtime_visibility.c | 180 +++++++++++++- .../test_cluster_undo_authority.c | 17 +- 18 files changed, 975 insertions(+), 97 deletions(-) diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index d290b159ab..b67154e81f 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -54,8 +54,10 @@ #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope */ #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (D4-6 owner range) */ #include "cluster/cluster_tt_durable.h" /* resolve_by_xid (D-i4 complete scan) */ #include "cluster/cluster_tt_slot.h" /* max_recycle_horizon (D-i4 bound) */ +#include "cluster/cluster_undo_authority.h" /* authority lookup + block0 prove (D4-6) */ #include "cluster/cluster_undo_record_api.h" /* tt_retention_rollover_count */ #include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block */ #include "cluster/cluster_xid_stripe.h" /* cluster_xid_is_mine (spec-6.15 D4) */ @@ -222,6 +224,7 @@ cluster_lms_undo_fetch_submit(const GcsBlockForwardPayload *fwd) slot->undo_segment_id = segment_id; slot->undo_block_no = block_no; slot->undo_xid = InvalidTransactionId; + slot->undo_owner = -1; /* D4-6: never inherit a recycled slot's owner */ memset(&slot->undo_auth, 0, sizeof(slot->undo_auth)); /* Publish the request fields before LMS can observe PENDING. */ @@ -236,20 +239,26 @@ cluster_lms_undo_fetch_submit(const GcsBlockForwardPayload *fwd) /* * cluster_lms_undo_verdict_submit — LMON dispatch side (spec-6.12i D-i4 / - * spec-6.15 D4). + * spec-6.15 D4 / spec-5.22d D4-6). * - * Park a complete-scan verdict request. The asked-for xid rides the - * widened watermark carrier: any non-zero upper 32 bits or a non-normal - * 32-bit value is a malformed carrier — refuse (the caller replies the + * Park a complete-scan verdict request (kinds 2/3) or a kind-4 dead-owner + * AUTHORITY verdict request. The asked-for xid rides the widened + * watermark carrier: any non-zero upper 32 bits or a non-normal 32-bit + * value is a malformed carrier — refuse (the caller replies the * fail-closed DENIED; the requester keeps 53R97, Rule 8.A). The synthetic - * tag is validated for shape only; the verdict scan is complete over ALL - * self-owned segments, so the tag's segment does not scope the answer. + * tag is validated for shape only; on kinds 2/3 the verdict scan is + * complete over ALL self-owned segments (the tag's segment does not scope + * the answer) and the owner carrier MUST be empty, on kind 4 the dead + * OWNER is decoded from tag.relNumber (range-checked, never self — the + * live-owner kinds answer own xids) and the serve-side triple check owns + * all trust decisions. */ bool cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd) { uint32 segment_id = 0; uint32 block_no = 0; + int32 wire_owner = -1; uint64 carrier; if (CrServerShared == NULL || fwd == NULL) @@ -259,6 +268,17 @@ cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd) if (!GcsBlockUndoFetchTagDecode(fwd->tag, &segment_id, &block_no)) return false; + if (GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(fwd)) { + /* kind 4: decode + range-check the owner carrier; a request naming + * US as the (dead) owner is malformed — our own xids are answered + * by the live-owner kinds, never by an authority detour. */ + if (!GcsBlockUndoAuthorityFetchTagDecodeOwner(fwd->tag, &wire_owner)) + return false; + if (wire_owner < 0 || wire_owner >= CLUSTER_MAX_NODES || wire_owner == cluster_node_id) + return false; + } else if (fwd->tag.relNumber != (RelFileNumber)0) + return false; /* owner-served kinds must leave the carrier empty */ + carrier = (uint64)GcsBlockForwardPayloadGetExpectedPiWatermarkScn(fwd); if (carrier > (uint64)PG_UINT32_MAX || !TransactionIdIsNormal((TransactionId)carrier)) return false; @@ -284,6 +304,7 @@ cluster_lms_undo_verdict_submit(const GcsBlockForwardPayload *fwd) slot->undo_block_no = block_no; slot->undo_xid = (TransactionId)carrier; slot->undo_authoritative = GcsBlockForwardPayloadIsUndoVerdictAuthoritative(fwd); + slot->undo_owner = wire_owner; /* -1 on kinds 2/3; the dead owner on kind 4 */ memset(&slot->undo_auth, 0, sizeof(slot->undo_auth)); /* Publish the request fields before LMS can observe PENDING. */ @@ -424,6 +445,83 @@ lms_undo_verdict_serve(ClusterLmsCrSlot *slot) return cluster_lms_undo_verdict_fill_page(xid, slot->undo_authoritative, v); } +/* + * lms_undo_authority_verdict_serve — LMS side of one kind-4 dead-owner + * AUTHORITY verdict slot (spec-5.22d D4-6). + * + * WE are asked to serve a verdict about a DEAD owner's xid from that + * owner's durable shared block0 — never from our own TT (the xid is not + * ours). The wire is never trusted: the spec-5.22d §2.4 triple check must + * pass before a byte is answered — + * + * (a) request epoch == our CURRENT epoch (slot->epoch carries fwd.epoch, + * which the requester stamped from its current epoch; any reconfig + * since then invalidates the election the request was routed under); + * (b) re-derive: the elected authority for (owner, current epoch) is + * US (this implicitly re-proves the owner is dead-decided — a live + * or undecided owner never elects an authority); + * (c) the shared block0 prove passes (cluster_undo_authority_block0_ + * prove — the SAME core the requester's self-authority leg runs, so + * the wire-served and self answers over one (owner, segment, xid) + * can never diverge, Rule 8.A; it re-checks the epoch axis, reads + * the owner's block0 under the AUTHORITY_BLOCK0 intent and demands + * the exact xid+wrap positive proof, bumping the undo_authority_* + * counters itself). + * + * The reply page carries the version-2 AUTHORITY provenance (an old + * requester's strict ==1 gate refuses it). slot->undo_auth: origin_epoch + * is our current epoch — the ship path copies it into hdr.epoch, which the + * requester binds strictly == its stamped epoch (8.A amend); live_hwm_lsn + * and tt_generation ride as ZERO — they describe an origin's OWN live TT + * plane, which does not exist for a dead owner; the prove already + * internalized generation/wrap coverage and the requester's authority leg + * ignores both. + * + * true = result_page holds the version-2 ClusterGcsUndoVerdictPage; + * false = refuse (caller ships DENIED — requester keeps 53R97). Runs + * under the drain's PG_TRY: any throw becomes a refusal, never an LMS + * exit. + */ +static bool +lms_undo_authority_verdict_serve(ClusterLmsCrSlot *slot) +{ + ClusterGcsUndoVerdictPage *v = (ClusterGcsUndoVerdictPage *)slot->result_page; + TransactionId xid = slot->undo_xid; + uint64 cur_epoch; + int32 authority = -1; + ClusterUndoVerdictResult r; + + if (!cluster_crossnode_runtime_visibility) + return false; + if (!TransactionIdIsNormal(xid)) + return false; + if (slot->undo_owner < 0 || slot->undo_owner == cluster_node_id) + return false; + + /* (a) the request's epoch must still be OUR current epoch */ + cur_epoch = cluster_epoch_get_current(); + if (slot->epoch != cur_epoch) + return false; + + /* (b) re-derive: the elected authority for (owner, cur) must be US */ + if (cluster_undo_serve_authority_lookup(slot->undo_owner, cur_epoch, &authority) + != CLUSTER_UNDO_AUTHORITY_OK + || authority != cluster_node_id) + return false; + + /* the reply's epoch carrier (see banner); hwm/tt_gen deliberately zero */ + slot->undo_auth.origin_epoch = cur_epoch; + slot->undo_auth.live_hwm_lsn = 0; + slot->undo_auth.tt_generation = 0; + + /* (c) shared block0 prove core (the D4-4 self leg's implementation) */ + r = cluster_undo_authority_block0_prove(slot->undo_owner, slot->undo_segment_id, xid, + cur_epoch); + + memset(slot->result_page, 0, BLCKSZ); + return cluster_undo_authority_verdict_page_fill(v, xid, &r); +} + /* * cluster_lms_undo_verdict_fill_page -- resolve an OWN xid into a verdict page * over the COMPLETE own durable TT (cluster_tt_slot_durable_resolve_by_xid) @@ -562,7 +660,14 @@ cluster_lms_cr_drain(void) if (!cluster_injection_should_skip("cluster-lms-undo-fetch")) { PG_TRY(); { - served = is_verdict ? lms_undo_verdict_serve(slot) : lms_undo_fetch_serve(slot); + /* D4-6: a verdict slot carrying a dead owner is the + * kind-4 authority serve (block0 prove), never the + * own-TT scan. */ + if (is_verdict) + served = slot->undo_owner >= 0 ? lms_undo_authority_verdict_serve(slot) + : lms_undo_verdict_serve(slot); + else + served = lms_undo_fetch_serve(slot); } PG_CATCH(); { diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index bb85e19fb4..c5ed597ced 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -2493,42 +2493,38 @@ cluster_gcs_block_undo_tt_fetch_and_wait(int32 origin_node, uint32 segment_id, u /* - * PGRAC: spec-6.12i D-i4 / spec-6.15 D4 — requester-side undo verdict fetch. + * gcs_block_undo_verdict_wire_exchange — shared TRANSPORT core of the two + * verdict fetch wrappers below (spec-6.12i D-i4 owner-served kinds 2/3 and + * spec-5.22d D4-6 authority-served kind 4): reserve slot, stamp, send, wait, + * verify status + trailer + checksum, copy the raw reply material out. ONE + * implementation so the two wire legs can never drift apart mechanically + * (Rule 8.A, the fill_page discipline); the acceptance POLICY — the page + * structural gate, the authority reply binding, the co-sample extraction — + * deliberately stays in the wrappers, so the v1 and authority policies can + * never cross-contaminate. * - * Ask origin_node for a COMPLETE own-TT by-xid verdict on `xid`, riding the - * same sub-case B wire shape as the undo-TT fetch above. The asked-for xid - * rides the widened watermark carrier; the synthetic tag keeps the ref's - * segment for tag validity + observability only (the verdict is complete - * over ALL origin segments). + * stamped_epoch is written to BOTH slot->request_epoch and fwd.epoch (one + * read, one value — the pre-refactor code read the clock twice, which + * could straddle an epoch bump; single-stamping is strictly tighter). * - * true -> *verdict_out holds the structurally validated verdict page - * (cluster_vis_undo_verdict_page_usable) and *auth_out the - * authority co-sampled with the scan. - * false -> fail-closed: timeout, DENIED, checksum failure, missing - * trailer, malformed page (Rule 8.A — the caller keeps its - * unchanged 53R97 refusal). + * true -> *hdr_out / *page_out / *tt_generation_out hold the checksum- + * verified reply material (page_out is the 48-byte verdict struct at the + * head of the BLCKSZ area; the checksum covered the whole area). + * false -> timeout / DENIED / wrong status / missing trailer / checksum + * mismatch (the caller keeps its 53R97 refusal, Rule 8.A). */ -bool -cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_id, - TransactionId xid, bool authoritative, - ClusterGcsUndoVerdictPage *verdict_out, - ClusterLiveAuthority *auth_out) +static bool +gcs_block_undo_verdict_wire_exchange(int32 dest_node, BufferTag tag, uint64 stamped_epoch, + TransactionId xid, bool authoritative, bool authority_kind, + GcsBlockReplyHeader *hdr_out, + ClusterGcsUndoVerdictPage *page_out, uint64 *tt_generation_out) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; - BufferTag tag; GcsBlockForwardPayload fwd; bool got_reply = false; bool fetched = false; - if (verdict_out == NULL || auth_out == NULL || origin_node < 0 || origin_node == cluster_node_id - || !TransactionIdIsNormal(xid)) - return false; - - memset(verdict_out, 0, sizeof(*verdict_out)); - memset(auth_out, 0, sizeof(*auth_out)); - tag = GcsBlockUndoFetchTagMake(segment_id, 0); - cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); @@ -2546,29 +2542,38 @@ cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_ cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); slot->reply_undo_trailer_valid = false; slot->reply_undo_tt_generation = 0; - slot->request_epoch = cluster_epoch_get_current(); + slot->request_epoch = stamped_epoch; slot->expected_master_node = cluster_node_id; slot->stale = false; LWLockRelease(&blk->lock.lock); memset(&fwd, 0, sizeof(fwd)); fwd.request_id = request_id; - fwd.epoch = cluster_epoch_get_current(); + fwd.epoch = stamped_epoch; fwd.tag = tag; fwd.original_requester_node = cluster_node_id; fwd.requester_backend_id = (int32)MyBackendId; fwd.master_node = cluster_node_id; fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, authoritative); + if (authority_kind) + GcsBlockForwardPayloadSetUndoAuthorityVerdictRequest(&fwd); + else + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, authoritative); /* The widened xid rides the watermark carrier (upper 32 bits zero). */ GcsBlockForwardPayloadSetExpectedPiWatermarkScn(&fwd, (SCN)(uint64)xid); if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, - (uint32)origin_node, &fwd, sizeof(fwd))) + (uint32)dest_node, &fwd, sizeof(fwd))) { + if (authority_kind) + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("cluster_gcs_block: failed to enqueue undo-verdict fetch " + "to authority node %d", + (int)dest_node))); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cluster_gcs_block: failed to enqueue undo-verdict fetch to " "origin node %d", - (int)origin_node))); + (int)dest_node))); + } deadline = GetCurrentTimestamp() + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; @@ -2603,19 +2608,13 @@ cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_ uint32 got = gcs_block_compute_checksum(slot->reply_block_data); if (expected == got) { - const ClusterGcsUndoVerdictPage *v - = (const ClusterGcsUndoVerdictPage *)slot->reply_block_data; - - if (cluster_vis_undo_verdict_page_usable(v, xid)) { - *verdict_out = *v; - auth_out->origin_epoch = slot->reply_header.epoch; - auth_out->live_hwm_lsn = (XLogRecPtr)slot->reply_header.page_lsn; - auth_out->tt_generation = slot->reply_undo_tt_generation; - /* spec-5.14 D2: the verdict is the origin's volatile - * co-sample — depend on it for fail-stop (D-i3). */ - gcs_block_stamp_touched(origin_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); - fetched = true; - } + if (hdr_out != NULL) + *hdr_out = slot->reply_header; + if (page_out != NULL) + memcpy(page_out, slot->reply_block_data, sizeof(*page_out)); + if (tt_generation_out != NULL) + *tt_generation_out = slot->reply_undo_tt_generation; + fetched = true; } else { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); } @@ -2633,6 +2632,131 @@ cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_ return fetched; /* false -> caller keeps the unchanged 53R97 refusal */ } +/* + * PGRAC: spec-6.12i D-i4 / spec-6.15 D4 — requester-side undo verdict fetch. + * + * Ask origin_node for a COMPLETE own-TT by-xid verdict on `xid`, riding the + * same sub-case B wire shape as the undo-TT fetch above. The asked-for xid + * rides the widened watermark carrier; the synthetic tag keeps the ref's + * segment for tag validity + observability only (the verdict is complete + * over ALL origin segments). + * + * true -> *verdict_out holds the structurally validated verdict page + * (cluster_vis_undo_verdict_page_usable) and *auth_out the + * authority co-sampled with the scan. + * false -> fail-closed: timeout, DENIED, checksum failure, missing + * trailer, malformed page (Rule 8.A — the caller keeps its + * unchanged 53R97 refusal). + */ +bool +cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uint32 segment_id, + TransactionId xid, bool authoritative, + ClusterGcsUndoVerdictPage *verdict_out, + ClusterLiveAuthority *auth_out) +{ + GcsBlockReplyHeader hdr; + ClusterGcsUndoVerdictPage page; + uint64 tt_generation = 0; + BufferTag tag; + + if (verdict_out == NULL || auth_out == NULL || origin_node < 0 || origin_node == cluster_node_id + || !TransactionIdIsNormal(xid)) + return false; + + memset(verdict_out, 0, sizeof(*verdict_out)); + memset(auth_out, 0, sizeof(*auth_out)); + tag = GcsBlockUndoFetchTagMake(segment_id, 0); + + if (!gcs_block_undo_verdict_wire_exchange(origin_node, tag, cluster_epoch_get_current(), xid, + authoritative, false /* owner-served kind */, &hdr, + &page, &tt_generation)) + return false; + + if (!cluster_vis_undo_verdict_page_usable(&page, xid)) + return false; + + *verdict_out = page; + auth_out->origin_epoch = hdr.epoch; + auth_out->live_hwm_lsn = (XLogRecPtr)hdr.page_lsn; + auth_out->tt_generation = tt_generation; + /* spec-5.14 D2: the verdict is the origin's volatile + * co-sample — depend on it for fail-stop (D-i3). */ + gcs_block_stamp_touched(origin_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + return true; +} + +/* + * PGRAC: spec-5.22d D4-6 — requester-side dead-owner AUTHORITY verdict fetch. + * + * Ask the elected serve authority (a live survivor, NOT the dead owner) for + * a block0-proven verdict on the dead owner_node's `xid`. Kind-4 wire: the + * owner rides in tag.relNumber (owner+1), the widened xid in the watermark + * carrier. The caller has already gated on the peer's HELLO D4 capability. + * + * Acceptance is the 8.A-amended FULL binding, strictly tighter than the v1 + * leg above: the transport core's status/trailer/checksum verify, PLUS + * sender == elected authority AND reply epoch == stamped epoch EXACTLY + * (cluster_vis_undo_authority_reply_binding_ok; the transport's HC100 >= is + * only a pre-filter), PLUS the version-2 authority structural gate + mapper + * (cluster_undo_verdict_from_authority_wire_page — refuses a v1 page, a + * smuggled horizon bound, an echo mismatch). The reply's hwm/tt_generation + * carriers are deliberately IGNORED: they describe an origin's own live TT + * plane, which does not exist for a dead owner — the block0 prove on the + * serve side already internalized generation/wrap coverage. + * + * true -> *out holds COMMITTED_EXACT{commit_scn, wrap} or ABORTED. The + * caller MUST Lamport-observe any commit_scn it consumes (AD-008). + * false -> fail-closed (caller keeps the 53R97 refusal, Rule 8.A). + */ +bool +cluster_gcs_block_undo_authority_verdict_fetch_and_wait(int32 authority_node, int32 owner_node, + uint32 segment_id, TransactionId xid, + ClusterUndoVerdictResult *out) +{ + GcsBlockReplyHeader hdr; + ClusterGcsUndoVerdictPage page; + uint64 tt_generation = 0; + BufferTag tag; + uint64 stamped_epoch; + ClusterUndoVerdictResult r; + + if (out == NULL) + return false; + out->kind = (uint8)CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED; + out->commit_scn = InvalidScn; + out->wrap = 0; + + if (authority_node < 0 || authority_node == cluster_node_id || owner_node < 0 + || owner_node == cluster_node_id || owner_node == authority_node + || !TransactionIdIsNormal(xid)) + return false; + + tag = GcsBlockUndoAuthorityFetchTagMake(segment_id, 0, owner_node); + stamped_epoch = cluster_epoch_get_current(); + + if (!gcs_block_undo_verdict_wire_exchange(authority_node, tag, stamped_epoch, xid, + false /* no owner-served sub-kind */, + true /* kind 4 */, &hdr, &page, &tt_generation)) + return false; + + /* 8.A amend: full reply binding — sender IS the elected authority and + * the reply epoch IS the stamped epoch, EXACTLY. */ + if (!cluster_vis_undo_authority_reply_binding_ok((int32)hdr.sender_node, authority_node, + hdr.epoch, stamped_epoch)) + return false; + + r = cluster_undo_verdict_from_authority_wire_page(&page, xid); + if (r.kind == (uint8)CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED) + return false; + + /* spec-5.14 D2: the verdict is the AUTHORITY's volatile derivation — + * depend on the authority for fail-stop (the dead owner cannot fail any + * further). */ + gcs_block_stamp_touched(authority_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + *out = r; + return true; +} + /* * PGRAC: spec-5.2 D11 — local-master writer-transfer (revoke) + wait. @@ -5021,16 +5145,19 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo } /* - * PGRAC: spec-6.12i D-i4 / spec-6.15 D4 — undo-verdict request. Same - * LMON shape as the two branches above (validate + park only; the - * complete durable-TT scan + CLOG cross-check runs in LMS, the LMON tick - * ships). MUST branch here, before any holder / GRD logic: the tag is a - * synthetic undo address and the SCN carrier holds the widened xid. A - * refused park (wave GUC off / malformed tag or carrier / no free slot) - * replies the fail-closed DENIED immediately so the requester keeps its - * unchanged 53R97 refusal (Rule 8.A). + * PGRAC: spec-6.12i D-i4 / spec-6.15 D4 — undo-verdict request; spec-5.22d + * D4-6 adds the kind-4 dead-owner AUTHORITY verdict on the same park. + * Same LMON shape as the two branches above (validate + park only; the + * complete durable-TT scan + CLOG cross-check — or the kind-4 authority + * triple check + block0 prove — runs in LMS, the LMON tick ships). MUST + * branch here, before any holder / GRD logic: the tag is a synthetic + * undo address and the SCN carrier holds the widened xid. A refused + * park (wave GUC off / malformed tag or carrier / bad owner carrier / no + * free slot) replies the fail-closed DENIED immediately so the requester + * keeps its unchanged 53R97 refusal (Rule 8.A). */ - if (GcsBlockForwardPayloadIsUndoVerdictRequest(fwd)) { + if (GcsBlockForwardPayloadIsUndoVerdictRequest(fwd) + || GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(fwd)) { if (!cluster_lms_undo_verdict_submit(fwd)) gcs_block_forward_reply_immediate_deny(fwd); return; diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index edec2444c7..b2b87168f2 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -606,6 +606,10 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version if (cluster_smart_fusion && cluster_interconnect_tier == cluster_smart_fusion_tier_min) capabilities |= PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2; + /* spec-5.22d D4-6: protocol capability, not runtime policy — this binary + * understands kind-4 / version-2 regardless of any GUC, so advertise + * unconditionally (the serve side's GUC gate refuses at run time). */ + capabilities |= PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); } diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index 9e5721afae..32fa488bd6 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -52,6 +52,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_mode.h" /* cluster_peer_mode_enabled (D3-2) */ #include "cluster/cluster_runtime_visibility.h" +#include "cluster/cluster_sf_dep.h" /* peer HELLO D4-capability gate (D4-6) */ #include "cluster/cluster_uba.h" #include "cluster/cluster_undo_authority.h" /* dead-owner serve authority (D4-4) */ #include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ @@ -592,14 +593,42 @@ cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, Transactio case CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0: /* self IS the elected authority for the dead owner */ return rtvis_authority_serve_block0(origin_node, undo_segment_id, raw_xid, &route); + case CLUSTER_UNDO_AUTHORITY_SERVE_PEER_BLOCK0: { + ClusterUndoVerdictResult r; + + /* + * D4-6: a PEER is the elected authority — kind-4 wire serve. + * Capability gate first (约束/A.1③): never route kind 4 to a + * peer that did not advertise the D4 protocol bit; the election + * is deterministic and NOT re-run against a different node, so + * a non-capable authority means fail closed. The fetch itself + * fails closed on timeout / DENIED / wrong sender / epoch moved + * / malformed or v1 page (8.A amend binding inside). + */ + if (!cluster_peer_supports_undo_authority_serve(route.destination_node) + || !cluster_gcs_block_undo_authority_verdict_fetch_and_wait( + route.destination_node, origin_node, undo_segment_id, raw_xid, &r)) { + cluster_undo_authority_note_failclosed(); + cluster_rtvis_resolve_note_failclosed(); + return unknown; + } + + /* Lamport-observe the SCN that crossed the wire (AD-008); + * observe of InvalidScn (ABORTED) is a no-op. */ + cluster_scn_observe(r.commit_scn); + if (r.kind == (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT) + cluster_rtvis_resolve_note_committed(); + else + cluster_rtvis_resolve_note_aborted(); + return r; + } case CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE: break; /* live owner: unchanged CP3 + CP5 path below */ case CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED: default: /* - * Dead owner with a peer/no authority (peer wire serve lands - * with D4-5/D4-6), or owner liveness unproven: fail closed, - * NEVER the native CLOG/hint path (Rule 8.A). + * Owner liveness unproven / no derivable authority: fail + * closed, NEVER the native CLOG/hint path (Rule 8.A). */ cluster_undo_authority_note_failclosed(); cluster_rtvis_resolve_note_failclosed(); diff --git a/src/backend/cluster/cluster_runtime_visibility_policy.c b/src/backend/cluster/cluster_runtime_visibility_policy.c index c99696cb09..daada4893a 100644 --- a/src/backend/cluster/cluster_runtime_visibility_policy.c +++ b/src/backend/cluster/cluster_runtime_visibility_policy.c @@ -259,6 +259,150 @@ cluster_undo_verdict_from_wire_page(const struct ClusterGcsUndoVerdictPage *v, } } +/* + * cluster_vis_undo_authority_verdict_page_usable + * + * spec-5.22d D4-6 structural validation of an AUTHORITY-served verdict page + * (version 2 provenance). Same field discipline as the v1 gate above with + * two deliberate tightenings: ONLY version 2 is evidence (the v1 gate's + * strict ==1 refuses version 2 and this gate refuses version 1, so neither + * provenance can masquerade as the other), and COMMITTED_BELOW_HORIZON is + * refused outright — the authority block0 prove has no horizon leg, so a + * bound-carrying page is malformed by construction. Pure: no I/O, no elog. + */ +bool +cluster_vis_undo_authority_verdict_page_usable(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid) +{ + if (v == NULL || !TransactionIdIsNormal(asked_xid)) + return false; + + if (v->magic != CLUSTER_GCS_UNDO_VERDICT_MAGIC + || v->version != CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY) + return false; + + if (v->xid_echo != (uint64)asked_xid) + return false; + + for (size_t i = 0; i < sizeof(v->reserved_0); i++) + if (v->reserved_0[i] != 0) + return false; + for (size_t i = 0; i < sizeof(v->reserved_1); i++) + if (v->reserved_1[i] != 0) + return false; + + switch (v->verdict) { + case (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT: + return SCN_VALID(v->commit_scn) && !SCN_VALID(v->horizon_scn); + case (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED: + return !SCN_VALID(v->commit_scn) && !SCN_VALID(v->horizon_scn); + default: + /* incl. BELOW_HORIZON: not producible by the block0 prove — refuse */ + return false; + } +} + +/* + * cluster_vis_undo_authority_reply_binding_ok + * + * spec-5.22d D4-6 (8.A amend): an authority reply is evidence ONLY under the + * full binding — the reply sender IS the elected authority and the reply + * epoch IS the stamped request epoch, EXACTLY. The transport's HC100 check + * admits hdr.epoch >= request_epoch (a general stale-reply pre-filter); on + * the authority leg a newer epoch means the election itself may have moved, + * so >= is deliberately not enough. Pure. + */ +bool +cluster_vis_undo_authority_reply_binding_ok(int32 sender_node, int32 authority_node, + uint64 reply_epoch, uint64 stamped_epoch) +{ + if (authority_node < 0 || sender_node < 0) + return false; + if (sender_node != authority_node) + return false; + return reply_epoch == stamped_epoch; +} + +/* + * cluster_undo_authority_verdict_page_fill + * + * spec-5.22d D4-6 serve-side wire-page fill: one prove result -> one version + * 2 page, the exact inverse of the requester's authority gate above so the + * two wire ends can never disagree about what an authority page carries. + * Refuses every kind the block0 prove cannot legitimately produce (UNKNOWN / + * BOUND / IN_PROGRESS — the caller ships DENIED and the requester keeps the + * 53R97 fail-closed, Rule 8.A). Zeroes the whole struct first, so a poison + * or stale byte can never leak onto the wire. Pure. + */ +bool +cluster_undo_authority_verdict_page_fill(struct ClusterGcsUndoVerdictPage *v, TransactionId xid, + const ClusterUndoVerdictResult *r) +{ + if (v == NULL || r == NULL || !TransactionIdIsNormal(xid)) + return false; + + switch (r->kind) { + case (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT: + if (!SCN_VALID(r->commit_scn)) + return false; /* an EXACT claim without a scn proves nothing */ + break; + case (uint8)CLUSTER_UNDO_VERDICT_ABORTED: + break; + default: + return false; + } + + memset(v, 0, sizeof(*v)); + v->magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; + v->version = CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY; + v->xid_echo = (uint64)xid; + v->commit_scn = InvalidScn; + v->horizon_scn = InvalidScn; + v->wrap = r->wrap; + if (r->kind == (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT) { + v->verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT; + v->commit_scn = (uint64)r->commit_scn; + } else + v->verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED; + return true; +} + +/* + * cluster_undo_verdict_from_authority_wire_page + * + * spec-5.22d D4-6 requester-side mapper: a version-2 AUTHORITY-served page + * -> the taxonomy. Fail-closed by construction: a page that does not pass + * the authority structural gate (wrong provenance/version, smuggled bound, + * echo mismatch) yields UNKNOWN_FAIL_CLOSED so the caller keeps the 53R97 + * boundary (Rule 8.A). Pure: no I/O, no elog. + */ +ClusterUndoVerdictResult +cluster_undo_verdict_from_authority_wire_page(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid) +{ + ClusterUndoVerdictResult r + = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; + + if (!cluster_vis_undo_authority_verdict_page_usable(v, asked_xid)) + return r; + + switch (v->verdict) { + case (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT: + r.kind = CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = (SCN)v->commit_scn; + r.wrap = v->wrap; + return r; + + case (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED: + r.kind = CLUSTER_UNDO_VERDICT_ABORTED; + return r; + + default: + /* the authority gate already fenced the kind range; defense in depth */ + return r; + } +} + /* * cluster_undo_verdict_from_block_proof * diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 4d18491eb1..98197953f7 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -157,6 +157,28 @@ cluster_sf_peer_supports_reply_v2(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2) != 0; } +/* + * cluster_peer_supports_undo_authority_serve + * + * spec-5.22d D4-6 capability gate: true iff the peer's verified HELLO + * advertised the kind-4 authority-serve protocol bit. NOT gated on any + * local GUC (see cluster_sf_dep.h) — an unknown/old peer reads as false and + * the caller's authority leg fails closed (Rule 8.A). + */ +bool +cluster_peer_supports_undo_authority_serve(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = ClusterSfDep->peer_capabilities[peer_id]; + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1) != 0; +} + void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload) { diff --git a/src/backend/cluster/cluster_undo_authority.c b/src/backend/cluster/cluster_undo_authority.c index d59f7f3ed3..555469d381 100644 --- a/src/backend/cluster/cluster_undo_authority.c +++ b/src/backend/cluster/cluster_undo_authority.c @@ -124,13 +124,15 @@ cluster_undo_route_decide(int32 owner_node, uint64 reconfig_epoch, } /* - * cluster_undo_authority_serve_decide -- pure consumer serve decision (D4-4). + * cluster_undo_authority_serve_decide -- pure consumer serve decision (D4-4; + * D4-6 adds the peer wire leg). * - * See cluster_undo_authority.h for the contract. The ONLY two provable - * actions are the live-owner path (D6, unchanged) and the self-authority - * block0 serve; every other route -- RECOVERING, UNKNOWN, a malformed - * destination, and an elected PEER authority (wire serve = D4-5/D4-6) -- - * fails closed. + * See cluster_undo_authority.h for the contract. The three provable + * actions are the live-owner path (D6, unchanged), the self-authority + * block0 serve, and the D4-6 kind-4 wire serve to an elected PEER + * authority; every unproven route -- RECOVERING, UNKNOWN, an invalid + * destination, an invalid self node (the consumer cannot prove it is not + * the destination) -- fails closed. */ ClusterUndoAuthorityServeDecision cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 self_node) @@ -139,10 +141,11 @@ cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 se case CLUSTER_UNDO_AUTHORITY_OWNER_LIVE: return CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE; case CLUSTER_UNDO_AUTHORITY_OK: - if (self_node >= 0 && route->destination_node == self_node) + if (route->destination_node < 0 || self_node < 0) + return CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED; + if (route->destination_node == self_node) return CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0; - /* peer authority: wire serve lands with D4-5/D4-6; fail closed */ - return CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED; + return CLUSTER_UNDO_AUTHORITY_SERVE_PEER_BLOCK0; case CLUSTER_UNDO_AUTHORITY_RECOVERING: case CLUSTER_UNDO_AUTHORITY_UNKNOWN: default: diff --git a/src/include/cluster/cluster_cr_server.h b/src/include/cluster/cluster_cr_server.h index 37bc1fef10..33bd6e42a0 100644 --- a/src/include/cluster/cluster_cr_server.h +++ b/src/include/cluster/cluster_cr_server.h @@ -61,7 +61,8 @@ #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_runtime_visibility.h" /* ClusterLiveAuthority (spec-6.12i) */ #include "cluster/cluster_scn.h" -#include "storage/buf_internals.h" /* BufferTag */ +#include "cluster/cluster_undo_verdict.h" /* ClusterUndoVerdictResult (spec-5.22d D4-6) */ +#include "storage/buf_internals.h" /* BufferTag */ /* Split verdict for the server-side construction (see banner). */ typedef enum ClusterCrServerSplit { @@ -135,6 +136,12 @@ typedef struct ClusterLmsCrSlot { * underivable own xid over its durable-TT + CLOG authority. false for the * derived (recycled) path, which keeps the self-check (6.12i P0 guard). */ bool undo_authoritative; + /* spec-5.22d D4-6: the dead OWNER a kind-4 AUTHORITY verdict request asks + * about, decoded from tag.relNumber at submit time (in-memory only, not + * wire ABI). -1 for every owner-served kind. >= 0 switches the drain to + * the authority serve (triple check + block0 prove), which never consults + * this node's own TT. */ + int32 undo_owner; ClusterLiveAuthority undo_auth; char result_page[BLCKSZ]; /* the constructed CR page (FULL/PARTIAL), the * fetched undo header block (UNDO_FETCH), or @@ -236,6 +243,21 @@ extern bool cluster_gcs_block_undo_verdict_fetch_and_wait(int32 origin_node, uin ClusterGcsUndoVerdictPage *verdict_out, ClusterLiveAuthority *auth_out); +/* Requester side (backend, spec-5.22d D4-6): ask the elected serve AUTHORITY + * (a live survivor — NOT the dead owner) for a block0-proven verdict on the + * dead owner_node's `xid` (kind-4 wire; owner rides in tag.relNumber). The + * caller has already gated on the peer's HELLO D4 capability. On success + * *out holds the mapped COMMITTED_EXACT / ABORTED verdict; false or an + * UNKNOWN_FAIL_CLOSED kind = fail-closed (timeout / DENIED / checksum / + * trailer missing / wrong sender / epoch moved / malformed or v1 page — + * caller keeps the 53R97 refusal, Rule 8.A). The caller MUST Lamport- + * observe any commit_scn it consumes (AD-008). */ +extern bool cluster_gcs_block_undo_authority_verdict_fetch_and_wait(int32 authority_node, + int32 owner_node, + uint32 segment_id, + TransactionId xid, + ClusterUndoVerdictResult *out); + #endif /* USE_PGRAC_CLUSTER */ #endif /* CLUSTER_CR_SERVER_H */ diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index c7f799d3e9..13c704b1f5 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1197,7 +1197,9 @@ GcsBlockForwardPayloadIsDirectLandArmed(const GcsBlockForwardPayload *p) * VALUE 1 ([0]=read-image, [1]=X-transfer, [2]=clean-eligible, * [3]=downgrade-request/nudge, [4]=CR-request, [5]=spec-6.13 direct-land * above; [6] is the last byte, value-multiplexed like [3]: value 1 = - * undo-TT fetch, value 2 = spec-6.15 D4 xid verdict). + * undo-TT fetch, values 2/3 = spec-6.15 D4 / spec-5.22f D6-7 xid verdict + * sub-kinds, value 4 = spec-5.22d D4-6 dead-owner authority verdict — see + * the per-kind banners below). * * Sent REQUESTER -> ORIGIN, riding the same 64B forward wire as the * spec-6.12b CR request. With this flag set the BufferTag is a SYNTHETIC @@ -1284,6 +1286,39 @@ GcsBlockForwardPayloadIsUndoVerdictAuthoritative(const GcsBlockForwardPayload *p return p->reserved_0[6] == (uint8)3; } +/* PGRAC: spec-5.22d D4-6 — reserved_0[6] VALUE 4 = dead-owner AUTHORITY + * verdict request (the byte's fourth value; 1 = undo-TT fetch, 2 = derived + * verdict, 3 = authoritative verdict above). + * + * Sent REQUESTER -> elected serve AUTHORITY (a live survivor, NOT the + * owner) when the undo OWNER of the asked-for xid is dead/absent: the + * owner's verdict wire has nobody behind it, so the deterministically + * elected survivor authority answers from the owner's durable shared + * block0 instead (spec-5.22d Route B). Identity vs destination are + * deliberately separate layers: the request's OWNER (whose xid / whose + * undo segment) never changes and rides in tag.relNumber (see + * GcsBlockUndoAuthorityFetchTagMake below); only the serve DESTINATION + * moves to the authority. The requester sends this kind ONLY to a peer + * that advertised PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 (an old + * binary never sees kind 4). The serve side NEVER blind-trusts the wire: + * it re-checks request epoch == its current epoch, re-derives the + * authority for (owner, current epoch) and only serves when that is + * itself, then proves the verdict on the owner's block0 bytes + * (cluster_undo_authority_block0_prove — the same core the requester's + * self-authority leg runs). Any failed check is a DENIED and the + * requester keeps the 53R97 fail-closed (Rule 8.A). */ +static inline void +GcsBlockForwardPayloadSetUndoAuthorityVerdictRequest(GcsBlockForwardPayload *p) +{ + p->reserved_0[6] = (uint8)4; +} + +static inline bool +GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(const GcsBlockForwardPayload *p) +{ + return p->reserved_0[6] == (uint8)4; +} + /* PGRAC: spec-6.12i D-i1 — synthetic undo-address tag for the fetch wire. * * An undo block has no BufferTag (undo segment files live outside shared @@ -1292,10 +1327,18 @@ GcsBlockForwardPayloadIsUndoVerdictAuthoritative(const GcsBlockForwardPayload *p * field to carry that address: spcOid holds a magic discriminator (so a * synthetic tag can never be confused with a real relation tag anywhere * it might leak into observability), dbOid carries the segment_id and - * blockNum the block_no. The OWNER is deliberately NOT carried: the - * origin only ever serves its own undo (owner_instance derives from the - * serving node's own id), so a forged owner field cannot redirect the - * read (fail-closed by construction). */ + * blockNum the block_no. On the owner-served kinds (reserved_0[6] values + * 1/2/3) the OWNER is deliberately NOT carried — the origin only ever + * serves its own undo (owner_instance derives from the serving node's own + * id), so a forged owner field cannot redirect the read (fail-closed by + * construction), and relNumber stays 0 (the LMON submit refuses a + * non-zero relNumber on those kinds). The spec-5.22d D4-6 AUTHORITY kind + * (value 4) is the ONE exception: the dead OWNER's node id rides in the + * otherwise-empty relNumber as owner+1 (0 stays "absent"), because the + * serve destination is NOT the owner. The serve side still never trusts + * it: the authority re-derivation + epoch check + block0 positive proof + * must all pass before a byte is answered (a forged owner is refused, + * never redirected-to). */ #define GCS_BLOCK_UNDO_FETCH_TAG_MAGIC ((Oid)0x50475549) /* "PGUI" */ static inline BufferTag @@ -1323,6 +1366,32 @@ GcsBlockUndoFetchTagDecode(BufferTag tag, uint32 *segment_id, uint32 *block_no) return true; } +/* PGRAC: spec-5.22d D4-6 — authority-kind tag: the dead OWNER rides in the + * otherwise-empty relNumber as owner+1 (see the banner above). Decode is + * shape-only (magic + non-zero owner carrier); range and self-exclusion are + * the LMON submit's job, and TRUST is nobody's until the serve-side triple + * check passes. */ +static inline BufferTag +GcsBlockUndoAuthorityFetchTagMake(uint32 segment_id, uint32 block_no, int32 owner_node) +{ + BufferTag tag = GcsBlockUndoFetchTagMake(segment_id, block_no); + + tag.relNumber = (RelFileNumber)(owner_node + 1); + return tag; +} + +static inline bool +GcsBlockUndoAuthorityFetchTagDecodeOwner(BufferTag tag, int32 *owner_node) +{ + if (tag.spcOid != GCS_BLOCK_UNDO_FETCH_TAG_MAGIC) + return false; + if (tag.relNumber == (RelFileNumber)0) + return false; /* owner absent: an owner-served-kind tag */ + if (owner_node != NULL) + *owner_node = (int32)tag.relNumber - 1; + return true; +} + /* PGRAC: spec-6.12i D-i1 — live-authority trailer appended after the BLCKSZ * page on UNDO_TT_FETCH_RESULT replies (wire size = 48B v1 header + 8192B * page + 16B trailer = 8256B; distinct from both the 8240B v1 and the 8504B @@ -1397,6 +1466,12 @@ ClusterGcsUndoAuthTrailerGetTtGeneration(const ClusterGcsUndoAuthTrailer *t) * failing leg (e) forever. */ #define CLUSTER_GCS_UNDO_VERDICT_MAGIC ((uint32)0x50475556) /* "PGUV" */ #define CLUSTER_GCS_UNDO_VERDICT_VERSION ((uint32)1) +/* PGRAC: spec-5.22d D4-6 — version 2 marks "dead-owner AUTHORITY-served" + * provenance (Route B block0 prove). An old requester's strict ==1 gate + * refuses it (fail-closed, never mistaken for owner-served), and the new + * authority leg accepts ONLY version 2 (an owner-served v1 page can never + * masquerade as an authority serve). Same 48-byte layout. */ +#define CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY ((uint32)2) typedef enum ClusterGcsUndoVerdictKind { CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT = 1, diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 981734a896..2402e422eb 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -228,6 +228,15 @@ extern const ClusterICOps *ClusterICOps_Active; #define PGRAC_IC_CLUSTER_NAME_MAX 24 #define PGRAC_IC_HELLO_CAPABILITIES_OFFSET 36 #define PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 ((uint32)0x00000001U) +/* PGRAC: spec-5.22d D4-6 — this binary understands the kind-4 dead-owner + * AUTHORITY verdict request and the version-2 authority-served verdict page. + * A PROTOCOL capability: advertised unconditionally (unlike the GUC/tier- + * gated smart-fusion bit above) — whether the serve actually runs is the + * serve side's runtime GUC gate, which refuses with DENIED and the requester + * stays fail-closed. A requester only routes kind 4 to a peer that + * advertised this bit; without it the authority leg fails closed (the + * election is NOT re-run against a different node). */ +#define PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 ((uint32)0x00000002U) typedef struct ClusterICHelloMsg { uint32 magic; /* PGRAC_IC_HELLO_MAGIC */ diff --git a/src/include/cluster/cluster_runtime_visibility.h b/src/include/cluster/cluster_runtime_visibility.h index 3c51c896b7..b1542df42a 100644 --- a/src/include/cluster/cluster_runtime_visibility.h +++ b/src/include/cluster/cluster_runtime_visibility.h @@ -149,6 +149,21 @@ struct ClusterGcsUndoVerdictPage; /* cluster_gcs_block.h */ extern bool cluster_vis_undo_verdict_page_usable(const struct ClusterGcsUndoVerdictPage *v, TransactionId asked_xid); +/* + * spec-5.22d D4-6: structural gate for an AUTHORITY-served verdict page + * (version 2 provenance) + the reply binding predicate. The authority gate + * mirrors the v1 discipline but accepts ONLY version 2 and refuses the + * BELOW_HORIZON kind (the block0 prove has no horizon leg); the binding + * predicate is the 8.A amend — reply sender == elected authority AND reply + * epoch == stamped request epoch EXACTLY (the transport HC100 >= is only a + * pre-filter). Pure: unit truth tables. + */ +extern bool +cluster_vis_undo_authority_verdict_page_usable(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid); +extern bool cluster_vis_undo_authority_reply_binding_ok(int32 sender_node, int32 authority_node, + uint64 reply_epoch, uint64 stamped_epoch); + /* * CP3 + CP5 orchestration (backend): active-runtime resolution of a RECYCLED * remote ITL ref. Two provable legs, both under the co-sampled live diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 06d8638f38..ccf4a5983d 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -141,6 +141,11 @@ extern XLogRecPtr cluster_sf_observed_origin_durable_lsn(int32 origin); extern void cluster_sf_publish_origin_durable_lsn(void); extern void cluster_sf_note_peer_hello_capabilities(int32 peer_id, uint32 capabilities); extern bool cluster_sf_peer_supports_reply_v2(int32 peer_id); +/* spec-5.22d D4-6: did the peer's HELLO advertise the kind-4 authority-serve + * protocol capability? Deliberately NOT gated on any local GUC (unlike the + * smart-fusion query above): the bit answers "can that binary parse kind 4"; + * the runtime arm gates live elsewhere. */ +extern bool cluster_peer_supports_undo_authority_serve(int32 peer_id); extern void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload); extern void cluster_sf_note_dep_touched(Buffer buffer); diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h index 664cbdf53a..4c9b564e28 100644 --- a/src/include/cluster/cluster_undo_authority.h +++ b/src/include/cluster/cluster_undo_authority.h @@ -162,22 +162,29 @@ extern ClusterUndoServeRoute cluster_undo_serve_authority(const ClusterResId *un * verdict consumer takes: * * SERVE_FAIL_CLOSED -- everything unproven: RECOVERING / UNKNOWN, a - * malformed route, AND an elected PEER authority. The - * peer wire serve lands with D4-5/D4-6 (LMS authority - * serve + capability gate); until then routing a - * request there would be an unproven path, so it fails - * closed (never native, Rule 8.A). Value 0 so a - * zeroed decision is the safe one. + * malformed route (invalid destination or an invalid + * self node). Fail closed (never native, Rule 8.A). + * Value 0 so a zeroed decision is the safe one. * SERVE_OWNER_LIVE -- owner is fresh-alive: stay on the D6 live-owner * path, byte-for-byte unchanged. * SERVE_SELF_BLOCK0 -- self IS the elected authority: read the dead * owner's shared block0 (AUTHORITY_BLOCK0 intent) and * serve the verdict locally. + * SERVE_PEER_BLOCK0 -- a PEER is the elected authority: route the kind-4 + * verdict request to it over the wire (D4-6). The + * consumer still gates on the peer's HELLO + * D4-capability BEFORE sending (an old binary never + * sees kind 4 — no capability means fail closed, the + * election is NOT re-run against a different node), + * and on the reply binding (sender == elected + * authority, reply epoch == stamped epoch, version 2 + * provenance) before trusting any answer. */ typedef enum ClusterUndoAuthorityServeDecision { CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED = 0, CLUSTER_UNDO_AUTHORITY_SERVE_OWNER_LIVE, - CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0 + CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0, + CLUSTER_UNDO_AUTHORITY_SERVE_PEER_BLOCK0 } ClusterUndoAuthorityServeDecision; extern ClusterUndoAuthorityServeDecision diff --git a/src/include/cluster/cluster_undo_verdict.h b/src/include/cluster/cluster_undo_verdict.h index 94cfd3ceb2..135218bd6e 100644 --- a/src/include/cluster/cluster_undo_verdict.h +++ b/src/include/cluster/cluster_undo_verdict.h @@ -86,6 +86,20 @@ extern ClusterUndoVerdictResult cluster_undo_verdict_from_wire_page(const struct ClusterGcsUndoVerdictPage *v, TransactionId asked_xid); +/* + * spec-5.22d D4-6 authority wire-page pair: the serve side fills a version-2 + * page from one block0 prove result (refusing every kind the prove cannot + * produce), the requester side maps it back through the authority structural + * gate. One fill + one mapper so the two wire ends can never disagree. + * Pure: no I/O, no shmem, no elog. + */ +extern bool cluster_undo_authority_verdict_page_fill(struct ClusterGcsUndoVerdictPage *v, + TransactionId xid, + const ClusterUndoVerdictResult *r); +extern ClusterUndoVerdictResult +cluster_undo_verdict_from_authority_wire_page(const struct ClusterGcsUndoVerdictPage *v, + TransactionId asked_xid); + /* * cluster_undo_verdict_from_block_proof -- map a CP3 single-block positive * proof to the taxonomy: COMMITTED -> COMMITTED_EXACT{commit_scn,wrap} (the diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..ebb6904913 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -511,10 +511,99 @@ UT_TEST(test_clean_xfer_stale_break_predicate) } +/* spec-5.22d D4-6: reserved_0[6] VALUE 4 = dead-owner AUTHORITY verdict + * request. Pins the value-multiplex against the three existing kinds + * (1 = undo-TT fetch, 2 = derived verdict, 3 = authoritative verdict): the + * kind-4 predicate must never match 1/2/3 and vice versa, and setting kind 4 + * must not perturb the widened-xid watermark carrier. ABI stays 64B. */ +UT_TEST(test_forward_payload_undo_authority_verdict_kind4) +{ + GcsBlockForwardPayload fwd; + + memset(&fwd, 0, sizeof(fwd)); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + + GcsBlockForwardPayloadSetUndoAuthorityVerdictRequest(&fwd); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 1); + /* kind 4 is NOT one of the owner-served verdict kinds (2/3), NOT the + * undo-TT fetch (1) */ + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictRequest(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictAuthoritative(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoTtFetchRequest(&fwd) ? 1 : 0, 0); + + /* and the owner-served kinds are NOT the authority kind */ + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, true /* value 3 */); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, false /* value 2 */); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + GcsBlockForwardPayloadSetUndoTtFetchRequest(&fwd, true /* value 1 */); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + + /* kind 4 must not perturb the widened-xid carrier bytes */ + memset(&fwd, 0, sizeof(fwd)); + GcsBlockForwardPayloadSetExpectedPiWatermarkScn(&fwd, (SCN)0x00000000AABBCCDDULL); + GcsBlockForwardPayloadSetUndoAuthorityVerdictRequest(&fwd); + UT_ASSERT_EQ((long long)GcsBlockForwardPayloadGetExpectedPiWatermarkScn(&fwd), + (long long)0x00000000AABBCCDDULL); + + UT_ASSERT_EQ((int)sizeof(GcsBlockForwardPayload), 64); +} + + +/* spec-5.22d D4-6: the authority fetch tag carries the dead OWNER in the + * previously-empty tag.relNumber as owner+1 (0 stays "absent" so the three + * owner-served kinds keep their strict empty-relNumber shape). The serve + * side NEVER blind-trusts this field — it re-derives the authority and + * only answers when the triple check passes — but the encode/decode + * roundtrip and the 0-absent boundary are pinned here. */ +UT_TEST(test_undo_authority_fetch_tag_owner_roundtrip) +{ + BufferTag legacy = GcsBlockUndoFetchTagMake(7, 0); + BufferTag tag; + uint32 segment_id = 0; + uint32 block_no = 99; + int32 owner = -1; + + /* legacy owner-served tag: relNumber empty, owner decode refuses */ + UT_ASSERT_EQ((int)legacy.relNumber, 0); + UT_ASSERT_EQ(GcsBlockUndoAuthorityFetchTagDecodeOwner(legacy, &owner) ? 1 : 0, 0); + + /* authority tag: owner 2 rides as relNumber 3; base fields unchanged */ + tag = GcsBlockUndoAuthorityFetchTagMake(7, 0, 2 /* owner */); + UT_ASSERT_EQ((int)tag.relNumber, 3); + UT_ASSERT_EQ(GcsBlockUndoFetchTagDecode(tag, &segment_id, &block_no) ? 1 : 0, 1); + UT_ASSERT_EQ((int)segment_id, 7); + UT_ASSERT_EQ((int)block_no, 0); + UT_ASSERT_EQ(GcsBlockUndoAuthorityFetchTagDecodeOwner(tag, &owner) ? 1 : 0, 1); + UT_ASSERT_EQ((int)owner, 2); + + /* owner 0 (node id 0) must survive the +1 bias roundtrip */ + tag = GcsBlockUndoAuthorityFetchTagMake(1, 0, 0); + UT_ASSERT_EQ((int)tag.relNumber, 1); + UT_ASSERT_EQ(GcsBlockUndoAuthorityFetchTagDecodeOwner(tag, &owner) ? 1 : 0, 1); + UT_ASSERT_EQ((int)owner, 0); + + /* wrong magic: never decodes, wherever the relNumber points */ + tag.spcOid = (Oid)0xDEADBEEF; + UT_ASSERT_EQ(GcsBlockUndoAuthorityFetchTagDecodeOwner(tag, &owner) ? 1 : 0, 0); +} + + +/* spec-5.22d D4-6: the authority-served verdict page version is a DISTINCT + * provenance value — an old requester's strict ==1 gate refuses it (fail + * closed) and the new authority leg accepts ONLY it (an owner-served v1 + * page can never masquerade as an authority serve). */ +UT_TEST(test_undo_verdict_version_authority_distinct) +{ + UT_ASSERT_EQ((int)CLUSTER_GCS_UNDO_VERDICT_VERSION, 1); + UT_ASSERT_EQ((int)CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY, 2); +} + + int main(void) { - UT_PLAN(21); + UT_PLAN(24); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -536,6 +625,9 @@ main(void) UT_RUN(test_request_payload_direct_land_flag_roundtrip_and_orthogonal); UT_RUN(test_clean_xfer_master_decision_5_branches); UT_RUN(test_clean_xfer_stale_break_predicate); + UT_RUN(test_forward_payload_undo_authority_verdict_kind4); + UT_RUN(test_undo_authority_fetch_tag_owner_roundtrip); + UT_RUN(test_undo_verdict_version_authority_distinct); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 6a08b4104f..ac8b93f78f 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -548,7 +548,12 @@ UT_TEST(test_hello_wire_reference_bytes) * Bytes 8-11: 04 03 02 01 source_node_id = 0x01020304 (LE) * Bytes 12-13: 41 42 "AB" * Bytes 14-35: 00..00 cluster_name NUL pad - * Bytes 36-63: 00..00 _pad (must be zero) + * Bytes 36-39: 02 00 00 00 capability bitmap (LE): the + * spec-5.22d D4-6 UNDO_AUTHORITY_ + * SERVE_V1 bit is a PROTOCOL + * capability, advertised + * unconditionally by this binary + * Bytes 40-63: 00..00 _pad (must be zero) * * Locking these exact bytes guards against compiler-pad drift, * unintended endian flips, and uninitialized memory leakage. @@ -577,8 +582,14 @@ UT_TEST(test_hello_wire_reference_bytes) UT_ASSERT_EQ(wire[13], 'B'); for (i = 14; i < 36; i++) UT_ASSERT_EQ(wire[i], 0); - /* _pad must be all zero */ - for (i = 36; i < PGRAC_IC_HELLO_BYTES; i++) + /* capability bitmap: exactly the unconditional D4-6 authority-serve bit + * (smart-fusion is off in this fixture) */ + UT_ASSERT_EQ(wire[36], 0x02); + UT_ASSERT_EQ(wire[37], 0x00); + UT_ASSERT_EQ(wire[38], 0x00); + UT_ASSERT_EQ(wire[39], 0x00); + /* remaining _pad must be all zero */ + for (i = 40; i < PGRAC_IC_HELLO_BYTES; i++) UT_ASSERT_EQ(wire[i], 0); } @@ -587,13 +598,17 @@ UT_TEST(test_hello_smart_fusion_capability_gate) uint8 wire[PGRAC_IC_HELLO_BYTES]; ClusterICHelloMsg parsed; + /* the spec-5.22d D4-6 authority-serve bit is unconditional, so it is + * the capability-word BASELINE in every row below; only the smart- + * fusion bit is GUC/tier-gated */ cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-off"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -601,7 +616,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-tier-mismatch"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), 0); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -609,7 +625,9 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-tier-match"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), + PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 + | PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; diff --git a/src/test/cluster_unit/test_cluster_runtime_visibility.c b/src/test/cluster_unit/test_cluster_runtime_visibility.c index 41dd707ddd..5d235abef0 100644 --- a/src/test/cluster_unit/test_cluster_runtime_visibility.c +++ b/src/test/cluster_unit/test_cluster_runtime_visibility.c @@ -26,6 +26,7 @@ #include "cluster/cluster_gcs_block.h" /* undo-fetch tag + auth trailer (CP2) */ #include "cluster/cluster_runtime_visibility.h" #include "cluster/cluster_undo_segment.h" /* fake TT header blocks (CP3) */ +#include "cluster/cluster_undo_verdict.h" /* ClusterUndoVerdictResult (D4-6) */ #include "unit_test.h" @@ -337,6 +338,180 @@ UT_TEST(test_undo_verdict_page_usable) UT_ASSERT_EQ(cluster_vis_undo_verdict_page_usable(&v, 1000), false); } +/* spec-5.22d D4-6: structural validation of an AUTHORITY-served verdict page + * (version 2 provenance). Same field discipline as the v1 table above, with + * two deliberate tightenings: ONLY version 2 is evidence (a v1 owner-served + * page can never masquerade as an authority serve, and vice versa — the v1 + * gate's strict ==1 already refuses version 2, pinned above), and the + * BELOW_HORIZON kind is refused outright — the authority block0 prove has no + * horizon leg, so a bound-carrying page is malformed by construction. */ +UT_TEST(test_undo_authority_verdict_page_usable) +{ + ClusterGcsUndoVerdictPage v; + + /* Baseline good authority EXACT page. */ + memset(&v, 0, sizeof(v)); + v.magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; + v.version = CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY; + v.xid_echo = 1000; + v.verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT; + v.commit_scn = 777; + v.horizon_scn = InvalidScn; + v.wrap = 5; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), true); + + /* NULL page / invalid asked xid. */ + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(NULL, 1000), false); + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, InvalidTransactionId), false); + + /* An owner-served v1 page is NOT authority evidence. */ + v.version = CLUSTER_GCS_UNDO_VERDICT_VERSION; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.version = CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY; + + /* Wrong magic / echo (incl. poisoned high word). */ + v.magic = 0xDEADBEEF; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1001), false); + v.xid_echo = (((uint64)1) << 32) + 1000; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.xid_echo = 1000; + + /* Non-zero reserved bytes poison the page. */ + v.reserved_0[5] = 1; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.reserved_0[5] = 0; + v.reserved_1[2] = 1; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.reserved_1[2] = 0; + + /* EXACT kind-field consistency: needs commit_scn, refuses a horizon. */ + v.commit_scn = InvalidScn; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.commit_scn = 777; + v.horizon_scn = 500; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.horizon_scn = InvalidScn; + + /* BELOW_HORIZON: refused OUTRIGHT on the authority leg (no horizon leg + * in the block0 prove), even in its v1-legal shape. */ + v.verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_BELOW_HORIZON; + v.commit_scn = InvalidScn; + v.horizon_scn = 500; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.horizon_scn = InvalidScn; + + /* ABORTED: carries no scn of any kind. */ + v.verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), true); + v.commit_scn = 777; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.commit_scn = InvalidScn; + + /* Unknown kinds refuse. */ + v.verdict = 0; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); + v.verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_ABORTED + 1; + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), false); +} + +/* spec-5.22d D4-6 (user 8.A amend #1): the authority reply is evidence ONLY + * under the full binding — the reply sender IS the elected authority and the + * reply epoch IS the stamped request epoch, exactly. The transport's HC100 + * >= check is a general pre-filter and deliberately NOT sufficient here; the + * epoch±1 rows pin the strict equality. */ +UT_TEST(test_undo_authority_reply_binding) +{ + /* the one admissible shape */ + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(3, 3, 42, 42), true); + + /* wrong sender: a peer that is not the elected authority (incl. the + * dead owner's id and a negative junk id) is never evidence */ + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(2, 3, 42, 42), false); + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(4, 3, 42, 42), false); + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(-1, 3, 42, 42), false); + + /* invalid elected authority: nothing can bind to it */ + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(3, -1, 42, 42), false); + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(-1, -1, 42, 42), false); + + /* epoch must match EXACTLY: ±1 both refuse (HC100 >= would admit +1) */ + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(3, 3, 41, 42), false); + UT_ASSERT_EQ(cluster_vis_undo_authority_reply_binding_ok(3, 3, 43, 42), false); +} + +/* spec-5.22d D4-6: serve-side page fill -> requester-side accept closes the + * loop over one prove result, so the two wire ends can never disagree about + * what an authority page carries. fill refuses every kind the block0 prove + * cannot legitimately produce (UNKNOWN, BOUND, IN_PROGRESS). */ +UT_TEST(test_undo_authority_verdict_page_fill_roundtrip) +{ + ClusterGcsUndoVerdictPage v; + ClusterUndoVerdictResult r; + ClusterUndoVerdictResult mapped; + + /* COMMITTED_EXACT roundtrip: scn + wrap survive, version is 2. */ + memset(&v, 0xAA, sizeof(v)); /* poison: fill must overwrite everything */ + r.kind = (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = 777; + r.wrap = 5; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), true); + UT_ASSERT_EQ(v.version, CLUSTER_GCS_UNDO_VERDICT_VERSION_AUTHORITY); + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), true); + mapped = cluster_undo_verdict_from_authority_wire_page(&v, 1000); + UT_ASSERT_EQ(mapped.kind, (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT); + UT_ASSERT_EQ((long long)mapped.commit_scn, 777); + UT_ASSERT_EQ(mapped.wrap, 5); + + /* ABORTED roundtrip: no scn of any kind on the page. */ + memset(&v, 0xAA, sizeof(v)); + r.kind = (uint8)CLUSTER_UNDO_VERDICT_ABORTED; + r.commit_scn = InvalidScn; + r.wrap = 3; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), true); + UT_ASSERT_EQ(cluster_vis_undo_authority_verdict_page_usable(&v, 1000), true); + mapped = cluster_undo_verdict_from_authority_wire_page(&v, 1000); + UT_ASSERT_EQ(mapped.kind, (uint8)CLUSTER_UNDO_VERDICT_ABORTED); + UT_ASSERT_EQ((long long)mapped.commit_scn, (long long)InvalidScn); + + /* fill refuses what the prove cannot produce: UNKNOWN / BOUND / + * IN_PROGRESS never become a wire page. */ + r.kind = (uint8)CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED; + r.commit_scn = InvalidScn; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), false); + r.kind = (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_BOUND; + r.commit_scn = 500; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), false); + r.kind = (uint8)CLUSTER_UNDO_VERDICT_IN_PROGRESS; + r.commit_scn = InvalidScn; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), false); + + /* EXACT without a valid scn is not fillable (nothing to prove). */ + r.kind = (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = InvalidScn; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, &r), false); + + /* fill guards its inputs: NULL page / invalid xid. */ + r.kind = (uint8)CLUSTER_UNDO_VERDICT_COMMITTED_EXACT; + r.commit_scn = 777; + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(NULL, 1000, &r), false); + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, InvalidTransactionId, &r), false); + UT_ASSERT_EQ(cluster_undo_authority_verdict_page_fill(&v, 1000, NULL), false); + + /* the mapper refuses a v1 owner-served page (wrong provenance). */ + memset(&v, 0, sizeof(v)); + v.magic = CLUSTER_GCS_UNDO_VERDICT_MAGIC; + v.version = CLUSTER_GCS_UNDO_VERDICT_VERSION; + v.xid_echo = 1000; + v.verdict = (uint8)CLUSTER_GCS_UNDO_VERDICT_COMMITTED_EXACT; + v.commit_scn = 777; + v.horizon_scn = InvalidScn; + mapped = cluster_undo_verdict_from_authority_wire_page(&v, 1000); + UT_ASSERT_EQ(mapped.kind, (uint8)CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED); + UT_ASSERT_EQ((long long)mapped.commit_scn, (long long)InvalidScn); +} + /* CP2: synthetic undo-address tag roundtrip + magic discrimination. */ UT_TEST(test_undo_fetch_tag_roundtrip) { @@ -362,7 +537,7 @@ UT_TEST(test_undo_fetch_tag_roundtrip) int main(void) { - UT_PLAN(13); + UT_PLAN(16); UT_RUN(test_covers_when_epoch_match_and_hwm_ge_anchor); UT_RUN(test_failclosed_when_epoch_differs); UT_RUN(test_failclosed_when_hwm_invalid); @@ -375,6 +550,9 @@ main(void) UT_RUN(test_ttproof_header_mismatch); UT_RUN(test_undo_auth_trailer_roundtrip); UT_RUN(test_undo_verdict_page_usable); + UT_RUN(test_undo_authority_verdict_page_usable); + UT_RUN(test_undo_authority_reply_binding); + UT_RUN(test_undo_authority_verdict_page_fill_roundtrip); UT_RUN(test_undo_fetch_tag_roundtrip); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_undo_authority.c b/src/test/cluster_unit/test_cluster_undo_authority.c index b788baefc1..0761346573 100644 --- a/src/test/cluster_unit/test_cluster_undo_authority.c +++ b/src/test/cluster_unit/test_cluster_undo_authority.c @@ -312,14 +312,16 @@ UT_TEST(test_serve_decide_self_block0) CLUSTER_UNDO_AUTHORITY_SERVE_SELF_BLOCK0); } -/* elected authority == a PEER -> fail closed (peer wire serve = D4-5/D4-6) */ -UT_TEST(test_serve_decide_peer_failclosed) +/* elected authority == a PEER -> route the verdict request over the wire to + * that peer (D4-6 kind-4 authority serve; the requester still gates on the + * HELLO capability + reply binding before trusting any answer) */ +UT_TEST(test_serve_decide_peer_block0) { ClusterUndoServeRoute r = cluster_undo_route_decide(0 /*dead owner*/, 42, CLUSTER_UNDO_AUTHORITY_OK, 3); UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1 /*self*/), - CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); + CLUSTER_UNDO_AUTHORITY_SERVE_PEER_BLOCK0); } /* RECOVERING / UNKNOWN -> fail closed (never native, never a guess) */ @@ -351,6 +353,13 @@ UT_TEST(test_serve_decide_invalid_dest_failclosed) CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, 1), CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); + + /* malformed: valid peer destination but an invalid SELF node — the + * consumer cannot prove it is not the destination, so neither the self + * leg nor the D4-6 peer wire leg may fire (fail closed, never a guess) */ + r.destination_node = 3; + UT_ASSERT_EQ(cluster_undo_authority_serve_decide(&r, -1), + CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); } int @@ -372,7 +381,7 @@ main(void) UT_RUN(test_coverage_three_way_and); UT_RUN(test_serve_decide_owner_live); UT_RUN(test_serve_decide_self_block0); - UT_RUN(test_serve_decide_peer_failclosed); + UT_RUN(test_serve_decide_peer_block0); UT_RUN(test_serve_decide_recovering_unknown_failclosed); UT_RUN(test_serve_decide_invalid_dest_failclosed); UT_DONE(); From 4bc3152e649194f26eacd854730c91577ee1377f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 20:21:56 +0800 Subject: [PATCH 22/38] feat(cluster): D4-8 A1 dead-owner prove complete-scan + e2e TAP t/369 (spec-5.22d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dead-owner authority verdict prove is widened from a single ref-segment block0 read to the owner's COMPLETE durable shared segment set — the CP5 complete-scan contract translated to durable state. D4-8's first true e2e proved the TT-slot cursor and the undo-record cursor roll independently, so a slot living outside the ref's segment made every such verdict fail closed forever (availability gap; fail-closed direction was already safe). Prove core (one implementation, self + kind-4 wire legs both covered): - enumerate /pg_undo/instance_/ — the single namespace the shared undo resolver ever writes; complete-or-refuse: any enumeration break, non-canonical entry name (strict parse + re-render equality; seg_0 included, aliases and over-range ids poison), unreadable or unparseable block0 refuses the WHOLE verdict, never skip-and-continue; - set-wide UNIQUE terminal match or refuse: 0-match and non-terminal keep the fail-closed shape, >=2 matches anywhere void the kept evidence; - claim epoch checked at entry AND re-checked after the scan window (a mid-scan reconfig voids the enumerated set's writer-exclusion argument); - writer exclusion is structural: the dead owner is dead-decided + write- fenced (revival forces an epoch bump), survivors' only foreign-owner intent is the read-only AUTHORITY_BLOCK0, and no undo unlink/rename path exists (the future D5 reclaimer must inherit this invariant). Pure layers: cluster_vis_tt_block_xid_scan core under the byte-identical positive-proof wrapper (nmatch reported, POISONED distinct), and the scan-fold aggregate with its admissibility predicate — both truth-table unit-pinned. Two new attribution counters (scan_incomplete_reject / multi_match_reject; cr 62->64, baselines updated) plus the D4-8 L3 injection point cluster-undo-authority-block0-prove (registry 158->159). TAP t/369 (renumbered from the workspace t/360 per the collision ruling): L1 committed rows served post-mortem with serve_hit attribution and materialized==0; L2a regular-abort xid stays in-doubt fail-closed (no durable ABORTED stamp exists outside 2PC/own-recovery); L2b 2PC-aborted row never serves visible (the positive ABORTED-serve e2e is unreachable here — ROLLBACK PREPARED reverts synchronously — and is forwarded to the D7 crash-window injection); L3/L3b injection-forced prove refusal with counter delta + unarmed control. Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (Amendment A1) --- src/backend/cluster/cluster_cr.c | 26 ++ src/backend/cluster/cluster_debug.c | 5 + src/backend/cluster/cluster_inject.c | 13 + .../cluster_runtime_visibility_policy.c | 103 +++-- src/backend/cluster/cluster_undo_authority.c | 53 +++ .../cluster/cluster_undo_authority_snapshot.c | 222 ++++++++-- src/include/cluster/cluster_cr.h | 7 +- .../cluster/cluster_runtime_visibility.h | 22 + src/include/cluster/cluster_undo_authority.h | 23 + src/test/cluster_tap/t/015_inject.pl | 8 +- .../t/215_cluster_3_9_cr_construction.pl | 4 +- .../t/216_cluster_3_10_cr_cache.pl | 4 +- .../t/254_pi_cr_recovery_acceptance.pl | 7 +- ...luster_5_22d_dead_owner_authority_serve.pl | 401 ++++++++++++++++++ src/test/cluster_unit/test_cluster_debug.c | 11 + .../test_cluster_runtime_visibility.c | 90 +++- .../test_cluster_undo_authority.c | 61 ++- 17 files changed, 973 insertions(+), 87 deletions(-) create mode 100644 src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl diff --git a/src/backend/cluster/cluster_cr.c b/src/backend/cluster/cluster_cr.c index 19b80bfaac..4533ab88f8 100644 --- a/src/backend/cluster/cluster_cr.c +++ b/src/backend/cluster/cluster_cr.c @@ -195,6 +195,11 @@ typedef struct ClusterCRShared { pg_atomic_uint64 undo_authority_serve_hit_count; pg_atomic_uint64 undo_authority_fail_closed_count; pg_atomic_uint64 undo_authority_epoch_stale_reject_count; + /* spec-5.22d A1 (D4-8): complete-scan refusal attribution — the durable + * segment-set enumeration/parse was incomplete (refuse-all), or the set + * held more than one match for the asked xid (ambiguity). */ + pg_atomic_uint64 undo_authority_scan_incomplete_reject_count; + pg_atomic_uint64 undo_authority_multi_match_reject_count; } ClusterCRShared; static ClusterCRShared *CRShared = NULL; @@ -282,6 +287,8 @@ cluster_cr_shmem_init(void) pg_atomic_init_u64(&CRShared->undo_authority_serve_hit_count, 0); pg_atomic_init_u64(&CRShared->undo_authority_fail_closed_count, 0); pg_atomic_init_u64(&CRShared->undo_authority_epoch_stale_reject_count, 0); + pg_atomic_init_u64(&CRShared->undo_authority_scan_incomplete_reject_count, 0); + pg_atomic_init_u64(&CRShared->undo_authority_multi_match_reject_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_resolved_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_recycled_invisible_count, 0); pg_atomic_init_u64(&CRShared->cr_xmax_invalid_or_ambiguous_count, 0); @@ -481,6 +488,21 @@ cluster_undo_authority_note_epoch_stale_reject(void) if (CRShared != NULL) pg_atomic_fetch_add_u64(&CRShared->undo_authority_epoch_stale_reject_count, 1); } + +/* PGRAC: spec-5.22d A1 (D4-8) — complete-scan refusal attribution. */ +void +cluster_undo_authority_note_scan_incomplete_reject(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->undo_authority_scan_incomplete_reject_count, 1); +} + +void +cluster_undo_authority_note_multi_match_reject(void) +{ + if (CRShared != NULL) + pg_atomic_fetch_add_u64(&CRShared->undo_authority_multi_match_reject_count, 1); +} CR_COUNTER_ACCESSOR(cluster_cr_corruption_count, cr_corruption_count) CR_COUNTER_ACCESSOR(cluster_cr_chain_walk_steps_sum, cr_chain_walk_steps_sum) CR_COUNTER_ACCESSOR(cluster_cr_inverse_insert_count, cr_inverse_insert_count) @@ -527,6 +549,10 @@ CR_COUNTER_ACCESSOR(cluster_undo_authority_serve_hit_count, undo_authority_serve CR_COUNTER_ACCESSOR(cluster_undo_authority_fail_closed_count, undo_authority_fail_closed_count) CR_COUNTER_ACCESSOR(cluster_undo_authority_epoch_stale_reject_count, undo_authority_epoch_stale_reject_count) +CR_COUNTER_ACCESSOR(cluster_undo_authority_scan_incomplete_reject_count, + undo_authority_scan_incomplete_reject_count) +CR_COUNTER_ACCESSOR(cluster_undo_authority_multi_match_reject_count, + undo_authority_multi_match_reject_count) /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ CR_COUNTER_ACCESSOR(cluster_cr_xmax_resolved_count, cr_xmax_resolved_count) CR_COUNTER_ACCESSOR(cluster_cr_xmax_recycled_invisible_count, cr_xmax_recycled_invisible_count) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index af81eb2e15..f93307da65 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2600,6 +2600,11 @@ dump_cr(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_undo_authority_fail_closed_count())); emit_row(rsinfo, "cr", "undo_authority_epoch_stale_reject_count", fmt_int64((int64)cluster_undo_authority_epoch_stale_reject_count())); + /* spec-5.22d A1 (D4-8): complete-scan refusal attribution. */ + emit_row(rsinfo, "cr", "undo_authority_scan_incomplete_reject_count", + fmt_int64((int64)cluster_undo_authority_scan_incomplete_reject_count())); + emit_row(rsinfo, "cr", "undo_authority_multi_match_reject_count", + fmt_int64((int64)cluster_undo_authority_multi_match_reject_count())); /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ emit_row(rsinfo, "cr", "cr_xmax_resolved_count", fmt_int64((int64)cluster_cr_xmax_resolved_count())); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..404d880ca8 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -368,6 +368,19 @@ static ClusterInjectPoint cluster_injection_points[] = { * 53R97) under a topology that would otherwise serve the block. */ { .name = "cluster-lms-undo-fetch" }, + /* + * spec-5.22d D4-8 — dead-owner authority block0-prove refusal injection. + * + * cluster-undo-authority-block0-prove: + * Fires at the head of the shared authority block0 prove core + * (cluster_undo_authority_block0_prove), which both the requester's + * SELF-authority leg and the kind-4 LMS wire leg run. SKIP forces + * the coverage-fail refusal (undo_authority_fail_closed bumps, the + * consumer keeps 53R97) under a topology that would otherwise serve + * the dead owner's verdict — TAP L3 pins the fail-closed arm with a + * counter delta (L408), never a native CLOG answer. + */ + { .name = "cluster-undo-authority-block0-prove" }, /* * spec-6.12e2 — holder-side BAST-nudge refusal injection. * diff --git a/src/backend/cluster/cluster_runtime_visibility_policy.c b/src/backend/cluster/cluster_runtime_visibility_policy.c index daada4893a..b38346867b 100644 --- a/src/backend/cluster/cluster_runtime_visibility_policy.c +++ b/src/backend/cluster/cluster_runtime_visibility_policy.c @@ -77,44 +77,51 @@ cluster_vis_live_authority_covers_policy(XLogRecPtr anchor_lsn, ClusterLiveAutho } /* - * cluster_vis_tt_block_positive_proof + * cluster_vis_tt_block_xid_scan * - * CP3 positive-proof scan over a D-i1-fetched TT header block. See the - * header comment for the full proof discipline (positive proof only; 0-match - * / multi-match / non-terminal / malformed all refuse). The block bytes are - * the origin's own shipped copy, so every refusal is a clean NONE — never an - * elog — and the caller keeps the 53R97 fail-closed boundary. + * spec-5.22d A1 (D4-8): the scan CORE under cluster_vis_tt_block_positive_ + * proof. One parse discipline for both consumers — the CP3 single-block + * wrapper below and the dead-owner complete-scan prove (which aggregates + * nmatch across the owner's whole durable segment set and must treat any + * unparseable block as refuse-ALL, never skip-and-continue). See the header + * for the contract. Pure; no I/O, no elog. */ -ClusterVisTtProof -cluster_vis_tt_block_positive_proof(const char *block, uint32 expected_segment_id, - uint8 expected_owner_instance, TransactionId xid, - SCN *out_commit_scn, uint16 *out_wrap) +ClusterVisTtBlockScanStatus +cluster_vis_tt_block_xid_scan(const char *block, uint32 expected_segment_id, + uint8 expected_owner_instance, TransactionId xid, int *out_nmatch, + ClusterVisTtProof *out_proof, SCN *out_commit_scn, uint16 *out_wrap) { const UndoSegmentHeaderData *hdr; int nmatch = 0; const TTSlot *match = NULL; + if (out_nmatch != NULL) + *out_nmatch = 0; + if (out_proof != NULL) + *out_proof = CLUSTER_VIS_TT_PROOF_NONE; if (out_commit_scn != NULL) *out_commit_scn = InvalidScn; if (out_wrap != NULL) *out_wrap = TT_WRAP_INVALID; + /* Caller-bug inputs are indistinguishable from unparseable evidence: + * refuse-all, never guess. */ if (block == NULL || !TransactionIdIsNormal(xid)) - return CLUSTER_VIS_TT_PROOF_NONE; + return CLUSTER_VIS_TT_BLOCK_SCAN_POISONED; /* * Header identity sanity: the bytes must provably be the asked-for TT. * A mismatched segment_id / owner (raced segment reuse on the origin, a * stale cache pairing, a mis-routed reply) or an over-range slot count - * means the scan below would not be evidence about `xid` — refuse. + * means the scan below would not be evidence about `xid` — poisoned. */ hdr = (const UndoSegmentHeaderData *)block; if (hdr->segment_id != expected_segment_id) - return CLUSTER_VIS_TT_PROOF_NONE; + return CLUSTER_VIS_TT_BLOCK_SCAN_POISONED; if (hdr->owner_instance != expected_owner_instance) - return CLUSTER_VIS_TT_PROOF_NONE; + return CLUSTER_VIS_TT_BLOCK_SCAN_POISONED; if (hdr->tt_slots_count > TT_SLOTS_PER_SEGMENT) - return CLUSTER_VIS_TT_PROOF_NONE; + return CLUSTER_VIS_TT_BLOCK_SCAN_POISONED; for (int i = 0; i < (int)hdr->tt_slots_count; i++) { const TTSlot *slot = &hdr->tt_slots[i]; @@ -122,10 +129,10 @@ cluster_vis_tt_block_positive_proof(const char *block, uint32 expected_segment_i /* * Any status byte outside the known range poisons the whole block: * we cannot claim to have understood a TT whose slots we cannot - * parse (this is a shipped COPY — refuse, never PANIC). + * parse (refuse, never PANIC). */ if (slot->status > (uint8)TT_SLOT_RECYCLABLE) - return CLUSTER_VIS_TT_PROOF_NONE; + return CLUSTER_VIS_TT_BLOCK_SCAN_POISONED; /* UNUSED carries no transaction; its xid bytes are placeholder. */ if (slot->status == (uint8)TT_SLOT_UNUSED) @@ -142,24 +149,54 @@ cluster_vis_tt_block_positive_proof(const char *block, uint32 expected_segment_i } } - if (nmatch != 1) - return CLUSTER_VIS_TT_PROOF_NONE; - - if (match->status == (uint8)TT_SLOT_COMMITTED && SCN_VALID(match->commit_scn)) { - if (out_commit_scn != NULL) - *out_commit_scn = match->commit_scn; - if (out_wrap != NULL) - *out_wrap = match->wrap; - return CLUSTER_VIS_TT_PROOF_COMMITTED; - } - if (match->status == (uint8)TT_SLOT_ABORTED) { - if (out_wrap != NULL) - *out_wrap = match->wrap; - return CLUSTER_VIS_TT_PROOF_ABORTED; + if (out_nmatch != NULL) + *out_nmatch = nmatch; + + if (nmatch == 1) { + if (match->status == (uint8)TT_SLOT_COMMITTED && SCN_VALID(match->commit_scn)) { + if (out_proof != NULL) + *out_proof = CLUSTER_VIS_TT_PROOF_COMMITTED; + if (out_commit_scn != NULL) + *out_commit_scn = match->commit_scn; + if (out_wrap != NULL) + *out_wrap = match->wrap; + } else if (match->status == (uint8)TT_SLOT_ABORTED) { + if (out_proof != NULL) + *out_proof = CLUSTER_VIS_TT_PROOF_ABORTED; + if (out_wrap != NULL) + *out_wrap = match->wrap; + } + /* ACTIVE / RECYCLABLE / COMMITTED-without-scn: found (nmatch=1) but + * in-doubt — proof stays NONE. */ } + return CLUSTER_VIS_TT_BLOCK_SCAN_OK; +} - /* ACTIVE / RECYCLABLE / COMMITTED-without-scn: not a terminal proof. */ - return CLUSTER_VIS_TT_PROOF_NONE; +/* + * cluster_vis_tt_block_positive_proof + * + * CP3 positive-proof scan over a D-i1-fetched TT header block. See the + * header comment for the full proof discipline (positive proof only; 0-match + * / multi-match / non-terminal / malformed all refuse). The block bytes are + * the origin's own shipped copy, so every refusal is a clean NONE — never an + * elog — and the caller keeps the 53R97 fail-closed boundary. Since + * spec-5.22d A1 a thin wrapper over the scan core above: POISONED and + * nmatch != 1 and non-terminal all collapse to NONE, byte-for-byte the + * pre-A1 behavior (the pre-A1 unit truth table passes unchanged). + */ +ClusterVisTtProof +cluster_vis_tt_block_positive_proof(const char *block, uint32 expected_segment_id, + uint8 expected_owner_instance, TransactionId xid, + SCN *out_commit_scn, uint16 *out_wrap) +{ + int nmatch = 0; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; + + if (cluster_vis_tt_block_xid_scan(block, expected_segment_id, expected_owner_instance, xid, + &nmatch, &proof, out_commit_scn, out_wrap) + != CLUSTER_VIS_TT_BLOCK_SCAN_OK) + return CLUSTER_VIS_TT_PROOF_NONE; + return proof; } /* diff --git a/src/backend/cluster/cluster_undo_authority.c b/src/backend/cluster/cluster_undo_authority.c index 555469d381..858fe8762c 100644 --- a/src/backend/cluster/cluster_undo_authority.c +++ b/src/backend/cluster/cluster_undo_authority.c @@ -165,4 +165,57 @@ cluster_undo_authority_coverage_ok(bool claimed_at_epoch, bool block0_readable, return claimed_at_epoch && block0_readable && wrap_match; } +/* + * A1 (D4-8) -- cross-segment scan aggregation (pure). See the header for + * the contract; the U truth table (test_scan_fold_truth_table) pins the + * shape. + */ +void +cluster_undo_authority_scan_agg_init(ClusterUndoAuthorityScanAgg *agg) +{ + agg->poisoned = false; + agg->total_match = 0; + agg->proof = CLUSTER_VIS_TT_PROOF_NONE; + agg->commit_scn = InvalidScn; + agg->wrap = 0; +} + +void +cluster_undo_authority_scan_fold(ClusterUndoAuthorityScanAgg *agg, bool block_ok, int nmatch, + ClusterVisTtProof proof, SCN commit_scn, uint16 wrap) +{ + if (agg == NULL) + return; + + if (!block_ok || nmatch < 0) { + agg->poisoned = true; /* latched: the set is not fully parseable */ + return; + } + + agg->total_match += nmatch; + + if (agg->total_match == nmatch && nmatch == 1) { + /* first (and so far only) match in the whole set: keep its evidence */ + agg->proof = proof; + agg->commit_scn = commit_scn; + agg->wrap = wrap; + } else if (agg->total_match > 1) { + /* ambiguity anywhere voids kept evidence (never serve two eras) */ + agg->proof = CLUSTER_VIS_TT_PROOF_NONE; + agg->commit_scn = InvalidScn; + agg->wrap = 0; + } +} + +bool +cluster_undo_authority_scan_admissible(const ClusterUndoAuthorityScanAgg *agg) +{ + if (agg == NULL || agg->poisoned) + return false; + if (agg->total_match != 1) + return false; + return agg->proof == CLUSTER_VIS_TT_PROOF_COMMITTED + || agg->proof == CLUSTER_VIS_TT_PROOF_ABORTED; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c index a22bc9225a..d1ab659b9a 100644 --- a/src/backend/cluster/cluster_undo_authority_snapshot.c +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -33,14 +33,17 @@ #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ #include "cluster/cluster_cr.h" /* undo_authority_* counters (D4-5) */ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_inject.h" /* D4-8 L3 prove-refusal injection */ #include "cluster/cluster_membership.h" /* cluster_membership_get_state */ #include "cluster/cluster_reconfig.h" /* cluster_reconfig_get_observed_fresh_alive */ #include "cluster/cluster_runtime_visibility.h" /* cluster_vis_tt_block_positive_proof */ #include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ #include "cluster/cluster_tt_slot.h" /* TT_WRAP_INVALID */ #include "cluster/cluster_undo_authority.h" -#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_master (D1) */ -#include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block (D4-5) */ +#include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_master (D1) */ +#include "cluster/cluster_undo_smgr.h" /* cluster_undo_smgr_read_block (D4-5) */ +#include "cluster/storage/cluster_shared_fs.h" /* undo_instance_dir_resolve (A1/D4-8) */ +#include "storage/fd.h" /* AllocateDir/ReadDirExtended (A1/D4-8) */ #ifdef USE_PGRAC_CLUSTER @@ -129,22 +132,140 @@ cluster_undo_serve_authority(const ClusterResId *undo_resid, uint64 reconfig_epo } /* - * cluster_undo_authority_block0_prove -- shared authority block0 prove core - * (D4-5). See cluster_undo_authority.h for the contract. + * authority_scan_owner_segments -- A1 (D4-8) durable segment-set scan. * - * The coverage three-way AND (§2.4), with the counters owned HERE so the - * requester self-serve leg (D4-4) and the wire-served LMS leg (D4-5/D4-6) - * cannot drift on the observability contract: - * (i) claimed_at_epoch: claim_epoch still IS the current accepted - * epoch; a stale claim bumps epoch_stale_reject + fail_closed. - * (ii) block0_readable: the AUTHORITY_BLOCK0 foreign-owner intent - * (D4-3) resolved + fully read the shared block0 bytes. - * (iii) wrap_match: the slot-level xid+wrap positive proof matched - * (content-based anti-ABA; the same proof the live CP3 leg - * trusts). block0 is synchronously durable (D2 Q3): a full read - * is crash-consistent, no LSN-cover window applies. - * Proof NONE has no CP5 fallback: the owner is dead, there is nobody to - * ask, so NONE proves nothing => fail closed (never a native fallback). + * Enumerate the dead owner's SHARED undo directory (the one namespace + * cluster_shared_fs_undo_path_resolve ever writes into — A1.1) and fold + * every segment's block0 xid scan into *agg. 完备-或-fail-closed: any + * enumeration break, non-canonical entry name, unreadable block0 or + * unparseable block poisons the whole set — the caller must refuse, never + * skip-and-continue (a uniqueness claim needs the full set visible). + * + * Writer exclusion (A1.1-bis #1) makes the enumerated set stable: the dead + * owner is dead-decided + write-fenced (its revival forces a reconfig ⇒ + * epoch bump ⇒ the caller's post-scan re-check refuses); survivors' only + * foreign-owner intent is AUTHORITY_BLOCK0, whose single I/O consumer is + * the read below (no writer, no unlink/rename path exists). D5's future + * reclaimer must inherit this invariant before deleting anything here. + * + * seg_0 is NOT skipped (A1.1-bis #3): "segment 0 never carries a xact TT + * slot" is not a verified structural invariant (the uba_encode Assert only + * fences record UBAs, and the slot cursor rolls independently), so if a + * shared seg_0 exists it is verified and counted like any other. + * + * true = the whole set was enumerated + parsed (agg holds the aggregate); + * false = the set is not provably complete (caller refuses; the caller + * owns the scan_incomplete counter so refusal attribution has one home). + */ +static bool +authority_scan_owner_segments(int32 owner_node, TransactionId raw_xid, + ClusterUndoAuthorityScanAgg *agg) +{ + char dirpath[MAXPGPATH]; + DIR *dir; + struct dirent *de; + uint8 owner_instance = (uint8)(owner_node + 1); + bool complete = true; + + if (cluster_shared_fs_undo_instance_dir_resolve(owner_instance, dirpath, sizeof(dirpath)) != 0) + return false; + + dir = AllocateDir(dirpath); + if (dir == NULL) + return false; + + errno = 0; + while ((de = ReadDirExtended(dir, dirpath, LOG)) != NULL) { + char canon[64]; + unsigned long id; + char *end; + PGAlignedBlock page; + int nmatch = 0; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; + SCN scn = InvalidScn; + uint16 wrap = TT_WRAP_INVALID; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) { + errno = 0; + continue; + } + + /* + * Canonical name gate (A1.1-bis #2): every other entry must be + * EXACTLY seg_.dat as the writer renders it — parse, then + * re-render and compare, so aliases (seg_007.dat), trailing junk + * and out-of-range ids all poison the set instead of hiding + * evidence under a name the scan would skip. + */ + if (strncmp(de->d_name, "seg_", 4) != 0) { + complete = false; + break; + } + errno = 0; + id = strtoul(de->d_name + 4, &end, 10); + if (errno != 0 || end == de->d_name + 4 || strcmp(end, ".dat") != 0 || id > UINT16_MAX) { + complete = false; + break; + } + snprintf(canon, sizeof(canon), "seg_%lu.dat", id); + if (strcmp(canon, de->d_name) != 0) { + complete = false; + break; + } + + /* Read + parse this segment's block0; any failure poisons the set. */ + if (!cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + (uint32)id, owner_instance, 0 /* block0 */, page.data)) { + complete = false; + break; + } + if (cluster_vis_tt_block_xid_scan(page.data, (uint32)id, owner_instance, raw_xid, &nmatch, + &proof, &scn, &wrap) + != CLUSTER_VIS_TT_BLOCK_SCAN_OK) { + complete = false; + break; + } + cluster_undo_authority_scan_fold(agg, true, nmatch, proof, scn, wrap); + errno = 0; + } + if (complete && errno != 0) + complete = false; /* readdir broke mid-enumeration: set not complete */ + + FreeDir(dir); + return complete; +} + +/* + * cluster_undo_authority_block0_prove -- shared authority prove core (D4-5; + * A1/D4-8 widened from single-segment to the owner's COMPLETE durable + * segment set). See cluster_undo_authority.h for the contract. + * + * The live CP5 verdict is a complete scan over the origin's LIVE TT + * because a single block0's 0-match proves nothing — the xid's slot may + * live in another segment (the slot cursor rolls independently of the + * record cursor the ref's UBA points at; D4-8 e2e evidence). For a DEAD + * owner the durable shared segment set is the same authority translated + * to durable state, so THIS is the complete scan: enumerate every shared + * segment (A1.1), demand the whole set parseable, and serve only a + * set-wide UNIQUE terminal match (truth table A1.2). + * + * Coverage discipline, counters owned HERE (self + wire legs can never + * drift): + * (i) claimed_at_epoch: claim_epoch is the current accepted epoch at + * entry AND STILL at scan end (A1 约束 3 — a reconfig mid-scan + * could otherwise stitch a cross-epoch verdict) — else + * epoch_stale_reject + fail_closed. + * (ii) set complete: enumeration + every block0 read + parse succeeded + * — else scan_incomplete_reject + fail_closed (A1 约束 1). + * (iii) unique terminal match: exactly one xid+wrap match across the + * whole set, terminal (COMMITTED-with-scn / ABORTED) — 0-match and + * non-terminal fail_closed; >=2 matches multi_match_reject + + * fail_closed (A1 约束 2). + * block0 bytes are synchronously durable (D2 Q3; the commit/abort stamps + * are targeted pre-commit pwrites), so full reads are crash-consistent — + * no LSN-cover window applies. A regular (non-2PC) abort leaves its slot + * durably ACTIVE (no durable ABORTED stamp exists outside 2PC and the + * owner's own recovery, A1.5): such xids stay in-doubt here by design. */ ClusterUndoVerdictResult cluster_undo_authority_block0_prove(int32 owner_node, uint32 segment_id, TransactionId raw_xid, @@ -152,12 +273,7 @@ cluster_undo_authority_block0_prove(int32 owner_node, uint32 segment_id, Transac { ClusterUndoVerdictResult unknown = { .kind = CLUSTER_UNDO_VERDICT_UNKNOWN_FAIL_CLOSED, .commit_scn = InvalidScn, .wrap = 0 }; - PGAlignedBlock page; - SCN scn = InvalidScn; - uint16 wrap = TT_WRAP_INVALID; - ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; - bool claimed; - bool readable = false; + ClusterUndoAuthorityScanAgg agg; /* malformed owner: never provable (defense in depth; the route layers * upstream already refuse it) */ @@ -166,37 +282,63 @@ cluster_undo_authority_block0_prove(int32 owner_node, uint32 segment_id, Transac return unknown; } - /* segment 0 is bootstrap-only (Assert-fenced in uba_encode): a ref - * carrying it is not resolvable evidence */ + /* Malformed-ref precondition ONLY (A1.2): the ref's segment_id no longer + * scopes the scan (the slot may live in any segment), but a ref carrying + * the bootstrap segment or an over-range id is not resolvable evidence. */ if (segment_id == 0 || segment_id > UINT16_MAX) { cluster_undo_authority_note_failclosed(); return unknown; } - /* (i) the claim must still be scoped to the CURRENT epoch */ - claimed = (cluster_epoch_get_current() == claim_epoch); - if (!claimed) + /* spec-5.22d D4-8 L3 (L408): SKIP forces the coverage-fail refusal under + * a topology that would otherwise serve — proves the fail-closed arm and + * its counter really fire (53R97, never a native answer). Inert unless + * armed; arm state is process-local, which reaches the SELF leg (this + * runs in the reader backend); the wire leg runs in LMS, where arming is + * the conf-time chaos-harness face (t/015 note). */ + CLUSTER_INJECTION_POINT("cluster-undo-authority-block0-prove"); + if (cluster_injection_should_skip("cluster-undo-authority-block0-prove")) { + cluster_undo_authority_note_failclosed(); + return unknown; + } + + /* (i) the claim must be scoped to the CURRENT epoch at entry */ + if (cluster_epoch_get_current() != claim_epoch) { cluster_undo_authority_note_epoch_stale_reject(); + cluster_undo_authority_note_failclosed(); + return unknown; + } - /* (ii) read the dead owner's shared block0 (read-only foreign intent) */ - if (claimed) - readable = cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, - segment_id, (uint8)(owner_node + 1), 0 /* block0 */, - page.data); + /* (ii) enumerate + parse the owner's COMPLETE durable segment set */ + cluster_undo_authority_scan_agg_init(&agg); + if (!authority_scan_owner_segments(owner_node, raw_xid, &agg) || agg.poisoned) { + cluster_undo_authority_note_scan_incomplete_reject(); + cluster_undo_authority_note_failclosed(); + return unknown; + } - /* (iii) slot-level xid+wrap positive proof on the read bytes */ - if (readable) - proof = cluster_vis_tt_block_positive_proof(page.data, segment_id, (uint8)(owner_node + 1), - raw_xid, &scn, &wrap); + /* (i-bis) the epoch must not have moved ACROSS the scan window (A1 约束 + * 3): the writer-exclusion argument for the enumerated set is scoped to + * one epoch, so a mid-scan reconfig voids the set's stability. */ + if (cluster_epoch_get_current() != claim_epoch) { + cluster_undo_authority_note_epoch_stale_reject(); + cluster_undo_authority_note_failclosed(); + return unknown; + } - if (!cluster_undo_authority_coverage_ok(claimed, readable, - proof != CLUSTER_VIS_TT_PROOF_NONE)) { + /* (iii) set-wide unique terminal match, or refuse */ + if (agg.total_match >= 2) { + cluster_undo_authority_note_multi_match_reject(); + cluster_undo_authority_note_failclosed(); + return unknown; + } + if (!cluster_undo_authority_scan_admissible(&agg)) { cluster_undo_authority_note_failclosed(); return unknown; } cluster_undo_authority_note_serve_hit(); - return cluster_undo_verdict_from_block_proof(proof, scn, wrap); + return cluster_undo_verdict_from_block_proof(agg.proof, agg.commit_scn, agg.wrap); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_cr.h b/src/include/cluster/cluster_cr.h index 66bcdd3e63..90c119b209 100644 --- a/src/include/cluster/cluster_cr.h +++ b/src/include/cluster/cluster_cr.h @@ -272,13 +272,18 @@ extern uint64 cluster_vis_freshref_verdict_failclosed_count(void); extern void cluster_vis_freshref_verdict_note_resolved(void); extern void cluster_vis_freshref_verdict_note_failclosed(void); -/* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve counters. */ +/* spec-5.22d D4-4/D4-5: dead-owner authority block0 serve counters. + * A1 (D4-8) adds the complete-scan refusal attribution pair. */ extern uint64 cluster_undo_authority_serve_hit_count(void); extern uint64 cluster_undo_authority_fail_closed_count(void); extern uint64 cluster_undo_authority_epoch_stale_reject_count(void); +extern uint64 cluster_undo_authority_scan_incomplete_reject_count(void); +extern uint64 cluster_undo_authority_multi_match_reject_count(void); extern void cluster_undo_authority_note_serve_hit(void); extern void cluster_undo_authority_note_failclosed(void); extern void cluster_undo_authority_note_epoch_stale_reject(void); +extern void cluster_undo_authority_note_scan_incomplete_reject(void); +extern void cluster_undo_authority_note_multi_match_reject(void); /* * spec-6.12i CP5 (D-i4): origin-side pieces of the cross-instance verdict diff --git a/src/include/cluster/cluster_runtime_visibility.h b/src/include/cluster/cluster_runtime_visibility.h index b1542df42a..a242951e8c 100644 --- a/src/include/cluster/cluster_runtime_visibility.h +++ b/src/include/cluster/cluster_runtime_visibility.h @@ -134,6 +134,28 @@ extern ClusterVisTtProof cluster_vis_tt_block_positive_proof(const char *block, TransactionId xid, SCN *out_commit_scn, uint16 *out_wrap); +/* + * spec-5.22d A1 (D4-8): the scan CORE under the positive-proof wrapper. + * Same parse discipline, exposed for the dead-owner complete-scan prove: + * the per-block match COUNT is reported (cross-segment uniqueness is the + * aggregate's to decide) and unparseable bytes are a DISTINCT status so the + * aggregate refuses the whole set instead of skip-and-continue (A1.1 + * 完备-或-fail-closed). OK + nmatch==1 fills out_proof (terminal COMMITTED + * with a valid scn / terminal ABORTED / NONE for in-doubt shapes) and, on a + * terminal proof, out_commit_scn / out_wrap. POISONED covers NULL block, + * invalid xid, header identity mismatch, over-range slot count, and any + * out-of-range slot status byte. Pure; unit truth table. + */ +typedef enum ClusterVisTtBlockScanStatus { + CLUSTER_VIS_TT_BLOCK_SCAN_OK = 0, + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED = 1 +} ClusterVisTtBlockScanStatus; + +extern ClusterVisTtBlockScanStatus +cluster_vis_tt_block_xid_scan(const char *block, uint32 expected_segment_id, + uint8 expected_owner_instance, TransactionId xid, int *out_nmatch, + ClusterVisTtProof *out_proof, SCN *out_commit_scn, uint16 *out_wrap); + /* * CP5 (D-i4) pure structural validation of a shipped verdict page (see * cluster_gcs_block.h for the wire struct and the verdict taxonomy). true diff --git a/src/include/cluster/cluster_undo_authority.h b/src/include/cluster/cluster_undo_authority.h index 4c9b564e28..1bfd40fb36 100644 --- a/src/include/cluster/cluster_undo_authority.h +++ b/src/include/cluster/cluster_undo_authority.h @@ -206,6 +206,29 @@ cluster_undo_authority_serve_decide(const ClusterUndoServeRoute *route, int32 se extern bool cluster_undo_authority_coverage_ok(bool claimed_at_epoch, bool block0_readable, bool wrap_match); +/* + * A1 (D4-8) -- cross-segment scan aggregation (pure). The complete-scan + * prove folds every enumerated block0's cluster_vis_tt_block_xid_scan result + * into this aggregate; the verdict is admissible ONLY when nothing poisoned + * the set (block_ok=false latches; 完备-或-fail-closed, A1.1) AND the whole + * owner set holds EXACTLY one match AND that match is terminal (truth table + * A1.2 #7-#12). A second match ANYWHERE voids the kept evidence (never + * serve an ambiguous identity). Pure; unit truth table. + */ +typedef struct ClusterUndoAuthorityScanAgg { + bool poisoned; /* latched: any unreadable/unparseable block */ + int total_match; /* xid matches across the whole enumerated set */ + ClusterVisTtProof proof; /* the unique match's terminal proof (or NONE) */ + SCN commit_scn; /* valid iff proof == COMMITTED */ + uint16 wrap; /* the unique match's wrap evidence */ +} ClusterUndoAuthorityScanAgg; + +extern void cluster_undo_authority_scan_agg_init(ClusterUndoAuthorityScanAgg *agg); +extern void cluster_undo_authority_scan_fold(ClusterUndoAuthorityScanAgg *agg, bool block_ok, + int nmatch, ClusterVisTtProof proof, SCN commit_scn, + uint16 wrap); +extern bool cluster_undo_authority_scan_admissible(const ClusterUndoAuthorityScanAgg *agg); + /* * D4-5 -- shared authority block0 prove core (heavy, snapshot object). The * ONE implementation both consumers run so the self-authority answer (D4-4 diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 9a62da52fa..b62e31c47c 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'pg_stat_cluster_injections returns 159 rows (spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '159 injection point names match the full registry (spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- diff --git a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl index 8c2aa8acee..d56b7d5e13 100644 --- a/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl +++ b/src/test/cluster_tap/t/215_cluster_3_9_cr_construction.pl @@ -94,8 +94,8 @@ # ---------- my $cr_rows = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}); -is($cr_rows, '62', - 'L2 cr category has 62 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 3 spec-5.22d D4 authority serve)'); +is($cr_rows, '64', + 'L2 cr category has 64 counter rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 5 spec-5.22d D4 authority serve incl. A1 scan attribution)'); # ---------- diff --git a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl index 0d226be5c0..58db737de1 100644 --- a/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl +++ b/src/test/cluster_tap/t/216_cluster_3_10_cr_cache.pl @@ -78,8 +78,8 @@ is( $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='cr'}), - '62', - 'L1d cr category has 62 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 3 spec-5.22d D4 authority serve)'); + '64', + 'L1d cr category has 64 rows (9 + 4 cache + 4 xmax + 5 spec-5.53 mismatch + 8 spec-5.54 tuple + 5 spec-5.56 lifecycle + 6 spec-6.12b cr-server + 16 spec-6.12i/6.15 runtime-visibility & verdict + 2 spec-5.22f D6 fresh-ref verdict + 5 spec-5.22d D4 authority serve incl. A1 scan attribution)'); } diff --git a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl index ddbecc30fc..8c9f102fcb 100644 --- a/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl +++ b/src/test/cluster_tap/t/254_pi_cr_recovery_acceptance.pl @@ -125,10 +125,11 @@ sub _new_single_node recovery => 39, # 3.16(4)+4.10 block(2)+4.11 thread(4)+4.3 plan(13)+4.4 worker(8)+4.5/4.7 merge(8) tt_recovery => 8, # 4.8 verdict counters gcs_recovery => 10, # 4.7 warm-recovery(8) + spec-2.41 D7 redo-coverage serve-gate(2) - cr => 62, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) + cr => 64, # 3.10/3.21/3.22 CR path(17) + 5.53 mismatch(5) + 5.54 tuple(8) # + ... + 6.12b(6) + 6.12i/6.15(16) + 2 spec-5.22f D6 fresh-ref verdict - # + 3 spec-5.22d D4 authority serve (undo_authority_ - # {serve_hit,fail_closed,epoch_stale_reject}, + # + 5 spec-5.22d D4 authority serve (undo_authority_ + # {serve_hit,fail_closed,epoch_stale_reject} + A1 + # {scan_incomplete,multi_match}_reject, # unconditional in dump_cr) # + 11 post-5.54 keys the baseline missed (stale on # main; first caught by a local full run): 6.12b diff --git a/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl b/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl new file mode 100644 index 0000000000..d2ca652bd5 --- /dev/null +++ b/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl @@ -0,0 +1,401 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 369_cluster_5_22d_dead_owner_authority_serve.pl +# spec-5.22d D4 + A1 — dead-owner undo verdict SERVE (Route B, +# complete-scan prove) end-to-end on a 2-node ClusterPair + shared +# cluster_fs root. This is the positive leg t/359 L6 honestly +# forward-linked: there the dead owner's verdict serve-gate DENIED +# (fail-closed); here the elected survivor authority SERVES the verdict +# from the dead owner's durable shared segment set. +# +# 2-node scope note (user-ruled, D4-6 approve): with two nodes the +# reader IS the survivor IS the elected authority, so this test covers +# the SELF-authority leg (route -> SERVE_SELF_BLOCK0 -> complete-scan +# prove) only. The kind-4 PEER-wire serve leg needs a 3+ node topology +# (reader != authority) and its e2e evidence is deliberately NOT +# claimed here (D7 / 3-node TAP). +# +# Substrate (t/359 family): node0 commits NORMAL-xid xacts while undo +# GCS coherence is ON, so the durable TT stamps land on the SHARED +# cluster_fs root. The A1 complete-scan prove enumerates the owner's +# whole shared segment set, so the TT-slot-cursor / record-cursor +# straddle (the D4-8 discovery) no longer strands evidence. Heap +# pages are phantom-shared and checkpointed clean before the crash. +# Four SUBJECT tables isolate attribution — each is touched by exactly +# ONE owner xact, so no later same-page access can clean the ITLs out +# from under the leg that needs a raw ref: +# s_t L1: one committed batch xact (8 rows); +# s_ab L2a: one regular-ROLLBACK xact (id=999) — NO durable +# ABORTED stamp exists for regular aborts (A1.5: only 2PC and +# the owner's own recovery stamp ABORTED), so under a dead +# never-recovered owner this xid is in-doubt: the read must +# FAIL CLOSED, never false-visible, never guessed-invisible; +# s_ab2 L2b: one 2PC PREPARE + ROLLBACK PREPARED xact (id=888) — +# the durable ABORTED stamp (cluster_tt_2pc.c) makes the +# terminal state provable from the dead owner's segment set; +# s_t2 L3: one committed batch xact, first-touched only under the +# armed injection (cold in every cache). +# +# PCM warmup (load-bearing): while node0 lives — and BEFORE +# crossnode_runtime_visibility is armed — node1 reads the tables once. +# The PCM layer works (node0's X grants downgrade for the clean-page S +# read under cluster.read_scache; the page bytes land S-held in node1's +# pool) but the visibility resolution fails closed 53R97, so NO xid +# resolves, NO memo/hint warms: every verdict below is provably +# post-mortem. Without this warmup the owner dies still X-holding the +# heap pages and node1's read dies upstream at "could not obtain read +# image from X holder" (dead-holder page recovery — out of D4 scope, +# registered follow-up). +# +# L1 committed batch -> owner DEAD -> node1 reads all 8 rows. HARD +# counter asserts: undo_authority_serve_hit_count moved (Route B +# complete-scan prove ran, not overlay/cache/native) AND +# materialized_remote_instances == 0 (no WAL-materialization +# dependence). +# L2a regular-abort row: read FAILS CLOSED 53R97 persistently + +# undo_authority_fail_closed_count moved (in-doubt by design — +# an on-disk ACTIVE slot is not proof of non-commit under the +# machine-crash model, and there is no live owner to ask). +# L2b 2PC-aborted row: NOT visible (never false-visible). EMPIRICAL +# FINDING (documented in spec A1.5): ROLLBACK PREPARED physically +# reverts the tuple synchronously — pre-kill warmup already reads +# count=0 natively — so the durably-ABORTED SERVE conjunction is +# unreachable through this vehicle (serve delta stays 0, printed +# honestly). The ABORTED serve arm is pinned at every pure layer +# (scan/fold/fill/map units); its e2e needs a crash window between +# the durable abort stamp and the undo apply -> D7 injection. +# L3 (L408) injection-forced prove refusal on the cold s_t2: +# fail-closed 53R97 + fail_closed counter delta; NEVER a native +# CLOG answer. +# L3b disarm control: a fresh backend re-reads s_t2 and succeeds via +# the serve — proving L3's refusal was the armed injection. +# +# Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (D4-8, §4.2 + A1) +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +sub state_val +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub write_retry +{ + my ($node, $sql) = @_; + for my $i (1 .. 10) + { + my $ok = eval { $node->safe_psql('postgres', $sql); 1 }; + return 1 if $ok; + usleep(500_000); + } + return 0; +} + +sub poll_until +{ + my ($node, $sql, $want, $secs) = @_; + my $deadline = time() + $secs; + while (time() < $deadline) + { + my $v = eval { $node->safe_psql('postgres', $sql); }; + return 1 if defined $v && $v eq $want; + usleep(250_000); + } + return 0; +} + +sub arm_guc_both +{ + my ($pair, $guc, $val) = @_; + for my $n ($pair->node0, $pair->node1) + { + $n->safe_psql('postgres', "ALTER SYSTEM SET $guc = $val"); + $n->safe_psql('postgres', 'SELECT pg_reload_conf()'); + } + usleep(1_000_000); +} + +# ============================================================ +# Boot. +# ============================================================ +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'spec_5_22d_authority', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 5', + 'cluster.undo_segments_max_per_instance = 256', + 'cluster.undo_segment_create_timeout_ms = 5000', + # L2b needs 2PC (the one regular path with a durable ABORTED stamp). + 'max_prepared_transactions = 10', + # read_scache (spec-6.12a ㉕): the PCM warmup read must actually + # DOWNGRADE the owner's X (quiescent X->S) instead of the default + # one-shot read-image ship (which leaves the dying owner X-holding + # the heap pages and strands every post-mortem read at "could not + # obtain read image from X holder", upstream of the verdict). + 'cluster.read_scache = on', + ]); +$pair->start_pair; +usleep(2_000_000); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'boot node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'boot node1 sees node0 connected'); + +my ($node0, $node1) = ($pair->node0, $pair->node1); + +# Phantom-shared heap tables (coincident relfilepath, like t/359/t/346). +# Created BEFORE coherence arms: the first local write must form the local +# pg_undo/instance_N tree (arming before any write hits the cold-formation +# PANIC — the known spec-5.22 gap, out of D4 scope). +for my $n ($node0, $node1) +{ + $n->safe_psql('postgres', 'CREATE TABLE s_t (id int, v int)'); + $n->safe_psql('postgres', 'CREATE TABLE s_ab (id int, v int)'); + $n->safe_psql('postgres', 'CREATE TABLE s_ab2 (id int, v int)'); + $n->safe_psql('postgres', 'CREATE TABLE s_t2 (id int, v int)'); +} +my $p0 = $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}); +my $p1 = $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}); +is($p0, $p1, 'boot s_t relfilepath coincidence holds (phantom-shared)'); +is($node1->safe_psql('postgres', 'SHOW cluster.crossnode_runtime_visibility'), + 'off', 'boot crossnode_runtime_visibility still off (warmup runs fail-closed)'); + +# Arm coherence, then write the subjects. The A1 complete-scan prove reads +# the owner's WHOLE shared segment set, so the TT-slot/record cursor straddle +# across the arming boundary no longer matters — the stamps land in SOME +# shared segment file and the scan will find them. +arm_guc_both($pair, 'cluster.undo_gcs_coherence', 'on'); +is($node0->safe_psql('postgres', 'SHOW cluster.undo_gcs_coherence'), + 'on', 'undo_gcs_coherence armed on'); + +# Force a fresh segment under coherence=on (the t/359 recipe position). +# Empirical device with TWO effects this test depends on: the subject +# xacts' records land in a shared-BORN segment, and — observed across the +# D4-8 iteration matrix — the subjects' ITLs stay RAW on the heap pages +# (without it the owner side hints/cleans them and every read below turns +# native + vacuous; the warmup HARD asserts below catch that drift loudly). +ok(write_retry($node0, q{SELECT cluster_undo_test_force_segment_end()}), + 'owner forced fresh undo segment under coherence=on'); + +# One owner xact per subject table (attribution isolation, see banner). +ok(write_retry($node0, 'INSERT INTO s_t SELECT g, g * 10 FROM generate_series(1, 8) g'), + 'owner committed 8 NORMAL-xid rows into s_t (L1 subject)'); +$node0->safe_psql('postgres', q{ + BEGIN; + INSERT INTO s_ab VALUES (999, -1); + ROLLBACK; +}); +$node0->safe_psql('postgres', q{ + BEGIN; + INSERT INTO s_ab2 VALUES (888, -2); + PREPARE TRANSACTION 'spec522d_l2b'; +}); +$node0->safe_psql('postgres', q{ROLLBACK PREPARED 'spec522d_l2b'}); +ok(write_retry($node0, 'INSERT INTO s_t2 SELECT g, g FROM generate_series(1, 5) g'), + 'owner committed 5 rows into s_t2 (L3 cold-cache subject)'); +ok(write_retry($node0, 'CHECKPOINT'), + 'owner checkpoint (heap pages clean on shared storage; no page crash-recovery needed)'); + +# ============================================================ +# PCM warmup (see banner): node1 pulls the heap pages S while node0 can +# still yield its X grants; crossnode is OFF so the visibility resolution +# fails closed and NOTHING about the xids is learned or cached. The +# aborted-subject pages may or may not error depending on rollback physics — +# the read attempt itself pulls the page either way (tolerant legs). +# ============================================================ +for my $t (qw(s_t s_t2)) +{ + my $warmed = 0; + my $err_shape = ''; + for my $try (1 .. 20) + { + my ($rc, $out, $err) = $node1->psql('postgres', "SELECT count(*) FROM $t"); + if ($rc != 0 && defined $err && $err =~ /cluster TT status unknown/) + { + $warmed = 1; + last; + } + $err_shape = ($err // '') . ' out=' . ($out // ''); + usleep(500_000); + } + diag("warmup $t last outcome: $err_shape") if !$warmed; + ok($warmed, + "warmup: node1 read of $t reached visibility and failed closed 53R97 (page now S-held, zero xid resolution)"); +} +for my $t (qw(s_ab s_ab2)) +{ + my ($rc, $out, $err) = $node1->psql('postgres', "SELECT count(*) FROM $t"); + diag("warmup $t (tolerant): rc=$rc out=" . ($out // '') . " err=" . ($err // '')); +} + +# Arm the crossnode consumer only now — the warmup above must never resolve. +arm_guc_both($pair, 'cluster.crossnode_runtime_visibility', 'on'); +is($node1->safe_psql('postgres', 'SHOW cluster.crossnode_runtime_visibility'), + 'on', 'crossnode_runtime_visibility armed on after warmup'); + +# Counter baselines on the survivor BEFORE the crash. +my $serve_hit0 = state_val($node1, 'cr', 'undo_authority_serve_hit_count'); +is(state_val($node1, 'recovery', 'materialized_remote_instances'), 0, + 'baseline: no materialized remote instances before the crash'); + +# ============================================================ +# Owner dies. node1 (the only survivor) becomes the deterministically +# elected serve authority once node0 is dead-DECIDED (accepted reconfig +# dead set, L419 — CSSD 'suspected' alone keeps the route fail-closed, +# which the read polls below absorb as retries). +# ============================================================ +$pair->kill_node9(0); +ok( poll_until($node1, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 40), + 'node1 CSSD marks node0 suspected/dead'); + +# ============================================================ +# L1: committed rows served from the dead owner's durable segment set. +# ============================================================ +{ + my $ok_rows = 0; + my $last_err = ''; + for my $try (1 .. 40) + { + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + if (defined $out && $out =~ /^8$/m) { $ok_rows = 1; last; } + $last_err = $err // ''; + usleep(1_000_000); + } + + for my $k (qw(undo_authority_serve_hit_count undo_authority_fail_closed_count + undo_authority_epoch_stale_reject_count undo_authority_scan_incomplete_reject_count + undo_authority_multi_match_reject_count rtvis_resolve_committed_count + rtvis_resolve_aborted_count rtvis_resolve_failclosed_count)) + { + diag("L1 node1 cr.$k = " . state_val($node1, 'cr', $k)); + } + diag("L1 node1 undo.smgr_pread_count = " . state_val($node1, 'undo', 'smgr_pread_count')); + diag("L1 last node1 read error: $last_err") if $last_err; + + ok($ok_rows, + 'L1 POSITIVE LEG: node1 fresh read sees the 8 committed rows after the owner died (authority serve)'); + + cmp_ok(state_val($node1, 'cr', 'undo_authority_serve_hit_count'), '>', $serve_hit0, + 'L1 HARD ASSERT: undo_authority_serve_hit_count moved (Route B complete-scan prove ran, not overlay/cache/native)'); + is(state_val($node1, 'recovery', 'materialized_remote_instances'), 0, + 'L1 HARD ASSERT: materialized_remote_instances == 0 (pure Route B, no WAL-materialization dependence)'); +} + +# ============================================================ +# L2a: the regular-abort row is IN-DOUBT under a dead never-recovered owner +# (no durable ABORTED stamp exists for regular aborts, A1.5): the read +# must FAIL CLOSED — never false-visible, never guessed-invisible. +# ============================================================ +{ + my $failclose_pre = state_val($node1, 'cr', 'undo_authority_fail_closed_count'); + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_ab'); + + isnt($rc, 0, 'L2a regular-abort subject: node1 read fails closed (in-doubt, dead owner)'); + unlike(($out // ''), qr/^\s*[01]\s*$/m, + 'L2a the read returned NO row count at all (neither false-visible 1 nor guessed-invisible 0)'); + like($err, qr/cluster TT status unknown|cluster TT slot recycled/, + 'L2a the failure is the 53R97 fail-closed boundary'); + cmp_ok(state_val($node1, 'cr', 'undo_authority_fail_closed_count'), '>', $failclose_pre, + 'L2a HARD ASSERT: undo_authority_fail_closed_count moved (the refusal came from the prove)'); +} + +# ============================================================ +# L2b: the 2PC-aborted row carries a durable ABORTED stamp — the one +# regular-path terminal abort a dead owner's segment set can prove. +# ============================================================ +{ + my $serve_pre = state_val($node1, 'cr', 'undo_authority_serve_hit_count'); + my $false_visible = 0; + my $outcome = 'error'; + for my $try (1 .. 15) + { + # Empirically this page can wedge PERMANENTLY in the post-reconfig + # block-protocol rebuild (the dead-X-holder strand family: some + # deferred 2PC-rollback activity re-takes X after the warmup + # downgrade, so the owner dies holding it — pre-existing wall, + # registered follow-up). A persistent fail-closed ERROR is a SAFE + # outcome; the ONLY failure mode this leg guards is the row being + # SERVED VISIBLE (count 1). + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_ab2'); + if (defined $out && $out =~ /^1$/m) { $false_visible = 1; last; } + if (defined $out && $out =~ /^0$/m) { $outcome = 'invisible'; last; } + usleep(1_000_000); + } + my $serve_delta = state_val($node1, 'cr', 'undo_authority_serve_hit_count') - $serve_pre; + diag("L2b outcome = $outcome, serve_hit delta = $serve_delta (expected 0: ROLLBACK PREPARED " + . "reverts synchronously, nothing left to resolve — the positive ABORTED-serve e2e is " + . "D7's crash-window injection)"); + + ok(!$false_visible, + 'L2b 2PC-aborted row is NEVER served visible on node1 (invisible or fail-closed are both safe; serve leg honestly vacuous, see banner)'); +} + +# ============================================================ +# L3: injection-forced refusal (L408). The SKIP point sits at the head of +# the complete-scan prove core; arming is process-local, and the SELF +# leg prove runs in the reader backend itself — so arm + first-touch +# read in ONE session. s_t2's xid is cold in every cache, so the read +# MUST reach the prove and MUST refuse fail-closed (53R97), never +# answer from a native path. +# ============================================================ +{ + my $failclose_pre = state_val($node1, 'cr', 'undo_authority_fail_closed_count'); + my ($rc, $out, $err) = $node1->psql('postgres', q{ + SELECT cluster_inject_fault('cluster-undo-authority-block0-prove', 'skip', 0); + SELECT count(*) FROM s_t2; + }); + isnt($rc, 0, 'L3 armed prove-refusal: node1 first-touch read of s_t2 errors (fail-closed)'); + unlike(($out // ''), qr/^\s*5\s*$/m, + 'L3 armed read did NOT return the 5 rows (no native/cached bypass of the refused prove)'); + like($err, qr/cluster TT status unknown|cluster TT slot recycled/, + 'L3 the failure is the 53R97 fail-closed boundary (never a native CLOG answer)'); + cmp_ok(state_val($node1, 'cr', 'undo_authority_fail_closed_count'), '>', $failclose_pre, + 'L3 HARD ASSERT: undo_authority_fail_closed_count moved (the refusal came from the prove arm)'); +} + +# ============================================================ +# L3b: disarm control — a fresh (unarmed) backend re-reads s_t2 and the +# serve succeeds, proving L3's refusal was the armed injection. +# ============================================================ +{ + my $ok_rows = 0; + for my $try (1 .. 20) + { + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t2'); + if (defined $out && $out =~ /^5$/m) { $ok_rows = 1; last; } + usleep(500_000); + } + ok($ok_rows, 'L3b unarmed control: fresh backend read of s_t2 sees the 5 rows (serve intact)'); +} + +$pair->stop_pair if $pair->can('stop_pair'); +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index b43ad7e52e..5b50738a69 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1870,6 +1870,17 @@ cluster_undo_authority_epoch_stale_reject_count(void) { return 0; } +/* spec-5.22d A1 (D4-8): complete-scan refusal attribution. */ +uint64 +cluster_undo_authority_scan_incomplete_reject_count(void) +{ + return 0; +} +uint64 +cluster_undo_authority_multi_match_reject_count(void) +{ + return 0; +} /* spec-3.22 D3: xmax recycled-slot resolve outcome buckets. */ uint64 cluster_cr_xmax_resolved_count(void) diff --git a/src/test/cluster_unit/test_cluster_runtime_visibility.c b/src/test/cluster_unit/test_cluster_runtime_visibility.c index 5d235abef0..59c1851c9e 100644 --- a/src/test/cluster_unit/test_cluster_runtime_visibility.c +++ b/src/test/cluster_unit/test_cluster_runtime_visibility.c @@ -237,6 +237,93 @@ UT_TEST(test_ttproof_ambiguity_and_garbage) CLUSTER_VIS_TT_PROOF_NONE); } +/* spec-5.22d A1 (D4-8): the scan CORE under the positive-proof wrapper. + * Same parse discipline, but nmatch is reported (the cross-segment + * aggregation needs it) and unparseable bytes are a distinct POISONED + * status (the aggregate must refuse-all, never skip-and-continue). */ +UT_TEST(test_tt_block_xid_scan_core) +{ + UndoSegmentHeaderData *hdr = mk_header(PROOF_SEG, PROOF_OWNER, TT_SLOTS_PER_SEGMENT); + int nmatch = -1; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_COMMITTED; + SCN scn = (SCN)1; + uint16 wrap = 1; + + /* OK + 0 matches: parseable block, no evidence; outs reset. */ + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 3000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_OK); + UT_ASSERT_EQ(nmatch, 0); + UT_ASSERT_EQ(proof, CLUSTER_VIS_TT_PROOF_NONE); + UT_ASSERT_EQ(scn, InvalidScn); + + /* OK + unique COMMITTED: terminal, scn/wrap out. */ + set_slot(hdr, 3, 1000, 5, (uint8)TT_SLOT_COMMITTED, (SCN)777); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 1000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_OK); + UT_ASSERT_EQ(nmatch, 1); + UT_ASSERT_EQ(proof, CLUSTER_VIS_TT_PROOF_COMMITTED); + UT_ASSERT_EQ(scn, (SCN)777); + UT_ASSERT_EQ(wrap, 5); + + /* OK + unique ABORTED: terminal. */ + set_slot(hdr, 9, 2000, 2, (uint8)TT_SLOT_ABORTED, InvalidScn); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 2000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_OK); + UT_ASSERT_EQ(nmatch, 1); + UT_ASSERT_EQ(proof, CLUSTER_VIS_TT_PROOF_ABORTED); + UT_ASSERT_EQ(wrap, 2); + + /* OK + unique ACTIVE: found but NOT terminal (in-doubt) — nmatch says + * "evidence lives here", proof says "not provable". */ + set_slot(hdr, 12, 2500, 1, (uint8)TT_SLOT_ACTIVE, InvalidScn); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 2500, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_OK); + UT_ASSERT_EQ(nmatch, 1); + UT_ASSERT_EQ(proof, CLUSTER_VIS_TT_PROOF_NONE); + + /* OK + 2 matches inside one block (RECYCLABLE residue counts). */ + set_slot(hdr, 7, 1000, 4, (uint8)TT_SLOT_RECYCLABLE, (SCN)555); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 1000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_OK); + UT_ASSERT_EQ(nmatch, 2); + UT_ASSERT_EQ(proof, CLUSTER_VIS_TT_PROOF_NONE); + UT_ASSERT_EQ(scn, InvalidScn); + + /* POISONED: header identity mismatch (segment / owner), count over. */ + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG + 1, PROOF_OWNER, 1000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER + 1, 1000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); + hdr->tt_slots_count = TT_SLOTS_PER_SEGMENT + 1; + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 1000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); + hdr->tt_slots_count = TT_SLOTS_PER_SEGMENT; + + /* POISONED: a garbage status byte anywhere poisons the whole block, + * whatever xid is asked. */ + set_slot(hdr, 40, 4000, 1, (uint8)7, InvalidScn); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, 2000, + &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); + set_slot(hdr, 40, 0, 0, (uint8)TT_SLOT_UNUSED, InvalidScn); + + /* POISONED: caller-bug inputs refuse-all (NULL block / invalid xid). */ + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(NULL, PROOF_SEG, PROOF_OWNER, 1000, &nmatch, &proof, + &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); + UT_ASSERT_EQ(cluster_vis_tt_block_xid_scan(proof_block.data, PROOF_SEG, PROOF_OWNER, + InvalidTransactionId, &nmatch, &proof, &scn, &wrap), + CLUSTER_VIS_TT_BLOCK_SCAN_POISONED); +} + /* Header identity mismatches: not provably the asked-for TT -> refuse. */ UT_TEST(test_ttproof_header_mismatch) { @@ -537,7 +624,7 @@ UT_TEST(test_undo_fetch_tag_roundtrip) int main(void) { - UT_PLAN(16); + UT_PLAN(17); UT_RUN(test_covers_when_epoch_match_and_hwm_ge_anchor); UT_RUN(test_failclosed_when_epoch_differs); UT_RUN(test_failclosed_when_hwm_invalid); @@ -547,6 +634,7 @@ main(void) UT_RUN(test_ttproof_committed_and_aborted); UT_RUN(test_ttproof_zero_match_active_invalid_scn); UT_RUN(test_ttproof_ambiguity_and_garbage); + UT_RUN(test_tt_block_xid_scan_core); UT_RUN(test_ttproof_header_mismatch); UT_RUN(test_undo_auth_trailer_roundtrip); UT_RUN(test_undo_verdict_page_usable); diff --git a/src/test/cluster_unit/test_cluster_undo_authority.c b/src/test/cluster_unit/test_cluster_undo_authority.c index 0761346573..bd7dca0d56 100644 --- a/src/test/cluster_unit/test_cluster_undo_authority.c +++ b/src/test/cluster_unit/test_cluster_undo_authority.c @@ -362,10 +362,68 @@ UT_TEST(test_serve_decide_invalid_dest_failclosed) CLUSTER_UNDO_AUTHORITY_SERVE_FAIL_CLOSED); } +/* spec-5.22d A1 (D4-8): the cross-segment scan aggregation fold — the pure + * half of the complete-scan prove (truth table A1.2 #7-#12). Admissible + * ONLY when nothing poisoned the set AND the whole owner set holds exactly + * one match AND that match is terminal. */ +UT_TEST(test_scan_fold_truth_table) +{ + ClusterUndoAuthorityScanAgg agg; + + /* init: empty set proves nothing (#8). */ + cluster_undo_authority_scan_agg_init(&agg); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + + /* one block, unique COMMITTED (#11): admissible, evidence kept. */ + cluster_undo_authority_scan_fold(&agg, true, 0, CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_COMMITTED, (SCN)777, 5); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 1); + UT_ASSERT_EQ(agg.proof, CLUSTER_VIS_TT_PROOF_COMMITTED); + UT_ASSERT_EQ((long long)agg.commit_scn, 777); + UT_ASSERT_EQ(agg.wrap, 5); + + /* a second match in ANOTHER block (#9): evidence voided, refuse. */ + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_ABORTED, InvalidScn, 4); + UT_ASSERT_EQ(agg.total_match, 2); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + UT_ASSERT_EQ(agg.proof, CLUSTER_VIS_TT_PROOF_NONE); + + /* unique ABORTED (#12): admissible. */ + cluster_undo_authority_scan_agg_init(&agg); + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_ABORTED, InvalidScn, 2); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 1); + UT_ASSERT_EQ(agg.proof, CLUSTER_VIS_TT_PROOF_ABORTED); + + /* unique but NON-terminal (ACTIVE-shaped, #10): found, in-doubt, refuse. */ + cluster_undo_authority_scan_agg_init(&agg); + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + UT_ASSERT_EQ(agg.total_match, 1); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + + /* multi-match INSIDE one block (#9): refuse. */ + cluster_undo_authority_scan_agg_init(&agg); + cluster_undo_authority_scan_fold(&agg, true, 2, CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + + /* poison latch (#7): a bad block ANYWHERE voids even a clean unique + * terminal match, before or after it. */ + cluster_undo_authority_scan_agg_init(&agg); + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_COMMITTED, (SCN)777, 5); + cluster_undo_authority_scan_fold(&agg, false, 0, CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + cluster_undo_authority_scan_agg_init(&agg); + cluster_undo_authority_scan_fold(&agg, false, 0, CLUSTER_VIS_TT_PROOF_NONE, InvalidScn, 0); + cluster_undo_authority_scan_fold(&agg, true, 1, CLUSTER_VIS_TT_PROOF_COMMITTED, (SCN)777, 5); + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(&agg) ? 1 : 0, 0); + + /* NULL agg is never admissible (defense in depth). */ + UT_ASSERT_EQ(cluster_undo_authority_scan_admissible(NULL) ? 1 : 0, 0); +} + int main(void) { - UT_PLAN(18); + UT_PLAN(19); UT_RUN(test_authority_dead_owner_elects_lowest_survivor); UT_RUN(test_authority_owner_live_no_election); UT_RUN(test_authority_owner_undecided_unknown); @@ -384,6 +442,7 @@ main(void) UT_RUN(test_serve_decide_peer_block0); UT_RUN(test_serve_decide_recovering_unknown_failclosed); UT_RUN(test_serve_decide_invalid_dest_failclosed); + UT_RUN(test_scan_fold_truth_table); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From fe6655df57c4856fb8cef19c6c188803e09c3a3f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 05:10:25 +0800 Subject: [PATCH 23/38] feat(cluster): D5-1 cluster undo horizon pure fold + import publish gap fix (spec-5.22e) The pure decision core of the cluster-wide undo retention brake: fold the local spec-3.12 horizon and every required MEMBER peer's report view into a {scn, epoch} recycle floor. A floor is only proven when each required peer's report is capable (current-connection UNDO_HORIZON_V1), present, seqlock-stable, current-epoch, well-formed (valid SCN + interval within [100,60000]ms), monotone (a same-epoch regression latch stalls: the peer contradicted its accepted lower bound, so the older higher value must not be consumed), and fresh (3 x max(sender, local) interval, receiver-local clock, future recv treated stale). Anything unproven STALLS with deterministic first-fail attribution (reason + blame node) -- deliberately NO fallback to local-horizon recycling on any edge: a required peer without capability may be an old binary still holding old snapshots. Empty required set folds to the local horizon (single node / cold formation stays on today's path). All SCN comparisons go through scn_time_cmp. Unit truth table U1-U16 (19 tests) pins every arm, including ghost-slot ignore, DEAD/REMOVED drop, blame ordering, and the reason-name table. Also closes the SetTransactionSnapshot publish gap found by the spec's S3.0 audit: the imported FirstXactSnapshot was registered without an immediate cluster_recompute_proc_read_scn(), so the exporter could release its pin before the importer's first push published the inherited read_scn -- a window where this node's retention floor (and its future horizon report) sat above a live imported snapshot. Mirror the GetTransactionSnapshot register path. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-1) --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_undo_horizon.c | 222 ++++++++ src/backend/utils/time/snapmgr.c | 13 + src/include/cluster/cluster_undo_horizon.h | 168 ++++++ src/test/cluster_unit/Makefile | 20 +- .../cluster_unit/test_cluster_undo_horizon.c | 501 ++++++++++++++++++ 6 files changed, 923 insertions(+), 2 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_horizon.c create mode 100644 src/include/cluster/cluster_undo_horizon.h create mode 100644 src/test/cluster_unit/test_cluster_undo_horizon.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 933c777a57..44447cb7b4 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -189,6 +189,7 @@ OBJS = \ cluster_undo_gcs.o \ cluster_undo_gcs_grant.o \ cluster_undo_gcs_stat.o \ + cluster_undo_horizon.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_undo_horizon.c b/src/backend/cluster/cluster_undo_horizon.c new file mode 100644 index 0000000000..c451ec870c --- /dev/null +++ b/src/backend/cluster/cluster_undo_horizon.c @@ -0,0 +1,222 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_horizon.c + * Cluster-wide undo retention horizon: pure fold from per-peer report + * views to a recycle floor (spec-5.22e D5-1). + * + * The fold implements the prevention half of the workstream hard + * invariant #4 ("the undo cleaner must not unilaterally recycle a + * slot a peer may still need"): the cleaner may only advance recycling + * past watermarks strictly below the CLUSTER floor, and the floor is + * only proven when every required MEMBER peer has a report that is + * capable, present, stable, current-epoch, well-formed, monotone and + * fresh. Anything unproven STALLS the fold -- the caller pauses + * reclaim entirely (a stall is always safe; using an unproven floor + * never is). There is deliberately NO fallback to the local horizon + * on any unproven edge (spec-5.22e Q3'': a required peer without a + * current-connection capability may be an old binary that still holds + * old snapshots -- falling back to local recycling would reopen the + * exact mis-recycle window this spec closes). + * + * Everything here is a pure decision layer: no shmem, no locks, no + * GUC reads, unit-pinned by the U1-U16 truth table. The heavy caller + * (the undo cleaner pass, D5-3) samples the per-peer seqlock slots and + * the required MEMBER bitmap from one membership/epoch snapshot and + * threads the returned {scn, epoch} floor through the pass, re-checking + * the epoch before every mutation (F-D2 epoch fence). + * + * SCN comparisons all go through scn_time_cmp (L457/AD-008: raw + * integer comparison of SCNs is forbidden; only the local_scn + * time-axis is comparable across nodes). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_horizon.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-1, §2.1/§3.0) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_undo_horizon.h" + +static inline bool +required_has(const uint8 *required, int node) +{ + return (required[node >> 3] & (uint8)(1u << (node & 7))) != 0; +} + +static inline uint32 +clamp_interval_ms(uint32 iv) +{ + if (iv < CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS) + return CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS; + if (iv > CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS) + return CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS; + return iv; +} + +/* + * cluster_undo_horizon_cluster_floor + * + * Fold the local horizon and every required peer's report view into the + * cluster recycle floor. See the header contract; the per-peer check + * order (NOCAP -> MISSING -> TORN -> EPOCH -> MALFORMED -> REGRESSION -> + * STALE, first failing peer in ascending node id wins the blame) is part + * of the contract so attribution is deterministic (U16b). + * + * Returns OK and fills *out_floor = {min, current_epoch}; or STALLED and + * fills *out_reason / *out_blame_node. On STALLED, *out_floor is set to + * {InvalidScn, current_epoch} so a caller that ignores the status cannot + * accidentally recycle against a live value (belt for rule 8.A; the + * cleaner checks the status first). + */ +ClusterUndoHorizonFoldStatus +cluster_undo_horizon_cluster_floor(SCN local_horizon, const ClusterUndoHorizonReportView *views, + int nviews, const uint8 *required, int32 self_node, + uint64 current_epoch, uint64 now_us, uint32 local_interval_ms, + ClusterUndoHorizonFloor *out_floor, + ClusterUndoHorizonStallReason *out_reason, int32 *out_blame_node) +{ + SCN floor_scn = local_horizon; + int node; + + Assert(out_floor != NULL && out_reason != NULL && out_blame_node != NULL); + + out_floor->scn = InvalidScn; + out_floor->epoch = current_epoch; + *out_reason = CLUSTER_UNDO_HORIZON_STALL_NONE; + *out_blame_node = -1; + + /* + * A fold with an unusable local horizon cannot prove anything: the + * caller is expected to hand us the spec-3.12 own-instance value, + * which is valid whenever the storage gate is on. Defensive stall, + * blamed on self (U7b) -- never "recycle a little anyway". + */ + if (local_horizon == InvalidScn) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_MALFORMED; + *out_blame_node = self_node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + + for (node = 0; node < nviews; node++) { + const ClusterUndoHorizonReportView *v = &views[node]; + uint64 window_us; + uint32 iv; + + if (node == self_node) + continue; /* local_horizon covers self */ + if (!required_has(required, node)) + continue; /* ghost / non-MEMBER slot: ignored (U8) */ + + /* + * Q3'' (F-D1): a required MEMBER without a CURRENT-connection + * capability stalls the fold. Capability gates sending and + * consumer admission elsewhere; here its absence just means this + * peer's coverage cannot be proven -- which is a stall, never a + * "fall back to local recycling". + */ + if (!v->has_capability) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_NOCAP; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + if (!v->valid) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_MISSING; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + if (!v->stable) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_TORN; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + if (v->epoch != current_epoch) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_EPOCH; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + + /* + * Well-formedness: the wire handler (D5-2) already rejects these + * before publishing; the fold re-checks defensively so a slot + * corrupted by any future path still fails closed. + */ + if (v->horizon_scn == InvalidScn + || v->sender_interval_ms < CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS + || v->sender_interval_ms > CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_MALFORMED; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + + /* + * Monotonicity violation latch (S3.0 corollary): the peer sent a + * same-epoch report BELOW its previously accepted value, i.e. a + * snapshot below the accepted lower bound exists over there. The + * old (higher) slot value is exactly what must NOT be consumed; + * stall until the peer publishes a conforming report again. + */ + if (v->regression_flagged) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_REGRESSION; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + + /* + * Freshness: window = FACTOR * max(sender, local) interval, in + * uint64 microseconds (no overflow: 3 * 60000 * 1000 << 2^64). + * The receiver-local recv_at makes this clock-skew safe; a + * recv_at in the future means the receiving clock went backwards + * -- treat as stale, never fresh (U10). + */ + iv = Max(clamp_interval_ms(v->sender_interval_ms), clamp_interval_ms(local_interval_ms)); + window_us = (uint64)CLUSTER_UNDO_HORIZON_FRESHNESS_FACTOR * (uint64)iv * (uint64)1000; + if (v->recv_at_us > now_us || now_us - v->recv_at_us > window_us) { + *out_reason = CLUSTER_UNDO_HORIZON_STALL_STALE; + *out_blame_node = node; + return CLUSTER_UNDO_HORIZON_FOLD_STALLED; + } + + /* proven: fold the peer's bound in (time-axis min, L457) */ + if (scn_time_cmp(v->horizon_scn, floor_scn) < 0) + floor_scn = v->horizon_scn; + } + + out_floor->scn = floor_scn; + out_floor->epoch = current_epoch; + return CLUSTER_UNDO_HORIZON_FOLD_OK; +} + +const char * +cluster_undo_horizon_stall_reason_name(ClusterUndoHorizonStallReason reason) +{ + switch (reason) { + case CLUSTER_UNDO_HORIZON_STALL_NONE: + return "none"; + case CLUSTER_UNDO_HORIZON_STALL_NOCAP: + return "nocap"; + case CLUSTER_UNDO_HORIZON_STALL_MISSING: + return "missing"; + case CLUSTER_UNDO_HORIZON_STALL_TORN: + return "torn"; + case CLUSTER_UNDO_HORIZON_STALL_EPOCH: + return "epoch"; + case CLUSTER_UNDO_HORIZON_STALL_MALFORMED: + return "malformed"; + case CLUSTER_UNDO_HORIZON_STALL_REGRESSION: + return "regression"; + case CLUSTER_UNDO_HORIZON_STALL_STALE: + return "stale"; + } + return "unknown"; +} diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index 4c922b7580..4a6378b488 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -707,6 +707,19 @@ SetTransactionSnapshot(Snapshot sourcesnap, VirtualTransactionId *sourcevxid, /* Mark it as "registered" in FirstXactSnapshot */ FirstXactSnapshot->regd_count++; pairingheap_add(&RegisteredSnapshots, &FirstXactSnapshot->ph_node); +#ifdef USE_PGRAC_CLUSTER + /* + * spec-5.22e D5-1 (S3.0 row 2): publish the imported snapshot's + * read_scn IMMEDIATELY, exactly as the GetTransactionSnapshot + * register path does. Without this, the exporter could release + * its pin before our first PushActiveSnapshot recompute, leaving + * a window where this node's published retention floor (and hence + * its cluster horizon report) sits above a live imported snapshot + * -- breaking the report lower-bound theorem the cluster-wide + * recycle brake is built on. + */ + cluster_recompute_proc_read_scn(); +#endif } FirstSnapshotSet = true; diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h new file mode 100644 index 0000000000..22b92c324d --- /dev/null +++ b/src/include/cluster/cluster_undo_horizon.h @@ -0,0 +1,168 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_horizon.h + * Cluster-wide undo retention horizon: per-peer report views and the + * pure fold that turns them into a recycle floor (spec-5.22e D5-1). + * + * The local retention horizon (cluster_undo_retention_horizon(), + * spec-3.12) covers only this node's ProcArray. Under cross-node undo + * consumption (live verdict / CP3 / fresh-ref / authority serve) a + * peer's snapshot can reference this node's TT slots, so the cleaner + * must not recycle past the CLUSTER floor: the scn_time_cmp-min of the + * local horizon and every required MEMBER peer's accepted horizon + * report. When any required peer's report cannot be proven fresh, + * consistent and monotone at the current reconfig epoch, the fold + * STALLS: the caller must not advance recycling at all (fail-closed + * direction: a stall pauses reclaim, it never mis-recycles). + * + * Everything in this header is a pure decision layer: no shmem, no + * locks, no GUC reads. The heavy caller (undo cleaner pass) samples + * the per-peer seqlock slots into ClusterUndoHorizonReportView[] and + * the required MEMBER bitmap from the same membership/epoch snapshot, + * then calls the fold. Mirrors the spec-5.22d D4-1 pure-layer shape + * (ClusterUndoAuthorityInput). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_undo_horizon.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-1, §2.1) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_UNDO_HORIZON_H +#define CLUSTER_UNDO_HORIZON_H + +#include "cluster/cluster_reconfig.h" /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ +#include "cluster/cluster_scn.h" /* SCN / scn_time_cmp */ + +/* + * Freshness factor: a required peer's report is fresh while + * now - recv_at <= FACTOR * max(sender_interval, local_interval). + * Fixed at compile time (spec-5.22e Q5': a tunable would add a + * misconfiguration surface -- window < interval = permanent stall -- with + * no benefit). Intervals are clamped to [MIN,MAX] ms before the window + * is computed; the window math is done in uint64 microseconds. + */ +#define CLUSTER_UNDO_HORIZON_FRESHNESS_FACTOR 3 +#define CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS 100 +#define CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS 60000 + +typedef enum ClusterUndoHorizonFoldStatus { + CLUSTER_UNDO_HORIZON_FOLD_OK = 0, + CLUSTER_UNDO_HORIZON_FOLD_STALLED +} ClusterUndoHorizonFoldStatus; + +/* + * Stall attribution, surfaced through the fold's out_reason/out_blame so + * the caller can count and LOG-once with a concrete culprit. Per-peer + * check order (first failing required peer, ascending node id, wins the + * blame): NOCAP -> MISSING -> TORN -> EPOCH -> MALFORMED -> REGRESSION -> + * STALE. + */ +typedef enum ClusterUndoHorizonStallReason { + CLUSTER_UNDO_HORIZON_STALL_NONE = 0, + CLUSTER_UNDO_HORIZON_STALL_NOCAP, /* required peer lacks a + * current-connection + * UNDO_HORIZON_V1 capability; + * NEVER a fallback to the + * local horizon (Q3'') */ + CLUSTER_UNDO_HORIZON_STALL_MISSING, /* no report this incarnation */ + CLUSTER_UNDO_HORIZON_STALL_TORN, /* seqlock double-read failed */ + CLUSTER_UNDO_HORIZON_STALL_EPOCH, /* report epoch != current */ + CLUSTER_UNDO_HORIZON_STALL_MALFORMED, /* invalid SCN / interval out + * of range / invalid local + * horizon */ + CLUSTER_UNDO_HORIZON_STALL_REGRESSION, /* peer violated same-epoch + * report monotonicity; the + * previously accepted (higher) + * value must NOT be used + * (spec-5.22e S3.0 corollary) */ + CLUSTER_UNDO_HORIZON_STALL_STALE /* report older than the + * freshness window, or + * recv_at in the future */ +} ClusterUndoHorizonStallReason; + +/* + * The recycle floor: scn paired with the reconfig epoch it was proven at. + * The cleaner threads this pair through the whole pass and re-verifies + * epoch equality before every TT-slot FREE and inside the segment + * mutation lock (spec-5.22e F-D2 epoch fence); a mismatch aborts the + * pass immediately. + */ +typedef struct ClusterUndoHorizonFloor { + SCN scn; + uint64 epoch; +} ClusterUndoHorizonFloor; + +/* + * One peer's report as sampled by the heavy caller from the seqlock slot. + * + * valid -- a report was accepted this incarnation. + * stable -- the seqlock double-read converged (false after + * bounded retries => TORN, U12). + * has_capability -- the peer advertised UNDO_HORIZON_V1 on its + * CURRENT connection (Q1' amend: capability dies + * with the connection, no stale reuse). + * regression_flagged -- the receive handler rejected a same-epoch + * regressing report and latched the violation; + * cleared when a conforming report is accepted. + * epoch -- sender's reconfig epoch at sampling. + * horizon_scn -- sender's published lower bound (S2.1 sampling + * rule: min of its ProcArray floor and its + * origin-bound). + * recv_at_us -- RECEIVER-local clock at accept (clock-skew safe). + * sender_interval_ms -- sender's lmon_main_loop_interval, carried in the + * wire payload so a slow-tick sender is not + * misjudged stale by a fast-tick receiver (U13). + */ +typedef struct ClusterUndoHorizonReportView { + bool valid; + bool stable; + bool has_capability; + bool regression_flagged; + uint64 epoch; + SCN horizon_scn; + uint64 recv_at_us; + uint32 sender_interval_ms; +} ClusterUndoHorizonReportView; + +/* + * Pure cluster-floor fold (spec-5.22e S2.1). + * + * required -- 128-bit bitmap of node ids whose reports are mandatory: + * the membership_state==MEMBER peer set sampled at + * current_epoch from the same membership snapshot + * (INV-J8; ABSENT/JOINING/DEAD/REMOVED are excluded -- + * their cross-node consumption is refused by the D5-8 + * read-admission gate, so they need no coverage here). + * self_node is skipped (local_horizon covers self). + * views -- indexed by node id, nviews entries; ghost entries for + * nodes outside `required` are ignored (U8). + * + * Returns OK with *out_floor = {scn_time_cmp-min(local, required + * reports), current_epoch} when every required peer has a stable, + * capable, current-epoch, well-formed, monotone, fresh report. An + * empty required set yields {local_horizon, current_epoch} (single + * node / cold formation: U1/U11). Any unproven required peer => + * STALLED with attribution; the caller MUST NOT advance recycling + * (rule 8.A: unproven is never treated as proven). + */ +extern ClusterUndoHorizonFoldStatus cluster_undo_horizon_cluster_floor( + SCN local_horizon, const ClusterUndoHorizonReportView *views, int nviews, + const uint8 *required, /* CLUSTER_RECONFIG_DEAD_BITMAP_BYTES */ + int32 self_node, uint64 current_epoch, uint64 now_us, uint32 local_interval_ms, + ClusterUndoHorizonFloor *out_floor, ClusterUndoHorizonStallReason *out_reason, + int32 *out_blame_node); + +/* Attribution name for LOG-once / dump lines ("nocap", "stale", ...). */ +extern const char *cluster_undo_horizon_stall_reason_name(ClusterUndoHorizonStallReason reason); + +#endif /* CLUSTER_UNDO_HORIZON_H */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 729bf4af39..7b0e594c96 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -91,7 +91,8 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_undo_authority \ test_cluster_undo_gcs \ test_cluster_undo_verdict \ - test_cluster_vis_undo_verdict_map + test_cluster_vis_undo_verdict_map \ + test_cluster_undo_horizon # Path to the cluster_version object (no PG deps, safe to link standalone). CLUSTER_VERSION_O = $(top_builddir)/src/backend/cluster/cluster_version.o @@ -183,7 +184,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_authority test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_undo_resid test_cluster_undo_authority test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map test_cluster_undo_horizon,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1872,6 +1873,21 @@ test_cluster_undo_authority: test_cluster_undo_authority.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_AUTHORITY_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-5.22e D5-1: test_cluster_undo_horizon — cluster-wide undo retention +# horizon pure fold (required-MEMBER report views -> recycle floor, U1-U16 +# truth table: every unproven edge STALLS with attribution, never a local +# fallback). Pure decision layer; scn_time_cmp is a byte-faithful local +# stub in the test (test_cluster_cr_tuple.c pattern) so cluster_scn.o's +# shmem/lwlock deps stay out. The seqlock slot sampling, wire handler +# validation and the LMON tick live in the heavy path (D5-2/D5-3) and are +# covered by the t/370 TAP. +CLUSTER_UNDO_HORIZON_O = $(top_builddir)/src/backend/cluster/cluster_undo_horizon.o +test_cluster_undo_horizon: test_cluster_undo_horizon.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_HORIZON_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_UNDO_HORIZON_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + # spec-5.22b D2-1/D2-3: test_cluster_undo_gcs — owner-as-master routing layer # (D2-1: cluster_undo_block_lookup_master / _master_is_self) + the pure reader # S-grant decision core (D2-3: cluster_undo_grant_armed / _admissible / diff --git a/src/test/cluster_unit/test_cluster_undo_horizon.c b/src/test/cluster_unit/test_cluster_undo_horizon.c new file mode 100644 index 0000000000..7641fa9a1c --- /dev/null +++ b/src/test/cluster_unit/test_cluster_undo_horizon.c @@ -0,0 +1,501 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_undo_horizon.c + * Unit tests for the cluster-wide undo retention horizon pure fold + * (spec-5.22e D5-1). + * + * Truth table U1-U16: the fold returns OK with the scn_time_cmp-min + * floor only when EVERY required MEMBER peer has a stable, capable, + * current-epoch, well-formed, monotone, fresh report; every unproven + * edge STALLS with first-fail attribution (reason + blame node) and + * never falls back to the local horizon (rule 8.A / Q3''). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_undo_horizon.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-1, §4.1) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_undo_horizon.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} + +/* + * scn_time_cmp -- byte-faithful local stub (mirrors test_cluster_cr_tuple.c): + * the fold compares peer bounds vs the local horizon through it. Linking + * cluster_scn.o would drag in shmem/atomics; the header-only scn_local() + * inline matches the real impl. + */ +int +scn_time_cmp(SCN a, SCN b) +{ + uint64 la = scn_local(a); + uint64 lb = scn_local(b); + + if (la < lb) + return -1; + if (la > lb) + return 1; + return 0; +} + +/* ---- helpers ------------------------------------------------------- */ + +#define TEST_EPOCH ((uint64)7) +#define TEST_NOW_US ((uint64)90 * 1000 * 1000) /* 90s, roomy */ +#define TEST_LOCAL_IV ((uint32)1000) /* 1s tick */ +#define TEST_SELF 0 + +/* fresh under window = 3 * max(1000,1000)ms = 3s */ +#define FRESH_RECV_US (TEST_NOW_US - (uint64)1 * 1000 * 1000) + +static void +bm_set(uint8 *bm, int node) +{ + bm[node >> 3] |= (uint8)(1u << (node & 7)); +} + +/* a fully healthy report view */ +static ClusterUndoHorizonReportView +mk_view(uint64 scn) +{ + ClusterUndoHorizonReportView v; + + memset(&v, 0, sizeof(v)); + v.valid = true; + v.stable = true; + v.has_capability = true; + v.regression_flagged = false; + v.epoch = TEST_EPOCH; + v.horizon_scn = (SCN)scn; + v.recv_at_us = FRESH_RECV_US; + v.sender_interval_ms = 1000; + return v; +} + +typedef struct FoldResult { + ClusterUndoHorizonFoldStatus st; + ClusterUndoHorizonFloor floor; + ClusterUndoHorizonStallReason reason; + int32 blame; +} FoldResult; + +static FoldResult +run_fold(SCN local, const ClusterUndoHorizonReportView *views, int nviews, const uint8 *required) +{ + FoldResult r; + + memset(&r, 0, sizeof(r)); + r.floor.scn = (SCN)0xdeadbeef; + r.floor.epoch = 0xdead; + r.reason = CLUSTER_UNDO_HORIZON_STALL_NONE; + r.blame = -2; + r.st = cluster_undo_horizon_cluster_floor(local, views, nviews, required, TEST_SELF, TEST_EPOCH, + TEST_NOW_US, TEST_LOCAL_IV, &r.floor, &r.reason, + &r.blame); + return r; +} + +/* ---- U1: empty required set => OK, floor = local ------------------- */ +UT_TEST(test_u1_no_required_peer) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + FoldResult r = run_fold((SCN)500, NULL, 0, req); + + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)500); + UT_ASSERT_EQ(r.floor.epoch, TEST_EPOCH); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_NONE); +} + +/* ---- U2: all required fresh + epoch match => min fold -------------- */ +UT_TEST(test_u2_all_fresh_min) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[3]; + FoldResult r; + + views[0] = mk_view(0); /* self slot: ignored */ + views[1] = mk_view(700); + views[2] = mk_view(900); + bm_set(req, 1); + bm_set(req, 2); + + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)700); + UT_ASSERT_EQ(r.floor.epoch, TEST_EPOCH); +} + +/* ---- U3: required peer never reported => MISSING + blame ----------- */ +UT_TEST(test_u3_missing) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].valid = false; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_MISSING); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U4: report older than 3 x max(sender, local) => STALE --------- */ +UT_TEST(test_u4_stale) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + /* window = 3s; age it 3.5s */ + views[1].recv_at_us = TEST_NOW_US - (uint64)3500 * 1000; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_STALE); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U5: report epoch != current => EPOCH --------------------------- */ +UT_TEST(test_u5_epoch_mismatch) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].epoch = TEST_EPOCH - 1; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_EPOCH); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U6: peer horizon below local => floor follows the peer -------- */ +UT_TEST(test_u6_peer_below_local) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(100); + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)100); +} + +/* ---- U7: InvalidScn report => MALFORMED ----------------------------- */ +UT_TEST(test_u7_invalid_scn) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(0); + views[1].horizon_scn = InvalidScn; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_MALFORMED); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U7b: invalid LOCAL horizon => MALFORMED, blame self ------------ */ +UT_TEST(test_u7b_invalid_local) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + bm_set(req, 1); + + r = run_fold(InvalidScn, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_MALFORMED); + UT_ASSERT_EQ(r.blame, TEST_SELF); +} + +/* ---- U8: ghost view for a non-required node is ignored -------------- */ +UT_TEST(test_u8_ghost_ignored) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[3]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[2] = mk_view(50); /* ghost: NOT required; lower than all */ + views[2].epoch = TEST_EPOCH - 3; /* and stale-epoch: must not matter */ + bm_set(req, 1); + + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)700); +} + +/* ---- U9: DEAD/REMOVED peer dropped from required => not consulted --- */ +UT_TEST(test_u9_dead_removed_dropped) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[3]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[2] = mk_view(0); + views[2].valid = false; /* node2 died; caller removed it from + * required -- its missing report must + * not stall the fold */ + bm_set(req, 1); + + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)700); +} + +/* ---- U10: recv_at in the future => conservative STALE --------------- */ +UT_TEST(test_u10_future_recv) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].recv_at_us = TEST_NOW_US + 1; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_STALE); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U11: multi-node declared but required empty (cold formation) --- */ +UT_TEST(test_u11_cold_formation) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[4]; + FoldResult r; + int i; + + /* declared 4 nodes, none MEMBER yet: caller passes an empty required + * set and whatever sits in the slots must be ignored */ + for (i = 0; i < 4; i++) { + views[i] = mk_view(1); + views[i].valid = false; + } + r = run_fold((SCN)800, views, 4, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)800); +} + +/* ---- U12: seqlock torn read => TORN --------------------------------- */ +UT_TEST(test_u12_torn) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].stable = false; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_TORN); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U13: slow-tick sender is not misjudged stale ------------------- */ +UT_TEST(test_u13_slow_sender_window) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].sender_interval_ms = 10000; /* 10s tick */ + /* age 20s: stale under 3 x local(1s), fresh under 3 x sender(10s) */ + views[1].recv_at_us = TEST_NOW_US - (uint64)20 * 1000 * 1000; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)700); +} + +/* ---- U13b: out-of-range sender interval => MALFORMED ---------------- */ +UT_TEST(test_u13b_bad_interval) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].sender_interval_ms = 60001; /* > MAX: handler rejects on + * the wire; the fold re-checks + * defensively */ + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_MALFORMED); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U14: same-epoch regression latch => REGRESSION ----------------- */ +UT_TEST(test_u14_regression) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); /* previously accepted value still in the + * slot -- it must NOT be used once the + * peer contradicted it */ + views[1].regression_flagged = true; + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_REGRESSION); + UT_ASSERT_EQ(r.blame, 1); +} + +/* U15 (wire validation matrix) lives with the D5-2 handler tests. */ + +/* ---- U16: required peer without current capability => NOCAP --------- */ +UT_TEST(test_u16_nocap) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); /* even with a fresh old report... */ + views[1].has_capability = false; /* ...no CURRENT-connection cap */ + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_NOCAP); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- U16b: first-fail attribution order is deterministic ------------ */ +UT_TEST(test_u16b_blame_order) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[3]; + FoldResult r; + + views[0] = mk_view(0); + views[1] = mk_view(700); + views[1].valid = false; /* node1: MISSING */ + views[2] = mk_view(700); + views[2].has_capability = false; /* node2: NOCAP */ + bm_set(req, 1); + bm_set(req, 2); + + /* lowest failing node id wins: node1 / MISSING */ + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_MISSING); + UT_ASSERT_EQ(r.blame, 1); +} + +/* ---- reason names cover every enum arm ------------------------------ */ +UT_TEST(test_reason_names) +{ + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_NONE), + "none"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_NOCAP), + "nocap"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_MISSING), + "missing"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_TORN), + "torn"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_EPOCH), + "epoch"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_MALFORMED), + "malformed"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_REGRESSION), + "regression"); + UT_ASSERT_STR_EQ(cluster_undo_horizon_stall_reason_name(CLUSTER_UNDO_HORIZON_STALL_STALE), + "stale"); +} + +int +main(void) +{ + UT_PLAN(19); + + UT_RUN(test_u1_no_required_peer); + UT_RUN(test_u2_all_fresh_min); + UT_RUN(test_u3_missing); + UT_RUN(test_u4_stale); + UT_RUN(test_u5_epoch_mismatch); + UT_RUN(test_u6_peer_below_local); + UT_RUN(test_u7_invalid_scn); + UT_RUN(test_u7b_invalid_local); + UT_RUN(test_u8_ghost_ignored); + UT_RUN(test_u9_dead_removed_dropped); + UT_RUN(test_u10_future_recv); + UT_RUN(test_u11_cold_formation); + UT_RUN(test_u12_torn); + UT_RUN(test_u13_slow_sender_window); + UT_RUN(test_u13b_bad_interval); + UT_RUN(test_u14_regression); + UT_RUN(test_u16_nocap); + UT_RUN(test_u16b_blame_order); + UT_RUN(test_reason_names); + + UT_DONE(); +} From aa6f770c02a07ee2e06df169c0528671f3378f0b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 05:29:32 +0800 Subject: [PATCH 24/38] feat(cluster): D5-2 undo horizon wire carrier + capability-gated publish + seqlock slots (spec-5.22e) The report plane of the cluster-wide undo retention brake: - New PGRAC_IC_MSG_UNDO_HORIZON (36; LMON-only producer, per-peer p2p, never broadcast) carrying {epoch, horizon_scn, sender_interval_ms}. The LMON tick samples the spec-3.12 own-instance floor bounded by the node's minimum future-snapshot origin (consistent_scn when MRP-active: the no-reader clock fallback would over-report an ADG standby whose future snapshots read far below the clock) and sends it to every declared peer whose CURRENT connection advertised the new HELLO capability UNDO_HORIZON_V1 (0x4) -- an old peer treats an unregistered msg_type as a peer-level failure and closes the connection, so no capability means no send, never a reconnect storm. - Capability state is now connection-bound (Q1' amend): the tier1 close funnel clears the peer's advertised HELLO capabilities (cluster_sf_note_peer_disconnected), so no consumer -- horizon sender/fold, D4 authority routing, smart fusion -- trusts a stale capability across a reconnect window. Clearing is uniformly the conservative direction for every existing consumer. - The receive handler validates strictly before publishing (exact length, declared non-self sender, valid SCN, interval within [100,60000]ms) and enforces same-epoch report monotonicity: a regressing report is rejected AND latches a violation flag -- the previously accepted higher value must not be consumed once the peer has contradicted it (the fold stalls on REGRESSION until a conforming report clears the latch). Invalid frames bump undo_horizon_wire_reject and are never published: the slot ages into a fold stall. - Reports live in a new shmem region 'pgrac cluster undo horizon' of per-peer seqlock slots (single LMON writer: odd/even seq + write barriers; cleaner-side sampling double-reads with bounded retries, torn => the fold's TORN stall). cluster.shmem_max_regions default raised 80 -> 96 (region +1 with headroom). - Two D5-7 injection points registered (registry 159 -> 161): cluster-undo-horizon-report-drop (stall leg) and cluster-undo-horizon-epoch-fence (F-D2 fence leg, consumed by D5-3). - Baseline sweep (L462): t/015 (161 rows + name list), t/020 (region count/list + L12/L13 96 + L15 161), t/021-023 (region count + L11). t/021-023's L11 was still asserting 158 -- a latent stale baseline from the D4 bump that only swept t/015; trued up to 161 here. Band t/015+t/020-023 rerun: 100/100 PASS. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-2) --- src/backend/cluster/Makefile | 1 + src/backend/cluster/cluster_guc.c | 4 +- src/backend/cluster/cluster_ic.c | 3 + src/backend/cluster/cluster_ic_tier1.c | 9 + src/backend/cluster/cluster_inject.c | 20 + src/backend/cluster/cluster_lmon.c | 19 + src/backend/cluster/cluster_sf_dep.c | 46 ++ src/backend/cluster/cluster_shmem.c | 6 + src/backend/cluster/cluster_undo_horizon_ic.c | 464 ++++++++++++++++++ src/include/cluster/cluster_ic.h | 10 + src/include/cluster/cluster_ic_envelope.h | 8 +- src/include/cluster/cluster_sf_dep.h | 4 + src/include/cluster/cluster_undo_horizon.h | 37 ++ src/test/cluster_tap/t/015_inject.pl | 8 +- src/test/cluster_tap/t/020_shmem_registry.pl | 14 +- src/test/cluster_tap/t/021_block_format.pl | 6 +- src/test/cluster_tap/t/022_itl_slot.pl | 6 +- .../cluster_tap/t/023_buffer_descriptor.pl | 6 +- src/test/cluster_unit/test_cluster_shmem.c | 6 +- 19 files changed, 650 insertions(+), 27 deletions(-) create mode 100644 src/backend/cluster/cluster_undo_horizon_ic.c diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 44447cb7b4..a5db441012 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -190,6 +190,7 @@ OBJS = \ cluster_undo_gcs_grant.o \ cluster_undo_gcs_stat.o \ cluster_undo_horizon.o \ + cluster_undo_horizon_ic.o \ cluster_undo_retention.o \ cluster_undo_srf.o \ cluster_cr.o \ diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 2f7978eeac..979db213f1 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -176,7 +176,7 @@ bool cluster_smgr_user_relations = false; */ bool cluster_shared_catalog = false; int cluster_oid_lease_size = 8192; -int cluster_shmem_max_regions = 80; /* spec-5.56: 64 -> 80 (cr relgen region; restore margin) */ +int cluster_shmem_max_regions = 96; /* spec-5.22e: 80 -> 96 (undo horizon region; restore margin); spec-5.56: 64 -> 80 */ /* spec-3.18 D1: undo block buffer pool slot count (0 = disabled). */ int cluster_undo_buffers = 2048; @@ -2167,7 +2167,7 @@ cluster_init_guc(void) "registers one region. Raise if FATAL on startup with " "errcode 53400 \"cluster shmem registry capacity " "exceeded\"."), - &cluster_shmem_max_regions, 80, CLUSTER_SHMEM_MIN_REGIONS, 256, + &cluster_shmem_max_regions, 96, CLUSTER_SHMEM_MIN_REGIONS, 256, PGC_POSTMASTER, /* registry array is palloc'd once at init */ 0, /* flags */ NULL, /* check_hook */ diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index b2b87168f2..3494f57a85 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -610,6 +610,9 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version * understands kind-4 / version-2 regardless of any GUC, so advertise * unconditionally (the serve side's GUC gate refuses at run time). */ capabilities |= PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1; + /* spec-5.22e D5-2: same protocol-capability discipline for the undo + * horizon report msg_type. */ + capabilities |= PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); } diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 2ab1516045..66e4eccb11 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -1837,6 +1837,15 @@ cluster_ic_tier1_close_peer(int32 peer_id, const char *reason) tier1_peer_fds[peer_id] = -1; } + /* + * spec-5.22e D5-2 (Q1' amend): HELLO capabilities are a property of the + * connection that carried them. Clear them in the close funnel so no + * consumer (horizon sender/fold, authority routing, smart fusion) trusts + * a stale capability across the reconnect window; the next verified + * HELLO repopulates. + */ + cluster_sf_note_peer_disconnected(peer_id); + if (Tier1Shmem != NULL) { if (reason != NULL && reason[0] != '\0') peer_record_error(peer_id, 0, "08006", "%s", reason); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 404d880ca8..ee51788510 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -381,6 +381,26 @@ static ClusterInjectPoint cluster_injection_points[] = { * counter delta (L408), never a native CLOG answer. */ { .name = "cluster-undo-authority-block0-prove" }, + /* + * spec-5.22e D5-7 — undo horizon report drop injection (TAP L2). + * + * cluster-undo-horizon-report-drop: + * Fires in the LMON receive handler before validation/publish; SKIP + * drops the peer's report so its slot ages into a fold stall + * (undo_horizon_stall_count bumps, recycling pauses) and self-heals + * once disarmed — pins the stall leg with a counter delta (L408). + */ + { .name = "cluster-undo-horizon-report-drop" }, + /* + * spec-5.22e D5-7 — undo horizon epoch fence injection (TAP L6). + * + * cluster-undo-horizon-epoch-fence: + * Fires in the cleaner pass between the fold and the first recycle + * mutation; SKIP simulates a mid-pass reconfig epoch bump so the + * fence aborts the pass (undo_horizon_pass_abort_count bumps, zero + * recycles this pass) — pins the F-D2 fence with a counter delta. + */ + { .name = "cluster-undo-horizon-epoch-fence" }, /* * spec-6.12e2 — holder-side BAST-nudge refusal injection. * diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..6449f8ac2a 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -57,6 +57,7 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR-server result ship */ #include "cluster/cluster_backup.h" /* cluster_backup_register_ic_msg_types + lmon_tick (spec-6.5) */ +#include "cluster/cluster_undo_horizon.h" /* cluster_undo_horizon_register_ic_msg_types + lmon_tick (spec-5.22e D5-2) */ #include "cluster/cluster_clean_leave.h" /* cluster_clean_leave_register_ic_msg_types (spec-5.13 D8) */ #include "cluster/cluster_node_remove.h" /* cluster_node_remove_lmon_tick + register (spec-5.18 D9/D10) */ #include "cluster/cluster_conf.h" @@ -448,6 +449,16 @@ cluster_lmon_shmem_init(void) smart_fusion_registered = true; } } + /* spec-5.22e D5-2: register the undo retention horizon report (LMON-only + * producer; capability-gated per-peer p2p, never broadcast). */ + { + static bool undo_horizon_registered = false; + + if (!undo_horizon_registered) { + cluster_undo_horizon_register_ic_msg_types(); + undo_horizon_registered = true; + } + } } @@ -1061,6 +1072,10 @@ LmonMain(void) * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ cluster_tt_status_hint_drain_outbound(); cluster_backup_lmon_tick(); + /* spec-5.22e D5-2: publish this node's undo retention horizon + * report per-peer (capability-gated; internally rate-limited to + * one send per lmon_main_loop_interval). */ + cluster_undo_horizon_lmon_tick(); /* * spec-2.34 D6 (HC93 leg a): TTL sweep of the GCS block @@ -1668,6 +1683,10 @@ LmonMain(void) * Fire-and-forget; L172 family — only LMON owns tier1 fds. */ cluster_tt_status_hint_drain_outbound(); cluster_backup_lmon_tick(); + /* spec-5.22e D5-2: publish this node's undo retention horizon + * report per-peer (capability-gated; internally rate-limited to + * one send per lmon_main_loop_interval). */ + cluster_undo_horizon_lmon_tick(); /* spec-2.34 D6 (HC93 leg a): TTL sweep GCS block dedup HTAB. */ cluster_gcs_block_dedup_sweep_expired(GetCurrentTimestamp()); diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 98197953f7..f2f113883f 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -179,6 +179,52 @@ cluster_peer_supports_undo_authority_serve(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1) != 0; } +/* + * cluster_sf_peer_supports_undo_horizon + * + * spec-5.22e D5-2 capability gate: true iff the peer's verified HELLO on the + * CURRENT connection advertised the undo-horizon report protocol bit. Same + * no-local-GUC discipline as the authority-serve query above. Combined with + * the disconnect reset below this is connection-bound (Q1' amend): a closed + * or not-yet-HELLOed connection reads as false, so the sender skips the peer + * and the fold stalls with NOCAP instead of trusting a stale capability. + */ +bool +cluster_sf_peer_supports_undo_horizon(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = ClusterSfDep->peer_capabilities[peer_id]; + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1) != 0; +} + +/* + * cluster_sf_note_peer_disconnected + * + * spec-5.22e D5-2 (Q1' amend): capability state is a property of the + * CONNECTION that carried the HELLO, not of the peer's identity. Called + * from the transport close funnel; clears every advertised bit so nothing + * consumes a stale capability across a reconnect window. The next verified + * HELLO repopulates. Clearing is uniformly the conservative direction for + * every existing consumer: smart-fusion falls back to v1 replies, the D4 + * authority leg fails closed, and the D5 horizon sender/fold skip/stall. + */ +void +cluster_sf_note_peer_disconnected(int32 peer_id) +{ + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return; + + LWLockAcquire(&ClusterSfDep->lock, LW_EXCLUSIVE); + ClusterSfDep->peer_capabilities[peer_id] = 0; + LWLockRelease(&ClusterSfDep->lock); +} + void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload) { diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index 2dc81089cf..bb86440af2 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -120,6 +120,7 @@ #include "cluster/cluster_tt_durable.h" /* cluster_tt_durable_shmem_register (spec-3.11 D7) */ #include "cluster/cluster_undo_gcs.h" /* cluster_undo_gcs_shmem_register (spec-5.22b D2-6) */ #include "cluster/cluster_sf_dep.h" /* cluster_sf_dep_shmem_register (spec-6.2 D6) */ +#include "cluster/cluster_undo_horizon.h" /* cluster_undo_horizon_shmem_register (spec-5.22e D5-2) */ #include "cluster/cluster_visibility_inject.h" /* cluster_visibility_inject_shmem_register (spec-3.2 D5b) */ #include "cluster/cluster_itl.h" /* cluster_lock_path_shmem_register (spec-3.4e D6) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_shmem_register (spec-2.6 Sprint A Step 1) */ @@ -564,6 +565,11 @@ cluster_init_shmem_module(void) if (cluster_shmem_lookup_region("pgrac cluster smart fusion deps") == NULL) cluster_sf_dep_shmem_register(); + /* spec-5.22e D5-2: per-peer undo retention horizon report slots + + * brake observability counters (seqlock + atomics; no LWLock). */ + if (cluster_shmem_lookup_region("pgrac cluster undo horizon") == NULL) + cluster_undo_horizon_shmem_register(); + /* * spec-3.2 D5b: register test-only visibility inject shmem. The * production build exports a no-op register helper; ENABLE_INJECTION diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c new file mode 100644 index 0000000000..7acef8d2c3 --- /dev/null +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -0,0 +1,464 @@ +/*------------------------------------------------------------------------- + * + * cluster_undo_horizon_ic.c + * Cluster-wide undo retention horizon: wire carrier, per-peer seqlock + * report slots and the LMON publish tick (spec-5.22e D5-2). + * + * Report flow: every node's LMON tick samples its own retention floor + * (spec-3.12 own-instance min, further bounded by the node's minimum + * future snapshot origin -- consistent_scn on an MRP-active node, + * S3.0 row 4) and sends it per-peer p2p over a dedicated msg_type, + * gated on the peer's CURRENT-connection UNDO_HORIZON_V1 capability + * (an old peer treats an unregistered msg_type as a peer-level + * failure and closes the connection, cluster_ic_router.c; so no + * capability = no send). The receive handler validates the payload + * strictly (length / sender / SCN / interval range / same-epoch + * monotonicity) and publishes it into a per-peer seqlock slot; any + * invalid frame is counted and NOT published, so the stale slot ages + * into a fold stall (fail-closed direction, spec-5.22e Q5' amend). + * A same-epoch regression additionally latches a violation flag on + * the slot: the previously accepted (higher) value must not be + * consumed once the peer has contradicted it (S3.0 corollary). + * + * Single writer: both the handler and the tick run in LMON. Readers + * (the undo cleaner pass) snapshot the slots through the seqlock + * double-read protocol into plain ClusterUndoHorizonReportView + * structs and hand them to the pure fold (cluster_undo_horizon.c). + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_undo_horizon_ic.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-2, §2.1/§2.2) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_conf.h" +#include "cluster/cluster_epoch.h" +#include "cluster/cluster_guc.h" +#include "cluster/cluster_ic.h" +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_ic_router.h" +#include "cluster/cluster_inject.h" +#include "cluster/cluster_membership.h" +#include "cluster/cluster_mrp.h" +#include "cluster/cluster_sf_dep.h" +#include "cluster/cluster_shmem.h" +#include "cluster/cluster_undo_horizon.h" +#include "cluster/cluster_undo_retention.h" +#include "storage/shmem.h" +#include "utils/timestamp.h" + +/* + * Per-peer report slot. Writer = LMON only (the IC dispatch loop is + * single-threaded), so the seqlock needs no writer-side CAS: seq goes + * odd -> fields -> even with write barriers; readers double-read with + * read barriers and retry (spec-5.22e F-C4: horizon values may lawfully + * differ between reports, and the slot is multi-field -- a torn read + * could pair an old higher horizon with a fresh recv_at and misjudge a + * dangerous floor as fresh). + */ +typedef struct ClusterUndoHorizonSlotShmem { + pg_atomic_uint32 seq; + uint64 epoch; + uint64 horizon_scn; + uint64 recv_at_us; + uint32 sender_interval_ms; + bool valid; + bool regression_flagged; + /* writer-private monotonicity base (LMON only; not part of the + * seqlock-published view) */ + uint64 accepted_scn; + uint64 accepted_epoch; + bool accepted_any; +} ClusterUndoHorizonSlotShmem; + +typedef struct ClusterUndoHorizonShmem { + ClusterUndoHorizonSlotShmem slots[CLUSTER_MAX_NODES]; + /* observability (D5-5); bumped via the note_* helpers below */ + pg_atomic_uint64 stall_count; + pg_atomic_uint64 peer_stale_count; + pg_atomic_uint64 pass_abort_count; + pg_atomic_uint64 wire_reject_count; + pg_atomic_uint64 admission_refuse_count; + pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ +} ClusterUndoHorizonShmem; + +static ClusterUndoHorizonShmem *UndoHorizonShmem = NULL; + +static Size +cluster_undo_horizon_shmem_size(void) +{ + return sizeof(ClusterUndoHorizonShmem); +} + +static void +cluster_undo_horizon_shmem_init(void) +{ + bool found; + ClusterUndoHorizonShmem *shmem; + + shmem = (ClusterUndoHorizonShmem *)ShmemInitStruct("pgrac cluster undo horizon", + cluster_undo_horizon_shmem_size(), &found); + if (!found) { + int i; + + memset(shmem, 0, sizeof(*shmem)); + for (i = 0; i < CLUSTER_MAX_NODES; i++) + pg_atomic_init_u32(&shmem->slots[i].seq, 0); + pg_atomic_init_u64(&shmem->stall_count, 0); + pg_atomic_init_u64(&shmem->peer_stale_count, 0); + pg_atomic_init_u64(&shmem->pass_abort_count, 0); + pg_atomic_init_u64(&shmem->wire_reject_count, 0); + pg_atomic_init_u64(&shmem->admission_refuse_count, 0); + pg_atomic_init_u64(&shmem->last_floor_scn, 0); + } + UndoHorizonShmem = shmem; +} + +static const ClusterShmemRegion cluster_undo_horizon_region = { + .name = "pgrac cluster undo horizon", + .size_fn = cluster_undo_horizon_shmem_size, + .init_fn = cluster_undo_horizon_shmem_init, + .lwlock_count = 0, /* seqlock + atomics only */ + .owner_subsys = "cluster_undo_horizon", + .reserved_flags = 0, +}; + +void +cluster_undo_horizon_shmem_register(void) +{ + cluster_shmem_register_region(&cluster_undo_horizon_region); +} + +/* ---------------- wire handler (LMON dispatch context) --------------- */ + +static void +undo_horizon_reject(const char *why, uint32 sender) +{ + if (UndoHorizonShmem != NULL) + pg_atomic_fetch_add_u64(&UndoHorizonShmem->wire_reject_count, 1); + ereport(DEBUG1, + (errmsg_internal("pgrac undo-horizon report rejected: %s (sender %u)", why, sender))); +} + +/* + * Publish one accepted report into the sender's slot (seqlock write). + * regression_flagged is part of the published view: an accept clears it, + * a same-epoch regression sets it WITHOUT updating the payload fields + * (the stale-but-once-accepted values stay visible for diagnostics; the + * flag makes the fold refuse them). + */ +static void +undo_horizon_slot_publish(ClusterUndoHorizonSlotShmem *slot, const ClusterUndoHorizonWire *wire, + uint64 now_us, bool regression) +{ + uint32 seq = pg_atomic_read_u32(&slot->seq); + + Assert((seq & 1) == 0); /* single writer: LMON */ + pg_atomic_write_u32(&slot->seq, seq + 1); + pg_write_barrier(); + if (!regression) { + slot->epoch = wire->epoch; + slot->horizon_scn = wire->horizon_scn; + slot->recv_at_us = now_us; + slot->sender_interval_ms = wire->sender_interval_ms; + slot->valid = true; + slot->regression_flagged = false; + } else + slot->regression_flagged = true; + pg_write_barrier(); + pg_atomic_write_u32(&slot->seq, seq + 2); +} + +static void +cluster_undo_horizon_report_handler(const ClusterICEnvelope *env, const void *payload) +{ + ClusterUndoHorizonWire wire; + ClusterUndoHorizonSlotShmem *slot; + uint32 sender = env->source_node_id; + uint64 now_us; + + if (UndoHorizonShmem == NULL) + return; + + /* t/370 L2: force-drop the report before publish (stall leg). */ + CLUSTER_INJECTION_POINT("cluster-undo-horizon-report-drop"); + if (cluster_injection_should_skip("cluster-undo-horizon-report-drop")) + return; + + /* + * Strict wire validation (Q5' amend): exact length, sender within the + * declared range and not self, valid SCN, interval within + * [100,60000]ms. Anything invalid is counted and NOT published: the + * old slot value keeps aging and the fold stalls on MISSING/STALE -- + * never publish a value we could not prove well-formed. + */ + if (env->payload_length != sizeof(ClusterUndoHorizonWire)) { + undo_horizon_reject("bad length", sender); + return; + } + if (sender >= CLUSTER_MAX_NODES || (int32)sender == cluster_node_id) { + undo_horizon_reject("bad sender", sender); + return; + } + memcpy(&wire, payload, sizeof(wire)); + if ((SCN)wire.horizon_scn == InvalidScn) { + undo_horizon_reject("invalid scn", sender); + return; + } + if (wire.sender_interval_ms < CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS + || wire.sender_interval_ms > CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS) { + undo_horizon_reject("interval out of range", sender); + return; + } + + slot = &UndoHorizonShmem->slots[sender]; + now_us = (uint64)GetCurrentTimestamp(); + + /* + * Same-epoch monotonicity (S3.0 corollary): an accepted report is a + * lower bound on the sender's future snapshots, so within one epoch + * the accepted sequence must be non-decreasing. A regression means a + * snapshot BELOW the accepted bound exists over there -- latch the + * violation (the fold stalls, U14) and do not move the payload. A + * later conforming report clears the latch. + */ + if (slot->accepted_any && slot->accepted_epoch == wire.epoch + && scn_time_cmp((SCN)wire.horizon_scn, (SCN)slot->accepted_scn) < 0) { + undo_horizon_reject("same-epoch regression", sender); + undo_horizon_slot_publish(slot, &wire, now_us, true); + return; + } + + undo_horizon_slot_publish(slot, &wire, now_us, false); + slot->accepted_scn = wire.horizon_scn; + slot->accepted_epoch = wire.epoch; + slot->accepted_any = true; +} + +void +cluster_undo_horizon_register_ic_msg_types(void) +{ + const ClusterICMsgTypeInfo report_info = { + .msg_type = PGRAC_IC_MSG_UNDO_HORIZON, + .name = "undo_horizon", + .allowed_producer_mask = CLUSTER_IC_PRODUCER_LMON, + .broadcast_ok = false, /* per-peer p2p, HEARTBEAT discipline */ + .handler = cluster_undo_horizon_report_handler, + }; + + cluster_ic_register_msg_type(&report_info); +} + +/* ---------------- LMON publish tick ---------------------------------- */ + +/* + * Sample this node's report value (S2.1 sampling rule): the spec-3.12 + * own-instance ProcArray floor, further bounded by the node's minimum + * future snapshot origin. On an MRP-active (ADG standby) node future + * snapshots read at consistent_scn -- which may sit far below the + * no-reader clock fallback -- so the report must not exceed it (S3.0 + * row 4; on a primary node the origin is the Lamport clock, which the + * ProcArray floor never exceeds). InvalidScn (e.g. consistent_scn not + * yet rebuilt after a restart) => no report this tick: peers stall on + * staleness, which is the conservative direction. + */ +static SCN +undo_horizon_sample_local_report(void) +{ + SCN floor = cluster_undo_retention_horizon(); + + if (floor == InvalidScn) + return InvalidScn; + if (cluster_mrp_should_start()) { + SCN bound = cluster_mrp_standby_consistent_scn(); + + if (bound == InvalidScn) + return InvalidScn; + if (scn_time_cmp(bound, floor) < 0) + floor = bound; + } + return floor; +} + +void +cluster_undo_horizon_lmon_tick(void) +{ + static uint64 last_sent_us = 0; + ClusterUndoHorizonWire wire; + uint64 now_us; + SCN report; + int pi; + + if (!cluster_enabled || cluster_node_id < 0 || UndoHorizonShmem == NULL) + return; + + /* one report per main-loop interval even when the latch churns */ + now_us = (uint64)GetCurrentTimestamp(); + if (last_sent_us != 0 && now_us - last_sent_us < (uint64)cluster_lmon_main_loop_interval * 1000) + return; + + report = undo_horizon_sample_local_report(); + if (report == InvalidScn) + return; + last_sent_us = now_us; + + wire.epoch = cluster_epoch_get_current(); + wire.horizon_scn = (uint64)report; + wire.sender_interval_ms = (uint32)cluster_lmon_main_loop_interval; + + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (pi == cluster_node_id) + continue; + if (cluster_conf_lookup_node(pi) == NULL) + continue; /* not declared */ + + /* + * Q1' amend hard gate: only send to a peer whose CURRENT + * connection advertised UNDO_HORIZON_V1. An old peer replies to + * an unregistered msg_type by closing the connection + * (cluster_ic_router.c inbound contract), so a blind send per + * tick would be a reconnect storm, not "ignored". + */ + if (!cluster_sf_peer_supports_undo_horizon(pi)) + continue; + + (void)cluster_ic_send_envelope(PGRAC_IC_MSG_UNDO_HORIZON, pi, &wire, sizeof(wire)); + /* fire-and-forget (L456): transport retention / errors surface + * through the transport's own paths; a lost report just ages the + * peer's view of us into a stall. */ + } +} + +/* ---------------- cleaner-side sampling (reader) ---------------------- */ + +#define UNDO_HORIZON_SEQLOCK_RETRIES 3 + +/* + * Snapshot every peer slot into plain views for the pure fold. Returns + * the number of views filled (CLUSTER_MAX_NODES). stable=false after + * bounded seqlock retries => the fold reports TORN for that peer if it + * is required. Capability is sampled here too, so the fold's NOCAP arm + * sees the CURRENT connection state (Q3''). + */ +int +cluster_undo_horizon_sample_views(ClusterUndoHorizonReportView *views, int maxviews) +{ + int n = Min(maxviews, CLUSTER_MAX_NODES); + int i; + + if (UndoHorizonShmem == NULL) + return 0; + + for (i = 0; i < n; i++) { + ClusterUndoHorizonSlotShmem *slot = &UndoHorizonShmem->slots[i]; + ClusterUndoHorizonReportView *v = &views[i]; + int attempt; + + memset(v, 0, sizeof(*v)); + v->has_capability = cluster_sf_peer_supports_undo_horizon(i); + v->stable = false; + + for (attempt = 0; attempt < UNDO_HORIZON_SEQLOCK_RETRIES; attempt++) { + uint32 seq1 = pg_atomic_read_u32(&slot->seq); + uint32 seq2; + + if (seq1 & 1) + continue; /* writer mid-publish */ + pg_read_barrier(); + v->epoch = slot->epoch; + v->horizon_scn = (SCN)slot->horizon_scn; + v->recv_at_us = slot->recv_at_us; + v->sender_interval_ms = slot->sender_interval_ms; + v->valid = slot->valid; + v->regression_flagged = slot->regression_flagged; + pg_read_barrier(); + seq2 = pg_atomic_read_u32(&slot->seq); + if (seq1 == seq2) { + v->stable = true; + break; + } + } + } + return n; +} + +/* + * Sample the required MEMBER peer set consistently with the reconfig + * epoch: epoch -> membership walk -> epoch re-check, bounded retries. + * Returns false when the epoch would not hold still (reconfig in + * flight): the caller treats that exactly like a fold stall. + */ +bool +cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch) +{ + int attempt; + + memset(required, 0, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + + for (attempt = 0; attempt < UNDO_HORIZON_SEQLOCK_RETRIES; attempt++) { + uint64 e1 = cluster_epoch_get_current(); + uint64 e2; + int i; + + memset(required, 0, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == cluster_node_id) + continue; + if (cluster_conf_lookup_node(i) == NULL) + continue; /* not declared */ + if (cluster_membership_get_state(i) == CLUSTER_MEMBER_MEMBER) + required[i >> 3] |= (uint8)(1u << (i & 7)); + } + e2 = cluster_epoch_get_current(); + if (e1 == e2) { + *out_epoch = e1; + return true; + } + } + return false; +} + +/* ---------------- observability (D5-5 accessors) ---------------------- */ + +#define UNDO_HORIZON_NOTE(field) \ + void cluster_undo_horizon_note_##field(void) \ + { \ + if (UndoHorizonShmem != NULL) \ + pg_atomic_fetch_add_u64(&UndoHorizonShmem->field##_count, 1); \ + } \ + uint64 cluster_undo_horizon_##field##_count(void) \ + { \ + return UndoHorizonShmem == NULL ? 0 \ + : pg_atomic_read_u64(&UndoHorizonShmem->field##_count); \ + } + +UNDO_HORIZON_NOTE(stall) +UNDO_HORIZON_NOTE(peer_stale) +UNDO_HORIZON_NOTE(pass_abort) +UNDO_HORIZON_NOTE(wire_reject) +UNDO_HORIZON_NOTE(admission_refuse) + +void +cluster_undo_horizon_note_floor(SCN scn) +{ + if (UndoHorizonShmem != NULL) + pg_atomic_write_u64(&UndoHorizonShmem->last_floor_scn, (uint64)scn); +} + +SCN +cluster_undo_horizon_last_floor(void) +{ + return UndoHorizonShmem == NULL ? InvalidScn + : (SCN)pg_atomic_read_u64(&UndoHorizonShmem->last_floor_scn); +} diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 2402e422eb..5c3e5ee9b1 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -237,6 +237,16 @@ extern const ClusterICOps *ClusterICOps_Active; * advertised this bit; without it the authority leg fails closed (the * election is NOT re-run against a different node). */ #define PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 ((uint32)0x00000002U) +/* PGRAC: spec-5.22e D5-2 — this binary registers PGRAC_IC_MSG_UNDO_HORIZON + * and publishes/consumes undo retention horizon reports. A PROTOCOL + * capability, advertised unconditionally. Send-side hard gate: a report is + * only sent to a peer whose CURRENT connection advertised this bit (an old + * peer treats the unregistered msg_type as a peer-level failure and closes + * the connection). Fold-side: a required MEMBER peer without this bit + * stalls recycling (NOCAP) — never a fallback to local-horizon recycling + * (Q3''). Capability state is connection-bound: cleared on peer close and + * only reinstated by the next HELLO (Q1' amend). */ +#define PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 ((uint32)0x00000004U) typedef struct ClusterICHelloMsg { uint32 magic; /* PGRAC_IC_HELLO_MAGIC */ diff --git a/src/include/cluster/cluster_ic_envelope.h b/src/include/cluster/cluster_ic_envelope.h index bfbbc8b420..310fd50cdd 100644 --- a/src/include/cluster/cluster_ic_envelope.h +++ b/src/include/cluster/cluster_ic_envelope.h @@ -241,9 +241,13 @@ typedef enum ClusterICMsgType { PGRAC_IC_MSG_BACKUP_ACK = 34, /* PGRAC: spec-6.5 D1/D4 — peer -> backup * coordinator (ClusterBackupWireAck): local thread REDO/checkpoint/stop-cut * metadata or fail-closed NAK reason. */ - PGRAC_IC_MSG_SMART_FUSION_DURABLE = 35 /* PGRAC: spec-6.2 D8 — origin durable-LSN + PGRAC_IC_MSG_SMART_FUSION_DURABLE = 35, /* PGRAC: spec-6.2 D8 — origin durable-LSN * gossip for Smart Fusion dependency release. */ - /* values 36..255 available for future sub-spec; never reuse 0..35 */ + PGRAC_IC_MSG_UNDO_HORIZON = 36 /* PGRAC: spec-5.22e D5-2 — per-peer undo + * retention horizon report (LMON-only producer; p2p, never + * broadcast; capability-gated on UNDO_HORIZON_V1 so an old + * peer never sees an unregistered msg_type). */ + /* values 37..255 available for future sub-spec; never reuse 0..36 */ } ClusterICMsgType; diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index ccf4a5983d..848383081f 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -146,6 +146,10 @@ extern bool cluster_sf_peer_supports_reply_v2(int32 peer_id); * smart-fusion query above): the bit answers "can that binary parse kind 4"; * the runtime arm gates live elsewhere. */ extern bool cluster_peer_supports_undo_authority_serve(int32 peer_id); +/* spec-5.22e D5-2: undo-horizon report capability (connection-bound; see + * cluster_sf_note_peer_disconnected) + the close-funnel reset hook. */ +extern bool cluster_sf_peer_supports_undo_horizon(int32 peer_id); +extern void cluster_sf_note_peer_disconnected(int32 peer_id); extern void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload); extern void cluster_sf_note_dep_touched(Buffer buffer); diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h index 22b92c324d..d0a962c4e4 100644 --- a/src/include/cluster/cluster_undo_horizon.h +++ b/src/include/cluster/cluster_undo_horizon.h @@ -165,4 +165,41 @@ extern ClusterUndoHorizonFoldStatus cluster_undo_horizon_cluster_floor( /* Attribution name for LOG-once / dump lines ("nocap", "stale", ...). */ extern const char *cluster_undo_horizon_stall_reason_name(ClusterUndoHorizonStallReason reason); +/* + * Wire payload (PGRAC_IC_MSG_UNDO_HORIZON, D5-2): 20 bytes packed. The + * sender's lmon_main_loop_interval rides along so a slow-tick sender is + * not misjudged stale by a fast-tick receiver (freshness window = + * FACTOR * max(sender, local)). The receive handler validates length, + * sender, SCN, interval range and same-epoch monotonicity before + * publishing (Q5' amend); invalid frames are counted, never published. + */ +typedef struct pg_attribute_packed() ClusterUndoHorizonWire { + uint64 epoch; /* sender's reconfig epoch at sampling */ + uint64 horizon_scn; /* S2.1 sampling rule output */ + uint32 sender_interval_ms; /* validated to [MIN,MAX] */ +} ClusterUndoHorizonWire; + +/* ---- heavy path (cluster_undo_horizon_ic.c; LMON + cleaner) --------- */ + +extern void cluster_undo_horizon_shmem_register(void); +extern void cluster_undo_horizon_register_ic_msg_types(void); +extern void cluster_undo_horizon_lmon_tick(void); + +extern int cluster_undo_horizon_sample_views(ClusterUndoHorizonReportView *views, int maxviews); +extern bool cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch); + +/* observability (D5-5) */ +extern void cluster_undo_horizon_note_stall(void); +extern uint64 cluster_undo_horizon_stall_count(void); +extern void cluster_undo_horizon_note_peer_stale(void); +extern uint64 cluster_undo_horizon_peer_stale_count(void); +extern void cluster_undo_horizon_note_pass_abort(void); +extern uint64 cluster_undo_horizon_pass_abort_count(void); +extern void cluster_undo_horizon_note_wire_reject(void); +extern uint64 cluster_undo_horizon_wire_reject_count(void); +extern void cluster_undo_horizon_note_admission_refuse(void); +extern uint64 cluster_undo_horizon_admission_refuse_count(void); +extern void cluster_undo_horizon_note_floor(SCN scn); +extern SCN cluster_undo_horizon_last_floor(void); + #endif /* CLUSTER_UNDO_HORIZON_H */ diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index b62e31c47c..b6242632dd 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '159', - 'pg_stat_cluster_injections returns 159 rows (spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '161', + 'pg_stat_cluster_injections returns 161 rows (spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '159 injection point names match the full registry (spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '161 injection point names match the full registry (spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index 0ed660eb50..7b232fe761 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -109,9 +109,9 @@ # between "pgrac cluster cr relgen" and "pgrac cluster cr tuple stats"). # spec-6.12h D-h3a: +1 "pgrac cluster pi shadow" (PI ship-SCN shadow table; # sorts between "pgrac cluster oid lease,pgrac cluster pcm grd" and "pgrac cluster qvotec"). -my $expected_region_count = $has_visibility_inject ? '80' : '79'; +my $expected_region_count = $has_visibility_inject ? '81' : '80'; my $expected_regions = - 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo gcs,pgrac cluster undo record cursor'; + 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo gcs,pgrac cluster undo horizon,pgrac cluster undo record cursor'; $expected_regions .= ',pgrac cluster visibility inject' if $has_visibility_inject; # spec-4.12 D7: cooperative write-fence region; always registered. Sorts after @@ -256,14 +256,14 @@ FROM pg_settings WHERE name = 'cluster.shmem_max_regions' }), - qr/^integer\|postmaster\|80\|(40|41)\|256$/, - 'L12 cluster.shmem_max_regions default 80 (spec-5.56: 64 -> 80 for cr relgen region)'); + qr/^integer\|postmaster\|96\|(40|41)\|256$/, + 'L12 cluster.shmem_max_regions default 96 (spec-5.22e: 80 -> 96 for undo horizon region + headroom)'); is($node->safe_psql( 'postgres', q{SELECT value FROM pg_cluster_state WHERE category = 'guc' AND key = 'cluster.shmem_max_regions'}), - '80', + '96', 'L13 pg_cluster_state.guc.cluster.shmem_max_regions reflects live GUC'); @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '161', + 'L15 total injection registry size is 161 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index 1356d80d35..5a911b2b22 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -56,7 +56,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '81' : '80'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L11 pg_stat_cluster_injections is 161 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 5bc3f49d9a..b4b2d30e3c 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -71,7 +71,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '81' : '80'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L12a pg_stat_cluster_injections is 161 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index 30385ba290..e1b4705666 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -61,7 +61,7 @@ # admission reason counters; +1 "pgrac cluster clean_leave" (spec-5.13); +1 # "pgrac cluster cr relgen" (spec-5.56 D4); +1 "pgrac cluster cr tuple stats" # (spec-5.54 D5); full list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '80' : '79'; # spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) + my $expected_region_count = $has_visibility_inject ? '81' : '80'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a) # ---------- @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '161', + 'L11 pg_stat_cluster_injections is 161 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index 7b98bdceac..dfea4a380e 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -1073,13 +1073,13 @@ UT_TEST(test_cluster_shmem_iter_regions_returns_false_when_uninit) UT_TEST(test_cluster_shmem_max_regions_default_value) { /* - * The boot-value of cluster.shmem_max_regions is 80 (spec-5.56 raised it - * 64 -> 80 for the per-relation CR generation region; cluster_guc.c static + * The boot-value of cluster.shmem_max_regions is 96 (spec-5.22e raised it + * 80 -> 96 for the undo horizon region + headroom; cluster_guc.c static * initializer). Unit test links cluster_guc.o but never calls * cluster_init_guc, so the C global retains its static-initializer default. */ extern int cluster_shmem_max_regions; - UT_ASSERT_EQ(cluster_shmem_max_regions, 80); + UT_ASSERT_EQ(cluster_shmem_max_regions, 96); } From dc8236e06fce64a3a06f4502849e3b9eef74032c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 06:14:36 +0800 Subject: [PATCH 25/38] feat(cluster): D5-3 cleaner cluster-floor consumption + F-D2 epoch fence (spec-5.22e) The undo cleaner pass input changes from the bare spec-3.12 local horizon to the cluster floor {scn, epoch}: sample the per-peer seqlock report views + the required MEMBER set (epoch-consistent walk) and run the pure fold. An empty required set folds to the local value (single node / cold formation = today's path, byte-equal behavior). Any unproven required peer STALLS the pass: the whole recycle stage -- shmem TT slot GC included (Q8: one horizon input) -- is skipped, with reason/blame attribution (undo_horizon_stall / undo_horizon_peer_stale counters + LOG-once mirroring the pinned-horizon pattern, re-armed on recovery). There is deliberately no fallback to local-horizon recycling on any stall reason (Q3''). F-D2 epoch fence: the folded floor's epoch is re-verified at every recycle mutation point -- - cluster_tt_slot_gc_current_pass: before each slot CTS_FREE (returns false = pass aborts); - cluster_undo_segment_advance_recyclable: INSIDE the lifecycle_lock, before the WAL/pwrite/fsync three-step (new result CLUSTER_SEG_RECYCLE_EPOCH_CHANGED; the fence sits in the caller's locked region rather than widening try_mark_recyclable's signature -- same guarantee, smaller ABI ripple). A mid-pass reconfig epoch bump (a join) voids the folded floor's member coverage, so a tripped fence aborts the WHOLE pass immediately (undo_horizon_pass_abort counter), never just waits for the next pass. The fence consults the cluster-undo-horizon-epoch-fence injection point so the t/370 L6 leg can force the tripped arm deterministically. Unit ripple: gc pass callers updated (test_cluster_tt_slot_allocator + fence/register/tick link stubs for standalone binaries). Full cluster_unit suite + PG219 regression pass. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-3) --- src/backend/cluster/cluster_tt_slot.c | 22 ++++- src/backend/cluster/cluster_undo_cleaner.c | 89 +++++++++++++++++-- src/backend/cluster/cluster_undo_horizon_ic.c | 18 ++++ src/backend/cluster/cluster_undo_record.c | 15 +++- src/include/cluster/cluster_undo_cleaner.h | 5 +- src/include/cluster/cluster_undo_horizon.h | 2 + src/include/cluster/cluster_undo_record_api.h | 5 +- .../cluster/storage/cluster_undo_alloc.h | 9 +- src/test/cluster_unit/test_cluster_lmon.c | 11 +++ src/test/cluster_unit/test_cluster_shmem.c | 7 ++ .../test_cluster_tt_slot_allocator.c | 20 +++-- 11 files changed, 185 insertions(+), 18 deletions(-) diff --git a/src/backend/cluster/cluster_tt_slot.c b/src/backend/cluster/cluster_tt_slot.c index 28234a4797..04bc8b74e0 100644 --- a/src/backend/cluster/cluster_tt_slot.c +++ b/src/backend/cluster/cluster_tt_slot.c @@ -53,6 +53,7 @@ #include "cluster/cluster_undo_cleaner.h" /* spec-3.13 D2-A gc pass + stats */ #include "storage/spin.h" /* protected-slot map lock (spec-3.15 D6) */ #include "cluster/storage/cluster_undo_alloc.h" /* CLUSTER_UNDO_SEGS_PER_INSTANCE */ +#include "cluster/cluster_undo_horizon.h" /* epoch fence (spec-5.22e F-D2) */ #include "miscadmin.h" #include "port/atomics.h" /* spec-3.12 D5 retention counters */ #include "storage/lwlock.h" @@ -474,22 +475,24 @@ cluster_tt_slot_alloc_ext(uint32 segment_id, TransactionId top_xid, bool *out_re * current segment), NOT crash residue — they are simply skipped and * NOT counted as stale (HC6 stale accounting is durable-side only). */ -void -cluster_tt_slot_gc_current_pass(SCN horizon, ClusterUndoCleanerPassStats *stats) +bool +cluster_tt_slot_gc_current_pass(SCN horizon, uint64 expected_epoch, + ClusterUndoCleanerPassStats *stats) { uint32 segment_id; ClusterTTSlotAllocPerSegment *seg; uint32 gcd = 0; + bool fence_ok = true; int i; Assert(stats != NULL); if (ClusterTTSlotShm == NULL || cluster_node_id < 0) - return; + return true; segment_id = cluster_tt_slot_current_segment(cluster_node_id); if (segment_id == 0) - return; /* no binding yet this incarnation */ + return true; /* no binding yet this incarnation */ seg = cluster_tt_slot_get_or_init(segment_id); @@ -507,6 +510,16 @@ cluster_tt_slot_gc_current_pass(SCN horizon, ClusterUndoCleanerPassStats *stats) continue; /* D5: ABA guard exhausted; wait for rollover/reuse */ } + /* + * spec-5.22e F-D2 epoch fence: re-verify the reconfig epoch before + * every slot FREE. A mid-pass epoch bump voids the folded floor's + * member coverage -- stop mutating and abort the whole pass. + */ + if (cluster_undo_horizon_epoch_fence_tripped(expected_epoch)) { + fence_ok = false; + break; + } + if (e->status == CTS_COMMITTED && ClusterTTSlotShm != NULL) { pg_atomic_fetch_add_u64(&ClusterTTSlotShm->retention_recycle_count, 1); /* spec-3.22: defense in depth -- this pass is meant to run only with a @@ -525,6 +538,7 @@ cluster_tt_slot_gc_current_pass(SCN horizon, ClusterUndoCleanerPassStats *stats) stats->shmem_tt_slots_gcd += gcd; stats->segments_scanned++; + return fence_ok; } diff --git a/src/backend/cluster/cluster_undo_cleaner.c b/src/backend/cluster/cluster_undo_cleaner.c index 50bc910bda..6d1de12fec 100644 --- a/src/backend/cluster/cluster_undo_cleaner.c +++ b/src/backend/cluster/cluster_undo_cleaner.c @@ -63,6 +63,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" /* cluster_storage_mode_enabled */ +#include "cluster/cluster_undo_horizon.h" /* cluster floor + fence (spec-5.22e D5-3) */ #include "cluster/cluster_undo_retention.h" /* horizon (C17: once per pass) */ #include "cluster/cluster_tt_slot.h" /* current TT segment (exclusion) */ #include "cluster/cluster_undo_record_api.h" /* active segment + advance (D3) */ @@ -366,12 +367,76 @@ undo_cleaner_run_pass(void) * D2 (step 3): horizon ONCE per pass, BEFORE any seg->lock (C17). * With the retention gate GUC off there is nothing to pre-free — * alloc Pass-2 recycles immediately (C6) — so the pass only ticks. + * + * spec-5.22e D5-3: the pass input is no longer the bare local horizon + * but the CLUSTER floor {scn, epoch}: the scn_time_cmp-min of the local + * horizon and every required MEMBER peer's accepted report. An empty + * required set (single node / cold formation) folds to the local value + * on today's path; any unproven required peer STALLS the pass — the + * whole recycle stage (shmem TT GC included, Q8: one horizon input) is + * skipped, never run against a floor we could not prove (rule 8.A / + * Q3'': NO fallback to local recycling, whatever the stall reason). + * The floor's epoch rides through the pass and is re-verified at every + * mutation (F-D2 fence); a mid-pass bump aborts the pass immediately. */ memset(&stats, 0, sizeof(stats)); if (cluster_undo_retention_horizon_enabled) { - SCN horizon = cluster_undo_retention_horizon(); + SCN local_horizon = cluster_undo_retention_horizon(); + ClusterUndoHorizonFloor floor; + ClusterUndoHorizonStallReason stall_reason = CLUSTER_UNDO_HORIZON_STALL_NONE; + int32 stall_blame = -1; + bool stalled = false; + bool fence_aborted = false; + static bool stall_logged = false; + SCN horizon; - cluster_tt_slot_gc_current_pass(horizon, &stats); + { + ClusterUndoHorizonReportView views[CLUSTER_MAX_NODES]; + uint8 required[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 fold_epoch; + int nviews; + + nviews = cluster_undo_horizon_sample_views(views, CLUSTER_MAX_NODES); + if (!cluster_undo_horizon_required_members(required, &fold_epoch)) { + /* reconfig in flight: the member set would not hold still */ + stalled = true; + stall_reason = CLUSTER_UNDO_HORIZON_STALL_EPOCH; + } else if (cluster_undo_horizon_cluster_floor( + local_horizon, views, nviews, required, cluster_node_id, fold_epoch, + (uint64)GetCurrentTimestamp(), (uint32)cluster_lmon_main_loop_interval, + &floor, &stall_reason, &stall_blame) + == CLUSTER_UNDO_HORIZON_FOLD_STALLED) + stalled = true; + } + + if (stalled) { + cluster_undo_horizon_note_stall(); + if (stall_blame >= 0 && stall_blame != cluster_node_id) + cluster_undo_horizon_note_peer_stale(); + if (!stall_logged) { + stall_logged = true; + ereport(LOG, + (errmsg("cluster undo cleaner: recycle stalled, cluster horizon " + "unproven (reason \"%s\", node %d)", + cluster_undo_horizon_stall_reason_name(stall_reason), stall_blame), + errhint("Recycling pauses until every MEMBER peer publishes a " + "fresh horizon report at the current epoch. Undo segment " + "pool may grow until then (53R9E at the hard cap)."))); + } + goto pass_account; /* 只扫不收: skip the whole recycle stage */ + } + if (stall_logged) { + stall_logged = false; + ereport(LOG, (errmsg("cluster undo cleaner: cluster horizon proven again; " + "recycling resumes"))); + } + horizon = floor.scn; + cluster_undo_horizon_note_floor(floor.scn); + + if (!cluster_tt_slot_gc_current_pass(horizon, floor.epoch, &stats)) { + cluster_undo_horizon_note_pass_abort(); + goto pass_account; /* F-D2: epoch moved mid-scan; abort the pass */ + } /* * D2-B + D3: walk this instance's rolled-away segment inventory, @@ -435,9 +500,19 @@ undo_cleaner_run_pass(void) if (cluster_undo_record_segment_commit_on_rollover) cluster_undo_segment_advance_committed(cur); - if (cluster_undo_segment_advance_recyclable(cur, horizon) - == CLUSTER_SEG_RECYCLE_ADVANCED) - stats.segments_marked_recyclable++; + { + ClusterUndoSegTryRecycle rr + = cluster_undo_segment_advance_recyclable(cur, horizon, floor.epoch); + + if (rr == CLUSTER_SEG_RECYCLE_EPOCH_CHANGED) { + /* F-D2: epoch moved inside the mutation lock; the + * mutation did not run. Abort the whole pass NOW. */ + fence_aborted = true; + break; + } + if (rr == CLUSTER_SEG_RECYCLE_ADVANCED) + stats.segments_marked_recyclable++; + } } /* @@ -446,9 +521,13 @@ undo_cleaner_run_pass(void) * is needed; same-process happens-before across passes). */ undo_cleaner_state->scan_resume_seg = seg; + + if (fence_aborted) + cluster_undo_horizon_note_pass_abort(); } } +pass_account: Assert(undo_cleaner_state != NULL); LWLockAcquire(&undo_cleaner_state->lwlock, LW_EXCLUSIVE); undo_cleaner_state->pass_count++; diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c index 7acef8d2c3..a018854e92 100644 --- a/src/backend/cluster/cluster_undo_horizon_ic.c +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -429,6 +429,24 @@ cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch) return false; } +/* + * F-D2 epoch fence: re-verify -- at every recycle mutation point -- that the + * reconfig epoch still equals the one the floor was folded at. A mid-pass + * join/epoch bump means the folded floor's member set no longer covers the + * cluster (the new MEMBER's snapshots were not required), so the caller must + * abort the whole pass immediately, not just wait for the next one. The + * injection point lets the t/370 L6 leg force the tripped arm at the first + * mutation without a real reconfig. + */ +bool +cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch) +{ + CLUSTER_INJECTION_POINT("cluster-undo-horizon-epoch-fence"); + if (cluster_injection_should_skip("cluster-undo-horizon-epoch-fence")) + return true; + return cluster_epoch_get_current() != expected_epoch; +} + /* ---------------- observability (D5-5 accessors) ---------------------- */ #define UNDO_HORIZON_NOTE(field) \ diff --git a/src/backend/cluster/cluster_undo_record.c b/src/backend/cluster/cluster_undo_record.c index 401f1277a1..d8e7a04295 100644 --- a/src/backend/cluster/cluster_undo_record.c +++ b/src/backend/cluster/cluster_undo_record.c @@ -90,6 +90,7 @@ #include "cluster/storage/cluster_undo_alloc.h" #include "cluster/storage/cluster_undo_xlog.h" /* spec-3.18 D2a XLOG_UNDO_BLOCK_WRITE */ #include "cluster/cluster_xnode_profile.h" /* PGRAC: spec-5.59 D7 profiling */ +#include "cluster/cluster_undo_horizon.h" /* epoch fence (spec-5.22e F-D2) */ #include "access/xlog.h" /* GetXLogWriteRecPtr */ @@ -2327,7 +2328,7 @@ cluster_undo_record_active_segment_id(void) * BEFORE this lock (C17). */ ClusterUndoSegTryRecycle -cluster_undo_segment_advance_recyclable(uint32 segment_id, SCN horizon) +cluster_undo_segment_advance_recyclable(uint32 segment_id, SCN horizon, uint64 expected_epoch) { ClusterUndoSegTryRecycle result; uint8 owner; @@ -2342,6 +2343,18 @@ cluster_undo_segment_advance_recyclable(uint32 segment_id, SCN horizon) LWLockRelease(&UndoRecordShared->lifecycle_lock.lock); return CLUSTER_SEG_RECYCLE_NOT_COMMITTED; /* writer-active: never a candidate */ } + + /* + * spec-5.22e F-D2 epoch fence, INSIDE the mutation lock and before the + * WAL/pwrite/fsync three-step: a reconfig epoch bump after the floor was + * folded voids its member coverage. Refuse the mutation; the caller + * aborts the whole pass. + */ + if (cluster_undo_horizon_epoch_fence_tripped(expected_epoch)) { + LWLockRelease(&UndoRecordShared->lifecycle_lock.lock); + return CLUSTER_SEG_RECYCLE_EPOCH_CHANGED; + } + result = cluster_undo_segment_try_mark_recyclable(segment_id, owner, horizon); LWLockRelease(&UndoRecordShared->lifecycle_lock.lock); diff --git a/src/include/cluster/cluster_undo_cleaner.h b/src/include/cluster/cluster_undo_cleaner.h index e720fd1e04..5cd77cb51a 100644 --- a/src/include/cluster/cluster_undo_cleaner.h +++ b/src/include/cluster/cluster_undo_cleaner.h @@ -220,7 +220,10 @@ typedef struct ClusterUndoCleanerPassStats { * any seg->lock — spec-3.12 C17). Shares the recycle transition * helper with alloc Pass-2 (C-R1: single typed implementation). */ -extern void cluster_tt_slot_gc_current_pass(SCN horizon, ClusterUndoCleanerPassStats *stats); +/* Returns false when the spec-5.22e F-D2 epoch fence tripped mid-scan: + * the caller must abort the whole pass immediately. */ +extern bool cluster_tt_slot_gc_current_pass(SCN horizon, uint64 expected_epoch, + ClusterUndoCleanerPassStats *stats); /* * D2-B: READ-ONLY scan of one segment's durable header TTSlot[] diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h index d0a962c4e4..c4c6703ac5 100644 --- a/src/include/cluster/cluster_undo_horizon.h +++ b/src/include/cluster/cluster_undo_horizon.h @@ -187,6 +187,8 @@ extern void cluster_undo_horizon_lmon_tick(void); extern int cluster_undo_horizon_sample_views(ClusterUndoHorizonReportView *views, int maxviews); extern bool cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch); +/* F-D2 epoch fence (recycle mutation points; injection-forceable, t/370 L6) */ +extern bool cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch); /* observability (D5-5) */ extern void cluster_undo_horizon_note_stall(void); diff --git a/src/include/cluster/cluster_undo_record_api.h b/src/include/cluster/cluster_undo_record_api.h index aca6719162..129d217710 100644 --- a/src/include/cluster/cluster_undo_record_api.h +++ b/src/include/cluster/cluster_undo_record_api.h @@ -230,8 +230,11 @@ extern bool cluster_undo_test_force_segment_end(void); /* spec-3.13 D3: cleaner-side segment lifecycle surface. */ extern uint32 cluster_undo_record_active_segment_id(void); +/* EPOCH_CHANGED when the spec-5.22e F-D2 fence tripped inside the mutation + * lock: the mutation was not performed; abort the whole pass. */ extern ClusterUndoSegTryRecycle cluster_undo_segment_advance_recyclable(uint32 segment_id, - SCN horizon); + SCN horizon, + uint64 expected_epoch); /* * spec-4.12a D1: record-segment ACTIVE -> COMMITTED drain. Called at the diff --git a/src/include/cluster/storage/cluster_undo_alloc.h b/src/include/cluster/storage/cluster_undo_alloc.h index 4212a3eff1..58b04242b5 100644 --- a/src/include/cluster/storage/cluster_undo_alloc.h +++ b/src/include/cluster/storage/cluster_undo_alloc.h @@ -273,7 +273,14 @@ typedef enum ClusterUndoSegTryRecycle { CLUSTER_SEG_RECYCLE_RETAINED = 2, /* predicate says a reader may need it */ CLUSTER_SEG_RECYCLE_NOT_COMMITTED = 3, /* ALLOCATED / ACTIVE: not a candidate */ CLUSTER_SEG_RECYCLE_READ_FAIL = 4, /* absent / I/O / identity mismatch */ - CLUSTER_SEG_RECYCLE_WRITE_FAIL = 5 /* pwrite / fsync failed; retry next pass */ + CLUSTER_SEG_RECYCLE_WRITE_FAIL = 5, /* pwrite / fsync failed; retry next pass */ + CLUSTER_SEG_RECYCLE_EPOCH_CHANGED = 6 /* spec-5.22e F-D2 epoch fence: the + * reconfig epoch moved after the + * recycle floor was folded; the + * mutation was NOT performed and the + * caller must abort the whole pass + * (the folded floor's member set no + * longer covers the cluster) */ } ClusterUndoSegTryRecycle; /* Caller holds undo lifecycle_lock and excluded the active record segment. */ diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..9b2d383e7c 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -485,6 +485,17 @@ void cluster_backup_lmon_tick(void) {} +/* spec-5.22e D5-2 stub: LmonMain also ticks the undo horizon publisher and + * registers its msg type (cluster_undo_horizon_ic.c not linked here). */ +void cluster_undo_horizon_lmon_tick(void); +void +cluster_undo_horizon_lmon_tick(void) +{} +void cluster_undo_horizon_register_ic_msg_types(void); +void +cluster_undo_horizon_register_ic_msg_types(void) +{} + /* spec-6.2 D7 stub: cluster_lmon_shmem_init registers Smart Fusion durable * gossip IC msg types; this standalone unit binary does not link * cluster_sf_dep.o. */ diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index dfea4a380e..99134373b4 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -675,6 +675,13 @@ void cluster_sf_dep_shmem_register(void) {} +/* spec-5.22e D5-2 stub: cluster_init_shmem_module also calls + * cluster_undo_horizon_shmem_register (cluster_undo_horizon_ic.c is not + * linked into this standalone test); link-only no-op stub. */ +void +cluster_undo_horizon_shmem_register(void) +{} + /* * spec-5.52 D9 stub: cluster_init_shmem_module also registers the independent * admission reason-counter region (cluster_cr_admit_stat.c is not linked into diff --git a/src/test/cluster_unit/test_cluster_tt_slot_allocator.c b/src/test/cluster_unit/test_cluster_tt_slot_allocator.c index 48aa16b03a..7a8df4d90b 100644 --- a/src/test/cluster_unit/test_cluster_tt_slot_allocator.c +++ b/src/test/cluster_unit/test_cluster_tt_slot_allocator.c @@ -80,6 +80,16 @@ UT_DEFINE_GLOBALS(); +/* spec-5.22e D5-3 stub: the gc pass consults the F-D2 epoch fence + * (cluster_undo_horizon_ic.c not linked here); never tripped in unit. */ +bool cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch); +bool +cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch) +{ + (void)expected_epoch; + return false; +} + /* ============================================================ * Shmem mock + LWLock stub @@ -840,7 +850,7 @@ UT_TEST(test_t40_gc_then_alloc_equals_direct_recycle) } /* path 2: cleaner GC frees sb, then owner 201 allocates it. */ - cluster_tt_slot_gc_current_pass((SCN)20, &stats); + cluster_tt_slot_gc_current_pass((SCN)20, (uint64)0, &stats); UT_ASSERT_EQ((int)stats.shmem_tt_slots_gcd, 1); /* only sb was recyclable */ { uint16 got = cluster_tt_slot_alloc(NODE0_SEG, (TransactionId)201); @@ -868,7 +878,7 @@ UT_TEST(test_t41_gc_respects_horizon_retained_slot_untouched) /* alloc_ext below consults the stubbed horizon provider. */ mock_retention_horizon = (SCN)20; - cluster_tt_slot_gc_current_pass((SCN)20, &stats); + cluster_tt_slot_gc_current_pass((SCN)20, (uint64)0, &stats); UT_ASSERT_EQ((int)stats.shmem_tt_slots_gcd, 0); UT_ASSERT_EQ((int)cluster_tt_slot_get_wrap(NODE0_SEG, sa), 0); @@ -902,7 +912,7 @@ UT_TEST(test_t42_gc_skips_inflight_active) sa = cluster_tt_slot_alloc(NODE0_SEG, (TransactionId)100); - cluster_tt_slot_gc_current_pass((SCN)1000, &stats); + cluster_tt_slot_gc_current_pass((SCN)1000, (uint64)0, &stats); UT_ASSERT_EQ((int)stats.shmem_tt_slots_gcd, 0); UT_ASSERT_EQ((int)stats.stale_active_skipped, 0); @@ -937,7 +947,7 @@ UT_TEST(test_t43_wrap_max_slot_is_retired_from_both_paths) /* Each gc+alloc pair bumps wrap by exactly 1 (T40 equivalence). */ for (cycles = 0; cycles < (uint32)TT_WRAP_MAX; cycles++) { memset(&stats, 0, sizeof(stats)); - cluster_tt_slot_gc_current_pass((SCN)1000, &stats); + cluster_tt_slot_gc_current_pass((SCN)1000, (uint64)0, &stats); if (stats.shmem_tt_slots_gcd != 1) break; /* retired: stop driving */ (void)cluster_tt_slot_alloc(NODE0_SEG, (TransactionId)(200 + cycles)); @@ -948,7 +958,7 @@ UT_TEST(test_t43_wrap_max_slot_is_retired_from_both_paths) /* cleaner path refuses + counts. */ retired_before = cluster_tt_slot_wrap_retired_count(); memset(&stats, 0, sizeof(stats)); - cluster_tt_slot_gc_current_pass((SCN)1000, &stats); + cluster_tt_slot_gc_current_pass((SCN)1000, (uint64)0, &stats); UT_ASSERT_EQ((int)stats.shmem_tt_slots_gcd, 0); UT_ASSERT_EQ((int)stats.slots_wrap_retired, 1); UT_ASSERT_EQ((int)(cluster_tt_slot_wrap_retired_count() - retired_before), 1); From 192aec8b7f162a17918730e11f87d85395d15d5e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 06:22:57 +0800 Subject: [PATCH 26/38] feat(cluster): D5-8 cross-node undo read admission v2 + mixed-capability hard gate (spec-5.22e) Closes the S3.0 condition (4) hole (F-C2/F-D3): a snapshot must either be covered by the retention fold or be unable to reference foreign undo at all. JOINING only blocks writes (xact.c INV-J9), reconfig interrupts absorb xid-less read-only transactions, and snapshots follow the storage gate -- so a not-yet-admitted node COULD run cross-node reads that no owner's required set covers, and a mere at-consumption MEMBER check would re-admit RR snapshots taken during JOINING. cluster_undo_horizon_read_admission_enforce, called on the FOREIGN arms of every cross-node undo consumption entry (cluster_undo_verdict_resolve origin!=self; try_resolve_remote entry -- which also covers the single acquire_shared caller, contract-noted there): - self not MEMBER -> 53R60 (retry-safe; local reads ungated) - snapshot read_epoch below self admission epoch -> 53R60 with a new-snapshot hint (pre-join snapshots are refused FOREVER, F-D3(3)) - any MEMBER peer without the current-connection UNDO_HORIZON_V1 capability -> false (mixed-version cluster: an old-binary owner has no brake, so a new consumer must not trust any owner retention); the caller keeps its UNKNOWN/53R97 fail-closed shape, LOG-once + admission counter here. Upgrade-window contract self-enforcing on new binaries (S3.3 matrix; old-old outside guarantee scope). Admission epoch captured exactly at the cluster_membership_set_state choke (self, ->MEMBER, prev!=MEMBER): bootstrap formation, online join and rejoin all funnel through it, so a founding member's post-admission snapshots are never mis-refused after unrelated epoch bumps (L470). Snapshot identity cross-checked against the resolver read_scn; any unprovable shape refuses conservatively. HELLO capability baseline sweep: wire[36] 0x02 -> 0x06 and the capability-gate truth table gain the unconditional 0x4 bit (test_cluster_ic). Link stubs for standalone membership/reconfig unit binaries. Full unit suite + PG219 pass. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-8) --- src/backend/cluster/cluster_membership.c | 19 ++- .../cluster/cluster_runtime_visibility.c | 23 +++- src/backend/cluster/cluster_undo_gcs_grant.c | 6 + src/backend/cluster/cluster_undo_horizon_ic.c | 123 +++++++++++++++++- src/include/cluster/cluster_undo_horizon.h | 5 + src/test/cluster_unit/test_cluster_ic.c | 18 +-- .../cluster_unit/test_cluster_membership.c | 11 ++ src/test/cluster_unit/test_cluster_reconfig.c | 10 ++ 8 files changed, 201 insertions(+), 14 deletions(-) diff --git a/src/backend/cluster/cluster_membership.c b/src/backend/cluster/cluster_membership.c index 3be77e1b3d..d22beb1583 100644 --- a/src/backend/cluster/cluster_membership.c +++ b/src/backend/cluster/cluster_membership.c @@ -26,8 +26,10 @@ */ #include "postgres.h" +#include "cluster/cluster_guc.h" /* cluster_node_id (spec-5.22e D5-8) */ #include "cluster/cluster_membership.h" -#include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (quorum sub-gate) */ +#include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (quorum sub-gate) */ +#include "cluster/cluster_undo_horizon.h" /* note_self_member (spec-5.22e D5-8) */ /* * Backing store. @@ -168,9 +170,24 @@ cluster_membership_get_state(int32 node_id) void cluster_membership_set_state(int32 node_id, ClusterMembershipState state) { + ClusterMembershipState prev; + if (!node_id_in_range(node_id)) return; + prev = (ClusterMembershipState)MembershipTable->membership_state[node_id]; MembershipTable->membership_state[node_id] = (uint8)state; + + /* + * spec-5.22e D5-8: capture the exact epoch at which THIS node becomes + * MEMBER (bootstrap formation, online join and rejoin all funnel through + * this choke). The read-admission gate compares snapshot->read_epoch + * against it so pre-join snapshots can never consume foreign undo, and + * a late capture would mis-refuse legitimate post-admission snapshots + * after an unrelated epoch bump. + */ + if (node_id == cluster_node_id && state == CLUSTER_MEMBER_MEMBER + && prev != CLUSTER_MEMBER_MEMBER) + cluster_undo_horizon_note_self_member(); } /* diff --git a/src/backend/cluster/cluster_runtime_visibility.c b/src/backend/cluster/cluster_runtime_visibility.c index 32fa488bd6..dac611a131 100644 --- a/src/backend/cluster/cluster_runtime_visibility.c +++ b/src/backend/cluster/cluster_runtime_visibility.c @@ -58,6 +58,7 @@ #include "cluster/cluster_undo_gcs.h" /* cluster_undo_block_acquire_shared (D3-2) */ #include "cluster/cluster_undo_resid.h" /* cluster_undo_resid_encode (D3-2) */ #include "cluster/cluster_undo_verdict.h" /* verdict taxonomy + entry (D3-3/D3-4) */ +#include "cluster/cluster_undo_horizon.h" /* D5-8 read admission (spec-5.22e) */ /* * cluster_vis_live_authority_covers @@ -226,8 +227,8 @@ cluster_undo_block_fetch_for_visibility(int origin_node, UBA uba, char *out_page */ static bool rtvis_try_origin_verdict(int origin_node, uint32 undo_segment_id, TransactionId raw_xid, - XLogRecPtr anchor_lsn, SCN read_scn, bool authoritative, bool *out_committed, - SCN *out_commit_scn, bool *out_commit_scn_is_bound) + XLogRecPtr anchor_lsn, SCN read_scn, bool authoritative, + bool *out_committed, SCN *out_commit_scn, bool *out_commit_scn_is_bound) { ClusterGcsUndoVerdictPage verdict; ClusterLiveAuthority auth; @@ -350,6 +351,11 @@ cluster_runtime_visibility_try_resolve_remote(int origin_node, uint32 undo_segme *out_committed = false; if (out_commit_scn != NULL) *out_commit_scn = InvalidScn; + + /* spec-5.22e D5-8: inherently remote -- same admission as the verdict + * entry (53R60 on not-member/pre-join; false = mixed-cap fail-closed). */ + if (!cluster_undo_horizon_read_admission_enforce(read_scn)) + return false; if (out_commit_scn_is_bound != NULL) *out_commit_scn_is_bound = false; if (out_committed == NULL || out_commit_scn == NULL || out_commit_scn_is_bound == NULL) @@ -571,6 +577,15 @@ cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, Transactio if (!TransactionIdIsNormal(raw_xid) || origin_node < 0) return unknown; + /* + * spec-5.22e D5-8 read admission, FOREIGN arm only (own-instance reads + * below are untouched): a non-MEMBER or pre-join snapshot must not + * consult foreign undo (53R60 inside), and a mixed-capability cluster + * fails closed here (false return keeps the UNKNOWN/53R97 shape). + */ + if (origin_node != cluster_node_id && !cluster_undo_horizon_read_admission_enforce(read_scn)) + return unknown; + /* master==self: own CLOG + own durable TT authority (Q5/D3-4). */ if (origin_node == cluster_node_id) return rtvis_resolve_own_xid(raw_xid, read_scn); @@ -639,8 +654,8 @@ cluster_undo_verdict_resolve(int origin_node, uint32 undo_segment_id, Transactio /* master!=self, owner live (or data plane unarmed): CP3 S-grant + CP5 * origin verdict, byte-for-byte the pre-D4 path. */ if (cluster_runtime_visibility_try_resolve_remote(origin_node, undo_segment_id, raw_xid, - anchor_lsn, read_scn, authoritative, &committed, - &commit_scn, &is_bound)) + anchor_lsn, read_scn, authoritative, + &committed, &commit_scn, &is_bound)) return cluster_undo_verdict_from_resolve(true, committed, commit_scn, is_bound); return unknown; } diff --git a/src/backend/cluster/cluster_undo_gcs_grant.c b/src/backend/cluster/cluster_undo_gcs_grant.c index c325be780e..8c629ed6f1 100644 --- a/src/backend/cluster/cluster_undo_gcs_grant.c +++ b/src/backend/cluster/cluster_undo_gcs_grant.c @@ -53,6 +53,12 @@ * miss / DENIED / doubt returns false so the caller keeps its 53R97 * fail-closed boundary (Rule 8.A -- MVCC/visibility never forward-links a * false-visible edge). + * + * spec-5.22e D5-8 contract: every caller must pass the read-admission + * gate (cluster_undo_horizon_read_admission_enforce) BEFORE reaching this + * primitive -- the sole current caller (try_resolve_remote) gates at its + * entry. A new caller that skips admission reopens the pre-join / + * mixed-capability consumption hole. */ bool cluster_undo_block_acquire_shared(const ClusterResId *undo_resid, uint32 expected_generation, diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c index a018854e92..74b2cb2fb2 100644 --- a/src/backend/cluster/cluster_undo_horizon_ic.c +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -56,6 +56,7 @@ #include "cluster/cluster_undo_horizon.h" #include "cluster/cluster_undo_retention.h" #include "storage/shmem.h" +#include "utils/snapmgr.h" /* ActiveSnapshotSet / GetActiveSnapshot (D5-8) */ #include "utils/timestamp.h" /* @@ -90,7 +91,8 @@ typedef struct ClusterUndoHorizonShmem { pg_atomic_uint64 pass_abort_count; pg_atomic_uint64 wire_reject_count; pg_atomic_uint64 admission_refuse_count; - pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ + pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ + pg_atomic_uint64 self_admitted_epoch; /* D5-8: epoch self last became MEMBER */ } ClusterUndoHorizonShmem; static ClusterUndoHorizonShmem *UndoHorizonShmem = NULL; @@ -121,6 +123,7 @@ cluster_undo_horizon_shmem_init(void) pg_atomic_init_u64(&shmem->wire_reject_count, 0); pg_atomic_init_u64(&shmem->admission_refuse_count, 0); pg_atomic_init_u64(&shmem->last_floor_scn, 0); + pg_atomic_init_u64(&shmem->self_admitted_epoch, 0); } UndoHorizonShmem = shmem; } @@ -429,6 +432,124 @@ cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch) return false; } +/* ---------------- D5-8 read admission (consumer side) ----------------- */ + +/* + * Self admission epoch: the reconfig epoch at which THIS node last became + * MEMBER (captured by the cluster_membership_set_state choke, so every + * admission path -- bootstrap formation, online join, rejoin -- records it + * exactly). 0 = never admitted this incarnation (conservative refuse). + */ +void +cluster_undo_horizon_note_self_member(void) +{ + if (UndoHorizonShmem == NULL) + return; + pg_atomic_write_u64(&UndoHorizonShmem->self_admitted_epoch, cluster_epoch_get_current()); +} + +/* + * cluster_undo_horizon_read_admission_enforce -- D5-8 gate at every + * cross-node undo consumption entry (D3 verdict wire leg, D2-3 CP3 + * acquire, 6.12i live resolve). Own-instance reads are NOT gated (the + * callers only invoke this on their foreign arms). + * + * Refusal arms (S3.0 (4): a snapshot must either be covered by the fold + * or be unable to reference foreign undo at all): + * - self not MEMBER: a JOINING/ABSENT node's reads are legal locally + * but must not consult foreign undo -- its snapshots are not covered + * by any owner's required set. 53R60, retry-safe: succeeds once the + * join commits and a NEW snapshot is taken. + * - pre-join snapshot: snapshot->read_epoch predates this node's + * admission -- it existed before owners were required to cover us + * (F-D3(3); a mere at-consumption MEMBER check would re-admit it). + * Also 53R60 with a new-snapshot hint. + * - mixed capability: an old-binary MEMBER owner has no brake, so a + * new consumer must not trust ANY owner's retention until the whole + * cluster is capable (S3.3 matrix). Returns false; the caller keeps + * its existing UNKNOWN/53R97 fail-closed shape (LOG-once here). + * + * The epoch is read from the ACTIVE snapshot (crossnode resolution runs + * under the snapshot being evaluated; catalog snapshots are forced LOCAL + * and never reach these paths). Any shape we cannot prove -- no active + * snapshot, or its read_scn disagreeing with the resolver's -- refuses + * conservatively (never "assume admissible"). + * + * Rejoin residual note (S3.0 (3)): a joiner adopts its admitted epoch off + * the voting disk (cluster_epoch_adopt_admitted), which carries no SCN + * observe; but every foreign consumption below rides IC envelopes whose + * reply observe (cluster_ic_envelope.c) advances the clock before any + * verdict is consumed, and a pre-observe snapshot that meets an + * already-recycled slot lands in the COMMITTED_BELOW_HORIZON catch-up + * bound arm -- fail-closed, never false-visible. + */ +bool +cluster_undo_horizon_read_admission_enforce(SCN read_scn) +{ + uint64 admitted; + Snapshot snap; + int pi; + + if (cluster_node_id < 0) + return true; /* cluster off: nothing to gate */ + + if (cluster_membership_get_state(cluster_node_id) != CLUSTER_MEMBER_MEMBER) { + if (UndoHorizonShmem != NULL) + pg_atomic_fetch_add_u64(&UndoHorizonShmem->admission_refuse_count, 1); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RECONFIG_IN_PROGRESS), + errmsg("cross-node undo access refused: this node is not an admitted " + "cluster member yet"), + errhint("Retry after the node's join completes; local reads are " + "unaffected."))); + } + + admitted + = UndoHorizonShmem == NULL ? 0 : pg_atomic_read_u64(&UndoHorizonShmem->self_admitted_epoch); + snap = ActiveSnapshotSet() ? GetActiveSnapshot() : NULL; + if (admitted == 0 || snap == NULL || snap->read_epoch < admitted + || (SCN_VALID(read_scn) && SCN_VALID(snap->read_scn) && snap->read_scn != read_scn)) { + if (UndoHorizonShmem != NULL) + pg_atomic_fetch_add_u64(&UndoHorizonShmem->admission_refuse_count, 1); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RECONFIG_IN_PROGRESS), + errmsg("cross-node undo access refused: snapshot predates this node's " + "cluster admission"), + errhint("Take a new snapshot (new statement or transaction) after the " + "join completed and retry."))); + } + + /* + * Mixed-capability hard gate (Q3''/S3.3): every MEMBER peer must be + * horizon-capable on its current connection before cross-node + * consumption may trust any owner's retention. + */ + for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + if (pi == cluster_node_id) + continue; + if (cluster_conf_lookup_node(pi) == NULL) + continue; + if (cluster_membership_get_state(pi) != CLUSTER_MEMBER_MEMBER) + continue; + if (!cluster_sf_peer_supports_undo_horizon(pi)) { + static bool nocap_logged = false; + + if (UndoHorizonShmem != NULL) + pg_atomic_fetch_add_u64(&UndoHorizonShmem->admission_refuse_count, 1); + if (!nocap_logged) { + nocap_logged = true; + ereport(LOG, (errmsg("cross-node undo access fail-closed: cluster member %d " + "lacks the undo-horizon capability (mixed-version " + "cluster)", + pi), + errhint("Upgrade every node or keep cross-node consumption " + "GUCs off during the upgrade window."))); + } + return false; /* caller keeps its UNKNOWN/53R97 fail-closed shape */ + } + } + + return true; +} + /* * F-D2 epoch fence: re-verify -- at every recycle mutation point -- that the * reconfig epoch still equals the one the floor was folded at. A mid-pass diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h index c4c6703ac5..7a61b82273 100644 --- a/src/include/cluster/cluster_undo_horizon.h +++ b/src/include/cluster/cluster_undo_horizon.h @@ -189,6 +189,11 @@ extern int cluster_undo_horizon_sample_views(ClusterUndoHorizonReportView *views extern bool cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch); /* F-D2 epoch fence (recycle mutation points; injection-forceable, t/370 L6) */ extern bool cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch); +/* D5-8 read admission: self-MEMBER capture + consumer-side enforce (53R60 on + * not-member / pre-join snapshot; false = mixed-capability, caller keeps its + * UNKNOWN/53R97 fail-closed shape). Foreign arms only; own reads ungated. */ +extern void cluster_undo_horizon_note_self_member(void); +extern bool cluster_undo_horizon_read_admission_enforce(SCN read_scn); /* observability (D5-5) */ extern void cluster_undo_horizon_note_stall(void); diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index ac8b93f78f..664c6a8d87 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -582,9 +582,10 @@ UT_TEST(test_hello_wire_reference_bytes) UT_ASSERT_EQ(wire[13], 'B'); for (i = 14; i < 36; i++) UT_ASSERT_EQ(wire[i], 0); - /* capability bitmap: exactly the unconditional D4-6 authority-serve bit + /* capability bitmap: the unconditional protocol bits -- D4-6 + * authority-serve (0x2) + spec-5.22e D5-2 undo-horizon (0x4) * (smart-fusion is off in this fixture) */ - UT_ASSERT_EQ(wire[36], 0x02); + UT_ASSERT_EQ(wire[36], 0x06); UT_ASSERT_EQ(wire[37], 0x00); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); @@ -598,9 +599,9 @@ UT_TEST(test_hello_smart_fusion_capability_gate) uint8 wire[PGRAC_IC_HELLO_BYTES]; ClusterICHelloMsg parsed; - /* the spec-5.22d D4-6 authority-serve bit is unconditional, so it is - * the capability-word BASELINE in every row below; only the smart- - * fusion bit is GUC/tier-gated */ + /* the spec-5.22d D4-6 authority-serve + spec-5.22e undo-horizon bits + * are unconditional, so they are the capability-word BASELINE in every + * row below; only the smart-fusion bit is GUC/tier-gated */ cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; @@ -608,7 +609,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) "sf-off"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -617,7 +618,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) "sf-tier-mismatch"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -627,7 +628,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 - | PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1); + | PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; diff --git a/src/test/cluster_unit/test_cluster_membership.c b/src/test/cluster_unit/test_cluster_membership.c index aec4eddb1b..b41789e561 100644 --- a/src/test/cluster_unit/test_cluster_membership.c +++ b/src/test/cluster_unit/test_cluster_membership.c @@ -54,6 +54,17 @@ UT_DEFINE_GLOBALS(); +/* spec-5.22e D5-8 stub: cluster_membership_set_state notes self-admission + * into the undo horizon shmem (cluster_undo_horizon_ic.c not linked here); + * also satisfies the cluster_node_id extern via cluster_guc.o linkage or + * local definition in this binary. */ +void cluster_undo_horizon_note_self_member(void); +void +cluster_undo_horizon_note_self_member(void) +{} +int cluster_node_id = -1; /* GUC global stub (cluster_guc.o not linked) */ + + void ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) { diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..fc0ce779d6 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,16 @@ UT_DEFINE_GLOBALS(); +/* spec-5.22e D5-8 stub: cluster_membership_set_state notes self-admission + * into the undo horizon shmem (cluster_undo_horizon_ic.c not linked here); + * also satisfies the cluster_node_id extern via cluster_guc.o linkage or + * local definition in this binary. */ +void cluster_undo_horizon_note_self_member(void); +void +cluster_undo_horizon_note_self_member(void) +{} + + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. From eef221890f78e78073b99371ceb5678de2271fce Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 06:27:28 +0800 Subject: [PATCH 27/38] feat(cluster): D5-4 writer-exclusion release guard + inscription; D5-5 brake observability (spec-5.22e) D5-4: the undo path intent/ownership pairing (plain RUNTIME_SHARED iff own instance; AUTHORITY_BLOCK0 iff foreign) was Assert-only -- a release build would silently resolve a violating pair into the shared tree. It is a writer-exclusion pillar for both the D4 complete-scan prove and the D5 retention brake, so it now fails closed in production too (ereport ERROR, DATA_CORRUPTED; resolvers run outside critical sections per the 53R9D discipline). The D4 prove-core inscription is updated: spec-5.22e RULED retention stays own-instance (dead-owner segments frozen); a future cross-node reclaimer (D7+) must revise A1.1-bis first. D5-5: pg_cluster_state 'undo' category grows 8 rows -- the five brake counters (horizon_stall / horizon_peer_stale / horizon_pass_abort / horizon_wire_reject / horizon_admission_refuse), the last proven floor gauge (horizon_last_floor_scn), a per-peer report summary line (horizon_peer_reports: valid/cap/epoch/scn/age/interval/regression per declared peer), and the previously accessor-less cleaner_header_tt_slots_below_horizon inventory counter. No TAP asserts an 'undo'-category row count (key-specific asserts only), so no baseline sweep beyond the test_cluster_debug link stubs. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-4/D5-5) --- src/backend/cluster/cluster_debug.c | 23 ++++++++++- .../cluster/cluster_undo_authority_snapshot.c | 8 +++- src/backend/cluster/cluster_undo_cleaner.c | 1 + src/backend/cluster/cluster_undo_horizon_ic.c | 36 ++++++++++++++++ .../cluster/storage/cluster_undo_alloc.c | 21 ++++++++++ src/include/cluster/cluster_undo_cleaner.h | 1 + src/include/cluster/cluster_undo_horizon.h | 1 + src/test/cluster_unit/test_cluster_debug.c | 41 +++++++++++++++++++ 8 files changed, 129 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index f93307da65..f4b424e08d 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -115,7 +115,8 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_grd_work_queue.h" #include "cluster/cluster_cssd.h" /* cluster_cssd_status (spec-2.5 D12) */ #include "cluster/cluster_stats.h" /* cluster_stats_status (spec-1.14 D12) */ -#include "cluster/cluster_undo_cleaner.h" /* dump_undo_cleaner (spec-3.13 D1) */ +#include "cluster/cluster_undo_cleaner.h" +#include "cluster/cluster_undo_horizon.h" /* D5-5 brake observability (spec-5.22e) */ /* dump_undo_cleaner (spec-3.13 D1) */ #include "cluster/cluster_undo_gcs.h" /* undo GCS grant-plane counters (spec-5.22b D2-6) */ #include "cluster/cluster_lmon.h" /* cluster_lmon_status (spec-1.11 Sprint B D12) */ #include "cluster/cluster_guc.h" @@ -2477,6 +2478,26 @@ dump_undo(ReturnSetInfo *rsinfo) emit_row(rsinfo, "undo", "tt_slot_wrap_retired_count", fmt_int64((int64)cluster_tt_slot_wrap_retired_count())); + /* spec-5.22e D5-5: cluster retention brake observability (32 -> 40 rows). + * Stall/abort/reject/admission counters + the last proven floor gauge + + * the previously accessor-less below-horizon inventory counter. */ + emit_row(rsinfo, "undo", "horizon_stall_count", + fmt_int64((int64)cluster_undo_horizon_stall_count())); + emit_row(rsinfo, "undo", "horizon_peer_stale_count", + fmt_int64((int64)cluster_undo_horizon_peer_stale_count())); + emit_row(rsinfo, "undo", "horizon_pass_abort_count", + fmt_int64((int64)cluster_undo_horizon_pass_abort_count())); + emit_row(rsinfo, "undo", "horizon_wire_reject_count", + fmt_int64((int64)cluster_undo_horizon_wire_reject_count())); + emit_row(rsinfo, "undo", "horizon_admission_refuse_count", + fmt_int64((int64)cluster_undo_horizon_admission_refuse_count())); + emit_row(rsinfo, "undo", "horizon_last_floor_scn", + fmt_int64((int64)cluster_undo_horizon_last_floor())); + emit_row(rsinfo, "undo", "horizon_peer_reports", + cluster_undo_horizon_peer_reports_summary()); + emit_row(rsinfo, "undo", "cleaner_header_tt_slots_below_horizon", + fmt_int64((int64)cluster_undo_cleaner_header_tt_slots_below_horizon())); + /* spec-4.8ab D7: 4 checkpoint-writeback boundary counters (grown into the * existing undo buffer pool region -- D0 finding-3, no new region). */ emit_row(rsinfo, "undo", "undo_buf_held_wal", diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c index d1ab659b9a..a12dab8748 100644 --- a/src/backend/cluster/cluster_undo_authority_snapshot.c +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -145,8 +145,12 @@ cluster_undo_serve_authority(const ClusterResId *undo_resid, uint64 reconfig_epo * owner is dead-decided + write-fenced (its revival forces a reconfig ⇒ * epoch bump ⇒ the caller's post-scan re-check refuses); survivors' only * foreign-owner intent is AUTHORITY_BLOCK0, whose single I/O consumer is - * the read below (no writer, no unlink/rename path exists). D5's future - * reclaimer must inherit this invariant before deleting anything here. + * the read below (no writer, no unlink/rename path exists). spec-5.22e + * (D5) RULED the reclaimer question: retention stays own-instance only + * (dead-owner segments are frozen; the path_resolve ownership guard is + * now enforced in release builds too), so this invariant holds unchanged. + * A future cross-node reclaimer (D7+) must revise the A1.1-bis argument + * (serialize against authority serve) BEFORE deleting anything here. * * seg_0 is NOT skipped (A1.1-bis #3): "segment 0 never carries a xact TT * slot" is not a verified structural invariant (the uba_encode Assert only diff --git a/src/backend/cluster/cluster_undo_cleaner.c b/src/backend/cluster/cluster_undo_cleaner.c index 6d1de12fec..f924fb1a87 100644 --- a/src/backend/cluster/cluster_undo_cleaner.c +++ b/src/backend/cluster/cluster_undo_cleaner.c @@ -681,6 +681,7 @@ UNDO_CLEANER_COUNTER_ACCESSOR(pass_count) UNDO_CLEANER_COUNTER_ACCESSOR(shmem_tt_slots_gcd) UNDO_CLEANER_COUNTER_ACCESSOR(segments_marked_recyclable) UNDO_CLEANER_COUNTER_ACCESSOR(stale_active_skipped) +UNDO_CLEANER_COUNTER_ACCESSOR(header_tt_slots_below_horizon) /* spec-5.22e D5-5 */ /* ============================================================ diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c index 74b2cb2fb2..87d3092fcb 100644 --- a/src/backend/cluster/cluster_undo_horizon_ic.c +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -568,6 +568,42 @@ cluster_undo_horizon_epoch_fence_tripped(uint64 expected_epoch) return cluster_epoch_get_current() != expected_epoch; } +/* + * Per-peer report summary for the dump surface (D5-5): one static line + * "n:v=,cap=,e=,scn=,age_ms=,iv=,fl=" + * per declared peer, space-separated. Diagnostic only (torn reads are + * acceptable here; the fold uses the seqlock-protocol sampler). + */ +const char * +cluster_undo_horizon_peer_reports_summary(void) +{ + static char buf[CLUSTER_MAX_NODES * 96]; + uint64 now_us = (uint64)GetCurrentTimestamp(); + size_t off = 0; + int i; + + buf[0] = '\0'; + if (UndoHorizonShmem == NULL) + return buf; + for (i = 0; i < CLUSTER_MAX_NODES && off + 96 < sizeof(buf); i++) { + ClusterUndoHorizonSlotShmem *slot = &UndoHorizonShmem->slots[i]; + int64 age_ms; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + age_ms = slot->recv_at_us == 0 || now_us < slot->recv_at_us + ? -1 + : (int64)((now_us - slot->recv_at_us) / 1000); + off += (size_t)snprintf( + buf + off, sizeof(buf) - off, + "%sn%d:v=%d,cap=%d,e=" UINT64_FORMAT ",scn=" UINT64_FORMAT ",age_ms=%lld,iv=%u,fl=%d", + off > 0 ? " " : "", i, slot->valid ? 1 : 0, + cluster_sf_peer_supports_undo_horizon(i) ? 1 : 0, slot->epoch, slot->horizon_scn, + (long long)age_ms, slot->sender_interval_ms, slot->regression_flagged ? 1 : 0); + } + return buf; +} + /* ---------------- observability (D5-5 accessors) ---------------------- */ #define UNDO_HORIZON_NOTE(field) \ diff --git a/src/backend/cluster/storage/cluster_undo_alloc.c b/src/backend/cluster/storage/cluster_undo_alloc.c index 73f8c3c808..358168378b 100644 --- a/src/backend/cluster/storage/cluster_undo_alloc.c +++ b/src/backend/cluster/storage/cluster_undo_alloc.c @@ -115,6 +115,27 @@ cluster_undo_path_resolve(ClusterUndoPathIntent intent, uint8 owner_instance, ui || (intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0 && owner_instance != (uint8)(cluster_node_id + 1))); + /* + * spec-5.22e D5-4: the pairing above is a WRITER-EXCLUSION pillar (the + * D4 complete-scan prove and the D5 retention brake both argue "no + * foreign writer can exist in an owner's shared namespace"), so an + * Assert alone is not enough -- production builds must fail closed too + * (rule 12: Assert never substitutes for a runtime guard). Callers + * resolve paths outside critical sections (the 53R9D discipline), so + * an ERROR here is safe. + */ + if (!((((intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED) + == (owner_instance == (uint8)(cluster_node_id + 1))) + || (intent == CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0 + && owner_instance != (uint8)(cluster_node_id + 1))))) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("undo path intent/ownership violation: intent %d, owner %u, self node %d", + (int)intent, (unsigned)owner_instance, cluster_node_id), + errhint("Foreign-owner shared undo is only reachable through the dead-owner " + "authority block0 route; anything else is a split-brain bug " + "(spec-5.22e writer-exclusion guard)."))); + /* * RUNTIME_SHARED own undo migrates to the shared cluster_fs root only * under peer-mode + cluster.undo_gcs_coherence; MATERIALIZED_LOCAL and diff --git a/src/include/cluster/cluster_undo_cleaner.h b/src/include/cluster/cluster_undo_cleaner.h index 5cd77cb51a..f9024d2a0d 100644 --- a/src/include/cluster/cluster_undo_cleaner.h +++ b/src/include/cluster/cluster_undo_cleaner.h @@ -189,6 +189,7 @@ extern int64 cluster_undo_cleaner_main_loop_iters(void); /* D6 counter accessors (dump_undo + tests). */ extern uint64 cluster_undo_cleaner_pass_count(void); +extern uint64 cluster_undo_cleaner_header_tt_slots_below_horizon(void); /* spec-5.22e D5-5 */ extern uint64 cluster_undo_cleaner_shmem_tt_slots_gcd(void); extern uint64 cluster_undo_cleaner_segments_marked_recyclable(void); extern uint64 cluster_undo_cleaner_stale_active_skipped(void); diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h index 7a61b82273..1ef2c593cc 100644 --- a/src/include/cluster/cluster_undo_horizon.h +++ b/src/include/cluster/cluster_undo_horizon.h @@ -206,6 +206,7 @@ extern void cluster_undo_horizon_note_wire_reject(void); extern uint64 cluster_undo_horizon_wire_reject_count(void); extern void cluster_undo_horizon_note_admission_refuse(void); extern uint64 cluster_undo_horizon_admission_refuse_count(void); +extern const char *cluster_undo_horizon_peer_reports_summary(void); extern void cluster_undo_horizon_note_floor(SCN scn); extern SCN cluster_undo_horizon_last_floor(void); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 5b50738a69..53f338df69 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2295,6 +2295,47 @@ cluster_undo_cleaner_pass_count(void) { return 0; } +/* spec-5.22e D5-5 stubs: retention brake observability rows. */ +uint64 +cluster_undo_cleaner_header_tt_slots_below_horizon(void) +{ + return 0; +} +uint64 +cluster_undo_horizon_stall_count(void) +{ + return 0; +} +uint64 +cluster_undo_horizon_peer_stale_count(void) +{ + return 0; +} +uint64 +cluster_undo_horizon_pass_abort_count(void) +{ + return 0; +} +uint64 +cluster_undo_horizon_wire_reject_count(void) +{ + return 0; +} +uint64 +cluster_undo_horizon_admission_refuse_count(void) +{ + return 0; +} +SCN +cluster_undo_horizon_last_floor(void) +{ + return (SCN)0; +} +const char * +cluster_undo_horizon_peer_reports_summary(void) +{ + return ""; +} uint64 cluster_undo_cleaner_shmem_tt_slots_gcd(void) { From b6e37a2fe5ff5f750d611ca608030150f7bd4c8a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 10:54:35 +0800 Subject: [PATCH 28/38] fix(cluster): sf_dep capability store always exists + connection-scoped cap reset; D5-7 t/370 (WIP, substrate-gated) (spec-5.22e) Two real fixes uncovered while wiring the D5 2-node brake TAP: 1. cluster_sf_dep sized the WHOLE region to zero under the default cluster.smart_fusion=off -- but the header carries the HELLO capability store that the undo-horizon sender/fold, the D4 authority-routing gate AND smart fusion all read. A zero-size region left ClusterSfDep == NULL, so every capability query silently returned false. The header (peer_capabilities + lock + counters) now exists in every cluster build; only the dep-vector slot array stays smart-fusion-gated. (D4's peer-authority query has the same latent dependency but its 2-node e2e only covers the self-authority leg, so it never noticed.) 2. cluster_ic_tier1_close_peer now clears the peer's HELLO capabilities only when tearing down an ESTABLISHED fd (Q1' amend), scoped inside the fd guard so a failed-dial / tie-break close does not wipe a surviving connection's capabilities. t/370 (D5-7) lands as WIP: legs L2 (stall), L6 (epoch fence), L3 (member-drop), L5 (read admission) exercise machinery that works on the current one-directional-HELLO substrate, but L0/L1 (bidirectional brake) need each node to know its PEER's capability -- and the tier1 mesh only learns capabilities one-directionally (the acceptor learns the dialer's; the active dialer never receives the peer's HELLO -- "No HELLO_ACK", cluster_ic_tier1.c). Making the brake work bidirectionally requires a symmetric HELLO exchange (spec-2.2 substrate); that lands in a separate focused change (user-approved direction) before t/370 is un-WIP'd. Unit suite 163/163 + PG219 219/219 stay green (the substrate gap only surfaces in the 2-node e2e). Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-7 WIP) --- src/backend/cluster/cluster_ic_tier1.c | 22 +- src/backend/cluster/cluster_sf_dep.c | 19 +- .../t/370_cluster_5_22e_retention_brake.pl | 378 ++++++++++++++++++ 3 files changed, 408 insertions(+), 11 deletions(-) create mode 100644 src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 66e4eccb11..53a95ac866 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -1835,16 +1835,20 @@ cluster_ic_tier1_close_peer(int32 peer_id, const char *reason) if (tier1_peer_fds[peer_id] >= 0) { (void)close(tier1_peer_fds[peer_id]); tier1_peer_fds[peer_id] = -1; - } - /* - * spec-5.22e D5-2 (Q1' amend): HELLO capabilities are a property of the - * connection that carried them. Clear them in the close funnel so no - * consumer (horizon sender/fold, authority routing, smart fusion) trusts - * a stale capability across the reconnect window; the next verified - * HELLO repopulates. - */ - cluster_sf_note_peer_disconnected(peer_id); + /* + * spec-5.22e D5-2 (Q1' amend): HELLO capabilities are a property of + * the connection that carried them. Clear them when an ESTABLISHED + * fd is torn down so no consumer (horizon sender/fold, authority + * routing, smart fusion) trusts a stale capability across the + * reconnect window; the next verified HELLO repopulates. Scoped + * INSIDE the fd guard: close_peer is also called defensively for + * failed dials and duplicate-connection tie-breaks where no + * established link existed -- wiping the surviving connection's + * capabilities there would zero them with no new HELLO to renote. + */ + cluster_sf_note_peer_disconnected(peer_id); + } if (Tier1Shmem != NULL) { if (reason != NULL && reason[0] != '\0') diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index f2f113883f..78d2c70d24 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -73,9 +73,23 @@ cluster_sf_dep_shmem_size(void) { int max_entries; - if (IsBootstrapProcessingMode() || !cluster_enabled || !cluster_smart_fusion) + if (IsBootstrapProcessingMode() || !cluster_enabled) return 0; + /* + * spec-5.22e D5-2 (latent-bug fix): the header carries the HELLO + * capability store (peer_capabilities + lock), which the undo-horizon + * sender/fold, the D4 authority routing gate AND smart fusion all + * consume -- it must exist in EVERY cluster build. Sizing the whole + * region to zero under smart_fusion=off (the default) silently killed + * every capability query (ClusterSfDep == NULL reads as "no + * capability"); the D4 peer-authority gate never noticed only because + * its 2-node e2e covers the SELF-authority leg. Only the dep-vector + * slot array stays smart-fusion-gated. + */ + if (!cluster_smart_fusion) + return MAXALIGN(offsetof(ClusterSfDepShared, slots)); + max_entries = NBuffers > 0 ? NBuffers : 1; return MAXALIGN(offsetof(ClusterSfDepShared, slots) + (Size)max_entries * sizeof(ClusterSfDepSlot)); @@ -96,7 +110,8 @@ cluster_sf_dep_shmem_init(void) int i; LWLockInitialize(&ClusterSfDep->lock, LWTRANCHE_CLUSTER_SMART_FUSION); - ClusterSfDep->max_entries = NBuffers > 0 ? NBuffers : 1; + /* slots exist only under smart_fusion (see shmem_size) */ + ClusterSfDep->max_entries = cluster_smart_fusion ? (NBuffers > 0 ? NBuffers : 1) : 0; for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) ClusterSfDep->origin_durable[i] = InvalidXLogRecPtr; for (i = 0; i < CLUSTER_MAX_NODES; i++) diff --git a/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl b/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl new file mode 100644 index 0000000000..6c79d6ca45 --- /dev/null +++ b/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl @@ -0,0 +1,378 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 370_cluster_5_22e_retention_brake.pl +# spec-5.22e D5 — cluster-wide undo retention brake end-to-end on a +# 2-node ClusterPair + shared cluster_fs root (t/369 family recipe). +# +# The brake arms whenever the MEMBER set has a peer (Q3'': independent +# of the consumption GUCs), so on this pair every cleaner pass folds +# the local horizon with node1's report. Legs: +# +# L0 report plumbing: node0's dump shows node1's report valid with a +# current-connection capability (and vice versa). +# L1 brake pin/release: node1 holds a REPEATABLE READ snapshot -> +# node0's proven floor freezes at node1's pinned read_scn and the +# churn commits' TT slots stay UNrecycled (shmem GC counter +# frozen); node1's cross-node read keeps succeeding (no 53R97 +# tail-lash); release -> floor advances past the pin and the +# held-back slots recycle (counter moves). Non-vacuous by +# construction: the same churn recycles once the pin lifts. +# L2 stall + self-heal (L408): the report-drop injection (armed via +# the conf face -- the handler runs in LMON, SQL arming is +# process-local) ages node1's slot out -> +# undo_horizon_stall_count moves, "recycle stalled" LOG-once, +# recycling pauses; disarm -> reports resume, stall stops +# growing, the floor advances again. +# L6 F-D2 epoch fence (L408): the epoch-fence injection (cleaner +# process, conf face) forces the tripped arm at the first +# mutation -> undo_horizon_pass_abort_count moves and the pass +# recycles nothing; disarm -> recycling resumes. +# L3 member-drop self-heal: node1 dies (immediate) -> dead-decide + +# reconfig drops it from the required set -> after a transient +# stall window node0's floor advances again WITHOUT node1's +# reports (fold set = MEMBER only). +# L5 read admission (Q9 condition): node1 restarts and rejoins +# online. (a) during the JOINING window its cross-node read of +# node0's xids is refused 53R60 (not-member arm) and eventually +# succeeds after admission; (b) a REPEATABLE READ snapshot taken +# DURING the join window is refused 53R60 (pre-join-snapshot arm) +# even after the node is MEMBER -- only a NEW snapshot may +# consume (F-D3(3)); local reads stay ungated throughout. +# +# Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-7, §4.2) +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +sub state_val +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub state_str +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) ? $v : ''; +} + +sub churn_commits +{ + my ($node, $n) = @_; + for my $i (1 .. $n) + { + $node->safe_psql('postgres', 'INSERT INTO churn VALUES (1)'); + } +} + +# Poll until $fn->() is true; returns 1 on success, 0 on deadline. +sub poll_ok +{ + my ($fn, $secs) = @_; + my $deadline = time() + $secs; + while (time() < $deadline) + { + return 1 if $fn->(); + usleep(500_000); + } + return 0; +} + +sub arm_conf_injection +{ + my ($node, $spec) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$spec'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(1_500_000); # let LMON / cleaner take the SIGHUP +} + +sub disarm_conf_injection +{ + my ($node) = @_; + $node->safe_psql('postgres', 'ALTER SYSTEM RESET cluster.injection_points'); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(1_500_000); +} + +# ============================================================ +# Boot (t/369 recipe: shared data + fast liveness + fast cleaner). +# ============================================================ +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'spec_5_22e_brake', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 5', + 'cluster.undo_segments_max_per_instance = 256', + 'cluster.undo_segment_create_timeout_ms = 5000', + 'cluster.read_scache = on', + # fast cleaner passes so the brake/stall/fence legs observe within + # seconds (default 30s would dominate the test wall clock) + 'cluster.undo_cleaner_interval_ms = 1000', + ]); +$pair->start_pair; +usleep(2_000_000); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'boot node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'boot node1 sees node0 connected'); + +my ($node0, $node1) = ($pair->node0, $pair->node1); + +# Phantom-shared subject + churn tables, created before coherence arms +# (t/369: the first local write must form the local pg_undo tree). +for my $n ($node0, $node1) +{ + $n->safe_psql('postgres', 'CREATE TABLE s_t (id int, v int)'); + $n->safe_psql('postgres', 'CREATE TABLE churn (i int)'); +} +is( $node0->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}), + $node1->safe_psql('postgres', q{SELECT pg_relation_filepath('s_t')}), + 'boot s_t relfilepath coincidence holds (phantom-shared)'); + +# Arm the data plane + cross-node visibility on both nodes (live-owner +# consumption legs need them; the brake itself is armed by membership). +for my $n ($node0, $node1) +{ + $n->safe_psql('postgres', 'ALTER SYSTEM SET cluster.undo_gcs_coherence = on'); + $n->safe_psql('postgres', + 'ALTER SYSTEM SET cluster.crossnode_runtime_visibility = on'); + $n->safe_psql('postgres', 'SELECT pg_reload_conf()'); +} +usleep(1_000_000); + +# node0 commits the cross-node subject rows AFTER coherence armed, so the +# durable TT stamps land on the shared root (t/369 substrate). +$node0->safe_psql('postgres', + 'INSERT INTO s_t SELECT g, g * 10 FROM generate_series(1, 8) g'); + +# ============================================================ +# L0: report plumbing both ways (valid + current-connection capability). +# ============================================================ +ok( poll_ok( + sub { state_str($node0, 'undo', 'horizon_peer_reports') =~ /n1:v=1,cap=1/ }, 30), + 'L0 node0 holds a valid, capability-backed report from node1') + or diag('node0 peer reports: ' . state_str($node0, 'undo', 'horizon_peer_reports')); +ok( poll_ok( + sub { state_str($node1, 'undo', 'horizon_peer_reports') =~ /n0:v=1,cap=1/ }, 30), + 'L0 node1 holds a valid, capability-backed report from node0'); + +# ============================================================ +# L1: brake pin / release. +# ============================================================ +{ + # settle: floor proven at least once + ok( poll_ok(sub { state_val($node0, 'undo', 'horizon_last_floor_scn') > 0 }, 20), + 'L1 node0 proved a cluster floor'); + + # node1 pins a REPEATABLE READ snapshot (published read_scn floors its + # reports); the first query fixes the snapshot. + my $pin = $node1->background_psql('postgres'); + $pin->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ'); + $pin->query_safe('SELECT count(*) FROM churn'); + + # give the pin one report round to reach node0 + usleep(3_000_000); + my $floor_pinned = state_val($node0, 'undo', 'horizon_last_floor_scn'); + my $gcd_pinned = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); + + # churn: committed xacts whose commit_scn now sits ABOVE node1's pin, + # so the cleaner must NOT recycle their TT slots while the pin holds. + churn_commits($node0, 12); + usleep(4_000_000); # >= a few cleaner passes + report rounds + + my $floor_held = state_val($node0, 'undo', 'horizon_last_floor_scn'); + my $gcd_held = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); + cmp_ok($floor_held, '<=', $floor_pinned + 0, + 'L1 HARD ASSERT: proven floor froze at the pinned read_scn (no advance past the pin)'); + is($gcd_held, $gcd_pinned, + 'L1 HARD ASSERT: churn TT slots stayed unrecycled while the peer pin held'); + + # the pinned reader's cross-node consumption keeps succeeding + is($pin->query_safe('SELECT count(*) FROM s_t'), '8', + 'L1 pinned RR snapshot still reads all 8 cross-node rows (no 53R97 tail-lash)'); + + $pin->query_safe('COMMIT'); + $pin->quit; + + # release: floor advances past the pin and the held slots recycle + ok( poll_ok( + sub { + state_val($node0, 'undo', 'horizon_last_floor_scn') > $floor_held + && state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd') > $gcd_held; + }, + 30), + 'L1 release: floor advanced past the pin and the held-back slots recycled') + or diag(sprintf('floor %s -> %s, gcd %s -> %s', + $floor_held, state_val($node0, 'undo', 'horizon_last_floor_scn'), + $gcd_held, state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'))); +} + +# ============================================================ +# L2: stall + self-heal (report-drop injection; LMON side => conf face). +# ============================================================ +{ + my $stall_pre = state_val($node0, 'undo', 'horizon_stall_count'); + + arm_conf_injection($node0, 'cluster-undo-horizon-report-drop:skip'); + + # node1's slot ages out after 3 x max(interval) = ~3s; cleaner passes + # every 1s, so stalls accumulate. + ok( poll_ok( + sub { state_val($node0, 'undo', 'horizon_stall_count') > $stall_pre }, 30), + 'L2 HARD ASSERT: undo_horizon_stall_count moved under the report-drop injection'); + ok( poll_ok( + sub { + my $log = PostgreSQL::Test::Utils::slurp_file($node0->logfile); + $log =~ /recycle stalled, cluster horizon unproven/; + }, + 10), + 'L2 the stall LOG-once fired with attribution'); + + # recycling is paused: churn slots pile up un-GCed + my $gcd_stalled = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); + churn_commits($node0, 6); + usleep(3_000_000); + is(state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'), + $gcd_stalled, 'L2 recycling paused while stalled (no local fallback, Q3\'\')'); + + disarm_conf_injection($node0); + + # self-heal: fresh reports land, the floor advances, recycling resumes + ok( poll_ok( + sub { state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd') > $gcd_stalled }, + 30), + 'L2 self-heal: reports resumed and the held slots recycled'); +} + +# ============================================================ +# L6: F-D2 epoch fence (cleaner side => conf face). +# ============================================================ +{ + my $abort_pre = state_val($node0, 'undo', 'horizon_pass_abort_count'); + my $gcd_pre = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); + + arm_conf_injection($node0, 'cluster-undo-horizon-epoch-fence:skip'); + churn_commits($node0, 6); + + ok( poll_ok( + sub { state_val($node0, 'undo', 'horizon_pass_abort_count') > $abort_pre }, 30), + 'L6 HARD ASSERT: undo_horizon_pass_abort_count moved (fence tripped, pass aborted)'); + is(state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'), + $gcd_pre, 'L6 the aborted passes recycled nothing (mutation refused at the fence)'); + + disarm_conf_injection($node0); + ok( poll_ok( + sub { state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd') > $gcd_pre }, 30), + 'L6 disarm: recycling resumed past the fence'); +} + +# ============================================================ +# L3: member-drop self-heal (fold set = MEMBER only). +# ============================================================ +{ + $node1->stop('immediate'); + + # node0 dead-decides node1 (heartbeat 2s x deadband 5 = ~10s) and the + # reconfig drops it from the required set; the floor then advances + # again WITHOUT node1's reports. + churn_commits($node0, 4); + my $floor_pre = state_val($node0, 'undo', 'horizon_last_floor_scn'); + ok( poll_ok( + sub { state_val($node0, 'undo', 'horizon_last_floor_scn') > $floor_pre }, 60), + 'L3 self-heal: floor advances again after the dead member left the required set'); +} + +# ============================================================ +# L5: read admission across the online rejoin (Q9 condition). +# ============================================================ +{ + $node1->start; + + # (a) not-member arm: while node1 is JOINING its cross-node read of + # node0's committed rows must refuse 53R60; once admitted it succeeds. + # Local reads stay ungated throughout. + my $saw_refusal = 0; + my $saw_success = 0; + my $join_deadline = time() + 60; + my $joining_snap; + while (time() < $join_deadline) + { + my ($rc, $out, $err) = $node1->psql('postgres', 'SELECT count(*) FROM s_t'); + if ($rc != 0 && ($err // '') =~ /not an admitted cluster member/) + { + $saw_refusal = 1; + + # (b) seed the pre-join snapshot NOW (inside the join window): + # a REPEATABLE READ snapshot fixed before admission must stay + # refused even after the node becomes MEMBER. + if (!defined $joining_snap) + { + $joining_snap = $node1->background_psql('postgres'); + $joining_snap->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ'); + $joining_snap->query_safe('SELECT count(*) FROM churn'); # local: ungated + } + } + elsif ($rc == 0 && ($out // '') =~ /^\s*8\s*$/m) + { + $saw_success = 1; + last; + } + usleep(250_000); + } + ok($saw_refusal, + 'L5a JOINING window: cross-node read refused 53R60 (not an admitted member)'); + ok($saw_success, 'L5a post-admission: a NEW snapshot reads all 8 cross-node rows'); + + SKIP: + { + skip 'join window closed before a pre-join RR snapshot could be seeded', 2 + unless defined $joining_snap; + + # (b) pre-join snapshot arm: the RR snapshot fixed during JOINING is + # refused FOREVER for cross-node consumption (F-D3(3)) even though + # the node is MEMBER now... + my ($rc, $out, $err) = + $node1->psql('postgres', 'SELECT 1'); # keep harness responsive + my $res = eval { $joining_snap->query('SELECT count(*) FROM s_t'); }; + my $qerr = $@ // ''; + ok( !defined($res) || $res !~ /^\s*8\s*$/m + || $qerr =~ /predates this node's cluster admission/, + 'L5b pre-join RR snapshot never consumes cross-node undo (refused, not 8 rows)'); + + # ...while its LOCAL reads still work inside the same transaction. + # (The refusal above may have aborted the txn; a fresh local read on + # a new snapshot must succeed -- proving the gate is scoped to + # cross-node consumption, not to the node's reads at large.) + $joining_snap->quit; + is($node1->safe_psql('postgres', 'SELECT count(*) FROM s_t'), + '8', 'L5b a fresh post-admission snapshot consumes cross-node rows fine'); + } +} + +done_testing(); From 0601ec580a161c4095f540672d6e69ce334e12cf Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 12:47:10 +0800 Subject: [PATCH 29/38] feat(cluster): bidirectional peer capability exchange via gated PEER_CAPS_REPLY The tier1 mesh performs a one-way HELLO (dialer -> acceptor), so only the acceptor ever learned the peer's capability word; the dialer's view of every peer it dialed stayed permanently empty, starving every capability consumer on the dialing side. Add an additive, capability- gated reply so both directions learn, with zero change to the HELLO 64-byte wire format and zero change to the active handshake sequence (send HELLO -> CONNECTED, no wait state). - HELLO capability bit 0x8 (CAPS_REPLY_V1): meta capability advertised unconditionally, meaning this binary registers PEER_CAPS_REPLY and can receive it. - msg_type 37 (PEER_CAPS_REPLY, LMON-only, p2p): after verifying a dialer's HELLO the acceptor replies with its own standard 64-byte HELLO -- only when the dialer advertised the meta bit, so an old binary is never sent a msg_type it would reject by closing the connection. A missing reply just leaves the peer's capabilities UNKNOWN, which every consumer treats as fail-closed; it never triggers reconnect or membership change. - Dialer-side handler revalidates the embedded HELLO (length, parse, versions, cluster_name, sender identity) and drops + counts on any mismatch -- never a close. - Capability records become generation-bound ({bits, generation, valid}; generation = the peer's reconnect_count while the connection was established). The close funnel only invalidates the record whose generation matches the closing connection, so defensive closes of failed dials / duplicate-connection tie-breaks can no longer wipe the surviving connection's record. RDMA passes generation 0: its CM lifecycle never consumed the tier1 clear, and RDMA HELLO exchange is already bidirectional via CONNECT_REQUEST/ESTABLISHED private_data. - Epoch-recheck resend: a rejoining acceptor's one-shot reply is stamped with its still-stale epoch and gets stale-epoch dropped by the survivor with no retry anywhere. Remember per-peer that a reply was sent and at which epoch; the recv drain resends after every local epoch advance (idempotent on the receiving store, no-op at equal epochs). - Test-only GUC cluster.ic_suppress_caps_reply (PGC_SIGHUP, default off) simulates a pre-amendment binary on both sides for the rolling- upgrade compat legs. - Observability: pg_cluster_state ic/peer_capabilities (per-peer bits/gen/validity) + ic/caps_reply_reject_count. Tests: unit hello meta-bit gate + reference bytes 0x0E + ClusterSfPeerCap generation matrix; t/386 3-node directed capability matrix + old/new compat both directions + generation invalidation across reconnects (29/29); t/370 L0/L1 bidirectional capability legs now pass. Local gates: cluster_unit suite green; cluster_regress 13/13; PG regress 219/219; TAP t/012 014 072 075-079 331 337 339 386 green. Pre-existing t/369 and t/370 tail failures reproduce identically with this change stashed (A/B verified); they belong to the D5 lane. Spec: spec-2.2-interconnect-tcp-listener-lmon-phase1.md (additive amendment); spec-5.22e-undo-cluster-retention-horizon.md (D5 prereq) --- src/backend/cluster/cluster_debug.c | 21 +- src/backend/cluster/cluster_guc.c | 24 +- src/backend/cluster/cluster_ic.c | 6 + src/backend/cluster/cluster_ic_rdma.c | 9 +- src/backend/cluster/cluster_ic_tier1.c | 242 +++++++++++++++++- src/backend/cluster/cluster_lmon.c | 11 + src/backend/cluster/cluster_sf_dep.c | 121 +++++++-- src/include/cluster/cluster_guc.h | 7 + src/include/cluster/cluster_ic.h | 11 + src/include/cluster/cluster_ic_envelope.h | 10 +- src/include/cluster/cluster_ic_tier1.h | 7 + src/include/cluster/cluster_sf_dep.h | 60 ++++- .../t/386_cluster_2_2_caps_reply_matrix.pl | 241 +++++++++++++++++ src/test/cluster_unit/test_cluster_debug.c | 13 + src/test/cluster_unit/test_cluster_ic.c | 77 ++++-- src/test/cluster_unit/test_cluster_ic_mock.c | 2 + src/test/cluster_unit/test_cluster_lmon.c | 7 + src/test/cluster_unit/test_cluster_sf_dep.c | 58 +++++ 18 files changed, 870 insertions(+), 57 deletions(-) create mode 100644 src/test/cluster_tap/t/386_cluster_2_2_caps_reply_matrix.pl diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index f4b424e08d..94d41e1a43 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -113,12 +113,12 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_grd_outbound.h" #include "cluster/cluster_grd_pending.h" #include "cluster/cluster_grd_work_queue.h" -#include "cluster/cluster_cssd.h" /* cluster_cssd_status (spec-2.5 D12) */ -#include "cluster/cluster_stats.h" /* cluster_stats_status (spec-1.14 D12) */ +#include "cluster/cluster_cssd.h" /* cluster_cssd_status (spec-2.5 D12) */ +#include "cluster/cluster_stats.h" /* cluster_stats_status (spec-1.14 D12) */ #include "cluster/cluster_undo_cleaner.h" #include "cluster/cluster_undo_horizon.h" /* D5-5 brake observability (spec-5.22e) */ /* dump_undo_cleaner (spec-3.13 D1) */ -#include "cluster/cluster_undo_gcs.h" /* undo GCS grant-plane counters (spec-5.22b D2-6) */ -#include "cluster/cluster_lmon.h" /* cluster_lmon_status (spec-1.11 Sprint B D12) */ +#include "cluster/cluster_undo_gcs.h" /* undo GCS grant-plane counters (spec-5.22b D2-6) */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_status (spec-1.11 Sprint B D12) */ #include "cluster/cluster_guc.h" #include "catalog/pg_control.h" /* DBState (spec-4.3 plan dump) */ #include "cluster/cluster_recovery_plan.h" @@ -404,6 +404,16 @@ dump_ic(ReturnSetInfo *rsinfo) emit_row(rsinfo, "ic", "tier1_listener_port", fmt_int32((int32)cluster_ic_tier1_get_listener_port())); } + + /* + * spec-2.2 additive amendment (spec-5.22e D5 prereq): per-peer learned + * HELLO capability records (generation-bound) + the PEER_CAPS_REPLY + * validation-drop counter. The directed capability matrix TAP legs and + * rolling-upgrade compat legs read these. + */ + emit_row(rsinfo, "ic", "peer_capabilities", cluster_sf_peer_capabilities_summary()); + emit_row(rsinfo, "ic", "caps_reply_reject_count", + psprintf(UINT64_FORMAT, cluster_sf_caps_reply_reject_count())); } static void @@ -2493,8 +2503,7 @@ dump_undo(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_undo_horizon_admission_refuse_count())); emit_row(rsinfo, "undo", "horizon_last_floor_scn", fmt_int64((int64)cluster_undo_horizon_last_floor())); - emit_row(rsinfo, "undo", "horizon_peer_reports", - cluster_undo_horizon_peer_reports_summary()); + emit_row(rsinfo, "undo", "horizon_peer_reports", cluster_undo_horizon_peer_reports_summary()); emit_row(rsinfo, "undo", "cleaner_header_tt_slots_below_horizon", fmt_int64((int64)cluster_undo_cleaner_header_tt_slots_below_horizon())); diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 979db213f1..d1fa76b18b 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -176,7 +176,8 @@ bool cluster_smgr_user_relations = false; */ bool cluster_shared_catalog = false; int cluster_oid_lease_size = 8192; -int cluster_shmem_max_regions = 96; /* spec-5.22e: 80 -> 96 (undo horizon region; restore margin); spec-5.56: 64 -> 80 */ +int cluster_shmem_max_regions + = 96; /* spec-5.22e: 80 -> 96 (undo horizon region; restore margin); spec-5.56: 64 -> 80 */ /* spec-3.18 D1: undo block buffer pool slot count (0 = disabled). */ int cluster_undo_buffers = 2048; @@ -407,6 +408,10 @@ int cluster_interconnect_heartbeat_interval_ms = 1000; int cluster_interconnect_connect_timeout_ms = 5000; int cluster_interconnect_recv_timeout_ms = 30000; +/* spec-2.2 additive amendment (spec-5.22e D5 prereq) -- test-only + * old-binary simulation for the PEER_CAPS_REPLY capability exchange. */ +bool cluster_ic_suppress_caps_reply = false; + /* spec-2.4 D9 -- chunked framing + TCP KeepAlive tuning (PGC_POSTMASTER). */ int cluster_interconnect_payload_max_bytes = 64 * 1024 * 1024; /* 64 MB */ int cluster_interconnect_chunk_reassembly_timeout_ms = 10000; /* 10s */ @@ -2802,6 +2807,23 @@ cluster_init_guc(void) &cluster_interconnect_connect_timeout_ms, 5000, 1000, 60000, PGC_POSTMASTER, GUC_UNIT_MS, NULL, NULL, NULL); + /* + * spec-2.2 additive amendment (spec-5.22e D5 prereq): test-only + * old-binary simulation for the PEER_CAPS_REPLY capability exchange. + * With it on, this node's HELLO omits the CAPS_REPLY_V1 meta bit and + * its acceptor never sends PEER_CAPS_REPLY -- exactly the two visible + * behaviors of a binary that predates the amendment. Consumed by LMON, + * so PGC_SIGHUP (postgresql.conf + reload); a session-level SET could + * never reach the process that builds HELLOs. + */ + DefineCustomBoolVariable( + "cluster.ic_suppress_caps_reply", + gettext_noop("Test-only: simulate a pre-CAPS_REPLY binary on this node."), + gettext_noop("Suppresses the CAPS_REPLY_V1 HELLO capability bit and the " + "accept-side PEER_CAPS_REPLY send. Rolling-upgrade compat " + "tests use it; production leaves it off."), + &cluster_ic_suppress_caps_reply, false, PGC_SIGHUP, GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + DefineCustomIntVariable("cluster.interconnect_recv_timeout_ms", gettext_noop("Tier1 IC per-peer recv read deadline in milliseconds."), gettext_noop("Per-peer recv(2) read deadline (HELLO handshake + " diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index 3494f57a85..e60d37d60f 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -613,6 +613,12 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version /* spec-5.22e D5-2: same protocol-capability discipline for the undo * horizon report msg_type. */ capabilities |= PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1; + /* spec-2.2 additive amendment (spec-5.22e D5 prereq): META capability -- + * "I can receive PEER_CAPS_REPLY". Same unconditional discipline; the + * test-only suppression GUC simulates an old binary (no bit on the + * wire, so a new acceptor never sends this node a reply). */ + if (!cluster_ic_suppress_caps_reply) + capabilities |= PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); } diff --git a/src/backend/cluster/cluster_ic_rdma.c b/src/backend/cluster/cluster_ic_rdma.c index 8b2fc19ff5..be3173ece1 100644 --- a/src/backend/cluster/cluster_ic_rdma.c +++ b/src/backend/cluster/cluster_ic_rdma.c @@ -686,8 +686,13 @@ rdma_verify_private_hello(const void *data, uint8 len, int32 expected_peer, uint *out_peer_id = msg.source_node_id; if (out_lane_type != NULL) *out_lane_type = private_data->lane_type; - cluster_sf_note_peer_hello_capabilities(msg.source_node_id, - cluster_ic_hello_capabilities(&msg)); + /* spec-2.2 additive amendment: generation 0 -- the RDMA lane lifecycle + * has no tier1 reconnect_count notion and never consumed the tier1 + * close-funnel clear (pre-existing; registered tier1-only boundary). + * RDMA HELLO exchange is CM-bidirectional (both sides carry HELLO in + * private_data), so no PEER_CAPS_REPLY leg is needed here. */ + cluster_sf_note_peer_hello_capabilities_gen(msg.source_node_id, + cluster_ic_hello_capabilities(&msg), 0); if (out_reason != NULL) *out_reason = NULL; return true; diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 53a95ac866..4a80832ed0 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -73,6 +73,7 @@ #include "cluster/cluster_conf.h" #include "cluster/cluster_elog.h" +#include "cluster/cluster_epoch.h" /* caps-reply epoch recheck (spec-2.2 amendment) */ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_chunk.h" /* cluster_ic_chunk_reset_peer (spec-2.4) */ #include "cluster/cluster_ic.h" @@ -184,6 +185,22 @@ static int tier1_hello_send_remaining[CLUSTER_MAX_NODES]; static uint8 tier1_anon_hello_buf[CLUSTER_MAX_NODES][PGRAC_IC_HELLO_BYTES]; static int tier1_anon_hello_len[CLUSTER_MAX_NODES]; +/* + * spec-2.2 additive amendment (spec-5.22e D5 prereq): accept-side + * PEER_CAPS_REPLY resend state (LMON process-local, like the fd table). + * The reply sent at HELLO-verify time is stamped with this node's CURRENT + * epoch; when this node is a rejoiner still at a stale epoch, the dialer's + * envelope verify drops that one-shot frame (spec-2.4 Invariant 2) and + * nothing would ever retry -- the dialer's view of this node's + * capabilities would stay UNKNOWN until the next reconnect. So remember, + * per accepted dialer, that it wants replies and which epoch the last + * reply was stamped with; the recv drain re-sends after every epoch + * advance (idempotent on the receiving store). Steady state (epochs + * equal) sends nothing. + */ +static bool tier1_caps_reply_wanted[CLUSTER_MAX_NODES]; +static uint64 tier1_caps_reply_epoch[CLUSTER_MAX_NODES]; + /* * spec-2.4 hardening v1.0.1 F2: outbound tail buffer is now dynamic * per-peer. Lazy-palloc on first partial-write up to PGRAC_IC_PAYLOAD_MAX @@ -1291,6 +1308,184 @@ cluster_ic_tier1_continue_hello_send(int32 peer_id, int peer_fd) return true; } +/* + * tier1_maybe_send_caps_reply -- accept-side leg of the spec-2.2 additive + * capability exchange (spec-5.22e D5 prereq, B3). + * + * Called after a dialer's HELLO fully verified and its capabilities were + * noted. Sends PEER_CAPS_REPLY (payload = this node's own standard 64-byte + * HELLO) back to the dialer ONLY when the dialer's HELLO advertised + * CAPS_REPLY_V1 -- an old dialer without the bit is never sent a frame + * whose msg_type it would reject by closing the connection. The test-only + * suppression GUC additionally simulates an old acceptor (no reply even to + * a capable dialer). + * + * Fire-and-forget: a lost or failed reply only leaves the dialer's view of + * this node's capabilities UNKNOWN, which every consumer treats as + * fail-closed (D4 authority leg refuses, D5 horizon fold stalls NOCAP); + * it must never trigger transport reconnect or membership change. + */ +static void +tier1_maybe_send_caps_reply(int32 peer_id, uint32 dialer_caps) +{ + uint8 self_hello[PGRAC_IC_HELLO_BYTES]; + const char *self_cluster_name; + + if (peer_id >= 0 && peer_id < CLUSTER_MAX_NODES) + tier1_caps_reply_wanted[peer_id] = false; + + if ((dialer_caps & PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1) == 0) + return; /* old dialer: never send it an unknown msg_type */ + if (cluster_ic_suppress_caps_reply) + return; /* test-only old-acceptor simulation */ + + self_cluster_name = (ClusterConfShmem != NULL) ? ClusterConfShmem->cluster_name : ""; + cluster_ic_build_hello(self_hello, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, + cluster_node_id, self_cluster_name); + (void)cluster_ic_send_envelope(PGRAC_IC_MSG_PEER_CAPS_REPLY, peer_id, self_hello, + PGRAC_IC_HELLO_BYTES); + if (peer_id >= 0 && peer_id < CLUSTER_MAX_NODES) { + tier1_caps_reply_wanted[peer_id] = true; + tier1_caps_reply_epoch[peer_id] = cluster_epoch_get_current(); + } + elog(DEBUG1, "cluster_ic tier1 sent PEER_CAPS_REPLY to peer %d", peer_id); +} + +/* + * tier1_caps_reply_epoch_recheck -- resend leg of the capability exchange. + * + * Called from the recv drain after every successfully dispatched envelope + * from peer_id. If this node previously sent that dialer a + * PEER_CAPS_REPLY and the local epoch has advanced past the epoch the + * last reply was stamped with, the old frame may have been stale-epoch + * dropped on the dialer (this node was a rejoiner still behind) -- resend + * stamped with the current epoch. Receiving a verified envelope is + * exactly the moment this node's epoch has caught up (envelope verify + * observe-advances it), so one resend per epoch advance suffices; equal + * epochs (steady state) send nothing. + */ +static void +tier1_caps_reply_epoch_recheck(int32 peer_id) +{ + uint8 self_hello[PGRAC_IC_HELLO_BYTES]; + const char *self_cluster_name; + uint64 now_epoch; + + if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return; + if (!tier1_caps_reply_wanted[peer_id]) + return; + if (cluster_ic_suppress_caps_reply) + return; /* test-only old-acceptor simulation */ + now_epoch = cluster_epoch_get_current(); + if (now_epoch == tier1_caps_reply_epoch[peer_id]) + return; + + self_cluster_name = (ClusterConfShmem != NULL) ? ClusterConfShmem->cluster_name : ""; + cluster_ic_build_hello(self_hello, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, + cluster_node_id, self_cluster_name); + (void)cluster_ic_send_envelope(PGRAC_IC_MSG_PEER_CAPS_REPLY, peer_id, self_hello, + PGRAC_IC_HELLO_BYTES); + tier1_caps_reply_epoch[peer_id] = now_epoch; + elog(DEBUG1, "cluster_ic tier1 resent PEER_CAPS_REPLY to peer %d after epoch advance", peer_id); +} + +/* + * tier1_peer_caps_reply_handler -- dialer-side leg of the spec-2.2 additive + * capability exchange (spec-5.22e D5 prereq, B2). + * + * Envelope handler for PGRAC_IC_MSG_PEER_CAPS_REPLY. The payload is the + * acceptor's standard 64-byte HELLO; it passes the same validation core as + * a first-class HELLO (parse + version + cluster_name + sender identity) + * before its capability word is noted, bound to the CURRENT connection's + * generation. Any mismatch drops the frame with a DEBUG1 + reject counter + * -- NEVER a close/reconnect (the peer stays CONNECTED with capabilities + * UNKNOWN, the fail-closed direction). Plane/channel note: today there is + * a single CONTROL plane, so envelope arrival on this connection IS the + * channel check; a future multi-plane split (7.3) revalidates here. + */ +static void +tier1_peer_caps_reply_handler(const ClusterICEnvelope *env, const void *payload) +{ + ClusterICHelloMsg msg; + const char *self_cluster_name; + int32 sender; + + if (env == NULL || payload == NULL || env->payload_length != PGRAC_IC_HELLO_BYTES) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: bad payload length"); + return; + } + + sender = (int32)env->source_node_id; + if (sender < 0 || sender >= CLUSTER_MAX_NODES) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: sender %d out of range", sender); + return; + } + + if (!cluster_ic_parse_hello((const uint8 *)payload, &msg)) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: embedded HELLO bad magic"); + return; + } + + if (msg.hello_version != PGRAC_IC_HELLO_VERSION_V1 + || msg.envelope_version != PGRAC_IC_ENVELOPE_VERSION_V1) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: version mismatch (hello=%u env=%u)", + msg.hello_version, msg.envelope_version); + return; + } + + self_cluster_name = (ClusterConfShmem != NULL) ? ClusterConfShmem->cluster_name : ""; + if (ClusterConfShmem != NULL && strcmp(msg.cluster_name, self_cluster_name) != 0) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: cluster_name mismatch (\"%s\")", + msg.cluster_name); + return; + } + + /* the embedded HELLO must be the envelope sender's own */ + if (msg.source_node_id != sender) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: embedded node %d != sender %d", + msg.source_node_id, sender); + return; + } + + if (Tier1Shmem == NULL) { + cluster_sf_note_caps_reply_rejected(); + elog(DEBUG1, "cluster_ic tier1 PEER_CAPS_REPLY dropped: no tier1 shmem"); + return; + } + + cluster_sf_note_peer_hello_capabilities_gen(sender, cluster_ic_hello_capabilities(&msg), + Tier1Shmem->peers[sender].reconnect_count); + elog(DEBUG1, "cluster_ic tier1 learned peer %d capabilities 0x%X via PEER_CAPS_REPLY", sender, + cluster_ic_hello_capabilities(&msg)); +} + +/* + * cluster_ic_tier1_register_caps_reply_msg_type -- register the + * PEER_CAPS_REPLY envelope msg_type (LMON-only producer; p2p, never + * broadcast). Called from LMON's phase-1 msg-type registration block + * alongside the other subsystem registrations. + */ +void +cluster_ic_tier1_register_caps_reply_msg_type(void) +{ + static const ClusterICMsgTypeInfo caps_reply_info = { + .msg_type = PGRAC_IC_MSG_PEER_CAPS_REPLY, + .name = "peer_caps_reply", + .allowed_producer_mask = CLUSTER_IC_PRODUCER_LMON, + .broadcast_ok = false, + .handler = tier1_peer_caps_reply_handler, + }; + + cluster_ic_register_msg_type(&caps_reply_info); +} + bool cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) { @@ -1364,7 +1559,9 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) Tier1Shmem->peers[peer_id].state = (int32)CLUSTER_IC_PEER_CONNECTED; Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(peer_id); /* cache addr in shmem for view */ - cluster_sf_note_peer_hello_capabilities(peer_id, cluster_ic_hello_capabilities(&msg)); + cluster_sf_note_peer_hello_capabilities_gen(peer_id, cluster_ic_hello_capabilities(&msg), + Tier1Shmem->peers[peer_id].reconnect_count); + tier1_maybe_send_caps_reply(peer_id, cluster_ic_hello_capabilities(&msg)); ereport(LOG, (errmsg("cluster_ic tier1 peer %d HELLO verified, state CONNECTED", peer_id))); return true; @@ -1527,7 +1724,9 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear Tier1Shmem->peers[learned].state = (int32)CLUSTER_IC_PEER_CONNECTED; Tier1Shmem->peers[learned].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(learned); - cluster_sf_note_peer_hello_capabilities(learned, cluster_ic_hello_capabilities(&msg)); + cluster_sf_note_peer_hello_capabilities_gen(learned, cluster_ic_hello_capabilities(&msg), + Tier1Shmem->peers[learned].reconnect_count); + tier1_maybe_send_caps_reply(learned, cluster_ic_hello_capabilities(&msg)); } if (out_learned_peer_id != NULL) @@ -1814,6 +2013,14 @@ cluster_ic_tier1_recv_heartbeat_drain(int32 peer_id, int peer_fd) return false; } + /* + * spec-2.2 additive amendment: a successfully dispatched envelope + * means verify observe-advanced our epoch to at least the peer's -- + * the moment a rejoining acceptor's earlier (stale-stamped, hence + * dialer-dropped) PEER_CAPS_REPLY becomes worth resending. + */ + tier1_caps_reply_epoch_recheck(peer_id); + /* Reset phase state for next frame. */ tier1_recv_buf_len[peer_id] = 0; tier1_recv_phase[peer_id] = 0; @@ -1837,17 +2044,28 @@ cluster_ic_tier1_close_peer(int32 peer_id, const char *reason) tier1_peer_fds[peer_id] = -1; /* - * spec-5.22e D5-2 (Q1' amend): HELLO capabilities are a property of - * the connection that carried them. Clear them when an ESTABLISHED - * fd is torn down so no consumer (horizon sender/fold, authority - * routing, smart fusion) trusts a stale capability across the - * reconnect window; the next verified HELLO repopulates. Scoped - * INSIDE the fd guard: close_peer is also called defensively for - * failed dials and duplicate-connection tie-breaks where no - * established link existed -- wiping the surviving connection's - * capabilities there would zero them with no new HELLO to renote. + * spec-5.22e D5-2 (Q1' amend) + spec-2.2 additive amendment: HELLO + * capabilities are a property of the connection that carried them. + * Clear them when an ESTABLISHED fd is torn down so no consumer + * (horizon sender/fold, authority routing, smart fusion) trusts a + * stale capability across the reconnect window; the next verified + * HELLO repopulates. Scoped INSIDE the fd guard AND generation- + * matched: close_peer is also called defensively for failed dials + * and duplicate-connection tie-breaks where no established link + * existed -- wiping the surviving connection's capabilities there + * would zero them with no new HELLO to renote. The generation of + * the closing connection is the peer's reconnect_count BEFORE the + * increment below (the same value the learn sites stamped while + * this connection was established), so only the matching record is + * invalidated. */ - cluster_sf_note_peer_disconnected(peer_id); + if (Tier1Shmem != NULL) + cluster_sf_note_peer_disconnected_gen(peer_id, + Tier1Shmem->peers[peer_id].reconnect_count); + else + cluster_sf_note_peer_disconnected(peer_id); + /* caps-reply resend state is per-connection too */ + tier1_caps_reply_wanted[peer_id] = false; } if (Tier1Shmem != NULL) { diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 6449f8ac2a..67e5e657fc 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -459,6 +459,17 @@ cluster_lmon_shmem_init(void) undo_horizon_registered = true; } } + /* spec-2.2 additive amendment (spec-5.22e D5 prereq): register the + * PEER_CAPS_REPLY capability exchange (LMON-only producer; p2p, + * capability-gated on the dialer's CAPS_REPLY_V1 HELLO bit). */ + { + static bool caps_reply_registered = false; + + if (!caps_reply_registered) { + cluster_ic_tier1_register_caps_reply_msg_type(); + caps_reply_registered = true; + } + } } diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 78d2c70d24..d100d64127 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -46,7 +46,10 @@ typedef struct ClusterSfDepShared { LWLock lock; int max_entries; XLogRecPtr origin_durable[CLUSTER_SF_DEP_MAX_ORIGINS]; - uint32 peer_capabilities[CLUSTER_MAX_NODES]; + /* generation-bound per-peer HELLO capability records (spec-2.2 additive + * amendment; spec-5.22e D5 prereq) */ + ClusterSfPeerCap peer_capabilities[CLUSTER_MAX_NODES]; + pg_atomic_uint64 caps_reply_reject_count; pg_atomic_uint64 install_count; pg_atomic_uint64 touch_count; pg_atomic_uint64 dbwr_brake_count; @@ -114,8 +117,12 @@ cluster_sf_dep_shmem_init(void) ClusterSfDep->max_entries = cluster_smart_fusion ? (NBuffers > 0 ? NBuffers : 1) : 0; for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) ClusterSfDep->origin_durable[i] = InvalidXLogRecPtr; - for (i = 0; i < CLUSTER_MAX_NODES; i++) - ClusterSfDep->peer_capabilities[i] = 0; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterSfDep->peer_capabilities[i].bits = 0; + ClusterSfDep->peer_capabilities[i].generation = 0; + ClusterSfDep->peer_capabilities[i].valid = false; + } + pg_atomic_init_u64(&ClusterSfDep->caps_reply_reject_count, 0); pg_atomic_init_u64(&ClusterSfDep->install_count, 0); pg_atomic_init_u64(&ClusterSfDep->touch_count, 0); pg_atomic_init_u64(&ClusterSfDep->dbwr_brake_count, 0); @@ -146,14 +153,24 @@ cluster_sf_dep_shmem_register(void) cluster_shmem_register_region(&cluster_sf_dep_region); } +/* + * cluster_sf_note_peer_hello_capabilities_gen + * + * spec-2.2 additive amendment (spec-5.22e D5 prereq): record the peer's + * verified HELLO capability word, bound to the connection generation that + * carried it (tier1: the peer's reconnect_count while the connection was + * established). Called from the acceptor's HELLO verify, from the dialer's + * PEER_CAPS_REPLY handler, and from RDMA verify (generation 0; tier1-only + * generation boundary, see cluster_sf_dep.h). + */ void -cluster_sf_note_peer_hello_capabilities(int32 peer_id, uint32 capabilities) +cluster_sf_note_peer_hello_capabilities_gen(int32 peer_id, uint32 capabilities, uint32 generation) { if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) return; LWLockAcquire(&ClusterSfDep->lock, LW_EXCLUSIVE); - ClusterSfDep->peer_capabilities[peer_id] = capabilities; + cluster_sf_peer_cap_note(&ClusterSfDep->peer_capabilities[peer_id], capabilities, generation); LWLockRelease(&ClusterSfDep->lock); } @@ -167,7 +184,7 @@ cluster_sf_peer_supports_reply_v2(int32 peer_id) return false; LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); - capabilities = ClusterSfDep->peer_capabilities[peer_id]; + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); LWLockRelease(&ClusterSfDep->lock); return (capabilities & PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2) != 0; } @@ -189,7 +206,7 @@ cluster_peer_supports_undo_authority_serve(int32 peer_id) return false; LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); - capabilities = ClusterSfDep->peer_capabilities[peer_id]; + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); LWLockRelease(&ClusterSfDep->lock); return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1) != 0; } @@ -213,21 +230,43 @@ cluster_sf_peer_supports_undo_horizon(int32 peer_id) return false; LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); - capabilities = ClusterSfDep->peer_capabilities[peer_id]; + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); LWLockRelease(&ClusterSfDep->lock); return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1) != 0; } +/* + * cluster_sf_note_peer_disconnected_gen + * + * spec-5.22e D5-2 (Q1' amend) + spec-2.2 additive amendment: capability + * state is a property of the CONNECTION that carried the HELLO, not of the + * peer's identity. Called from the transport close funnel with the CLOSING + * connection's generation; clears the record only when the generations + * match, so a defensive close of a failed dial or of an older connection + * never wipes a record the surviving connection installed. The next + * verified HELLO repopulates. Clearing is uniformly the conservative + * direction for every existing consumer: smart-fusion falls back to v1 + * replies, the D4 authority leg fails closed, and the D5 horizon + * sender/fold skip/stall. + */ +void +cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation) +{ + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return; + + LWLockAcquire(&ClusterSfDep->lock, LW_EXCLUSIVE); + (void)cluster_sf_peer_cap_invalidate_gen(&ClusterSfDep->peer_capabilities[peer_id], generation); + LWLockRelease(&ClusterSfDep->lock); +} + /* * cluster_sf_note_peer_disconnected * - * spec-5.22e D5-2 (Q1' amend): capability state is a property of the - * CONNECTION that carried the HELLO, not of the peer's identity. Called - * from the transport close funnel; clears every advertised bit so nothing - * consumes a stale capability across a reconnect window. The next verified - * HELLO repopulates. Clearing is uniformly the conservative direction for - * every existing consumer: smart-fusion falls back to v1 replies, the D4 - * authority leg fails closed, and the D5 horizon sender/fold skip/stall. + * Unconditional clear -- the fail-closed fallback for close paths that have + * no connection-generation notion (tier1 keeps it for the Tier1Shmem==NULL + * defensive branch only). Losing a live capability record is always the + * conservative direction; trusting a stale one never is. */ void cluster_sf_note_peer_disconnected(int32 peer_id) @@ -236,10 +275,60 @@ cluster_sf_note_peer_disconnected(int32 peer_id) return; LWLockAcquire(&ClusterSfDep->lock, LW_EXCLUSIVE); - ClusterSfDep->peer_capabilities[peer_id] = 0; + ClusterSfDep->peer_capabilities[peer_id].bits = 0; + ClusterSfDep->peer_capabilities[peer_id].valid = false; LWLockRelease(&ClusterSfDep->lock); } +/* + * cluster_sf_peer_capabilities_summary + * + * Dump-surface line for pg_cluster_state ("ic" / "peer_capabilities"): + * one "n:bits=0x,gen=,v=" token per declared peer, + * space-separated. Diagnostic only; takes the store lock shared. + */ +const char * +cluster_sf_peer_capabilities_summary(void) +{ + static char buf[CLUSTER_MAX_NODES * 48]; + size_t off = 0; + int i; + + buf[0] = '\0'; + if (ClusterSfDep == NULL) + return buf; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + for (i = 0; i < CLUSTER_MAX_NODES && off + 48 < sizeof(buf); i++) { + const ClusterSfPeerCap *cap = &ClusterSfDep->peer_capabilities[i]; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + off += (size_t)snprintf(buf + off, sizeof(buf) - off, "%sn%d:bits=0x%X,gen=%u,v=%d", + off > 0 ? " " : "", i, cap->valid ? cap->bits : 0, cap->generation, + cap->valid ? 1 : 0); + } + LWLockRelease(&ClusterSfDep->lock); + return buf; +} + +/* PEER_CAPS_REPLY frames dropped by the dialer-side validation core + * (spec-2.2 additive amendment §3.4: drop + count, never reconnect). */ +void +cluster_sf_note_caps_reply_rejected(void) +{ + if (ClusterSfDep != NULL) + pg_atomic_fetch_add_u64(&ClusterSfDep->caps_reply_reject_count, 1); +} + +uint64 +cluster_sf_caps_reply_reject_count(void) +{ + if (ClusterSfDep == NULL) + return 0; + return pg_atomic_read_u64(&ClusterSfDep->caps_reply_reject_count); +} + void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload) { diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index bd975372b7..3ec21cbe10 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -570,6 +570,13 @@ extern int cluster_interconnect_heartbeat_interval_ms; extern int cluster_interconnect_connect_timeout_ms; extern int cluster_interconnect_recv_timeout_ms; +/* spec-2.2 additive amendment (spec-5.22e D5 prereq): test-only old-binary + * simulation — suppress the CAPS_REPLY_V1 HELLO meta bit AND the acceptor's + * PEER_CAPS_REPLY send on this node. Consumed by LMON (build_hello + the + * accept-side reply gate), hence PGC_SIGHUP (conf + reload), never a + * session SET. */ +extern bool cluster_ic_suppress_caps_reply; + /* spec-2.4 D9: chunked framing + TCP KeepAlive 5 GUC. */ extern int cluster_interconnect_payload_max_bytes; extern int cluster_interconnect_chunk_reassembly_timeout_ms; diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 5c3e5ee9b1..5f3de99c52 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -247,6 +247,17 @@ extern const ClusterICOps *ClusterICOps_Active; * (Q3''). Capability state is connection-bound: cleared on peer close and * only reinstated by the next HELLO (Q1' amend). */ #define PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 ((uint32)0x00000004U) +/* PGRAC: spec-2.2 additive amendment (spec-5.22e D5 prereq) — META + * capability: "this binary registers PGRAC_IC_MSG_PEER_CAPS_REPLY and can + * receive it". Advertised unconditionally (suppressible only by the + * test-only cluster.ic_suppress_caps_reply old-binary simulation GUC). + * The acceptor sends PEER_CAPS_REPLY back to a verified dialer ONLY when + * the dialer's HELLO carried this bit, so an old binary is never sent a + * frame whose msg_type it would reject by closing the connection. The + * active handshake sequence is unchanged (send HELLO -> CONNECTED, no + * wait): a missing reply just leaves the dialer's view of the peer's + * capabilities UNKNOWN, which every consumer treats as fail-closed. */ +#define PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 ((uint32)0x00000008U) typedef struct ClusterICHelloMsg { uint32 magic; /* PGRAC_IC_HELLO_MAGIC */ diff --git a/src/include/cluster/cluster_ic_envelope.h b/src/include/cluster/cluster_ic_envelope.h index 310fd50cdd..e045de5c86 100644 --- a/src/include/cluster/cluster_ic_envelope.h +++ b/src/include/cluster/cluster_ic_envelope.h @@ -243,11 +243,17 @@ typedef enum ClusterICMsgType { * metadata or fail-closed NAK reason. */ PGRAC_IC_MSG_SMART_FUSION_DURABLE = 35, /* PGRAC: spec-6.2 D8 — origin durable-LSN * gossip for Smart Fusion dependency release. */ - PGRAC_IC_MSG_UNDO_HORIZON = 36 /* PGRAC: spec-5.22e D5-2 — per-peer undo + PGRAC_IC_MSG_UNDO_HORIZON = 36, /* PGRAC: spec-5.22e D5-2 — per-peer undo * retention horizon report (LMON-only producer; p2p, never * broadcast; capability-gated on UNDO_HORIZON_V1 so an old * peer never sees an unregistered msg_type). */ - /* values 37..255 available for future sub-spec; never reuse 0..36 */ + PGRAC_IC_MSG_PEER_CAPS_REPLY = 37 /* PGRAC: spec-2.2 additive amendment + * (spec-5.22e D5 prereq) — acceptor -> dialer capability + * reply carrying the acceptor's own standard 64-byte HELLO + * as payload (LMON-only producer; p2p, never broadcast; + * capability-gated on the dialer's CAPS_REPLY_V1 HELLO bit + * so an old dialer never sees an unregistered msg_type). */ + /* values 38..255 available for future sub-spec; never reuse 0..37 */ } ClusterICMsgType; diff --git a/src/include/cluster/cluster_ic_tier1.h b/src/include/cluster/cluster_ic_tier1.h index 0bab9a7a23..2a8275b2d6 100644 --- a/src/include/cluster/cluster_ic_tier1.h +++ b/src/include/cluster/cluster_ic_tier1.h @@ -174,6 +174,13 @@ extern bool cluster_ic_tier1_finish_connect(int32 peer_id, int peer_fd); */ extern bool cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd); +/* + * spec-2.2 additive amendment (spec-5.22e D5 prereq): register the + * PEER_CAPS_REPLY msg_type + dialer-side handler (LMON-only producer; + * p2p, never broadcast). Called from LMON's phase-1 registration block. + */ +extern void cluster_ic_tier1_register_caps_reply_msg_type(void); + /* * Send a HEARTBEAT message to a CONNECTED peer (no-op for non-CONNECTED). * Updates heartbeat_send_count + last_heartbeat_sent_at on DONE. diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 848383081f..4c7cbffbe3 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -54,6 +54,50 @@ typedef struct ClusterSfDepEntry { uint64 installed_scn; } ClusterSfDepEntry; +/* + * Per-peer HELLO capability record (spec-2.2 additive amendment; spec-5.22e + * D5 prereq). Capability state is a property of the CONNECTION that carried + * the HELLO / PEER_CAPS_REPLY, so the record binds the learned bits to the + * transport connection generation (tier1: the peer's reconnect_count while + * that connection was established). A clear only applies when the caller's + * generation matches the recorded one: a defensive close of a failed dial or + * of an OLDER connection can never wipe the surviving connection's record + * (forward hazard for the 7.3 multi-channel plane). The helpers are pure so + * cluster_unit can lock the matrix; cluster_sf_dep.c wraps them in the store + * LWLock. + */ +typedef struct ClusterSfPeerCap { + uint32 bits; /* learned HELLO capability word (0 when !valid) */ + uint32 generation; /* connection generation at learn time */ + bool valid; /* record live? false reads as "no capability" */ +} ClusterSfPeerCap; + +static inline void +cluster_sf_peer_cap_note(ClusterSfPeerCap *cap, uint32 bits, uint32 generation) +{ + cap->bits = bits; + cap->generation = generation; + cap->valid = true; +} + +static inline uint32 +cluster_sf_peer_cap_bits(const ClusterSfPeerCap *cap) +{ + return cap->valid ? cap->bits : 0; +} + +/* Returns true iff the record was live for exactly this generation and got + * cleared; a mismatch (older/newer generation, already invalid) is a no-op. */ +static inline bool +cluster_sf_peer_cap_invalidate_gen(ClusterSfPeerCap *cap, uint32 generation) +{ + if (!cap->valid || cap->generation != generation) + return false; + cap->valid = false; + cap->bits = 0; + return true; +} + static inline void cluster_sf_dep_vec_reset(ClusterSfDepVec *vec) { @@ -139,7 +183,15 @@ extern int cluster_sf_dep_suspect_origin_dead(int32 origin); extern void cluster_sf_observe_origin_durable_lsn(int32 origin, XLogRecPtr durable_lsn); extern XLogRecPtr cluster_sf_observed_origin_durable_lsn(int32 origin); extern void cluster_sf_publish_origin_durable_lsn(void); -extern void cluster_sf_note_peer_hello_capabilities(int32 peer_id, uint32 capabilities); +/* spec-2.2 additive amendment (spec-5.22e D5 prereq): capability learn + + * clear are generation-bound (see ClusterSfPeerCap above). Learn sites pass + * the connection generation that carried the HELLO / PEER_CAPS_REPLY; the + * tier1 close funnel passes the closing connection's generation so only the + * matching record is invalidated. RDMA verify passes generation 0 (its + * lifecycle never consumed the tier1 clear even before this amendment; + * registered tier1-only boundary). */ +extern void cluster_sf_note_peer_hello_capabilities_gen(int32 peer_id, uint32 capabilities, + uint32 generation); extern bool cluster_sf_peer_supports_reply_v2(int32 peer_id); /* spec-5.22d D4-6: did the peer's HELLO advertise the kind-4 authority-serve * protocol capability? Deliberately NOT gated on any local GUC (unlike the @@ -147,9 +199,13 @@ extern bool cluster_sf_peer_supports_reply_v2(int32 peer_id); * the runtime arm gates live elsewhere. */ extern bool cluster_peer_supports_undo_authority_serve(int32 peer_id); /* spec-5.22e D5-2: undo-horizon report capability (connection-bound; see - * cluster_sf_note_peer_disconnected) + the close-funnel reset hook. */ + * cluster_sf_note_peer_disconnected_gen) + the close-funnel reset hooks. */ extern bool cluster_sf_peer_supports_undo_horizon(int32 peer_id); +extern void cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation); extern void cluster_sf_note_peer_disconnected(int32 peer_id); +extern const char *cluster_sf_peer_capabilities_summary(void); +extern uint64 cluster_sf_caps_reply_reject_count(void); +extern void cluster_sf_note_caps_reply_rejected(void); extern void cluster_sf_handle_durable_gossip(const ClusterICEnvelope *env, const void *payload); extern void cluster_sf_note_dep_touched(Buffer buffer); diff --git a/src/test/cluster_tap/t/386_cluster_2_2_caps_reply_matrix.pl b/src/test/cluster_tap/t/386_cluster_2_2_caps_reply_matrix.pl new file mode 100644 index 0000000000..fadb9a58d7 --- /dev/null +++ b/src/test/cluster_tap/t/386_cluster_2_2_caps_reply_matrix.pl @@ -0,0 +1,241 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 386_cluster_2_2_caps_reply_matrix.pl +# spec-2.2 additive amendment (spec-5.22e D5 prereq) — bidirectional +# peer capability exchange on a 3-node ClusterTriple. Legs: +# +# M1 directed capability matrix: every ordered pair (i,j) holds a +# valid record for its peer — acceptor legs learn from the +# dialer's HELLO, dialer legs from the capability-gated +# PEER_CAPS_REPLY (the mesh dials low-id -> high-id, so node0 +# never receives a HELLO from anyone; only the reply can teach +# it). Every record carries the CAPS_REPLY_V1 meta bit (0x8) +# and the UNDO_HORIZON_V1 bit (0x4). +# C1 new dialer -> old acceptor: node1 restarts with the test-only +# suppression GUC (simulated old binary). node0 redials and +# stays CONNECTED with zero reconnect churn, but its record for +# node1 stays UNKNOWN (v=0) — the old acceptor never replies; +# fail-closed, no transport impact. +# C2 old dialer -> new acceptor: the same suppressed node1 redials +# node2. node2 learns node1's HELLO word WITHOUT the meta bit +# (the suppressed wire really lacks 0x8) and, gated on that, +# sends no reply — node1's record for node2 stays UNKNOWN while +# the connection stays CONNECTED and quiet. +# G1 generation binding: node1's restarts invalidate the peers' +# old records during the down window (v=0, matched-generation +# clear) and the re-learned records carry a strictly higher +# generation; the validation core rejects nothing end-to-end. +# +# Spec: spec-2.2-ic-tier1-tcp.md (additive amendment); +# spec-5.22e-undo-cluster-retention-horizon.md (D5 prereq) +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/386_cluster_2_2_caps_reply_matrix.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterTriple; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +use constant CAP_UNDO_HORIZON => 0x4; +use constant CAP_CAPS_REPLY => 0x8; + +# Parse node $i's "ic"/"peer_capabilities" dump for peer $j: +# "n:bits=0x,gen=,v=<0|1>" -> (bits, gen, valid); (0, 0, 0) +# when the peer token is absent. +sub cap_rec +{ + my ($node, $peer) = @_; + my $line = $node->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='ic' AND key='peer_capabilities'}); + return (0, 0, 0) unless defined $line; + if ($line =~ /\bn$peer:bits=0x([0-9A-Fa-f]+),gen=(\d+),v=([01])\b/) + { + return (hex($1), $2 + 0, $3 + 0); + } + return (0, 0, 0); +} + +sub reject_count +{ + my ($node) = @_; + my $v = $node->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='ic' AND key='caps_reply_reject_count'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub reconnects +{ + my ($node, $peer) = @_; + my $v = $node->safe_psql('postgres', + "SELECT reconnect_count FROM pg_cluster_ic_peers WHERE node_id = $peer"); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub poll_ok +{ + my ($fn, $secs) = @_; + my $deadline = time() + $secs; + while (time() < $deadline) + { + return 1 if $fn->(); + usleep(300_000); + } + return 0; +} + +# ============================================================ +# Boot: plain 3-node transport mesh (no shared storage needed — +# capability exchange is pure tier1 substrate). +# ============================================================ +my $triple = PostgreSQL::Test::ClusterTriple->new_triple( + 'spec_2_2_caps_matrix', + extra_conf => [ 'autovacuum = off' ]); +$triple->start_triple; + +my @nodes = ($triple->node0, $triple->node1, $triple->node2); + +for my $i (0 .. 2) +{ + for my $j (0 .. 2) + { + next if $i == $j; + ok($triple->wait_for_peer_state($i, $j, 'connected', 30), + "boot node$i sees node$j connected"); + } +} + +# ============================================================ +# M1: directed capability matrix. +# ============================================================ +for my $i (0 .. 2) +{ + for my $j (0 .. 2) + { + next if $i == $j; + ok( poll_ok( + sub { + my ($bits, $gen, $v) = cap_rec($nodes[$i], $j); + $v == 1 + && ($bits & CAP_CAPS_REPLY) + && ($bits & CAP_UNDO_HORIZON); + }, + 30), + "M1 node$i holds a valid capability record for node$j (meta + horizon bits)") + or diag("node$i dump: " + . $nodes[$i]->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='ic' AND key='peer_capabilities'})); + } +} + +my (undef, $gen_n0_for_1_before, undef) = cap_rec($nodes[0], 1); + +# ============================================================ +# C1 + C2: rolling-upgrade compat — node1 becomes the "old binary". +# node0 -> node1 exercises new dialer -> old acceptor; node1 -> node2 +# exercises old dialer -> new acceptor. One restart covers both +# directions because node1 is acceptor for node0 and dialer for node2. +# ============================================================ +$nodes[1]->append_conf('postgresql.conf', 'cluster.ic_suppress_caps_reply = on'); +$nodes[1]->restart; + +ok($triple->wait_for_peer_state(0, 1, 'connected', 30), 'C1 node0 redialed old node1: connected'); +ok($triple->wait_for_peer_state(1, 2, 'connected', 30), 'C2 old node1 redialed node2: connected'); + +# C2: node2 relearns node1's HELLO word and it really lacks the meta bit +# (poll on generation-bearing validity first: the record repopulates when +# the redial's HELLO verifies). +ok( poll_ok( + sub { + my ($bits, $gen, $v) = cap_rec($nodes[2], 1); + $v == 1 && !($bits & CAP_CAPS_REPLY) && ($bits & CAP_UNDO_HORIZON); + }, + 30), + 'C2 node2 learned suppressed node1 HELLO: horizon bit present, meta bit absent') + or diag('node2 dump: ' + . $nodes[2]->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='ic' AND key='peer_capabilities'})); + +# Let a few heartbeat rounds pass, then assert the fail-closed steady state. +usleep(4_000_000); + +{ + my ($bits, $gen, $v) = cap_rec($nodes[0], 1); + is($v, 0, 'C1 node0 record for old node1 stays UNKNOWN (acceptor never replied)'); +} +{ + my ($bits, $gen, $v) = cap_rec($nodes[1], 2); + is($v, 0, 'C2 old node1 record for node2 stays UNKNOWN (reply gated on the absent meta bit)'); +} + +# Zero reconnect churn in the steady state: sample, wait 2+ heartbeat +# rounds, sample again. +{ + my $rc01 = reconnects($nodes[0], 1); + my $rc12 = reconnects($nodes[1], 2); + usleep(5_000_000); + is(reconnects($nodes[0], 1), $rc01, 'C1 no reconnect storm on the 0->1 link'); + is(reconnects($nodes[1], 2), $rc12, 'C2 no reconnect storm on the 1->2 link'); + ok($triple->wait_for_peer_state(0, 1, 'connected', 5), 'C1 0->1 still connected'); + ok($triple->wait_for_peer_state(1, 2, 'connected', 5), 'C2 1->2 still connected'); +} + +# ============================================================ +# G1: generation binding across reconnects. +# ============================================================ +$nodes[1]->append_conf('postgresql.conf', 'cluster.ic_suppress_caps_reply = off'); +$nodes[1]->restart; + +ok($triple->wait_for_peer_state(0, 1, 'connected', 30), 'G1 node0 redialed new node1: connected'); +ok($triple->wait_for_peer_state(1, 2, 'connected', 30), 'G1 new node1 redialed node2: connected'); + +ok( poll_ok( + sub { + my ($bits, $gen, $v) = cap_rec($nodes[0], 1); + $v == 1 && ($bits & CAP_CAPS_REPLY) && $gen > $gen_n0_for_1_before; + }, + 30), + 'G1 node0 relearned node1 via PEER_CAPS_REPLY with a strictly higher generation') + or diag(sprintf('gen before=%d dump now: %s', + $gen_n0_for_1_before, + $nodes[0]->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='ic' AND key='peer_capabilities'}))); + +ok( poll_ok( + sub { + my ($bits, $gen, $v) = cap_rec($nodes[1], 2); + $v == 1 && ($bits & CAP_CAPS_REPLY); + }, + 30), + 'G1 un-suppressed node1 relearned node2 via PEER_CAPS_REPLY'); + +ok( poll_ok( + sub { + my ($bits, $gen, $v) = cap_rec($nodes[2], 1); + $v == 1 && ($bits & CAP_CAPS_REPLY); + }, + 30), + 'G1 node2 relearned node1 HELLO with the meta bit back'); + +# The validation core rejected nothing anywhere, end to end. +for my $i (0 .. 2) +{ + is(reject_count($nodes[$i]), 0, "G1 node$i PEER_CAPS_REPLY reject count is zero"); +} + +$triple->stop_triple; + +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 53f338df69..4748ea7895 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2085,6 +2085,19 @@ cluster_sf_dep_retry_failclosed_count(void) { return 0; } +/* spec-2.2 additive amendment (spec-5.22e D5 prereq): the ic dump rows read + * the per-peer capability summary + PEER_CAPS_REPLY reject counter from + * cluster_sf_dep.c, which is not linked here; stub both. */ +const char * +cluster_sf_peer_capabilities_summary(void) +{ + return ""; +} +uint64 +cluster_sf_caps_reply_reject_count(void) +{ + return 0; +} /* spec-4.8: tt_recovery counter accessors (cluster_tt_durable_stat.c) are not * linked here; stub the 8 accessors the tt_recovery dump rows read. */ uint64 diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 664c6a8d87..baa55dcd1a 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -85,6 +85,9 @@ int cluster_node_id = -1; int cluster_interconnect_tier = 0; /* CLUSTER_IC_TIER_STUB */ bool cluster_smart_fusion = false; int cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; +/* spec-2.2 additive amendment (spec-5.22e D5 prereq): test-only old-binary + * simulation gate consumed by cluster_ic.c::cluster_ic_build_hello. */ +bool cluster_ic_suppress_caps_reply = false; /* spec-5.59 D1 stubs: cluster_ic.o now carries GUC-gated profiling probes * (cluster_xnode_profile.h); the unit harness links neither cluster_guc.o @@ -548,11 +551,13 @@ UT_TEST(test_hello_wire_reference_bytes) * Bytes 8-11: 04 03 02 01 source_node_id = 0x01020304 (LE) * Bytes 12-13: 41 42 "AB" * Bytes 14-35: 00..00 cluster_name NUL pad - * Bytes 36-39: 02 00 00 00 capability bitmap (LE): the - * spec-5.22d D4-6 UNDO_AUTHORITY_ - * SERVE_V1 bit is a PROTOCOL - * capability, advertised - * unconditionally by this binary + * Bytes 36-39: 0E 00 00 00 capability bitmap (LE): the + * PROTOCOL capabilities advertised + * unconditionally by this binary -- + * D4-6 authority-serve (0x2), D5-2 + * undo-horizon (0x4) and the + * spec-2.2 additive-amendment + * CAPS_REPLY_V1 meta bit (0x8) * Bytes 40-63: 00..00 _pad (must be zero) * * Locking these exact bytes guards against compiler-pad drift, @@ -583,9 +588,9 @@ UT_TEST(test_hello_wire_reference_bytes) for (i = 14; i < 36; i++) UT_ASSERT_EQ(wire[i], 0); /* capability bitmap: the unconditional protocol bits -- D4-6 - * authority-serve (0x2) + spec-5.22e D5-2 undo-horizon (0x4) - * (smart-fusion is off in this fixture) */ - UT_ASSERT_EQ(wire[36], 0x06); + * authority-serve (0x2) + spec-5.22e D5-2 undo-horizon (0x4) + + * CAPS_REPLY_V1 meta bit (0x8) (smart-fusion is off in this fixture) */ + UT_ASSERT_EQ(wire[36], 0x0E); UT_ASSERT_EQ(wire[37], 0x00); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); @@ -599,17 +604,19 @@ UT_TEST(test_hello_smart_fusion_capability_gate) uint8 wire[PGRAC_IC_HELLO_BYTES]; ClusterICHelloMsg parsed; - /* the spec-5.22d D4-6 authority-serve + spec-5.22e undo-horizon bits - * are unconditional, so they are the capability-word BASELINE in every - * row below; only the smart-fusion bit is GUC/tier-gated */ + /* the spec-5.22d D4-6 authority-serve + spec-5.22e undo-horizon + + * CAPS_REPLY_V1 meta bits are unconditional, so they are the + * capability-word BASELINE in every row below; only the smart-fusion + * bit is GUC/tier-gated */ cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_3; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-off"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 + | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -617,8 +624,9 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-tier-mismatch"); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 + | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -629,13 +637,49 @@ UT_TEST(test_hello_smart_fusion_capability_gate) UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 | PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 - | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1); + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; } +/* + * spec-2.2 additive amendment (spec-5.22e D5 prereq, B1): the CAPS_REPLY_V1 + * meta bit ("I can receive PEER_CAPS_REPLY") is advertised unconditionally, + * UNLESS the test-only old-binary simulation GUC suppresses it. Old-binary + * compat rests on this bit: an acceptor only sends PEER_CAPS_REPLY to a + * dialer whose HELLO carried the bit, so a peer without it (an actual old + * binary, or a node simulating one) is never sent a frame it cannot parse. + */ +UT_TEST(test_hello_caps_reply_meta_gate) +{ + uint8 wire[PGRAC_IC_HELLO_BYTES]; + ClusterICHelloMsg parsed; + + /* wire id + bit value are frozen protocol constants */ + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1, (uint32)0x00000008U); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PEER_CAPS_REPLY, 37); + + /* default: meta bit advertised */ + cluster_ic_suppress_caps_reply = false; + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, + "caps-on"); + UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); + UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1) != 0); + + /* suppressed (old-binary simulation): meta bit absent, the other + * protocol bits untouched */ + cluster_ic_suppress_caps_reply = true; + cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, + "caps-off"); + UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); + UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1) == 0); + UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1) != 0); + + cluster_ic_suppress_caps_reply = false; +} + UT_TEST(test_hello_parse_rejects_bad_magic) { uint8 wire[PGRAC_IC_HELLO_BYTES]; @@ -695,6 +739,7 @@ main(void) UT_RUN(test_hello_wire_roundtrip); UT_RUN(test_hello_wire_reference_bytes); UT_RUN(test_hello_smart_fusion_capability_gate); + UT_RUN(test_hello_caps_reply_meta_gate); UT_RUN(test_hello_parse_rejects_bad_magic); UT_RUN(test_hello_build_truncates_long_name); UT_DONE(); diff --git a/src/test/cluster_unit/test_cluster_ic_mock.c b/src/test/cluster_unit/test_cluster_ic_mock.c index 3c15a6b388..c94b813058 100644 --- a/src/test/cluster_unit/test_cluster_ic_mock.c +++ b/src/test/cluster_unit/test_cluster_ic_mock.c @@ -69,6 +69,8 @@ int cluster_node_id = -1; int cluster_interconnect_tier = 0; bool cluster_smart_fusion = false; int cluster_smart_fusion_tier_min = CLUSTER_IC_TIER_3; +/* spec-2.2 additive amendment (spec-5.22e D5 prereq): build_hello gate. */ +bool cluster_ic_suppress_caps_reply = false; /* spec-5.59 D6 stubs: cluster_ic.o now carries GUC-gated profiling probes * (cluster_xnode_profile.h); the unit harness links neither cluster_guc.o diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index 9b2d383e7c..4fd5359611 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -503,6 +503,13 @@ void cluster_sf_dep_register_ic_msg_types(void) {} +/* spec-2.2 additive amendment (spec-5.22e D5 prereq) stub: + * cluster_lmon_shmem_init registers the PEER_CAPS_REPLY msg type; this + * standalone unit binary does not link cluster_ic_tier1.o. */ +void +cluster_ic_tier1_register_caps_reply_msg_type(void) +{} + /* spec-2.2 D5 LMON drive references cluster_conf_lookup_node + cluster_node_id. */ const struct ClusterNodeInfo * cluster_conf_lookup_node(int32 node_id pg_attribute_unused()) diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index e410d165d6..fefb3f6050 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -162,6 +162,62 @@ UT_TEST(test_gcs_block_reply_v2_dep_extract_accepts_empty_v2_no_dep) UT_ASSERT(cluster_sf_dep_vec_is_empty(&vec)); } +/* + * spec-2.2 additive amendment (spec-5.22e D5 prereq, B4): the per-peer HELLO + * capability record is generation-bound. A clear only applies when the + * caller's connection generation matches the generation recorded at learn + * time, so a defensive close of a failed dial or of an OLDER connection can + * never wipe the surviving connection's capability record. + */ +UT_TEST(test_peer_cap_gen_note_query_invalidate) +{ + ClusterSfPeerCap cap; + + memset(&cap, 0, sizeof(cap)); + + /* an unset record reads as "no capability" (UNKNOWN) */ + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0); + + /* note stamps bits + generation and turns the record valid */ + cluster_sf_peer_cap_note(&cap, (uint32)0x0E, (uint32)3); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x0E); + + /* mismatched-generation invalidate must NOT clear the record */ + UT_ASSERT(!cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)2)); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x0E); + UT_ASSERT(!cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)4)); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x0E); + + /* matching generation clears exactly once */ + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)3)); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0); + UT_ASSERT(!cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)3)); +} + +UT_TEST(test_peer_cap_gen_renote_after_reconnect) +{ + ClusterSfPeerCap cap; + + memset(&cap, 0, sizeof(cap)); + + /* connection gen 3 learns, closes (matched clear), gen 4 relearns */ + cluster_sf_peer_cap_note(&cap, (uint32)0x0E, (uint32)3); + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)3)); + cluster_sf_peer_cap_note(&cap, (uint32)0x04, (uint32)4); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x04); + + /* a straggler clear for the OLD generation must not touch gen 4 */ + UT_ASSERT(!cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)3)); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x04); + + /* same-generation re-note (HELLO then CAPS_REPLY on one connection) + * is a plain overwrite, not an error */ + cluster_sf_peer_cap_note(&cap, (uint32)0x0E, (uint32)4); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0x0E); + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, (uint32)4)); + UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0); +} + int main(void) { @@ -172,5 +228,7 @@ main(void) UT_RUN(test_gcs_block_reply_v2_dep_extract_valid); UT_RUN(test_gcs_block_reply_v2_dep_extract_rejects_malformed); UT_RUN(test_gcs_block_reply_v2_dep_extract_accepts_empty_v2_no_dep); + UT_RUN(test_peer_cap_gen_note_query_invalidate); + UT_RUN(test_peer_cap_gen_renote_after_reconnect); UT_DONE(); } From 7d99cc5005e1d32cbe3c32a14993f7dafe42c943 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:18:40 +0800 Subject: [PATCH 30/38] fix(cluster): bias the undo-horizon self-admission epoch so cold formation is not the never-admitted sentinel The D5-8 read-admission gate stored the epoch at which this node last became MEMBER directly, with 0 doubling as the never-admitted sentinel. Cold formation admits at the initial epoch -- which IS 0 -- so on any freshly formed cluster the gate conflated an admitted member with a never-admitted node and refused every cross-node undo consumption with "snapshot predates this node's cluster admission" (all seven t/369 dead-owner authority-serve legs regressed: the positive serve leg, both 53R97 fail-closed shape asserts, their prove counters, and the unarmed control leg; t/370's post-rejoin admission leg failed the same way). Store (epoch + 1) instead: 0 stays the conservative never-admitted refusal, N+1 means admitted at epoch N, and the pre-join snapshot check compares read_epoch against the unbiased value. Verified: t/369 27/27 green again; t/370 L5 admission legs green. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-8) --- src/backend/cluster/cluster_undo_horizon_ic.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c index 87d3092fcb..1b375d8240 100644 --- a/src/backend/cluster/cluster_undo_horizon_ic.c +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -92,7 +92,11 @@ typedef struct ClusterUndoHorizonShmem { pg_atomic_uint64 wire_reject_count; pg_atomic_uint64 admission_refuse_count; pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ - pg_atomic_uint64 self_admitted_epoch; /* D5-8: epoch self last became MEMBER */ + pg_atomic_uint64 self_admitted_epoch; /* D5-8: (epoch self last became + * MEMBER) + 1; 0 = never admitted. + * Biased so a cold-formation + * admission at epoch 0 is distinct + * from the never-admitted sentinel. */ } ClusterUndoHorizonShmem; static ClusterUndoHorizonShmem *UndoHorizonShmem = NULL; @@ -438,14 +442,19 @@ cluster_undo_horizon_required_members(uint8 *required, uint64 *out_epoch) * Self admission epoch: the reconfig epoch at which THIS node last became * MEMBER (captured by the cluster_membership_set_state choke, so every * admission path -- bootstrap formation, online join, rejoin -- records it - * exactly). 0 = never admitted this incarnation (conservative refuse). + * exactly). Stored BIASED BY ONE: 0 = never admitted this incarnation + * (conservative refuse), N+1 = admitted at epoch N. Cold formation admits + * at CLUSTER_EPOCH_INITIAL (= 0), so the raw epoch value cannot double as + * the sentinel -- conflating them refused every cross-node read on a + * freshly formed cluster with "snapshot predates admission" (t/369 L1-L3b + * regression driver). */ void cluster_undo_horizon_note_self_member(void) { if (UndoHorizonShmem == NULL) return; - pg_atomic_write_u64(&UndoHorizonShmem->self_admitted_epoch, cluster_epoch_get_current()); + pg_atomic_write_u64(&UndoHorizonShmem->self_admitted_epoch, cluster_epoch_get_current() + 1); } /* @@ -503,10 +512,11 @@ cluster_undo_horizon_read_admission_enforce(SCN read_scn) "unaffected."))); } + /* biased store: 0 = never admitted, N+1 = admitted at epoch N */ admitted = UndoHorizonShmem == NULL ? 0 : pg_atomic_read_u64(&UndoHorizonShmem->self_admitted_epoch); snap = ActiveSnapshotSet() ? GetActiveSnapshot() : NULL; - if (admitted == 0 || snap == NULL || snap->read_epoch < admitted + if (admitted == 0 || snap == NULL || snap->read_epoch < admitted - 1 || (SCN_VALID(read_scn) && SCN_VALID(snap->read_scn) && snap->read_scn != read_scn)) { if (UndoHorizonShmem != NULL) pg_atomic_fetch_add_u64(&UndoHorizonShmem->admission_refuse_count, 1); From 0bf5bc027e567f419cd74242526bcaff37ee01a9 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:18:55 +0800 Subject: [PATCH 31/38] test(cluster): un-WIP t/370 retention-brake tail legs (22/22 green) Three test-side defects kept the L2-L6 tail red after the substrate and admission fixes landed: - Injection disarm used a bare ALTER SYSTEM RESET, but colon-typed arms (':skip') survive the not-in-list reclaim -- the report-drop stayed armed in LMON forever, so the L2 self-heal leg (and everything after it) only recovered when L3 removed node1 from the required set ~100s later. Disarm now re-arms the point to ':none' explicitly before clearing the GUC (t/338 precedent). - L3 churned before sampling floor_pre while the dead peer's report was still inside its freshness window, so the sampled floor already contained the churn-advanced clock and the post-drop self-only fold pinned at exactly floor_pre forever. Sample first, churn after. - Floor comparisons used raw gauge values, but the SCN wire encoding is (8-bit node_id || 56-bit local_scn) and the fold's scn_time_cmp min flips between node encodings across samples -- numeric comparison across encodings is meaningless (and the L1 legs passed only while the fold happened to stay on one encoding). Compare the 56-bit time component via the new scn_time() helper. Spec: spec-5.22e-undo-cluster-retention-horizon.md (D5-7) --- .../t/370_cluster_5_22e_retention_brake.pl | 59 +++++++++++++++---- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl b/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl index 6c79d6ca45..3ade2d51d5 100644 --- a/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl +++ b/src/test/cluster_tap/t/370_cluster_5_22e_retention_brake.pl @@ -100,6 +100,19 @@ sub poll_ok return 0; } +# SCN wire encoding is (8-bit node_id || 56-bit local_scn), so the raw +# horizon_last_floor_scn gauge value carries the pinning node's id in the +# top byte (diagnostic feature). Cross-sample floor comparisons must use +# the 56-bit TIME component only -- the fold's scn_time_cmp min can flip +# between node encodings across samples, and a raw numeric compare across +# encodings is meaningless (node1-encoded values are always numerically +# larger than node0-encoded ones). +sub scn_time +{ + my ($v) = @_; + return $v & 0x00FFFFFFFFFFFFFF; +} + sub arm_conf_injection { my ($node, $spec) = @_; @@ -110,10 +123,18 @@ sub arm_conf_injection sub disarm_conf_injection { - my ($node) = @_; - $node->safe_psql('postgres', 'ALTER SYSTEM RESET cluster.injection_points'); + # L477: a colon-typed arm (':skip' etc.) survives the not-in-list + # reclaim of a bare RESET (only WARNING arms are reclaimed), so the + # disarm must explicitly re-arm the point to ':none' in every process + # (t/338 precedent) before clearing the GUC text. + my ($node, $point) = @_; + $node->safe_psql('postgres', + "ALTER SYSTEM SET cluster.injection_points = '$point:none'"); $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); usleep(1_500_000); + $node->safe_psql('postgres', 'ALTER SYSTEM RESET cluster.injection_points'); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(500_000); } # ============================================================ @@ -198,7 +219,7 @@ sub disarm_conf_injection # give the pin one report round to reach node0 usleep(3_000_000); - my $floor_pinned = state_val($node0, 'undo', 'horizon_last_floor_scn'); + my $floor_pinned = scn_time(state_val($node0, 'undo', 'horizon_last_floor_scn')); my $gcd_pinned = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); # churn: committed xacts whose commit_scn now sits ABOVE node1's pin, @@ -206,7 +227,7 @@ sub disarm_conf_injection churn_commits($node0, 12); usleep(4_000_000); # >= a few cleaner passes + report rounds - my $floor_held = state_val($node0, 'undo', 'horizon_last_floor_scn'); + my $floor_held = scn_time(state_val($node0, 'undo', 'horizon_last_floor_scn')); my $gcd_held = state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'); cmp_ok($floor_held, '<=', $floor_pinned + 0, 'L1 HARD ASSERT: proven floor froze at the pinned read_scn (no advance past the pin)'); @@ -223,7 +244,7 @@ sub disarm_conf_injection # release: floor advances past the pin and the held slots recycle ok( poll_ok( sub { - state_val($node0, 'undo', 'horizon_last_floor_scn') > $floor_held + scn_time(state_val($node0, 'undo', 'horizon_last_floor_scn')) > $floor_held && state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd') > $gcd_held; }, 30), @@ -261,7 +282,7 @@ sub disarm_conf_injection is(state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'), $gcd_stalled, 'L2 recycling paused while stalled (no local fallback, Q3\'\')'); - disarm_conf_injection($node0); + disarm_conf_injection($node0, 'cluster-undo-horizon-report-drop'); # self-heal: fresh reports land, the floor advances, recycling resumes ok( poll_ok( @@ -286,7 +307,7 @@ sub disarm_conf_injection is(state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd'), $gcd_pre, 'L6 the aborted passes recycled nothing (mutation refused at the fence)'); - disarm_conf_injection($node0); + disarm_conf_injection($node0, 'cluster-undo-horizon-epoch-fence'); ok( poll_ok( sub { state_val($node0, 'undo', 'cleaner_shmem_tt_slots_gcd') > $gcd_pre }, 30), 'L6 disarm: recycling resumed past the fence'); @@ -301,11 +322,29 @@ sub disarm_conf_injection # node0 dead-decides node1 (heartbeat 2s x deadband 5 = ~10s) and the # reconfig drops it from the required set; the floor then advances # again WITHOUT node1's reports. + # + # Sample floor_pre BEFORE the churn: node1's last report stays fresh + # for ~3s after the kill, so a fold sampled after the churn would + # already contain the churn-advanced clock -- and with no commits + # after that the self-only fold pins at exactly floor_pre forever + # (the no-reader local-horizon fallback is the static Lamport clock). + # Churning after the sample guarantees the post-drop fold lands + # strictly above it. + my $floor_pre = scn_time(state_val($node0, 'undo', 'horizon_last_floor_scn')); churn_commits($node0, 4); - my $floor_pre = state_val($node0, 'undo', 'horizon_last_floor_scn'); ok( poll_ok( - sub { state_val($node0, 'undo', 'horizon_last_floor_scn') > $floor_pre }, 60), - 'L3 self-heal: floor advances again after the dead member left the required set'); + sub { + scn_time(state_val($node0, 'undo', 'horizon_last_floor_scn')) > $floor_pre; + }, + 60), + 'L3 self-heal: floor advances again after the dead member left the required set') + or diag(sprintf( + 'floor_pre=%s floor_now=%s local_horizon=%s stall=%s reports=%s', + $floor_pre, + state_val($node0, 'undo', 'horizon_last_floor_scn'), + state_val($node0, 'undo', 'retention_horizon_scn'), + state_val($node0, 'undo', 'horizon_stall_count'), + state_str($node0, 'undo', 'horizon_peer_reports'))); } # ============================================================ From 65176890b7b47e82af4c4ba41cdc96edc800731f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 18:54:06 +0800 Subject: [PATCH 32/38] fix(cluster): fail closed on a truncated dead-owner undo enumeration authority_scan_owner_segments detected a mid-enumeration directory read break via errno after ReadDirExtended(dir, path, LOG). But ReadDirExtended calls ereport(LOG) before returning NULL on a readdir error, and the LOG output path can clobber errno -- so an EIO-truncated scan could read errno == 0, be judged "complete", and serve a definite (false-unique) verdict on a partial segment set. PostgreSQL's own ReadDirExtended comment notes a LOG-level error looks like end-of-directory, tolerable only for best-effort cleanup scans. Switch to ReadDir (ERROR elevel) so a readdir break throws instead of masquerading as EOF, wrap the enumeration loop in PG_TRY/PG_CATCH that folds any throw into the existing scan_incomplete coverage failure, and move FreeDir past PG_END_TRY so the AllocateDir handle is freed on every path (no leak on the LMS drain's error-swallowing envelope). Both the self reader leg and the wire LMS leg now fail closed identically; drop the unreliable errno probe. New injection point cluster-undo-authority-scan models the readdir fault; t/369 L3c arms it and pins the scan-incomplete fail-closed with a counter delta (53R97), distinct from the head refusal. Injection registry 167 -> 168. Spec: spec-5.22d-undo-dead-owner-verdict-serve.md (Hardening v1.0.1) --- src/backend/cluster/cluster_inject.c | 15 ++ .../cluster/cluster_undo_authority_snapshot.c | 138 +++++++++++------- src/test/cluster_tap/t/015_inject.pl | 8 +- src/test/cluster_tap/t/017_debug.pl | 8 +- src/test/cluster_tap/t/020_shmem_registry.pl | 4 +- src/test/cluster_tap/t/021_block_format.pl | 4 +- src/test/cluster_tap/t/022_itl_slot.pl | 4 +- .../cluster_tap/t/023_buffer_descriptor.pl | 4 +- src/test/cluster_tap/t/030_acceptance.pl | 6 +- ...luster_5_22d_dead_owner_authority_serve.pl | 26 ++++ 10 files changed, 147 insertions(+), 70 deletions(-) diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index cc34d8b2ba..e1b2f53ade 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -421,6 +421,21 @@ static ClusterInjectPoint cluster_injection_points[] = { * counter delta (L408), never a native CLOG answer. */ { .name = "cluster-undo-authority-block0-prove" }, + /* + * spec-5.22d Hardening — dead-owner authority durable-segment ENUMERATION + * fault injection. + * + * cluster-undo-authority-scan: + * Fires inside authority_scan_owner_segments' enumeration loop, once + * per directory entry, modelling a mid-scan directory read failure + * (ReadDir throws at the ERROR elevel). ERROR arms the throw; the + * scan's PG_TRY folds it into a coverage failure so the prove fails + * closed via undo_authority_scan_incomplete_reject -- never a truncated + * "complete" scan that could serve a false-unique verdict. TAP L3c + * pins the scan-incomplete arm with a counter delta (53R97), proving an + * un-completable enumeration is refused, not swallowed. + */ + { .name = "cluster-undo-authority-scan" }, /* * spec-5.22e D5-7 — undo horizon report drop injection (TAP L2). * diff --git a/src/backend/cluster/cluster_undo_authority_snapshot.c b/src/backend/cluster/cluster_undo_authority_snapshot.c index a12dab8748..60e5699692 100644 --- a/src/backend/cluster/cluster_undo_authority_snapshot.c +++ b/src/backend/cluster/cluster_undo_authority_snapshot.c @@ -169,7 +169,8 @@ authority_scan_owner_segments(int32 owner_node, TransactionId raw_xid, DIR *dir; struct dirent *de; uint8 owner_instance = (uint8)(owner_node + 1); - bool complete = true; + volatile bool complete = true; + MemoryContext scan_ctx = CurrentMemoryContext; if (cluster_shared_fs_undo_instance_dir_resolve(owner_instance, dirpath, sizeof(dirpath)) != 0) return false; @@ -178,62 +179,97 @@ authority_scan_owner_segments(int32 owner_node, TransactionId raw_xid, if (dir == NULL) return false; - errno = 0; - while ((de = ReadDirExtended(dir, dirpath, LOG)) != NULL) { - char canon[64]; - unsigned long id; - char *end; - PGAlignedBlock page; - int nmatch = 0; - ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; - SCN scn = InvalidScn; - uint16 wrap = TT_WRAP_INVALID; + /* + * 完备-或-fail-closed enumeration (spec-5.22d Hardening, errno-fragility): + * ReadDir uses the ERROR elevel, so a directory read that breaks + * mid-enumeration THROWS rather than returning NULL indistinguishably from + * end-of-directory. The pre-fix errno-after-loop probe was unreliable -- + * ereport can clobber errno on the LOG path, so an EIO-truncated scan could + * be judged "complete" and serve a false-unique verdict. The PG_TRY folds + * any throw into the same coverage failure the parse/read paths below + * already signal, so BOTH the SELF reader leg and the LMS wire leg fail + * closed identically (Rule 8.A); FreeDir runs on every path (no AllocateDir + * leak on the wire leg's error-swallowing drain envelope). + */ + PG_TRY(); + { + while ((de = ReadDir(dir, dirpath)) != NULL) { + char canon[64]; + unsigned long id; + char *end; + PGAlignedBlock page; + int nmatch = 0; + ClusterVisTtProof proof = CLUSTER_VIS_TT_PROOF_NONE; + SCN scn = InvalidScn; + uint16 wrap = TT_WRAP_INVALID; - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) { + /* + * spec-5.22d Hardening (errno-fragility): model a mid-enumeration + * directory read failure. Armed ERROR throws here exactly as a real + * ReadDir I/O error would; the enclosing PG_TRY folds it into a + * coverage failure, never a truncated "complete" scan. + */ + CLUSTER_INJECTION_POINT("cluster-undo-authority-scan"); + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + + /* + * Canonical name gate (A1.1-bis #2): every other entry must be + * EXACTLY seg_.dat as the writer renders it — parse, then + * re-render and compare, so aliases (seg_007.dat), trailing junk + * and out-of-range ids all poison the set instead of hiding + * evidence under a name the scan would skip. + */ + if (strncmp(de->d_name, "seg_", 4) != 0) { + complete = false; + break; + } errno = 0; - continue; - } + id = strtoul(de->d_name + 4, &end, 10); + if (errno != 0 || end == de->d_name + 4 || strcmp(end, ".dat") != 0 + || id > UINT16_MAX) { + complete = false; + break; + } + snprintf(canon, sizeof(canon), "seg_%lu.dat", id); + if (strcmp(canon, de->d_name) != 0) { + complete = false; + break; + } + /* Read + parse this segment's block0; any failure poisons the set. */ + if (!cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, + (uint32)id, owner_instance, 0 /* block0 */, + page.data)) { + complete = false; + break; + } + if (cluster_vis_tt_block_xid_scan(page.data, (uint32)id, owner_instance, raw_xid, + &nmatch, &proof, &scn, &wrap) + != CLUSTER_VIS_TT_BLOCK_SCAN_OK) { + complete = false; + break; + } + cluster_undo_authority_scan_fold(agg, true, nmatch, proof, scn, wrap); + } + } + PG_CATCH(); + { /* - * Canonical name gate (A1.1-bis #2): every other entry must be - * EXACTLY seg_.dat as the writer renders it — parse, then - * re-render and compare, so aliases (seg_007.dat), trailing junk - * and out-of-range ids all poison the set instead of hiding - * evidence under a name the scan would skip. + * A directory read fault (or any throw) during enumeration means the + * set is not provably complete: fold it to a coverage failure and keep + * the reader backend / LMS worker alive (the caller bumps + * scan_incomplete_reject + fail_closed). cluster_undo_smgr_read_block + * never throws -- it returns false, handled above -- so in practice + * this catches the ReadDir enumeration fault; either way an + * un-completable scan can never masquerade as a complete one. */ - if (strncmp(de->d_name, "seg_", 4) != 0) { - complete = false; - break; - } - errno = 0; - id = strtoul(de->d_name + 4, &end, 10); - if (errno != 0 || end == de->d_name + 4 || strcmp(end, ".dat") != 0 || id > UINT16_MAX) { - complete = false; - break; - } - snprintf(canon, sizeof(canon), "seg_%lu.dat", id); - if (strcmp(canon, de->d_name) != 0) { - complete = false; - break; - } - - /* Read + parse this segment's block0; any failure poisons the set. */ - if (!cluster_undo_smgr_read_block(CLUSTER_UNDO_PATH_RUNTIME_SHARED_AUTHORITY_BLOCK0, - (uint32)id, owner_instance, 0 /* block0 */, page.data)) { - complete = false; - break; - } - if (cluster_vis_tt_block_xid_scan(page.data, (uint32)id, owner_instance, raw_xid, &nmatch, - &proof, &scn, &wrap) - != CLUSTER_VIS_TT_BLOCK_SCAN_OK) { - complete = false; - break; - } - cluster_undo_authority_scan_fold(agg, true, nmatch, proof, scn, wrap); - errno = 0; + MemoryContextSwitchTo(scan_ctx); + FlushErrorState(); + complete = false; } - if (complete && errno != 0) - complete = false; /* readdir broke mid-enumeration: set not complete */ + PG_END_TRY(); FreeDir(dir); return complete; diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index da18a8d961..3db21d0c16 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', - 'pg_stat_cluster_injections returns 167 rows (spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', + 'pg_stat_cluster_injections returns 168 rows (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '167 injection point names match the full registry (spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '168 injection point names match the full registry (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index ea13193aef..ead4ed54ee 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '167', - 'all 167 injection points have a .fault_type entry (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', + 'all 168 injection points have a .fault_type entry (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '167', - 'all 167 injection points have a .hits entry (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', + 'all 168 injection points have a .hits entry (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index f955cf5148..8b2a8044cf 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', - 'L15 total injection registry size is 167 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '168', + 'L15 total injection registry size is 168 (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index bb3beda2cd..27a72f7de9 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', - 'L11 pg_stat_cluster_injections is 167 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '168', + 'L11 pg_stat_cluster_injections is 168 (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 126d415115..d5744f0dfb 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', - 'L12a pg_stat_cluster_injections is 167 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '168', + 'L12a pg_stat_cluster_injections is 168 (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index c132f4f8ef..241c8f82db 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', - 'L11 pg_stat_cluster_injections is 167 (spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '168', + 'L11 pg_stat_cluster_injections is 168 (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 9309965b79..6837c9aa7e 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '167', 'M1 167 injection points (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', 'M1 168 injection points (spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '334', - 'M5 inject category has 167×2 = 334 sub-keys (.fault_type + .hits; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '336', + 'M5 inject category has 168×2 = 336 sub-keys (.fault_type + .hits; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl b/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl index d2ca652bd5..bf38f6aa85 100644 --- a/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl +++ b/src/test/cluster_tap/t/369_cluster_5_22d_dead_owner_authority_serve.pl @@ -382,6 +382,32 @@ sub arm_guc_both 'L3 HARD ASSERT: undo_authority_fail_closed_count moved (the refusal came from the prove arm)'); } +# ============================================================ +# L3c: enumeration-fault refusal (spec-5.22d Hardening, errno-fragility). +# A directory read that breaks mid-enumeration must fail closed via the +# SCAN-INCOMPLETE path (the durable segment set is not provably complete), +# NEVER a raw error abort and never a truncated "complete" scan that could +# serve a false-unique verdict. Arm 'error' models the ReadDir throw; the +# read of the still-cold s_t2 must reach the prove's enumeration loop and +# refuse via the scan-incomplete counter, not a generic head refusal. +# ============================================================ +{ + my $incpl_pre = + state_val($node1, 'cr', 'undo_authority_scan_incomplete_reject_count'); + my ($rc, $out, $err) = $node1->psql('postgres', q{ + SELECT cluster_inject_fault('cluster-undo-authority-scan', 'error', 0); + SELECT count(*) FROM s_t2; + }); + isnt($rc, 0, 'L3c armed enum-fault: node1 read of s_t2 errors (fail-closed, not served)'); + unlike(($out // ''), qr/^\s*5\s*$/m, + 'L3c armed read did NOT return the 5 rows (an incomplete enumeration never serves a verdict)'); + like($err, qr/cluster TT status unknown|cluster TT slot recycled/, + 'L3c the failure is the 53R97 fail-closed boundary (enumeration folded to a coverage failure, never a raw abort)'); + cmp_ok(state_val($node1, 'cr', 'undo_authority_scan_incomplete_reject_count'), + '>', $incpl_pre, + 'L3c HARD ASSERT: undo_authority_scan_incomplete_reject_count moved (refusal came from the enum-incomplete path, not a generic head refusal)'); +} + # ============================================================ # L3b: disarm control — a fresh (unarmed) backend re-reads s_t2 and the # serve succeeds, proving L3's refusal was the armed injection. From 96f2c39dc261e8058e2249a6f680cb29552aa9c5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 18:54:19 +0800 Subject: [PATCH 33/38] fix(cluster): fail closed on invalid undo resid identity in production builds The undo resid derived primitives guarded their invariants with Assert only: decode / master / generation_matches asserted the class byte, and encode asserted the owner range. Under --disable-cassert these are no-ops, so a production build could decode / route a non-undo-class resid, or encode an out-of-range owner (truncated into the uint16 owner field), and silently misroute the undo authority. The consuming side (cluster_grd_lookup_master) already fails closed with an ereport; the producing side did not -- an asymmetry that only bites production. Replace the four no-op Asserts with real runtime ereport(ERROR) guards that fire in both cassert and production builds, using the new SQLSTATE 53R9R ERRCODE_CLUSTER_UNDO_RESID_INVALID (53R9Q hash_routed covers only the reverse misroute). All current callers pass validated undo-class resids and in-range owners, so the guards are defense-in-depth and never fire on a valid path. test_cluster_undo_resid gains a setjmp trampoline and four guard-fire cases requiring the ereport arm (rc 1), so the guard is exercised at runtime even in the cassert unit build; the two sibling tests that link the object gain elog link stubs. Spec: spec-5.22a-undo-block-resource-identity.md (Hardening v1.0.1) --- src/backend/cluster/cluster_undo_resid.c | 57 ++++-- src/backend/utils/errcodes.txt | 9 + src/test/cluster_unit/test_cluster_undo_gcs.c | 37 ++++ .../cluster_unit/test_cluster_undo_resid.c | 190 +++++++++++++++++- .../cluster_unit/test_cluster_undo_verdict.c | 37 ++++ 5 files changed, 316 insertions(+), 14 deletions(-) diff --git a/src/backend/cluster/cluster_undo_resid.c b/src/backend/cluster/cluster_undo_resid.c index ddb0af925c..cf17d76cbb 100644 --- a/src/backend/cluster/cluster_undo_resid.c +++ b/src/backend/cluster/cluster_undo_resid.c @@ -6,11 +6,13 @@ * * This file ships the backend-pure layer: the undo resid * encoder/decoder, the class discriminator, the owner-as-master - * routing function, and the anti-ABA generation predicate. None of - * these touch elog / shmem / locks, so the cluster_unit test links the - * object standalone. The data plane that consumes this identity - * (grant / PI / block serving / recovery materialization / retention) - * lands with later deliverables. + * routing function, and the anti-ABA generation predicate. These touch + * only elog (the spec-5.22d Hardening fail-closed identity guards below + * ereport 53R9R on a non-undo-class resid or an out-of-range owner) -- + * no shmem / locks -- so the cluster_unit test still links the object + * standalone against a small errstart/errfinish trampoline. The data + * plane that consumes this identity (grant / PI / block serving / + * recovery materialization / retention) lands with later deliverables. * * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group @@ -30,8 +32,9 @@ */ #include "postgres.h" -#include "cluster/cluster_scn.h" /* SCN_NODE_ID_VALID */ +#include "cluster/cluster_scn.h" /* SCN_NODE_ID_VALID / SCN_MAX_VALID_NODE_ID */ #include "cluster/cluster_undo_resid.h" +#include "utils/errcodes.h" /* ERRCODE_CLUSTER_UNDO_RESID_INVALID (fail-closed guards) */ /* * cluster_undo_resid_encode -- build the undo-block resource id. @@ -49,7 +52,16 @@ cluster_undo_resid_encode(int32 owner_node, uint32 undo_segment, uint32 block_no Assert(dst != NULL); if (dst == NULL) return; - Assert(SCN_NODE_ID_VALID(owner_node)); + /* + * Fail closed on an out-of-range owner (spec-5.22d Hardening): a no-op + * Assert let --disable-cassert builds truncate the owner into the uint16 + * field4 and silently misroute the undo authority. Symmetric with the + * GRD-side 53R9Q hash-route guard. + */ + if (!SCN_NODE_ID_VALID(owner_node)) + ereport(ERROR, (errcode(ERRCODE_CLUSTER_UNDO_RESID_INVALID), + errmsg("undo resid owner_node %d out of range [0, %d]", owner_node, + SCN_MAX_VALID_NODE_ID))); dst->field1 = undo_segment; dst->field2 = block_no; @@ -62,9 +74,9 @@ cluster_undo_resid_encode(int32 owner_node, uint32 undo_segment, uint32 block_no /* * cluster_undo_resid_decode -- split an undo resid back into its fields. * - * The caller must pass an undo-class resid (Assert enforced); the - * decoder is the wire-ABI boundary, so it never guesses at foreign - * classes. + * The caller must pass an undo-class resid (fail-closed 53R9R on a + * foreign class); the decoder is the wire-ABI boundary, so it never + * guesses at foreign classes. */ void cluster_undo_resid_decode(const ClusterResId *rid, int32 *owner_node, uint32 *undo_segment, @@ -74,7 +86,13 @@ cluster_undo_resid_decode(const ClusterResId *rid, int32 *owner_node, uint32 *un if (rid == NULL || owner_node == NULL || undo_segment == NULL || block_no == NULL || generation == NULL) return; - Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + /* Fail closed on a foreign class (spec-5.22d Hardening): the decoder is the + * wire-ABI boundary, so a non-undo resid must never be reinterpreted as + * undo fields (a no-op Assert did this silently under --disable-cassert). */ + if (rid->type != CLUSTER_UNDO_RESID_TYPE) + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_UNDO_RESID_INVALID), + errmsg("undo resid decode on non-undo class 0x%02x", (unsigned int)rid->type))); *owner_node = (int32)rid->field4; *undo_segment = rid->field1; @@ -110,7 +128,14 @@ cluster_undo_resid_master(const ClusterResId *rid) Assert(rid != NULL); if (rid == NULL) return -1; - Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + /* Fail closed on a foreign class (spec-5.22d Hardening): returning field4 + * of a non-undo resid would route the authority to a bogus node (a no-op + * Assert did this silently under --disable-cassert). Symmetric with the + * GRD-side 53R9Q hash-route guard. */ + if (rid->type != CLUSTER_UNDO_RESID_TYPE) + ereport(ERROR, (errcode(ERRCODE_CLUSTER_UNDO_RESID_INVALID), + errmsg("undo resid master lookup on non-undo class 0x%02x", + (unsigned int)rid->type))); return (int32)rid->field4; } @@ -128,7 +153,13 @@ cluster_undo_resid_generation_matches(const ClusterResId *rid, uint32 expected_g Assert(rid != NULL); if (rid == NULL) return false; - Assert(rid->type == CLUSTER_UNDO_RESID_TYPE); + /* Fail closed on a foreign class (spec-5.22d Hardening): comparing field3 + * of a non-undo resid would fabricate an anti-ABA verdict (a no-op Assert + * did this silently under --disable-cassert). */ + if (rid->type != CLUSTER_UNDO_RESID_TYPE) + ereport(ERROR, (errcode(ERRCODE_CLUSTER_UNDO_RESID_INVALID), + errmsg("undo resid generation check on non-undo class 0x%02x", + (unsigned int)rid->type))); return rid->field3 == expected_generation; } diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index 5c1544767f..e404ad7e09 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -754,6 +754,15 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # X-transfer band below). 53R9Q E ERRCODE_CLUSTER_UNDO_RESID_HASH_ROUTED cluster_undo_resid_hash_routed +# spec-5.22d Hardening: the undo resid derived primitives (decode / master / +# generation_matches / encode) previously guarded their class byte and owner +# range with Assert only -- a no-op under --disable-cassert, so production could +# decode / route a non-undo-class resid, or encode an out-of-range owner (uint16 +# field4 wrap), and silently misroute the undo authority. They now fail closed +# with 53R9R, symmetric with the GRD-side 53R9Q hash-route guard above. R is the +# next free slot in the 53R9 family. +53R9R E ERRCODE_CLUSTER_UNDO_RESID_INVALID cluster_undo_resid_invalid + # spec-5.2a D6: clean-page X-transfer enabler terminal fail-closed. A clean # (sequence) page X-transfer could not complete safely and there is no proven- # safe fallback: a 3-node third-party master clean transfer (out of the 2-node diff --git a/src/test/cluster_unit/test_cluster_undo_gcs.c b/src/test/cluster_unit/test_cluster_undo_gcs.c index 4c27ffc761..bfff5a4422 100644 --- a/src/test/cluster_unit/test_cluster_undo_gcs.c +++ b/src/test/cluster_unit/test_cluster_undo_gcs.c @@ -59,6 +59,43 @@ ExceptionalCondition(const char *conditionName, const char *fileName, int lineNu abort(); } +/* + * spec-5.22d Hardening — cluster_undo_resid.o now fail-closes with + * ereport(53R9R) on an invalid-identity resid; these link stubs satisfy the + * reference. Every resid this test builds is a valid undo resid, so the guard + * never fires (errfinish aborts if it ever does, surfacing the surprise). + */ +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + return elevel >= 21; +} + +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} + +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{ + abort(); +} + +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} + +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + /* spec-7.1a D3 spread scn_time_cmp into the linked objects; SCNs here are * plain monotonic test values, so the total-order stub is a raw compare * (test_cluster_runtime_visibility.c pattern). */ diff --git a/src/test/cluster_unit/test_cluster_undo_resid.c b/src/test/cluster_unit/test_cluster_undo_resid.c index 9f85cf3328..4f1e5beb17 100644 --- a/src/test/cluster_unit/test_cluster_undo_resid.c +++ b/src/test/cluster_unit/test_cluster_undo_resid.c @@ -33,6 +33,7 @@ #include "cluster/cluster_scn.h" /* SCN_MAX_VALID_NODE_ID */ #include "cluster/cluster_undo_resid.h" #include "storage/lock.h" +#include "utils/errcodes.h" /* ERRCODE_CLUSTER_UNDO_RESID_INVALID (guard-fire) */ #undef printf #undef fprintf @@ -42,13 +43,95 @@ UT_DEFINE_GLOBALS(); +/* + * spec-5.22d Hardening — fail-closed guard-fire trampoline (mirrors + * test_cluster_grd.c / test_cluster_fence.c). When armed, an Assert trip + * (assert builds) jumps back with rc 2, and an ereport(ERROR) jumps back with + * rc 1 after the errcode stub captured the SQLSTATE. The derived-primitive + * guards are runtime ereports (NOT Asserts), so rc 1 is the pass arm in BOTH + * cassert and production builds (an Assert trip, rc 2, means the guard is still + * the pre-fix no-op-in-production Assert). + */ +static sigjmp_buf ut_trap_jump; +static bool ut_trap_jump_armed = false; +static int ut_ereport_last_errcode = 0; +static int ut_current_elevel = 0; + void ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) { + if (ut_trap_jump_armed) + siglongjmp(ut_trap_jump, 2); printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); abort(); } +bool +errstart(int elevel, const char *d pg_attribute_unused()) +{ + ut_current_elevel = elevel; + /* PG: ERROR = 21 (elog.h); >= ERROR runs the ereport body so the errcode + * stub can capture the SQLSTATE before errfinish jumps. */ + return elevel >= 21; +} + +bool +errstart_cold(int elevel, const char *d) +{ + return errstart(elevel, d); +} + +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{ + if (ut_current_elevel >= 21 && ut_trap_jump_armed) + siglongjmp(ut_trap_jump, 1); +} + +int +errcode(int s) +{ + if (ut_trap_jump_armed) + ut_ereport_last_errcode = s; + return 0; +} + +int +errmsg(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +int +errmsg_internal(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +int +errdetail(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +int +errhint(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + if (PG_exception_stack != NULL) + siglongjmp(*PG_exception_stack, 1); + abort(); +} + /* ====================================================================== * U2 -- class byte: 0xF9, above PG lock types, no collision with any * existing resid class (spec-5.22a §2.1) @@ -228,10 +311,111 @@ UT_TEST(test_undo_resid_generation_matches) UT_ASSERT(!cluster_undo_resid_generation_matches(&r, 1)); } +/* ====================================================================== + * spec-5.22d Hardening — derived-primitive fail-closed guards. A non-undo + * class resid passed to decode / master / generation_matches, or an + * out-of-range owner_node passed to encode, must ereport(ERROR) 53R9R (a real + * runtime guard, symmetric with the GRD-side 53R9Q hash-route guard), NEVER a + * silently misrouted answer. Pre-fix these were Assert-only (a no-op under + * --disable-cassert): the trampoline requires rc 1 (the ereport arm), so a + * cassert-only Assert trip (rc 2) fails the test -- proving the guard now fires + * in production too. + * ====================================================================== */ +static void +make_non_undo_resid(ClusterResId *r) +{ + /* a real other class (CF), never the undo class -- only the type byte + * matters to the discriminator guard */ + memset(r, 0, sizeof(*r)); + r->type = CLUSTER_CF_RESID_TYPE; + r->lockmethodid = DEFAULT_LOCKMETHOD; + r->field4 = 1; +} + +UT_TEST(test_undo_resid_master_rejects_non_undo_class) +{ + ClusterResId r; + int rc; + + make_non_undo_resid(&r); + ut_ereport_last_errcode = 0; + rc = sigsetjmp(ut_trap_jump, 0); + if (rc == 0) { + ut_trap_jump_armed = true; + (void)cluster_undo_resid_master(&r); + ut_trap_jump_armed = false; + UT_ASSERT(false); /* reaching here = the misroute was not refused */ + } + ut_trap_jump_armed = false; + UT_ASSERT_EQ(rc, 1); /* ereport arm, not the cassert-only Assert (rc 2) */ + UT_ASSERT_EQ(ut_ereport_last_errcode, ERRCODE_CLUSTER_UNDO_RESID_INVALID); +} + +UT_TEST(test_undo_resid_decode_rejects_non_undo_class) +{ + ClusterResId r; + int32 owner = 0; + uint32 seg = 0; + uint32 block = 0; + uint32 gen = 0; + int rc; + + make_non_undo_resid(&r); + ut_ereport_last_errcode = 0; + rc = sigsetjmp(ut_trap_jump, 0); + if (rc == 0) { + ut_trap_jump_armed = true; + cluster_undo_resid_decode(&r, &owner, &seg, &block, &gen); + ut_trap_jump_armed = false; + UT_ASSERT(false); + } + ut_trap_jump_armed = false; + UT_ASSERT_EQ(rc, 1); + UT_ASSERT_EQ(ut_ereport_last_errcode, ERRCODE_CLUSTER_UNDO_RESID_INVALID); +} + +UT_TEST(test_undo_resid_generation_matches_rejects_non_undo_class) +{ + ClusterResId r; + int rc; + + make_non_undo_resid(&r); + ut_ereport_last_errcode = 0; + rc = sigsetjmp(ut_trap_jump, 0); + if (rc == 0) { + ut_trap_jump_armed = true; + (void)cluster_undo_resid_generation_matches(&r, 0); + ut_trap_jump_armed = false; + UT_ASSERT(false); + } + ut_trap_jump_armed = false; + UT_ASSERT_EQ(rc, 1); + UT_ASSERT_EQ(ut_ereport_last_errcode, ERRCODE_CLUSTER_UNDO_RESID_INVALID); +} + +UT_TEST(test_undo_resid_encode_rejects_out_of_range_owner) +{ + ClusterResId r; + int rc; + + ut_ereport_last_errcode = 0; + rc = sigsetjmp(ut_trap_jump, 0); + if (rc == 0) { + ut_trap_jump_armed = true; + /* one past the valid node range would truncate into the uint16 field4 */ + cluster_undo_resid_encode(SCN_MAX_VALID_NODE_ID + 1, 3, 5, 1, &r); + ut_trap_jump_armed = false; + UT_ASSERT(false); + } + ut_trap_jump_armed = false; + UT_ASSERT_EQ(rc, 1); + UT_ASSERT_EQ(ut_ereport_last_errcode, ERRCODE_CLUSTER_UNDO_RESID_INVALID); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(12); UT_RUN(test_undo_resid_class_byte); UT_RUN(test_undo_resid_encode_decode_roundtrip); UT_RUN(test_undo_resid_tt_header_block_zero); @@ -240,6 +424,10 @@ main(void) UT_RUN(test_undo_resid_owner_bounds); UT_RUN(test_undo_resid_master_is_owner); UT_RUN(test_undo_resid_generation_matches); + UT_RUN(test_undo_resid_master_rejects_non_undo_class); + UT_RUN(test_undo_resid_decode_rejects_non_undo_class); + UT_RUN(test_undo_resid_generation_matches_rejects_non_undo_class); + UT_RUN(test_undo_resid_encode_rejects_out_of_range_owner); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_undo_verdict.c b/src/test/cluster_unit/test_cluster_undo_verdict.c index 25841c3a4a..a08bfb9ceb 100644 --- a/src/test/cluster_unit/test_cluster_undo_verdict.c +++ b/src/test/cluster_unit/test_cluster_undo_verdict.c @@ -75,6 +75,43 @@ ExceptionalCondition(const char *conditionName, const char *fileName, int lineNu abort(); } +/* + * spec-5.22d Hardening — cluster_undo_resid.o now fail-closes with + * ereport(53R9R) on an invalid-identity resid; these link stubs satisfy the + * reference. Every resid this test builds is a valid undo resid, so the guard + * never fires (errfinish aborts if it ever does, surfacing the surprise). + */ +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + return elevel >= 21; +} + +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} + +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{ + abort(); +} + +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} + +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + /* * Build a structurally valid verdict page for the asked xid, so the mapper's * internal cluster_vis_undo_verdict_page_usable() gate accepts it and the From 7b601b2630e72188035d95ba40ef5f6a8bcabce8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 19:10:35 +0800 Subject: [PATCH 34/38] test(cluster): lock the fresh-ref recycled-slot ABA "evidence not verdict" guard A single-block CP3 positive proof classifies COMMITTED by matching on xid and reporting the slot's CURRENT wrap/scn -- it does not compare against the ref's expected wrap, so a recycled slot reused by the same raw xid (bumped wrap, an unrelated COMMITTED stamp) still reads COMMITTED with the recycled commit's wrap/scn. A single-block COMMITTED proof is therefore evidence, not a verdict; the anti-ABA belongs downstream (the D4 dead-owner complete-scan uniqueness, or the CP5 origin verdict leg's own-CLOG cross-check + wrap-suspect gate), and the fetch / fresh-ref fast leg breaks to the verdict leg (C1b) rather than concluding visible. Harden the ungated cluster_undo_verdict_from_block_proof banner to state it applies no anti-ABA of its own and must only be reached behind a caller that already did, and add test_ttproof_recycled_same_xid_reports_bumped_wrap_not_gated pinning the report-not-gate contract for the same-xid/bumped-wrap case (distinct from the recycled different-xid -> NONE case already covered). No behaviour change: the routing invariant is already locked e2e by the verdict-wire counter assertions in the 7.1a and 5.22f TAP legs. Spec: spec-5.22f-shared-catalog-seed-visibility-consumer.md (Hardening v1.0.1) --- .../cluster_runtime_visibility_policy.c | 24 ++++++++--- .../test_cluster_runtime_visibility.c | 40 ++++++++++++++++++- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/backend/cluster/cluster_runtime_visibility_policy.c b/src/backend/cluster/cluster_runtime_visibility_policy.c index 93caf558a3..dd719cf15c 100644 --- a/src/backend/cluster/cluster_runtime_visibility_policy.c +++ b/src/backend/cluster/cluster_runtime_visibility_policy.c @@ -464,11 +464,25 @@ cluster_undo_verdict_from_authority_wire_page(const struct ClusterGcsUndoVerdict /* * cluster_undo_verdict_from_block_proof * - * D3-1 pure mapper: a CP3 single-block positive proof (exact xid+wrap slot - * match on the shipped block0 bytes) -> the taxonomy. COMMITTED carries the - * true commit SCN (EXACT); ABORTED is terminal; NONE is UNKNOWN_FAIL_CLOSED, - * which the orchestrator reads as "fall to the CP5 origin verdict", never as - * a terminal answer to the caller (Rule 8.A). Pure. + * D3-1 pure mapper: a CP3 single-block positive proof (an xid slot match on the + * shipped block0 bytes) -> the taxonomy. COMMITTED carries the true commit SCN + * (EXACT); ABORTED is terminal; NONE is UNKNOWN_FAIL_CLOSED, which the + * orchestrator reads as "fall to the CP5 origin verdict", never as a terminal + * answer to the caller (Rule 8.A). Pure. + * + * UNGATED (spec-5.22c/5.22f Hardening, root-cause-#1 review): this mapper takes + * COMMITTED at face value and passes the proof's wrap/scn straight through -- it + * has no horizon / expected-wrap / CLOG inputs and applies NO anti-ABA of its + * own. A single-block COMMITTED proof is EVIDENCE, not a verdict: a recycled + * slot reused by the same xid reports a bumped wrap and an unrelated commit_scn + * (test_cluster_runtime_visibility recycled-same-xid case), and a durable + * COMMITTED stamp also lands at 2PC pre-commit, so a stamped-then-crashed xid is + * in-doubt. It is therefore safe to reach ONLY behind a caller that already + * applied the anti-ABA + commit-finality: the D4 dead-owner authority + * complete-scan (uniqueness over the full durable set), or the CP5 origin + * verdict leg (own-CLOG cross-check + wrap-suspect gate). The single-block + * fetch / fresh-ref fast leg MUST NOT call this for COMMITTED -- it breaks to + * the verdict leg (C1b, cluster_runtime_visibility.c). */ ClusterUndoVerdictResult cluster_undo_verdict_from_block_proof(ClusterVisTtProof proof, SCN commit_scn, uint16 wrap) diff --git a/src/test/cluster_unit/test_cluster_runtime_visibility.c b/src/test/cluster_unit/test_cluster_runtime_visibility.c index 61aae40f2c..ebb4fd5c3f 100644 --- a/src/test/cluster_unit/test_cluster_runtime_visibility.c +++ b/src/test/cluster_unit/test_cluster_runtime_visibility.c @@ -258,6 +258,43 @@ UT_TEST(test_ttproof_committed_is_evidence_not_verdict) CLUSTER_VIS_TT_PROOF_COMMITTED); } +/* spec-5.22c/5.22f Hardening (root-cause-#1 integration review, #1) -- + * "recycled-ref-disguised-as-fresh". A slot recycled and reused by the SAME + * raw xid bytes but a BUMPED wrap, carrying a COMMITTED stamp for a DIFFERENT + * commit, still classifies COMMITTED: the scan matches on xid and REPORTS the + * slot's current wrap/scn -- it does NOT compare against the ref's expected + * wrap (it has none to compare). So a single-block COMMITTED proof is EVIDENCE + * only; the anti-ABA belongs to the caller, applied downstream (D4 + * complete-scan uniqueness, or the CP5 verdict leg's wrap-suspect gate -- see + * test_cluster_tt_durable test_wrap_suspect_below_horizon_unreliable_is_suspect + * and the C1b routing pinned by t/365 L4 / t/359 L3). This case (same xid, + * bumped wrap) is distinct from the recycled DIFFERENT-xid -> NONE case pinned + * by test_cluster_undo_verdict U12: here the proof is COMMITTED but the + * reported scn/wrap are the RECYCLED commit's, so trusting them terminally + * would false-commit. Pin the report-not-gate contract so no future change + * lets the fast leg conclude EXACT from a proof's COMMITTED. */ +UT_TEST(test_ttproof_recycled_same_xid_reports_bumped_wrap_not_gated) +{ + UndoSegmentHeaderData *hdr = mk_header(PROOF_SEG, PROOF_OWNER, TT_SLOTS_PER_SEGMENT); + SCN scn = InvalidScn; + uint16 wrap = 0; + + /* A fresh ref bound (xid 1000, wrap 5); the slot's wrap was bumped to 99 by + * a whole-segment recycle, then the bytes reused by the same raw xid with a + * COMMITTED stamp for an unrelated commit (scn 9999). */ + set_slot(hdr, 4, 1000, 99, (uint8)TT_SLOT_COMMITTED, (SCN)9999); + + /* Still COMMITTED (matches on xid), and it reports the CURRENT slot wrap + * (99) and scn (9999) -- the recycled commit's, NOT the ref's (wrap 5). + * The scan cannot self-detect the ABA; the caller must route to the verdict + * leg so the wrap-suspect gate can refuse. */ + UT_ASSERT_EQ(cluster_vis_tt_block_positive_proof(proof_block.data, PROOF_SEG, PROOF_OWNER, 1000, + &scn, &wrap), + CLUSTER_VIS_TT_PROOF_COMMITTED); + UT_ASSERT_EQ(wrap, 99); + UT_ASSERT_EQ((uint64)scn, 9999); +} + /* 0-match / ACTIVE / COMMITTED-without-scn: never a proof (user boundary: * a single fetched block's 0-match is NOT recycled/aborted evidence). */ UT_TEST(test_ttproof_zero_match_active_invalid_scn) @@ -823,7 +860,7 @@ UT_TEST(test_undo_fetch_tag_roundtrip) int main(void) { - UT_PLAN(22); + UT_PLAN(23); UT_RUN(test_covers_when_epoch_match_and_scn_ge_demand); UT_RUN(test_covers_ignores_cross_thread_lsn); UT_RUN(test_failclosed_when_epoch_differs); @@ -835,6 +872,7 @@ main(void) UT_RUN(test_failclosed_epoch_differs_above_32bit); UT_RUN(test_ttproof_committed_and_aborted); UT_RUN(test_ttproof_committed_is_evidence_not_verdict); + UT_RUN(test_ttproof_recycled_same_xid_reports_bumped_wrap_not_gated); UT_RUN(test_ttproof_zero_match_active_invalid_scn); UT_RUN(test_ttproof_ambiguity_and_garbage); UT_RUN(test_tt_block_xid_scan_core); From e5e232cf0de9b027af941d03e8c800fb094e7422 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 19:16:22 +0800 Subject: [PATCH 35/38] ci(cppcheck): suppress the PG-Assert nullPointerRedundantCheck in cluster_undo_horizon.c cluster_undo_horizon_cluster_floor guards its out-params with Assert(out_floor != NULL && out_reason != NULL && out_blame_node != NULL) then writes through them. PG's Assert has no noreturn attribute on the failure path, so cppcheck reads the four post-Assert out-param writes as redundant null checks (nullPointerRedundantCheck). Same false positive already suppressed for cluster_shmem.c / cluster_dl_lock.c / cluster_oid_lease.c; add the same per-file suppression to keep the policy consistent. Verified: run-cppcheck.sh reports 0 findings (matches baseline). --- scripts/ci/run-cppcheck.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/ci/run-cppcheck.sh b/scripts/ci/run-cppcheck.sh index 261db3314d..379c5a4efa 100755 --- a/scripts/ci/run-cppcheck.sh +++ b/scripts/ci/run-cppcheck.sh @@ -153,6 +153,12 @@ SHAREDFS_SUPP=( # Reason: same PG-Assert-non-trapping pattern -- Assert(lease != NULL); # lease->next in cluster_oid_lease_consume (spec-6.14 D6). --suppress=nullPointerRedundantCheck:src/backend/cluster/cluster_oid_lease.c + # Reason: spec-5.22e D5 cluster_undo_horizon_cluster_floor uses the same + # Assert(out_floor != NULL && out_reason != NULL && out_blame_node != NULL); + # then writes *out_floor / *out_reason / *out_blame_node -- PG Assert is + # non-trapping to cppcheck, so the four post-Assert out-param writes read as + # redundant null checks (identical false positive to cluster_shmem.c above). + --suppress=nullPointerRedundantCheck:src/backend/cluster/cluster_undo_horizon.c # Reason: spec-2.3 D1 cluster_ic_envelope.h uses pg_attribute_packed() # macro from c.h to honor spec-2.0 §4 frozen offsets (uint64 epoch at # offset 12 / scn at offset 20 are non-8-aligned naturally; without From 88c0a8552fffbf079f4c432b8ce0751ea3e28942 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 19:34:33 +0800 Subject: [PATCH 36/38] test(cluster): refresh nightly count assertions the D5 lane missed The D5 undo-horizon lane updated the injection-registry count in t/015 / t/017 / t/020-023 / t/030 to reflect D4-8 (+1 authority-block0-prove) and D5-7 (+2 horizon) but missed t/018 and t/024, which stayed at 164; and it added 8 undo-category brake-observability keys (horizon_stall/peer_stale/pass_abort/ wire_reject/admission_refuse/last_floor_scn/peer_reports + cleaner below-horizon) without refreshing t/214's undo-category row count (60 -> 68). These are the nightly foundation-core and stage3-core reds. Refresh all three to the current registry (168 injection points incl. this branch's cluster-undo-authority-scan; 68 undo rows). Verified locally: t/018/t/024 168 PASS, t/214 68 PASS. --- src/test/cluster_tap/t/018_shared_fs.pl | 4 ++-- src/test/cluster_tap/t/024_pcm_lock.pl | 4 ++-- src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index ee4dfb7794..99bbad138a 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L9 total injection registry size is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', + 'L9 total injection registry size is 168 (spec-5.22d Hardening +1 cluster-undo-authority-scan; D5 lane 漏更本文件含 spec-5.22d D4-8 +1 authority-block0-prove + spec-5.22e D5-7 +2 horizon; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 4181f4b730..f9583ae6cf 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -163,8 +163,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L6a pg_stat_cluster_injections has 164 entries (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '168', + 'L6a pg_stat_cluster_injections has 168 entries (spec-5.22d Hardening +1 cluster-undo-authority-scan; D5 lane 漏更本文件含 spec-5.22d D4-8 +1 + spec-5.22e D5-7 +2; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl index ab66119dc8..2d5741c13d 100644 --- a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl +++ b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl @@ -87,8 +87,8 @@ # ---------- my $undo_row_count = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='undo'}); -is($undo_row_count, '60', - "L2 undo category has 60 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4] + 6 undo GCS grant-plane [spec-5.22b D2-6])" +is($undo_row_count, '68', + "L2 undo category has 68 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4] + 6 undo GCS grant-plane [spec-5.22b D2-6] + 8 horizon brake observability [spec-5.22e D5-5, D5 lane 漏更 t/214])" ); From 284d33a712725f5c8232da4264fe043bde25d24b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 19:57:12 +0800 Subject: [PATCH 37/38] fix(cluster): give the authoritative undo-verdict request its own wire value (was colliding with multi-verdict) spec-5.22f D6-7 introduced the AUTHORITATIVE single-verdict request sub-kind by reusing reserved_0[6] value 3 -- the value already assigned to the spec-7.1 D3-b undo-MULTI-verdict request. GcsBlockForwardPayloadIsUndoVerdictRequest then matched (2 || 3), so it accepted a multi-verdict request, and the block-forward handler's single-verdict branch (which runs before the multi branch) stole it. A cross-node multixact member serve was routed to the single-xid verdict path, which cannot resolve the widened MXID, so the requester fail-closed with 53R97 and never converged (t/359_mxid G5: green on main, red on the branch). Move the authoritative single verdict to the next free value 5; multi keeps its shipped value 3. Byte legend is now 1=undo-TT fetch, 2=derived verdict, 3=MULTI verdict, 4=dead-owner authority verdict, 5=authoritative single verdict -- all five request kinds mutually exclusive across every Is* predicate. Only this sub-kind (unshipped on main) moves; the wire value for every kind that main already ships is unchanged. Add test_forward_payload_undo_verdict_kinds_no_collision pinning the mutual exclusion (the pre-existing kind-4 test never checked the multi predicate, so it missed the collision). Verified: t/359_mxid 19/19, and the authoritative-flag consumers t/365 / t/369 / t/370 / t/359_seed all pass. Spec: spec-5.22f-shared-catalog-seed-visibility-consumer.md --- src/include/cluster/cluster_gcs_block.h | 19 +++++-- .../cluster_unit/test_cluster_gcs_block.c | 55 +++++++++++++++++-- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index f49fcf379c..f525ac27a0 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1291,28 +1291,39 @@ GcsBlockForwardPayloadSetDirectLandFromRequest(GcsBlockForwardPayload *fwd, * into two sub-kinds: VALUE 2 = a DERIVED verdict (the spec-6.15 D4 recycled * path, whose origin was derived from the xid value; the serve keeps the * cluster_xid_is_mine self-check that guards the 6.12i P0 wrong-origin match), - * VALUE 3 = an AUTHORITATIVE verdict (the spec-5.22f fresh-ref path, whose + * VALUE 5 = an AUTHORITATIVE verdict (the spec-5.22f fresh-ref path, whose * origin is the tuple page's PHYSICAL ITL binding — the requester already * proved this is the correct owner, so the serve skips the stripe pre-filter * and answers underivable own xids over its own durable-TT + CLOG authority; * the positive-proof gates are unchanged, Rule 8.A). + * + * spec-5.22f Hardening (RC#1 integration review): the AUTHORITATIVE sub-kind + * originally reused VALUE 3, which COLLIDES with the spec-7.1 D3-b + * undo-MULTI-verdict request (also VALUE 3 below). IsUndoVerdictRequest then + * matched a multi request and the forward handler's single-verdict branch stole + * it before the multi branch, so a cross-node multixact member serve refused and + * the requester fail-closed (t/359_mxid G5 red on the branch, green on main). + * The byte legend is now 0=none, 1=undo-TT fetch, 2=derived verdict, 3=MULTI + * verdict (7.1 D3-b, unchanged), 4=dead-owner authority verdict (5.22d D4-6), + * 5=authoritative single verdict (moved off the multi value). Multi keeps its + * shipped value 3; only this unshipped-on-main sub-kind moves. */ static inline void GcsBlockForwardPayloadSetUndoVerdictRequest(GcsBlockForwardPayload *p, bool authoritative) { - p->reserved_0[6] = authoritative ? (uint8)3 : (uint8)2; + p->reserved_0[6] = authoritative ? (uint8)5 : (uint8)2; } static inline bool GcsBlockForwardPayloadIsUndoVerdictRequest(const GcsBlockForwardPayload *p) { - return p->reserved_0[6] == (uint8)2 || p->reserved_0[6] == (uint8)3; + return p->reserved_0[6] == (uint8)2 || p->reserved_0[6] == (uint8)5; } static inline bool GcsBlockForwardPayloadIsUndoVerdictAuthoritative(const GcsBlockForwardPayload *p) { - return p->reserved_0[6] == (uint8)3; + return p->reserved_0[6] == (uint8)5; } /* PGRAC: spec-5.22d D4-6 — reserved_0[6] VALUE 4 = dead-owner AUTHORITY diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 24c4c20073..5144f836ba 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -513,10 +513,10 @@ UT_TEST(test_clean_xfer_stale_break_predicate) /* spec-5.22d D4-6: reserved_0[6] VALUE 4 = dead-owner AUTHORITY verdict - * request. Pins the value-multiplex against the three existing kinds - * (1 = undo-TT fetch, 2 = derived verdict, 3 = authoritative verdict): the - * kind-4 predicate must never match 1/2/3 and vice versa, and setting kind 4 - * must not perturb the widened-xid watermark carrier. ABI stays 64B. */ + * request. Pins the value-multiplex against the other kinds (1 = undo-TT + * fetch, 2 = derived verdict, 3 = MULTI verdict, 5 = authoritative verdict): + * the kind-4 predicate must never match 1/2/3/5 and vice versa, and setting + * kind 4 must not perturb the widened-xid watermark carrier. ABI stays 64B. */ UT_TEST(test_forward_payload_undo_authority_verdict_kind4) { GcsBlockForwardPayload fwd; @@ -533,7 +533,7 @@ UT_TEST(test_forward_payload_undo_authority_verdict_kind4) UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoTtFetchRequest(&fwd) ? 1 : 0, 0); /* and the owner-served kinds are NOT the authority kind */ - GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, true /* value 3 */); + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, true /* value 5 */); UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, false /* value 2 */); UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); @@ -551,6 +551,48 @@ UT_TEST(test_forward_payload_undo_authority_verdict_kind4) } +/* spec-5.22f Hardening (RC#1 integration review): the AUTHORITATIVE single + * verdict sub-kind (spec-5.22f D6-7) must NOT share reserved_0[6] value 3 with + * the spec-7.1 D3-b MULTI verdict request. It originally did, so + * IsUndoVerdictRequest matched a multi request and the forward handler's + * single-verdict branch stole it before the multi branch -> a cross-node + * multixact member serve refused and the requester fail-closed 53R97 + * (t/359_mxid G5 red on the branch, green on main). Lock the full byte legend + * (1 fetch / 2 derived / 3 MULTI / 4 authority / 5 authoritative) so the five + * request kinds stay mutually exclusive across every Is* predicate. */ +UT_TEST(test_forward_payload_undo_verdict_kinds_no_collision) +{ + GcsBlockForwardPayload fwd; + + /* MULTI verdict (value 3): matched ONLY by the multi predicate. */ + memset(&fwd, 0, sizeof(fwd)); + GcsBlockForwardPayloadSetUndoMultiVerdictRequest(&fwd, true); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoMultiVerdictRequest(&fwd) ? 1 : 0, 1); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictRequest(&fwd) ? 1 : 0, 0); /* the collision */ + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictAuthoritative(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoTtFetchRequest(&fwd) ? 1 : 0, 0); + + /* AUTHORITATIVE single verdict (value 5): a verdict request, authoritative, + * but NOT a multi and NOT the dead-owner authority kind. */ + memset(&fwd, 0, sizeof(fwd)); + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, true); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictRequest(&fwd) ? 1 : 0, 1); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictAuthoritative(&fwd) ? 1 : 0, 1); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoMultiVerdictRequest(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoAuthorityVerdictRequest(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoTtFetchRequest(&fwd) ? 1 : 0, 0); + + /* DERIVED single verdict (value 2): a verdict request, NOT authoritative, + * NOT a multi. */ + memset(&fwd, 0, sizeof(fwd)); + GcsBlockForwardPayloadSetUndoVerdictRequest(&fwd, false); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictRequest(&fwd) ? 1 : 0, 1); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoVerdictAuthoritative(&fwd) ? 1 : 0, 0); + UT_ASSERT_EQ(GcsBlockForwardPayloadIsUndoMultiVerdictRequest(&fwd) ? 1 : 0, 0); +} + + /* spec-5.22d D4-6: the authority fetch tag carries the dead OWNER in the * previously-empty tag.relNumber as owner+1 (0 stays "absent" so the three * owner-served kinds keep their strict empty-relNumber shape). The serve @@ -604,7 +646,7 @@ UT_TEST(test_undo_verdict_version_authority_distinct) int main(void) { - UT_PLAN(24); + UT_PLAN(25); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -627,6 +669,7 @@ main(void) UT_RUN(test_clean_xfer_master_decision_5_branches); UT_RUN(test_clean_xfer_stale_break_predicate); UT_RUN(test_forward_payload_undo_authority_verdict_kind4); + UT_RUN(test_forward_payload_undo_verdict_kinds_no_collision); UT_RUN(test_undo_authority_fetch_tag_owner_roundtrip); UT_RUN(test_undo_verdict_version_authority_distinct); UT_DONE(); From aa9c25b28083601867f9a84b0f9a2cc8e56d34d0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 20:31:17 +0800 Subject: [PATCH 38/38] fix(cluster): keep peer capabilities a CONTROL-plane property so a DATA-plane reset does not wipe them Peer HELLO capabilities live in the shared ClusterSfDep store but were recorded AND cleared by whichever tier1 plane verified the HELLO or closed the fd. Since spec-7.2 tier1 is multi-plane (CONTROL = LMON, DATA = LMS workers), a transient DATA-plane worker reconnect participated in the caps lifecycle: a same-epoch DATA-plane conn-reset cleared the CONTROL-established capabilities, and the DATA reconnect could not re-send the CONTROL-only PEER_CAPS_REPLY (and the still-up CONTROL plane had no reason to), so the peer read NOCAP forever. The spec-5.22e D5-8 admission gate then fail-closed every cross-node undo access, so a cross-node write after a DATA-plane reset failed permanently (t/360 L5.5). Make the capability lifecycle a CONTROL-plane property: only the CONTROL plane records the peer's HELLO caps / sends PEER_CAPS_REPLY, and only a CONTROL-plane close clears the shared record. A DATA-plane worker only READS the store. A genuine peer loss still tears down the CONTROL connection, which clears as before, so the fail-closed safety is preserved. Verified: t/360 38/38 (RED->GREEN), and the caps / D5 / membership band t/386 / t/369 / t/370 / t/359_mxid / t/359_seed / t/365 / t/364 / t/367 all pass. Spec: spec-5.22e-undo-cluster-retention-horizon.md --- src/backend/cluster/cluster_ic_tier1.c | 52 ++++++++++++++++++++------ 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/src/backend/cluster/cluster_ic_tier1.c b/src/backend/cluster/cluster_ic_tier1.c index 9557eccd81..ce887af9b5 100644 --- a/src/backend/cluster/cluster_ic_tier1.c +++ b/src/backend/cluster/cluster_ic_tier1.c @@ -1865,9 +1865,21 @@ cluster_ic_tier1_recv_and_verify_hello(int32 peer_id, int peer_fd) Tier1Shmem->peers[peer_id].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[peer_id].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(peer_id); /* cache addr in shmem for view */ - cluster_sf_note_peer_hello_capabilities_gen(peer_id, cluster_ic_hello_capabilities(&msg), - Tier1Shmem->peers[peer_id].reconnect_count); - tier1_maybe_send_caps_reply(peer_id, cluster_ic_hello_capabilities(&msg)); + /* + * spec-5.22e Hardening (RC#1 integration review): peer HELLO capabilities + * live in the SHARED ClusterSfDep store, but their lifecycle is a + * CONTROL-plane (LMON) property. A transient DATA-plane worker reconnect + * must NOT record/overwrite them here (nor clear them on close, below) -- + * otherwise a same-epoch DATA-plane reset wipes the CONTROL-established caps + * and the DATA reconnect cannot re-send the CONTROL-only PEER_CAPS_REPLY, so + * the peer reads NOCAP forever and the D5-8 admission gate fail-closes every + * cross-node write (t/360 L5.5). CONTROL owns caps; DATA only reads them. + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_CONTROL) { + cluster_sf_note_peer_hello_capabilities_gen(peer_id, cluster_ic_hello_capabilities(&msg), + Tier1Shmem->peers[peer_id].reconnect_count); + tier1_maybe_send_caps_reply(peer_id, cluster_ic_hello_capabilities(&msg)); + } /* spec-7.3 D3 — tag the DATA-plane CONNECTED log with this worker's * channel so a 2-node test can prove the shard-aligned i<->i mesh formed @@ -2071,9 +2083,14 @@ cluster_ic_tier1_continue_hello_recv(int anon_slot, int peer_fd, int32 *out_lear Tier1Shmem->peers[learned].conn_epoch = cluster_epoch_get_current(); Tier1Shmem->peers[learned].last_connect_at = GetCurrentTimestamp(); (void)peer_addr(learned); - cluster_sf_note_peer_hello_capabilities_gen(learned, cluster_ic_hello_capabilities(&msg), - Tier1Shmem->peers[learned].reconnect_count); - tier1_maybe_send_caps_reply(learned, cluster_ic_hello_capabilities(&msg)); + /* CONTROL owns caps lifecycle (see the named-peer path above); a + * DATA-plane worker only reads the shared store. */ + if (tier1_my_plane == CLUSTER_IC_PLANE_CONTROL) { + cluster_sf_note_peer_hello_capabilities_gen(learned, + cluster_ic_hello_capabilities(&msg), + Tier1Shmem->peers[learned].reconnect_count); + tier1_maybe_send_caps_reply(learned, cluster_ic_hello_capabilities(&msg)); + } } if (out_learned_peer_id != NULL) @@ -2414,11 +2431,24 @@ cluster_ic_tier1_close_peer(int32 peer_id, const char *reason) * this connection was established), so only the matching record is * invalidated. */ - if (Tier1Shmem != NULL) - cluster_sf_note_peer_disconnected_gen(peer_id, - Tier1Shmem->peers[peer_id].reconnect_count); - else - cluster_sf_note_peer_disconnected(peer_id); + /* + * spec-5.22e Hardening (RC#1 integration review): ONLY a CONTROL-plane + * (LMON) close clears the shared peer-capability record. Caps are a + * CONTROL property (see the HELLO-verify record path); a DATA-plane + * worker's transient same-epoch reset must not wipe the + * CONTROL-established caps -- doing so left the peer NOCAP forever (the + * DATA reconnect cannot re-send the CONTROL-only PEER_CAPS_REPLY) and + * the D5-8 admission gate fail-closed every cross-node write (t/360). + * A genuine peer loss still tears down the CONTROL connection, which + * clears here as before (fail-closed preserved). + */ + if (tier1_my_plane == CLUSTER_IC_PLANE_CONTROL) { + if (Tier1Shmem != NULL) + cluster_sf_note_peer_disconnected_gen(peer_id, + Tier1Shmem->peers[peer_id].reconnect_count); + else + cluster_sf_note_peer_disconnected(peer_id); + } /* caps-reply resend state is per-connection too */ tier1_caps_reply_wanted[peer_id] = false; }