From b73ebe794cc5fce1df101bb6708d692761b3cdd0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 12:15:53 +0800 Subject: [PATCH 1/7] perf(cluster): spec-7.4 D0 commit-path census probes + census runner Five new commit-decomposition xnode_profile buckets (undo precommit flush / ITL stamp / TT stamp / commit-record WAL flush / quorum read), instrumented at the xact.c commit sequence and gated by the existing cluster.xnode_profile GUC (zero cost while off). Buckets are APPENDED at the enum tail: bucket indexes address a shmem array, so the enum is append-only -- an insertion renumbers later buckets and any stale .o keeps writing old indexes (probe-verified failure mode). New REPORT-ONLY census runner scripts/perf/run_74_commit_census.pl: commit component table (pair, fsync on/off), single-node serialization ceiling ladder with crash detection + wait-event attribution, BOC staleness differencing (idle/busy), hwm publication-channel audit, and a t/328-methodology version-comparison leg. Baselines synced: unit U1 23->28 buckets, U5 51->61 dump keys, t/334 key-count 61, 5.59 runner bucket table. Spec: spec-7.4-commit-scn-propagation-freshness.md (D0) --- scripts/perf/run_2node_xnode_profile.pl | 14 +- scripts/perf/run_74_commit_census.pl | 934 ++++++++++++++++++ src/backend/access/transam/xact.c | 29 + src/backend/cluster/cluster_xnode_profile.c | 5 + src/include/cluster/cluster_xnode_profile.h | 16 + .../cluster_tap/t/334_stage5_xnode_profile.pl | 2 +- .../cluster_unit/test_cluster_xnode_profile.c | 7 +- 7 files changed, 1000 insertions(+), 7 deletions(-) create mode 100644 scripts/perf/run_74_commit_census.pl diff --git a/scripts/perf/run_2node_xnode_profile.pl b/scripts/perf/run_2node_xnode_profile.pl index c705d7ef3d..6baa177a68 100755 --- a/scripts/perf/run_2node_xnode_profile.pl +++ b/scripts/perf/run_2node_xnode_profile.pl @@ -8,7 +8,7 @@ # shared_buffers 64MB): (a) native, (b) solo cluster node, (c) 2-node # ClusterPair (shared_data + 3 voting disks) -- cluster tiers with # cluster.xnode_profile = on. Drives the four axis scenarios against -# the pair, snapshotting the full xnode_profile dump (all 51 keys) per +# the pair, snapshotting the full xnode_profile dump (all 61 keys) per # node before/after each scenario and reporting per-bucket deltas: # W pgbench -n -N -c 1 on node0 with peer online (M3 single-writer), # interleaved with native + solo rounds (three-tier medians => @@ -158,7 +158,8 @@ BEGIN die "XP_STORAGE must be '' or 'block_device' (got '$XP_STORAGE')\n" unless $XP_STORAGE eq '' || $XP_STORAGE eq 'block_device'; -# The 23 buckets of cluster_xnode_profile.h, dump-key order. +# The 28 buckets of cluster_xnode_profile.h, dump-key order (23 spec-5.59 +# originals + 5 spec-7.4 D0 commit-decomposition buckets). my @ALL_BUCKETS = qw( w_gcs_x_request w_gcs_x_receive w_gcs_x_install w_gcs_x_invalidate w_ges_enqueue w_ges_convert w_ges_wait w_ges_wake w_hw_extend @@ -168,6 +169,8 @@ BEGIN c_scn_commit_advance c_scn_boc_broadcast ic_send_service ic_inbound_dispatch local_undo_itl_wal + c_commit_undo_flush c_commit_itl_stamp c_commit_tt_stamp + c_commit_wal_flush c_commit_quorum_read ); # Table 1: requester decision buckets -- the ONLY set pp-folding is valid @@ -194,6 +197,11 @@ BEGIN i_index_block_xfer => 'overlay (also counted in GCS buckets)', i_rightmost_leaf_ping => 'count-only probe (GUC-gated)', c_scn_boc_broadcast => 'service-side (BOC fanout send)', + c_commit_undo_flush => 'spec-7.4 commit census (folded by run_74 only)', + c_commit_itl_stamp => 'spec-7.4 commit census (folded by run_74 only)', + c_commit_tt_stamp => 'spec-7.4 commit census (folded by run_74 only)', + c_commit_wal_flush => 'spec-7.4 commit census (folded by run_74 only)', + c_commit_quorum_read => 'spec-7.4 commit census (folded by run_74 only)', ic_send_service => 'service-side (IC one-way send)', ic_inbound_dispatch => 'service-side (IC inbound dispatch)', ); @@ -264,7 +272,7 @@ sub poll_sql_eq } # ---------- -# xnode_profile dump snapshots (all 51 keys) + deltas +# xnode_profile dump snapshots (all 61 keys) + deltas # ---------- sub xp_snapshot { diff --git a/scripts/perf/run_74_commit_census.pl b/scripts/perf/run_74_commit_census.pl new file mode 100644 index 0000000000..3ccb29b948 --- /dev/null +++ b/scripts/perf/run_74_commit_census.pl @@ -0,0 +1,934 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# run_74_commit_census.pl +# spec-7.4 D0 -- commit/SCN propagation census (REPORT-ONLY). +# +# Decomposes the pgrac commit path into measured components and +# measures the propagation staleness that spec-7.4's levers target. +# Five legs, selectable via CS_LEGS (default "abcd"; "e" is the +# version-comparison leg driven separately per install prefix): +# +# A commit component table -- 2-node pair (shared_data + voting +# disks), node0 runs a single-row-UPDATE commit workload under +# cluster.xnode_profile = on, once with fsync=off (CPU-side +# attribution, t/328-comparable) and once with fsync=on (real +# flush weights). Per-component nanos/commit from the +# c_commit_* / c_scn_commit_advance bucket deltas; the +# denominator is the client-side END-statement latency from +# pgbench -r (native tier measured alongside for the same +# workload). Residual = denominator - sum(components). +# +# B single-node serialization-ceiling matrix -- native vs solo +# cluster node (t/328 topology, fsync=off), interleaved short +# windows over a client ladder (1..64). Per-rung median TPS + +# tax%; at the configured attribution rung the runner samples +# pg_stat_activity wait events and per-commit bucket means to +# attribute the ceiling (undo cursor LWLock vs SCN atomics vs +# aux IPC -- the three suspects named by the spec amendment). +# +# C BOC staleness -- 2-node pair; node0 commits via a background +# psql session, the runner then polls node1's observed SCN +# (scn_max_observed_remote) on a tight interval and records +# t(observe) - t(commit-return) per sample, idle (no other +# traffic: the Lamport-clock-stall case) and busy (background +# pgbench on node0). Histogram buckets 1/2/5/10/20/50/100/200ms +# (spec-7.4 §2.4; same-host same-clock differencing). +# +# D hwm freshness co-sample audit -- counters snapshot over the +# Leg A window: how often the UNDO_TT_FETCH piggyback path (the +# only live_hwm publication channel today) actually fires +# relative to the commit rate. +# +# E version comparison -- exact t/328 methodology (fsync=off, +# TPC-B -N, 4 clients, short interleaved native/solo windows, +# median) intended to be invoked once per install prefix +# (CS_INSTALL=/tmp/pgrac-{74,v128,v129}-install CS_LEGS=e) to +# rule the "did 6.14 add write tax" question on one runner. +# +# REPORT-ONLY: trend datapoints, never a gate; exits 0 on completion, +# dies only on harness failure (mirrors run_2node_xnode_profile.pl). +# +# Env knobs: CS_LEGS (abcd) legs to run; CS_SECS (10) Leg A seconds; +# CS_ROUNDS (3) Leg A rounds; CS_SCALE (10) pgbench scale; CS_LADDER +# (1,2,4,8,16,32,64) Leg B client rungs; CS_LADDER_SECS (2) window +# seconds; CS_LADDER_PAIRS (6) interleaved pairs per rung; CS_ATTR_C +# (32) attribution rung; CS_STALE_N (100) Leg C samples per mode; +# CS_E_ROUNDS (30) Leg E interleaved pairs; CS_OUT report path +# (default scripts/perf/results/74-commit-census-.md); +# CS_TSV_OUT optional machine-readable dump; CS_INSTALL optional +# install prefix prepended to PATH (Leg E per-version invocations); +# PGBENCH pgbench binary override. +# +# Run from repo root in a built tree: +# CS_INSTALL=/tmp/pgrac-74-install perl scripts/perf/run_74_commit_census.pl +# +# Author: SqlRush +# Portions Copyright (c) 2026, pgrac contributors +# +# Spec: spec-7.4-commit-scn-propagation-freshness.md (D0) +# +# IDENTIFICATION +# scripts/perf/run_74_commit_census.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../src/test/perl"; + +use File::Basename qw(dirname); +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use POSIX (); +use Time::HiRes (); + +our ($REAL_STDOUT, $have_fixtures, $fixtures_err); + +BEGIN +{ + open($REAL_STDOUT, '>&', \*STDOUT) + or die "cannot dup STDOUT: $!\n"; + my $prev = select($REAL_STDOUT); + $| = 1; + select($prev); + + if (defined $ENV{CS_INSTALL} && length $ENV{CS_INSTALL}) + { + $ENV{PATH} = "$ENV{CS_INSTALL}/bin:$ENV{PATH}"; + for my $var (qw(LD_LIBRARY_PATH DYLD_LIBRARY_PATH)) + { + $ENV{$var} = join(':', "$ENV{CS_INSTALL}/lib", + grep { defined && length } $ENV{$var}); + } + } + + { + require FindBin; + FindBin->import(); + my $repo = File::Spec->rel2abs("$FindBin::RealBin/../.."); + + $ENV{PG_REGRESS} = "$repo/src/test/regress/pg_regress" + unless defined $ENV{PG_REGRESS} && length $ENV{PG_REGRESS}; + $ENV{TESTDATADIR} = "$repo/tmp_check" + unless defined $ENV{TESTDATADIR} && length $ENV{TESTDATADIR}; + $ENV{TESTLOGDIR} = "$repo/tmp_check/log" + unless defined $ENV{TESTLOGDIR} && length $ENV{TESTLOGDIR}; + } + + eval { + require PostgreSQL::Test::Utils; + PostgreSQL::Test::Utils->import(); + require PostgreSQL::Test::Cluster; + PostgreSQL::Test::Cluster->import(); + require PostgreSQL::Test::ClusterPair; + PostgreSQL::Test::ClusterPair->import(); + $have_fixtures = 1; + 1; + } or $fixtures_err = $@; +} + +use Test::More; + +die "TAP fixtures unavailable (need a built tree; PERL5LIB reaches " + . "src/test/perl via FindBin): $fixtures_err\n" + unless $have_fixtures; + +# ---------- +# knobs + constants +# ---------- +my $CS_LEGS = lc($ENV{CS_LEGS} // 'abcd'); +my $CS_SECS = $ENV{CS_SECS} // 10; +my $CS_ROUNDS = $ENV{CS_ROUNDS} // 3; +my $CS_SCALE = $ENV{CS_SCALE} // 10; +my @CS_LADDER = split /,/, ($ENV{CS_LADDER} // '1,2,4,8,16,32,64'); +my $CS_LADDER_SECS = $ENV{CS_LADDER_SECS} // 2; +my $CS_LADDER_PAIRS = $ENV{CS_LADDER_PAIRS} // 6; +my $CS_ATTR_C = $ENV{CS_ATTR_C} // 16; +my $CS_STALE_N = $ENV{CS_STALE_N} // 100; +my $CS_E_ROUNDS = $ENV{CS_E_ROUNDS} // 30; +my $PGBENCH = $ENV{PGBENCH} // 'pgbench'; + +my $REPO_ROOT = File::Spec->rel2abs("$FindBin::RealBin/../.."); +my $CS_OUT = $ENV{CS_OUT} + // File::Spec->catfile($REPO_ROOT, 'scripts', 'perf', 'results', + '74-commit-census-' . time() . '.md'); +my $CS_TSV_OUT = $ENV{CS_TSV_OUT} // ''; + +# Commit-decomposition buckets (spec-7.4 D0) + the pre-existing SCN +# advance bucket; nanos are requester-exclusive so nanos/commit means are +# comparable against the client END latency denominator. +my @COMMIT_BUCKETS = qw( + c_commit_quorum_read c_commit_undo_flush c_scn_commit_advance + c_commit_itl_stamp c_commit_tt_stamp c_commit_wal_flush +); + +# Leg C histogram bucket upper bounds, ms (spec-7.4 §2.4). +my @STALE_BOUNDS_MS = (1, 2, 5, 10, 20, 50, 100, 200); + +my @REPORT; +my %TSV; + +sub progress +{ + my ($msg) = @_; + print $REAL_STDOUT "[census] $msg\n"; +} + +sub report +{ + my ($line) = @_; + push @REPORT, $line; +} + +sub median +{ + my @s = sort { $a <=> $b } grep { defined } @_; + return undef unless @s; + return $s[int(@s / 2)]; +} + +sub pctile +{ + my ($p, @vals) = @_; + my @s = sort { $a <=> $b } @vals; + return undef unless @s; + my $idx = int($p / 100 * (@s - 1) + 0.5); + return $s[$idx]; +} + +sub poll_sql_eq +{ + my ($node, $sql, $want, $timeout_s) = @_; + my $deadline = Time::HiRes::time() + ($timeout_s // 15); + while (Time::HiRes::time() < $deadline) + { + my $got = eval { $node->safe_psql('postgres', $sql); }; + return 1 if defined $got && $got eq $want; + select(undef, undef, undef, 0.25); + } + return 0; +} + +# ---------- +# xnode_profile dump snapshots + deltas (run_2node_xnode_profile.pl) +# ---------- +sub xp_snapshot +{ + my ($node) = @_; + my %snap; + my $rows = $node->safe_psql('postgres', + "SELECT key, value FROM pg_cluster_state WHERE category='xnode_profile'"); + for my $line (split /\n/, ($rows // '')) + { + my ($k, $v) = split /\|/, $line, 2; + $snap{$k} = ($v // 0) + 0 if defined $k && length $k; + } + return \%snap; +} + +sub xp_delta +{ + my ($before, $after) = @_; + my %d; + $d{$_} = ($after->{$_} // 0) - ($before->{$_} // 0) for keys %$after; + return \%d; +} + +sub bkt_n { my ($d, $b) = @_; return $d->{"bucket.$b.n_events"} // 0; } +sub bkt_ns { my ($d, $b) = @_; return $d->{"bucket.$b.total_nanos"} // 0; } + +# Generic pg_cluster_state counter snapshot for one category. +sub cat_snapshot +{ + my ($node, $cat) = @_; + my %snap; + my $rows = $node->safe_psql('postgres', + "SELECT key, value FROM pg_cluster_state WHERE category='$cat'"); + for my $line (split /\n/, ($rows // '')) + { + my ($k, $v) = split /\|/, $line, 2; + next unless defined $k && length $k; + $snap{$k} = ($v =~ /^-?\d+$/) ? $v + 0 : $v; + } + return \%snap; +} + +# ---------- +# conf + tier boots +# ---------- +# Census isolation knobs. Unlike the 5.59 profile conf, fsync is a LEG +# PARAMETER here: the commit census must see real flush weights in its +# fsync=on leg, while the fsync=off legs stay t/328-comparable. +sub census_conf +{ + my ($fsync) = @_; + return ( + "autovacuum = off\n", + "fsync = $fsync\n", + "shared_buffers = 256MB\n", + "max_wal_size = 4GB\n", + ); +} + +sub boot_native +{ + my ($fsync, $tag) = @_; + $tag //= $fsync; + my $native = PostgreSQL::Test::Cluster->new('census_native_' . $tag); + $native->init; + $native->append_conf('postgresql.conf', $_) for census_conf($fsync); + $native->start; + return $native; +} + +sub boot_solo +{ + my ($fsync, $tag) = @_; + $tag //= $fsync; + my $ic_port = PostgreSQL::Test::Cluster::get_free_port(); + my $solo = PostgreSQL::Test::Cluster->new('census_solo_' . $tag); + $solo->init; + $solo->append_conf('postgresql.conf', "$_\n") + for ('cluster.enabled = on', 'cluster.interconnect_tier = tier1', + 'cluster.allow_single_node = on', 'cluster.node_id = 0', + 'cluster.xnode_profile = on'); + $solo->append_conf('postgresql.conf', $_) for census_conf($fsync); + PostgreSQL::Test::Utils::append_to_file( + $solo->data_dir . '/pgrac.conf', + "[cluster]\nname = census_solo\n\n[node.0]\ninterconnect_addr = 127.0.0.1:$ic_port\n\n" + ); + $solo->start; + return $solo; +} + +sub boot_pair +{ + my ($fsync, $tag) = @_; + $tag //= $fsync; + my @pair_conf = map { my $l = $_; chomp $l; $l } census_conf($fsync); + my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'census_pair_' . $tag, + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + @pair_conf, + 'cluster.xnode_profile = on', + 'cluster.gcs_block_retransmit_max_retries = 8', + 'cluster.gcs_block_dedup_max_entries = 16384', + # The Leg A commit storm leaves a GES/GCS backlog; a census + # harness prefers patience over spurious lock timeouts. + 'cluster.ges_request_timeout_ms = 20000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 10', + ]); + $pair->start_pair; + my $ready = + $pair->wait_for_peer_state(0, 1, 'connected', 30) + && $pair->wait_for_peer_state(1, 0, 'connected', 30) + && poll_sql_eq($pair->node0, + 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20) + && poll_sql_eq($pair->node1, + 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20); + die "harness failure: ClusterPair did not reach connected + in_quorum\n" + unless $ready; + return $pair; +} + +# ---------- +# pgbench drivers +# ---------- +sub conn_args +{ + my ($node) = @_; + return '-h ' . $node->host . ' -p ' . $node->port . ' postgres'; +} + +sub pgbench_init +{ + my ($node) = @_; + system("$PGBENCH -i -s $CS_SCALE -q " . conn_args($node) . " >/dev/null 2>&1"); + return $? == 0; +} + +# TPC-B (-N) window; returns tps or undef (t/328 pgbench_one). +sub pgbench_tpcb +{ + my ($node, $secs, $clients) = @_; + my $out = `$PGBENCH -n -T $secs -c $clients -j 2 -N @{[conn_args($node)]} 2>&1`; + return $1 + 0.0 if $out =~ /tps\s*=\s*([\d.]+)\s*\(without initial/; + return $1 + 0.0 if $out =~ /tps\s*=\s*([\d.]+)/; + return undef; +} + +# Commit-census workload: single-row UPDATE in an explicit transaction, +# reported per-statement (-r) so the END line is the client-side commit +# latency. Returns (tps, xacts, end_latency_ms) or (). +my $COMMIT_SCRIPT_DIR = tempdir(CLEANUP => 1); +my $COMMIT_SCRIPT = "$COMMIT_SCRIPT_DIR/commit_census.sql"; +{ + open my $fh, '>', $COMMIT_SCRIPT or die "cannot write $COMMIT_SCRIPT: $!"; + print $fh <<'EOS'; +\set aid random(1, 100000 * :scale) +BEGIN; +UPDATE pgbench_accounts SET abalance = abalance + 1 WHERE aid = :aid; +END; +EOS + close $fh; +} + +sub pgbench_commit_run +{ + my ($node, $secs, $clients) = @_; + my $out = `$PGBENCH -n -T $secs -c $clients -j 2 -r -D scale=$CS_SCALE -f $COMMIT_SCRIPT @{[conn_args($node)]} 2>&1`; + my ($tps) = $out =~ /tps\s*=\s*([\d.]+)/; + my ($xacts) = $out =~ /number of transactions actually processed:\s*(\d+)/; + my ($end_ms); + # "statement latencies in milliseconds[ and failures]:" table; the + # END; row is the commit. + for my $line (split /\n/, $out) + { + $end_ms = $1 + 0.0 if $line =~ /^\s*([\d.]+)\s+(?:\d+\s+)?END;/; + } + return () unless defined $tps && defined $xacts && $xacts > 0; + return ($tps + 0.0, $xacts, $end_ms); +} + +# ---------- +# Leg A -- commit component table (pair node0 + native denominator) +# ---------- +sub leg_a +{ + my ($fsync) = @_; + progress("Leg A (fsync=$fsync): boot native + pair"); + my $native = boot_native($fsync); + my $pair = boot_pair($fsync); + my $node0 = $pair->node0; + + pgbench_init($native) or die "pgbench init (native) failed\n"; + pgbench_init($node0) or die "pgbench init (pair node0) failed\n"; + + my (@nat_end, @clu_end, @nat_tps, @clu_tps); + my ($xacts_total, %ns_total, %n_total) = (0); + my $d_snap; + + # Leg D pre-window counter snapshot (publication-channel audit). + my %d_pre = $CS_LEGS =~ /d/ + ? (scn0 => cat_snapshot($node0, 'scn'), + scn1 => cat_snapshot($pair->node1, 'scn'), + gcs1 => cat_snapshot($pair->node1, 'gcs')) + : (); + + for my $round (1 .. $CS_ROUNDS) + { + progress("Leg A (fsync=$fsync): round $round/$CS_ROUNDS"); + # c=1 single-writer on the pair (5.59 W-axis / M3 precedent): + # multi-client UPDATE churn on a peer-online pair trips GES + # handoff drain fail-closed storms and poisons the census; + # queueing effects are Leg B's job (solo ladder). + my ($ntps, $nx, $nend) = pgbench_commit_run($native, $CS_SECS, 1); + my $before = xp_snapshot($node0); + my ($ctps, $cx, $cend) = pgbench_commit_run($node0, $CS_SECS, 1); + my $after = xp_snapshot($node0); + next unless defined $ntps && defined $ctps; + push @nat_tps, $ntps; push @nat_end, $nend; + push @clu_tps, $ctps; push @clu_end, $cend; + my $d = xp_delta($before, $after); + # Denominator: the SCN-advance event count IS the commit count + # in this window (exactly 1/commit, probe-verified); pgbench's + # xact number would also include warm-up outside the snapshot. + $xacts_total += bkt_n($d, 'c_scn_commit_advance'); + for my $b (@COMMIT_BUCKETS) + { + $ns_total{$b} += bkt_ns($d, $b); + $n_total{$b} += bkt_n($d, $b); + } + $d_snap = $d; + } + + die "Leg A produced no valid rounds\n" unless @clu_tps; + + my $nat_end_med = median(@nat_end) // 0; + my $clu_end_med = median(@clu_end) // 0; + my $denom_ns = $clu_end_med * 1e6; # ms -> ns per commit + + report(""); + report("## Leg A -- commit component table (fsync=$fsync)"); + report(""); + report(sprintf( + "workload: single-row UPDATE + explicit commit, 1 client " + . "(5.59 M3 single-writer precedent on the pair), " + . "%ds x %d rounds, scale %d; pair node0 (shared_data), medians", + $CS_SECS, $CS_ROUNDS, $CS_SCALE)); + report(""); + report(sprintf("- native END latency (median): %.3f ms (tps %.0f)", + $nat_end_med, median(@nat_tps) // 0)); + report(sprintf("- cluster END latency (median): %.3f ms (tps %.0f)", + $clu_end_med, median(@clu_tps) // 0)); + report(sprintf("- commits measured (xp window): %d", $xacts_total)); + report(""); + report("| component | ns/commit | us/event | events/commit | % of END |"); + report("|---|---:|---:|---:|---:|"); + + my $sum_ns_per_commit = 0; + for my $b (@COMMIT_BUCKETS) + { + my $ns_per_commit = $xacts_total ? $ns_total{$b} / $xacts_total : 0; + my $us_per_event = $n_total{$b} ? $ns_total{$b} / $n_total{$b} / 1000 : 0; + my $ev_per_commit = $xacts_total ? $n_total{$b} / $xacts_total : 0; + my $pct = $denom_ns ? 100 * $ns_per_commit / $denom_ns : 0; + $sum_ns_per_commit += $ns_per_commit; + report(sprintf("| %s | %.0f | %.2f | %.3f | %.2f%% |", + $b, $ns_per_commit, $us_per_event, $ev_per_commit, $pct)); + $TSV{"a.$fsync.$b.ns_per_commit"} = sprintf("%.0f", $ns_per_commit); + $TSV{"a.$fsync.$b.pct_of_end"} = sprintf("%.2f", $pct); + } + my $resid_ns = $denom_ns - $sum_ns_per_commit; + my $resid_pct = $denom_ns ? 100 * $resid_ns / $denom_ns : 0; + report(sprintf("| (sum of components) | %.0f | | | %.2f%% |", + $sum_ns_per_commit, $denom_ns ? 100 * $sum_ns_per_commit / $denom_ns : 0)); + report(sprintf("| residual (client rtt, executor, CLOG, ProcArray, ...) " + . "| %.0f | | | %.2f%% |", $resid_ns, $resid_pct)); + $TSV{"a.$fsync.end_latency_ms"} = sprintf("%.3f", $clu_end_med); + $TSV{"a.$fsync.native_end_latency_ms"} = sprintf("%.3f", $nat_end_med); + $TSV{"a.$fsync.residual_pct"} = sprintf("%.2f", $resid_pct); + + # Leg D piggyback: publication-channel counter audit over the same + # window (no extra cross-node traffic: a post-storm GES request on + # this pair can stall 20s+ -- pre-existing, logged as follow-up). + if ($CS_LEGS =~ /d/) + { + my %d_post = ( + scn0 => cat_snapshot($node0, 'scn'), + scn1 => cat_snapshot($pair->node1, 'scn'), + gcs1 => cat_snapshot($pair->node1, 'gcs')); + leg_d($xacts_total, \%d_pre, \%d_post, $fsync); + } + + $pair->stop_pair if $pair->can('stop_pair'); + $native->stop; + return; +} + +# ---------- +# Leg B -- serialization ceiling matrix (native vs solo, t/328 topology) +# ---------- +sub leg_b +{ + progress("Leg B: boot native + solo (fsync=off, t/328 topology)"); + my $native = boot_native('off', 'b'); + my $solo = boot_solo('off', 'b'); + + pgbench_init($native) or die "pgbench init (native) failed\n"; + pgbench_init($solo) or die "pgbench init (solo) failed\n"; + + report(""); + report("## Leg B -- single-node serialization ceiling (native vs solo cluster)"); + report(""); + report(sprintf( + "TPC-B -N interleaved windows, %ds x %d pairs per rung, medians; fsync=off", + $CS_LADDER_SECS, $CS_LADDER_PAIRS)); + report(""); + report("| clients | native tps | cluster tps | tax %% |"); + report("|---:|---:|---:|---:|"); + + my $solo_crashed_at; + for my $c (@CS_LADDER) + { + my (@nat, @clu); + for my $i (1 .. $CS_LADDER_PAIRS) + { + my $n = pgbench_tpcb($native, $CS_LADDER_SECS, $c); + my $s = pgbench_tpcb($solo, $CS_LADDER_SECS, $c); + push @nat, $n if defined $n && $n > 0; + push @clu, $s if defined $s && $s > 0; + } + # Ceiling-by-crash detection: if the solo server died under this + # rung (undo buffer pool PANIC family), that IS the ceiling + # finding -- record it and stop climbing. + my $alive = eval { $solo->safe_psql('postgres', 'SELECT 1'); 1 }; + my $nm = median(@nat); + my $cm = median(@clu); + my $tax = (defined $nm && defined $cm && $nm > 0) + ? 100 * ($nm - $cm) / $nm : undef; + my $note = $alive ? '' : ' (SERVER CRASHED under this rung)'; + report(sprintf("| %d | %.0f | %.0f | %s |", + $c, $nm // 0, $cm // 0, + (defined $tax ? sprintf("%.2f%%", $tax) : 'n/a') . $note)); + $TSV{"b.c$c.native_tps"} = sprintf("%.0f", $nm // 0); + $TSV{"b.c$c.cluster_tps"} = sprintf("%.0f", $cm // 0); + $TSV{"b.c$c.tax_pct"} = defined $tax ? sprintf("%.2f", $tax) : ''; + progress(sprintf("Leg B: c=%d native=%.0f cluster=%.0f tax=%s%s", + $c, $nm // 0, $cm // 0, + defined $tax ? sprintf("%.2f%%", $tax) : 'n/a', $note)); + if (!$alive) + { + $solo_crashed_at = $c; + report(""); + report(sprintf( + "**ceiling-by-crash**: the solo cluster server died under " + . "c=%d (see census_solo_b.log for the PANIC signature); " + . "higher rungs skipped.", $c)); + last; + } + } + if (defined $solo_crashed_at) + { + progress("Leg B: rebooting solo for the attribution rung"); + $solo = boot_solo('off', 'b2'); + pgbench_init($solo) or die "pgbench re-init (solo b2) failed\n"; + } + + # Attribution rung: per-commit bucket means at c=1 vs c=CS_ATTR_C + + # wait-event sampling during a sustained c=CS_ATTR_C window. + progress("Leg B: attribution rung c=$CS_ATTR_C"); + my %attr; + for my $c (1, $CS_ATTR_C) + { + my $before = xp_snapshot($solo); + my ($tps, $xacts) = pgbench_commit_run($solo, $CS_LADDER_SECS * 3, $c); + my $after = xp_snapshot($solo); + next unless defined $xacts && $xacts > 0; + my $d = xp_delta($before, $after); + for my $b (@COMMIT_BUCKETS, 'local_undo_itl_wal', 'ic_send_service') + { + $attr{$c}{$b} = { + ns_per_commit => bkt_ns($d, $b) / $xacts, + us_per_event => bkt_n($d, $b) + ? bkt_ns($d, $b) / bkt_n($d, $b) / 1000 : 0, + }; + } + } + report(""); + report(sprintf( + "### attribution: per-commit component means, c=1 vs c=%d (solo, commit workload)", + $CS_ATTR_C)); + report(""); + report("| component | c=1 us/event | c=$CS_ATTR_C us/event | inflation x |"); + report("|---|---:|---:|---:|"); + for my $b (@COMMIT_BUCKETS, 'local_undo_itl_wal', 'ic_send_service') + { + my $u1 = $attr{1}{$b}{us_per_event} // 0; + my $uN = $attr{$CS_ATTR_C}{$b}{us_per_event} // 0; + report(sprintf("| %s | %.2f | %.2f | %s |", + $b, $u1, $uN, $u1 > 0 ? sprintf("%.1f", $uN / $u1) : 'n/a')); + $TSV{"b.attr.$b.c1_us"} = sprintf("%.2f", $u1); + $TSV{"b.attr.$b.cN_us"} = sprintf("%.2f", $uN); + } + + # Wait-event sampling during a sustained attribution window. + progress("Leg B: wait-event sampling under c=$CS_ATTR_C"); + my $sampler_secs = $CS_LADDER_SECS * 3; + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + if ($pid == 0) + { + # child: exec (NOT perl exit -- that would fire Cluster.pm's END + # teardown and stop the nodes under the parent) + exec("$PGBENCH -n -T $sampler_secs -c $CS_ATTR_C -j 2 " + . "-D scale=$CS_SCALE -f $COMMIT_SCRIPT " + . conn_args($solo) + . " >/dev/null 2>&1") + or do { POSIX::_exit(1) }; + } + my %we; + my $deadline = Time::HiRes::time() + $sampler_secs - 0.5; + my $samples = 0; + while (Time::HiRes::time() < $deadline) + { + my $rows = eval { + $solo->safe_psql('postgres', + "SELECT coalesce(wait_event_type,'CPU') || ':' || " + . "coalesce(wait_event,'running'), count(*) " + . "FROM pg_stat_activity WHERE state='active' " + . "AND backend_type='client backend' " + . "AND query NOT LIKE '%pg_stat_activity%' " + . "GROUP BY 1"); + }; + if (defined $rows) + { + $samples++; + for my $line (split /\n/, $rows) + { + my ($k, $n) = split /\|/, $line, 2; + $we{$k} += $n if defined $k && length $k; + } + } + Time::HiRes::sleep(0.05); + } + waitpid($pid, 0); + report(""); + report(sprintf( + "### wait-event census under c=%d commit workload (%d samples x 50ms)", + $CS_ATTR_C, $samples)); + report(""); + report("| wait event | backend-samples | share %% |"); + report("|---|---:|---:|"); + my $we_total = 0; + $we_total += $_ for values %we; + for my $k (sort { $we{$b} <=> $we{$a} } keys %we) + { + my $share = $we_total ? 100 * $we{$k} / $we_total : 0; + next if $share < 0.5; + report(sprintf("| %s | %d | %.1f%% |", $k, $we{$k}, $share)); + (my $tk = $k) =~ s/[^A-Za-z0-9]+/_/g; + $TSV{"b.we.$tk.share_pct"} = sprintf("%.1f", $share); + } + + $solo->stop; + $native->stop; + return; +} + +# ---------- +# Leg C -- BOC staleness (pair; idle + busy) +# ---------- +sub stale_histogram +{ + my ($label, @ms) = @_; + my @counts = (0) x (@STALE_BOUNDS_MS + 1); + for my $v (@ms) + { + my $i = 0; + $i++ while $i < @STALE_BOUNDS_MS && $v > $STALE_BOUNDS_MS[$i]; + $counts[$i]++; + } + report(""); + report("### $label (n=" . scalar(@ms) . ")"); + report(""); + report(sprintf("p50 %.1f ms | p99 %.1f ms | max %.1f ms", + median(@ms) // 0, pctile(99, @ms) // 0, + (sort { $b <=> $a } @ms)[0] // 0)); + report(""); + report("| bucket | count |"); + report("|---|---:|"); + for my $i (0 .. $#STALE_BOUNDS_MS) + { + report(sprintf("| <= %d ms | %d |", $STALE_BOUNDS_MS[$i], $counts[$i])); + } + report(sprintf("| > %d ms | %d |", $STALE_BOUNDS_MS[-1], $counts[-1])); + (my $tl = lc $label) =~ s/[^a-z0-9]+/_/g; + $TSV{"c.$tl.p50_ms"} = sprintf("%.1f", median(@ms) // 0); + $TSV{"c.$tl.p99_ms"} = sprintf("%.1f", pctile(99, @ms) // 0); + return; +} + +sub leg_c +{ + progress("Leg C: boot pair for BOC staleness"); + my $pair = boot_pair('off', 'c'); + my ($node0, $node1) = ($pair->node0, $pair->node1); + + $node0->safe_psql('postgres', + 'CREATE TABLE census_stale (id int primary key, v bigint); ' + . 'INSERT INTO census_stale VALUES (1, 0)'); + + my $committer = $node0->background_psql('postgres'); + my $observer = $node1->background_psql('postgres'); + + my $scn_sql = + "SELECT value FROM pg_cluster_state WHERE category='scn' AND key='scn_max_observed_remote'"; + + my $run_mode = sub { + my ($mode) = @_; + my @samples_ms; + # Busy mode: background TPC-B churn on node0 keeps BOC traffic + # flowing; idle mode measures the clock-stall case the spec + # calls out (no traffic -> remote observes only via sweep). + my $bgpid; + if ($mode eq 'busy') + { + $bgpid = fork(); + die "fork failed: $!" unless defined $bgpid; + if ($bgpid == 0) + { + # exec, not perl exit (Cluster.pm END teardown hazard) + exec("$PGBENCH -n -T " . (60 + $CS_STALE_N) + . " -c 1 -j 1 -N " . conn_args($node0) + . " >/dev/null 2>&1") + or do { POSIX::_exit(1) }; + } + sleep 2; # let the churn establish + } + for my $i (1 .. $CS_STALE_N) + { + # One committed write on node0 (autocommit), then read the + # post-commit local SCN in the same session. t0 is the + # commit return; the extra SELECT round trip biases scn0 + # upward under concurrent traffic (staleness reads high -- + # the conservative direction; exact in idle mode). + $committer->query_safe( + 'UPDATE census_stale SET v = v + 1 WHERE id = 1'); + my $t0 = Time::HiRes::time(); + my $scn0 = $committer->query_safe( + "SELECT value FROM pg_cluster_state WHERE category='scn' AND key='scn_current_local'" + ); + $scn0 =~ s/\s+//g; + my $deadline = $t0 + 2.0; + my $dt_ms; + while (Time::HiRes::time() < $deadline) + { + my $obs = $observer->query_safe($scn_sql); + $obs =~ s/\s+//g; + if (length $obs && length $scn0 && $obs + 0 >= $scn0 + 0) + { + $dt_ms = (Time::HiRes::time() - $t0) * 1000; + last; + } + Time::HiRes::sleep(0.002); + } + push @samples_ms, $dt_ms if defined $dt_ms; + # Idle spacing: let the sweep window fully close between + # samples so each commit lands in a fresh window. + Time::HiRes::sleep($mode eq 'idle' ? 0.15 : 0.02); + } + waitpid($bgpid, 0) if defined $bgpid && $bgpid > 0; + return @samples_ms; + }; + + report(""); + report("## Leg C -- BOC staleness: commit(node0) -> observe(node1)"); + report(""); + report("same-host same-clock differencing; polling granularity ~2ms + one " + . "background-psql round trip -- treat sub-5ms readings as bounds, " + . "not exact values (spec-7.4 H-2)"); + + my @idle = $run_mode->('idle'); + stale_histogram('idle (no other traffic; Lamport clock-stall case)', @idle); + + my @busy = $run_mode->('busy'); + stale_histogram('busy (background TPC-B churn on node0)', @busy); + + $committer->quit; + $observer->quit; + $pair->stop_pair if $pair->can('stop_pair'); + return; +} + +# ---------- +# Leg D -- hwm publication-channel audit (counter deltas over Leg A) +# ---------- +sub leg_d +{ + my ($commit_count, $pre, $post, $fsync) = @_; + progress("Leg D: hwm co-sample channel audit"); + + report(""); + report("## Leg D -- live_hwm publication channel audit (fsync=$fsync window)"); + report(""); + report("today's only publication path is the UNDO_TT_FETCH reply-header " + . "co-sample (cluster_gcs_block.c) consumed into per-backend " + . "memos; there is no node-level hwm cache and no proactive " + . "broadcast. Counter deltas across the Leg A commit window:"); + report(""); + report("| node | category.key | delta |"); + report("|---|---|---:|"); + for my $side (sort keys %$pre) + { + my ($cat) = $side =~ /^([a-z]+)/; + for my $k (sort keys %{ $post->{$side} }) + { + next unless ($post->{$side}{$k} // '') =~ /^-?\d+$/; + my $d = $post->{$side}{$k} - ($pre->{$side}{$k} // 0); + next unless $d != 0; + report(sprintf("| %s | %s.%s | %d |", $side, $cat, $k, $d)); + (my $tk = "$side.$k") =~ s/[^A-Za-z0-9]+/_/g; + $TSV{"d.$fsync.$tk"} = $d; + } + } + report(""); + report(sprintf( + "commit rate context: %d commits on node0 in this window; node1's " + . "hwm view moved only by BOC observe bumps (SCN freshness), " + . "never by an hwm publication -- the channel is passive " + . "(fetch-piggyback) only.", $commit_count // 0)); + return; +} + +# ---------- +# Leg E -- version-comparison leg (exact t/328 methodology, one install) +# ---------- +sub leg_e +{ + my $label = $ENV{CS_E_LABEL} // ($ENV{CS_INSTALL} // 'in-path'); + progress("Leg E ($label): boot native + solo (t/328 parameters)"); + my $native = boot_native('off', 'e'); + my $solo = boot_solo('off', 'e'); + + pgbench_init($native) or die "pgbench init (native) failed\n"; + pgbench_init($solo) or die "pgbench init (solo) failed\n"; + + my (@nat, @clu, @taxes); + for my $i (1 .. $CS_E_ROUNDS) + { + my $n = pgbench_tpcb($native, 2, 4); + my $c = pgbench_tpcb($solo, 2, 4); + next unless defined $n && defined $c && $n > 0 && $c > 0; + push @nat, $n; + push @clu, $c; + push @taxes, 100 * ($n - $c) / $n; + } + die "Leg E produced no valid rounds\n" unless @taxes; + + my ($nm, $cm, $tm) = (median(@nat), median(@clu), median(@taxes)); + report(""); + report("## Leg E -- version leg: $label"); + report(""); + report(sprintf( + "t/328 methodology (TPC-B -N, 4 clients, 2s x %d interleaved pairs, " + . "fsync=off, medians)", scalar(@taxes))); + report(""); + report(sprintf("- native tps median: %.0f", $nm)); + report(sprintf("- cluster tps median: %.0f", $cm)); + report(sprintf("- write tax median: %.2f%%", $tm)); + (my $tl = lc $label) =~ s/[^a-z0-9]+/_/g; + $TSV{"e.$tl.native_tps"} = sprintf("%.0f", $nm); + $TSV{"e.$tl.cluster_tps"} = sprintf("%.0f", $cm); + $TSV{"e.$tl.tax_pct"} = sprintf("%.2f", $tm); + + $solo->stop; + $native->stop; + return; +} + +# ---------- +# main +# ---------- +report("# spec-7.4 D0 commit census -- " . scalar(localtime)); +report(""); +report(sprintf("host: %s | legs: %s | install: %s", + `hostname -s` =~ s/\s+//gr, $CS_LEGS, $ENV{CS_INSTALL} // '(PATH)')); + +if ($CS_LEGS =~ /a/) +{ + leg_a('off'); + leg_a('on'); +} +leg_b() if $CS_LEGS =~ /b/; +leg_c() if $CS_LEGS =~ /c/; +# Leg D runs inside leg_a (shares the pair); standalone only with A. +leg_e() if $CS_LEGS =~ /e/; + +make_path(dirname($CS_OUT)); +open my $out, '>', $CS_OUT or die "cannot write $CS_OUT: $!"; +print $out "$_\n" for @REPORT; +close $out; +progress("report written: $CS_OUT"); + +if (length $CS_TSV_OUT) +{ + make_path(dirname($CS_TSV_OUT)); + open my $tsv, '>', $CS_TSV_OUT or die "cannot write $CS_TSV_OUT: $!"; + print $tsv "$_\t$TSV{$_}\n" for sort keys %TSV; + close $tsv; + progress("tsv written: $CS_TSV_OUT"); +} + +print $REAL_STDOUT join("\n", @REPORT), "\n"; + +ok(1, 'census completed (report-only)'); +done_testing(); diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 11591002fd..c32621305a 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -162,6 +162,7 @@ #include "cluster/cluster_node_remove.h" /* PGRAC: spec-5.18 INV-LF9 self-demote gate */ #include "cluster/cluster_reconfig.h" /* PGRAC: spec-5.15 §2.4 joiner write gate */ #include "cluster/cluster_sf_dep.h" /* PGRAC: spec-6.2 Smart Fusion commit brake */ +#include "cluster/cluster_xnode_profile.h" /* PGRAC: spec-7.4 D0 commit census probes */ #include "cluster/storage/cluster_undo_xlog.h" /* PGRAC: spec-3.18 D4.1 TT fold redo stamp */ #endif #endif @@ -1607,6 +1608,10 @@ RecordTransactionCommit(void) { if (cluster_storage_mode_enabled()) { + /* PGRAC: spec-7.4 D0 -- commit-path decomposition probes + * (GUC-gated, zero cost while cluster.xnode_profile is off). */ + ClusterXpScope commit_xps; + /* * P0 perf hardening (2026-05-31): make this xact's undo durable * BEFORE the commit becomes visible. Durable ordering: @@ -1617,7 +1622,9 @@ RecordTransactionCommit(void) * START_CRIT_SECTION, so an fsync failure ereport(ERROR)s into a * clean abort (an aborted xact's un-fsync'd undo is irrelevant). */ + cluster_xp_begin(&commit_xps, CLXP_C_COMMIT_UNDO_FLUSH); cluster_undo_xact_precommit_flush(); + cluster_xp_end(&commit_xps); cluster_backup_pending_commit_enter(); commit_scn = cluster_scn_advance_for_commit(); @@ -1637,7 +1644,12 @@ RecordTransactionCommit(void) * transactions, autovacuum, etc.). */ if (cluster_itl_touch_has_pending()) + { + /* PGRAC: spec-7.4 D0 census component. */ + cluster_xp_begin(&commit_xps, CLXP_C_COMMIT_ITL_STAMP); cluster_itl_xact_precommit_finish(xid, tt_commit_scn); + cluster_xp_end(&commit_xps); + } /* * PGRAC modifications by SqlRush : @@ -1652,8 +1664,11 @@ RecordTransactionCommit(void) * cluster_tt_local_record_commit() overlay install (C1b: this only * stamps commit_scn; committed-ness stays CLOG's authority). */ + /* PGRAC: spec-7.4 D0 census component. */ + cluster_xp_begin(&commit_xps, CLXP_C_COMMIT_TT_STAMP); has_tt_fold = cluster_tt_local_precommit_durable_finish(xid, tt_commit_scn, &tt_fold); + cluster_xp_end(&commit_xps); } /* @@ -1746,8 +1761,16 @@ RecordTransactionCommit(void) synchronous_commit > SYNCHRONOUS_COMMIT_OFF) || forceSyncCommit || nrels > 0) { +#ifdef USE_PGRAC_CLUSTER + /* PGRAC: spec-7.4 D0 -- commit-record flush component (includes + * any group-commit queueing inside XLogFlush). GUC-gated. */ + ClusterXpScope commit_flush_xps; + + cluster_xp_begin(&commit_flush_xps, CLXP_C_COMMIT_WAL_FLUSH); +#endif XLogFlush(XactLastRecEnd); #ifdef USE_PGRAC_CLUSTER + cluster_xp_end(&commit_flush_xps); if (cluster_smart_fusion) cluster_sf_publish_origin_durable_lsn(); #endif @@ -2641,6 +2664,11 @@ CommitTransaction(void) && cluster_voting_disks != NULL && cluster_voting_disks[0] != '\0') { + /* PGRAC: spec-7.4 D0 -- quorum/lease-read census component (the + * ERROR path simply discards the started scope). */ + ClusterXpScope quorum_xps; + + cluster_xp_begin(&quorum_xps, CLXP_C_COMMIT_QUORUM_READ); if (!cluster_qvotec_in_quorum()) ereport(ERROR, (errcode(ERRCODE_CLUSTER_QUORUM_LOST), @@ -2648,6 +2676,7 @@ CommitTransaction(void) " rejecting commit to prevent split-brain"), errhint("check pg_cluster_quorum_state.in_quorum and" " pg_cluster_voting_disks for disk health"))); + cluster_xp_end(&quorum_xps); } /* diff --git a/src/backend/cluster/cluster_xnode_profile.c b/src/backend/cluster/cluster_xnode_profile.c index 863c0a4d3c..60025f21f4 100644 --- a/src/backend/cluster/cluster_xnode_profile.c +++ b/src/backend/cluster/cluster_xnode_profile.c @@ -68,6 +68,11 @@ static const char *const cluster_xp_bucket_names[CLXP_NBUCKETS] = { [CLXP_I_RIGHTMOST_LEAF_PING] = "i_rightmost_leaf_ping", [CLXP_C_SCN_COMMIT_ADVANCE] = "c_scn_commit_advance", [CLXP_C_SCN_BOC_BROADCAST] = "c_scn_boc_broadcast", + [CLXP_C_COMMIT_UNDO_FLUSH] = "c_commit_undo_flush", + [CLXP_C_COMMIT_ITL_STAMP] = "c_commit_itl_stamp", + [CLXP_C_COMMIT_TT_STAMP] = "c_commit_tt_stamp", + [CLXP_C_COMMIT_WAL_FLUSH] = "c_commit_wal_flush", + [CLXP_C_COMMIT_QUORUM_READ] = "c_commit_quorum_read", [CLXP_IC_SEND_SERVICE] = "ic_send_service", [CLXP_IC_INBOUND_DISPATCH] = "ic_inbound_dispatch", [CLXP_LOCAL_UNDO_ITL_WAL] = "local_undo_itl_wal", diff --git a/src/include/cluster/cluster_xnode_profile.h b/src/include/cluster/cluster_xnode_profile.h index 28b41dac66..2fd5da82d0 100644 --- a/src/include/cluster/cluster_xnode_profile.h +++ b/src/include/cluster/cluster_xnode_profile.h @@ -78,6 +78,22 @@ typedef enum ClusterXnodeBucket { CLXP_IC_INBOUND_DISPATCH, /* recv side: verify + dispatch */ /* --- local write-side machinery (undo + ITL + WAL, per DML) --- */ CLXP_LOCAL_UNDO_ITL_WAL, + /* + * Commit-path decomposition (spec-7.4 D0 census; D4 makes them a + * permanent surface). All are requester-exclusive-wait: the + * committing backend alone spends the interval, so per-commit + * nanos/event means are directly comparable against the client-side + * end-to-end commit latency denominator. + * + * APPENDED at the tail on purpose: bucket indexes address a shmem + * array, so the enum is append-only (an insertion renumbers every + * later bucket and any stale .o keeps writing the old index). + */ + CLXP_C_COMMIT_UNDO_FLUSH, /* pre-commit undo segment fsync */ + CLXP_C_COMMIT_ITL_STAMP, /* pre-commit ITL COMMITTED stamp + WAL */ + CLXP_C_COMMIT_TT_STAMP, /* pre-commit durable TT slot commit_scn stamp */ + CLXP_C_COMMIT_WAL_FLUSH, /* commit-record XLogFlush (incl. group-commit queue) */ + CLXP_C_COMMIT_QUORUM_READ, /* PRE_COMMIT local quorum/lease read */ CLXP_NBUCKETS } ClusterXnodeBucket; diff --git a/src/test/cluster_tap/t/334_stage5_xnode_profile.pl b/src/test/cluster_tap/t/334_stage5_xnode_profile.pl index 025d0d2a8b..386536e20b 100644 --- a/src/test/cluster_tap/t/334_stage5_xnode_profile.pl +++ b/src/test/cluster_tap/t/334_stage5_xnode_profile.pl @@ -156,7 +156,7 @@ sub xp_decision_nanos # Dump surface exists and is all-zero while the GUC is off. my $key_count = $solo->safe_psql('postgres', "SELECT count(*) FROM pg_cluster_state WHERE category='xnode_profile'"); -is($key_count, '51', 'xnode_profile dump surface: 51 keys (23 buckets x2 + 5 probes)'); +is($key_count, '61', 'xnode_profile dump surface: 61 keys (28 buckets x2 + 5 probes)'); my $nonzero_off = $solo->safe_psql('postgres', "SELECT count(*) FROM pg_cluster_state WHERE category='xnode_profile' " . "AND key LIKE 'bucket.%' AND value <> '0'"); diff --git a/src/test/cluster_unit/test_cluster_xnode_profile.c b/src/test/cluster_unit/test_cluster_xnode_profile.c index 3ac63d0f08..6da5357bc1 100644 --- a/src/test/cluster_unit/test_cluster_xnode_profile.c +++ b/src/test/cluster_unit/test_cluster_xnode_profile.c @@ -135,7 +135,8 @@ ut_bucket_events(ClusterXnodeBucket b) /* ---------- * U1 — bucket enum completeness: every bucket has a unique non-NULL - * name; out-of-range returns NULL; the enum has the spec'd 23 buckets. + * name; out-of-range returns NULL; the enum has the spec'd 23 buckets + * (spec-5.59) + 5 commit-decomposition buckets (spec-7.4 D0) = 28. * ---------- */ UT_TEST(test_u1_bucket_enum_complete) @@ -143,7 +144,7 @@ UT_TEST(test_u1_bucket_enum_complete) int i; int j; - UT_ASSERT_EQ(CLXP_NBUCKETS, 23); + UT_ASSERT_EQ(CLXP_NBUCKETS, 28); for (i = 0; i < CLXP_NBUCKETS; i++) { const char *name = cluster_xp_bucket_name((ClusterXnodeBucket)i); @@ -287,7 +288,7 @@ UT_TEST(test_u4_off_path_zero_syscall) UT_TEST(test_u5_dump_key_surface) { UT_ASSERT_EQ(CLUSTER_XP_N_PROBE_KEYS, 5); - UT_ASSERT_EQ(CLXP_NBUCKETS * 2 + CLUSTER_XP_N_PROBE_KEYS, 51); + UT_ASSERT_EQ(CLXP_NBUCKETS * 2 + CLUSTER_XP_N_PROBE_KEYS, 61); } /* ---------- From 66105346912d168bf9c67a87554327aa3ddafe0a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:22:15 +0800 Subject: [PATCH 2/7] feat(cluster): spec-7.4 D1 durable_safe_scn frontier registry + BOC payload v1 receive side Contiguous durable frontier replaces any atomic-max watermark notion: every own-origin commit SCN <= the published frontier is durable on WAL. Pending registry registers at commit-SCN allocation (reservation loop now runs for every commit advance, extending the INV-ADG8 register-before-observable invariant to the frontier), fills in the commit-record LSN at insert, and discharges on three paths: sync commit after END_CRIT_SECTION, walwriter behind XLogBackgroundFlush (cluster_scn_boc_tick), abort cleanup (lsn-filled entries are real commits and only detach). Registry overflow freezes the frontier stickily instead of publishing an unsafe claim. Receive side: BOC payload_length contract widens to {0, 8} (v1 = 8B origin_durable_safe_scn, explicit LE). Handler validates length, decodability, and SCN node bits == env.source_node_id (fail-closed counted drops), then updates a per-(origin, epoch) seqlock cache; same-epoch regressions are counted rejects, epoch changes rebuild. Send-side payload attach + event wakeup + GUC arrive with the next deliverable slice; 0-byte frames stay valid pulse-only traffic. Unit: test_cluster_scn_frontier (20 tests) covers the out-of-order flush P0 heart, walwriter horizon discharge, abort guard, sticky overflow freeze, monotonic publish, LE codec, and handler validation matrix. Spec: spec-7.4-commit-scn-propagation-freshness.md Spec: spec-2.9-scn-broadcast-service-activation.md --- src/backend/access/transam/twophase.c | 24 + src/backend/access/transam/xact.c | 42 + src/backend/cluster/cluster_scn.c | 611 ++++++++++++- src/include/cluster/cluster_scn.h | 52 ++ src/test/cluster_unit/Makefile | 12 +- src/test/cluster_unit/test_cluster_scn.c | 17 +- .../cluster_unit/test_cluster_scn_frontier.c | 861 ++++++++++++++++++ 7 files changed, 1599 insertions(+), 20 deletions(-) create mode 100644 src/test/cluster_unit/test_cluster_scn_frontier.c diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 90620887f5..8863bc11db 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -93,6 +93,13 @@ * XactLog{Commit,Abort}Record with the same byte-identical-WAL * guarantee when --disable-cluster (InvalidScn suppresses section). * + * What changed (spec-7.4 D1): + * - RecordTransactionCommitPrepared(): durable-frontier LSN fill-in + * after XactLogCommitRecord; frontier discharge after + * END_CRIT_SECTION (prepared commits always XLogFlush). + * - FinishPreparedTransaction() commit-path PG_CATCH: unblock the + * durable frontier on a precommit ERROR. + * * Why: * Spec-1.16 v0.2 Q5 REVISED -- PG's durable commit point for 2PC is * RecordTransactionCommitPrepared, not PREPARE. PREPARE leaves the @@ -1544,6 +1551,9 @@ FinishPreparedTransaction(const char *gid, bool isCommit) PG_CATCH(); { cluster_backup_pending_commit_abort(0, (Datum)0); + /* PGRAC: spec-7.4 D1 -- unblock the durable frontier; the + * allocated commit SCN never became a commit record. */ + (void)cluster_scn_durable_pending_abort_self(); PG_RE_THROW(); } PG_END_TRY(); @@ -2387,6 +2397,15 @@ RecordTransactionCommitPrepared(TransactionId xid, int nchildren, TransactionId commit_scn, /* PGRAC: spec-1.18 */ NULL); /* PGRAC: spec-3.18 D4.1: 2PC keeps standalone 0x30 */ +#ifdef USE_PGRAC_CLUSTER + /* PGRAC: spec-7.4 D1 -- record the commit-record LSN on the durable + * pending entry (crit-section-safe shmem write). Prepared commits + * always flush below, so the discharge happens after END_CRIT_SECTION + * in this same function. */ + if (SCN_VALID(commit_scn)) + (void) cluster_scn_durable_pending_fill_lsn(commit_scn, recptr); +#endif + if (replorigin) /* Move LSNs forward for this replication origin */ replorigin_session_advance(replorigin_session_origin_lsn, XactLastRecEnd); @@ -2425,7 +2444,12 @@ RecordTransactionCommitPrepared(TransactionId xid, int nchildren, TransactionId #ifdef USE_PGRAC_CLUSTER (void)cluster_scn_pending_commit_clear(commit_scn); if (SCN_VALID(commit_scn)) + { (void)cluster_adg_emit_thread_barrier(); + /* PGRAC: spec-7.4 D1 -- prepared commits always XLogFlush above; + * discharge the durable frontier outside the critical section. */ + (void)cluster_scn_durable_pending_discharge_scn(commit_scn); + } #endif /* diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index c32621305a..8eda1e6230 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -31,6 +31,15 @@ * path) deliberately skipped -- only top-level abort decisions * advance SCN per Q4. * + * What changed (spec-7.4 D1): + * - RecordTransactionCommit(): durable-frontier wiring. LSN fill-in + * after XactLogCommitRecord (crit-section-safe shmem write); a + * local commit_record_flushed flag set after our own XLogFlush; + * frontier discharge AFTER END_CRIT_SECTION (sync commits only -- + * async commits leave the entry for the walwriter flush horizon). + * Precommit PG_CATCH + AbortTransaction() unblock the frontier via + * cluster_scn_durable_pending_abort_self(). + * * What changed (spec-2.6 v0.2 D6 — Sprint A Step 3): * - CommitTransaction(): added cluster fail-closed commit-boundary * check after PreCommit_Portals loop, before CallXactCallbacks @@ -1476,6 +1485,13 @@ RecordTransactionCommit(void) * additional SCN allocation. */ SCN tt_commit_scn = InvalidScn; + bool commit_record_flushed = false; /* PGRAC: spec-7.4 D1 -- set + * inside the commit critical + * section after our own + * XLogFlush; consumed AFTER + * END_CRIT_SECTION (no publish + * work inside the crit + * section) */ #endif /* @@ -1685,6 +1701,11 @@ RecordTransactionCommit(void) PG_CATCH(); { cluster_backup_pending_commit_abort(0, (Datum)0); + /* PGRAC: spec-7.4 D1 -- an ERROR escaped the precommit gap; + * the allocated commit SCN never became a commit record. + * Unblock the durable frontier (lsn-filled entries are kept + * by the guard inside). */ + (void)cluster_scn_durable_pending_abort_self(); PG_RE_THROW(); } PG_END_TRY(); @@ -1707,6 +1728,12 @@ RecordTransactionCommit(void) has_tt_fold ? &tt_fold : NULL); /* PGRAC: spec-3.18 D4.1 */ #ifdef USE_PGRAC_CLUSTER cluster_backup_pending_commit_exit(); + /* PGRAC: spec-7.4 D1 -- the commit record now exists; record its + * end LSN on the durable-pending entry so the walwriter flush + * horizon can discharge it (async commits never flush here). Pure + * shmem write under the SCN lwlock; crit-section safe. */ + if (SCN_VALID(tt_commit_scn)) + (void) cluster_scn_durable_pending_fill_lsn(tt_commit_scn, XactLastRecEnd); #endif if (replorigin) @@ -1773,6 +1800,11 @@ RecordTransactionCommit(void) cluster_xp_end(&commit_flush_xps); if (cluster_smart_fusion) cluster_sf_publish_origin_durable_lsn(); + /* PGRAC: spec-7.4 D1 -- our commit record is durable; only mark + * a local flag here. The frontier discharge + publish happens + * after END_CRIT_SECTION (mini-plan v1.1 ruling #3: no publish + * step may run inside the commit critical section). */ + commit_record_flushed = true; #endif /* @@ -1821,6 +1853,11 @@ RecordTransactionCommit(void) (void)cluster_scn_pending_commit_clear(tt_commit_scn); if (SCN_VALID(tt_commit_scn)) (void)cluster_adg_emit_thread_barrier(); + /* PGRAC: spec-7.4 D1 -- sync-commit frontier discharge, outside the + * critical section. Async commits leave their (lsn-filled) entry + * for the walwriter flush horizon. */ + if (commit_record_flushed && SCN_VALID(tt_commit_scn)) + (void)cluster_scn_durable_pending_discharge_scn(tt_commit_scn); } #endif @@ -3313,6 +3350,11 @@ AbortTransaction(void) * freeze permanently on an aborted local transaction. */ (void)cluster_scn_pending_commit_clear_my_pending(); + /* spec-7.4 D1: same discipline for the durable frontier registry -- + * an aborted allocation must not block the frontier forever. The + * lsn-filled guard inside keeps real (async) commits pending. */ + (void)cluster_scn_durable_pending_abort_self(); + /* spec-6.14 D8: an ERROR escaping mid-resolve unwinds past the resolver's * no-recursion depth decrement; reset it so the catalog-snapshot guard * does not false-positive on the next catalog scan. */ diff --git a/src/backend/cluster/cluster_scn.c b/src/backend/cluster/cluster_scn.c index b290141fef..f4241616ce 100644 --- a/src/backend/cluster/cluster_scn.c +++ b/src/backend/cluster/cluster_scn.c @@ -56,6 +56,7 @@ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ #include "cluster/cluster_shmem.h" #include "cluster/cluster_xnode_profile.h" /* PGRAC: spec-5.59 D2 profiling */ +#include "access/xlog.h" /* GetFlushRecPtr (spec-7.4 D1 walwriter discharge) */ #include "miscadmin.h" #include "storage/lwlock.h" #include "storage/shmem.h" @@ -163,6 +164,47 @@ typedef struct ClusterScnSharedState { pg_atomic_uint64 adg_pending_register_count; pg_atomic_uint64 adg_pending_clear_count; pg_atomic_uint64 adg_pending_overflow_count; + /* + * spec-7.4 D1: durable_safe_scn frontier registry (own origin). + * + * Tracks commit SCNs allocated but not yet proven durable, so the + * published frontier stays a CONTIGUOUS durability claim: every + * own-origin commit SCN <= durable_safe_scn is durable on WAL. A + * plain atomic-max of flushed commit SCNs would falsely cover a + * stalled earlier allocation (P0; mini-plan v1.1 §1.1). + * + * durable_pending_scn/lsn are parallel arrays under lwlock (lsn is + * InvalidXLogRecPtr until the commit record is inserted). Overflow + * freezes the frontier stickily (until restart); consumers fall + * back to the fetch-reply piggyback sampling. durable_safe_scn is + * read lock-free (single u64; monotonic under the lock). + */ + uint16 durable_pending_count; + bool durable_frozen; + SCN durable_pending_scn[CLUSTER_SCN_DURABLE_PENDING_MAX]; + XLogRecPtr durable_pending_lsn[CLUSTER_SCN_DURABLE_PENDING_MAX]; + SCN durable_last_allocated; + pg_atomic_uint64 durable_safe_scn; + pg_atomic_uint64 durable_overflow_count; + pg_atomic_uint64 durable_regression_count; + /* + * spec-7.4 D1: per-origin remote durable frontier cache. + * + * {epoch, scn} pairs consistency-protected by a per-origin seqlock + * (remote_durable_seq; odd = write in progress). Single writer: + * the LMON dispatch running the BOC handler. Readers bounded-spin + * and fall back (return false) rather than blocking -- the write + * side has no failure point between seq bumps, and a crashed LMON + * takes the whole cluster through PG crash-restart (shmem rebuilt), + * so a stuck-odd seqlock cannot outlive its readers. + */ + pg_atomic_uint32 remote_durable_seq[CLUSTER_MAX_NODES]; + uint64 remote_durable_epoch[CLUSTER_MAX_NODES]; + SCN remote_durable_scn[CLUSTER_MAX_NODES]; + pg_atomic_uint64 boc_payload_accept_count; + pg_atomic_uint64 boc_payload_bad_length_count; + pg_atomic_uint64 boc_payload_node_mismatch_count; + pg_atomic_uint64 boc_payload_regression_count; } ClusterScnSharedState; /* spec-2.12 D2: defensive — raw-bits storage of TimestampTz in @@ -174,12 +216,17 @@ StaticAssertDecl(sizeof(TimestampTz) == sizeof(uint64), static ClusterScnSharedState *cluster_scn_state = NULL; static SCN cluster_scn_backend_pending_commit_scn = InvalidScn; +/* spec-7.4 D1: this backend's in-flight durable-pending commit SCN. */ +static SCN cluster_scn_backend_durable_pending_scn = InvalidScn; /* WARNING throttle: at most 1/min */ static TimestampTz last_warn_emitted_at = 0; static bool cluster_scn_pending_commit_register_locked(SCN commit_scn); static bool cluster_scn_pending_commit_remove_locked(SCN commit_scn); +static void cluster_scn_durable_pending_register_locked(SCN commit_scn); +static bool cluster_scn_durable_pending_remove_locked(SCN commit_scn); +static void cluster_scn_durable_frontier_publish_locked(void); /* @@ -627,7 +674,7 @@ cluster_scn_advance_for_commit(void) CLUSTER_INJECTION_POINT("cluster-scn-commit-pre-advance"); - if (cluster_enable_adg) { + { NodeId node; Assert(cluster_scn_state != NULL); @@ -638,6 +685,13 @@ cluster_scn_advance_for_commit(void) * cluster_scn_advance() hot path publishes with fetch_add first, so * ADG commit advance uses a register-before-CAS reservation loop. * + * spec-7.4 D1 extends the same invariant to the durable frontier: + * an allocated commit SCN must be in the durable pending registry + * BEFORE it becomes observable, or a concurrent discharge could + * publish a frontier past it (the atomic-max P0). The reservation + * loop therefore runs for EVERY commit advance; the ADG registry + * registration inside it stays gated on cluster_enable_adg. + * * Other, non-commit SCN advances may still race through the lock-free * path. A failed CAS removes only this backend's tentative entry and * retries with the new current value; a barrier seeing the tentative @@ -669,7 +723,9 @@ cluster_scn_advance_for_commit(void) continue; } - (void)cluster_scn_pending_commit_register_locked(scn); + if (cluster_enable_adg) + (void)cluster_scn_pending_commit_register_locked(scn); + cluster_scn_durable_pending_register_locked(scn); expected = old_local; if (pg_atomic_compare_exchange_u64(&cluster_scn_state->current_local_scn, &expected, @@ -678,11 +734,12 @@ cluster_scn_advance_for_commit(void) break; } - (void)cluster_scn_pending_commit_remove_locked(scn); + if (cluster_enable_adg) + (void)cluster_scn_pending_commit_remove_locked(scn); + (void)cluster_scn_durable_pending_remove_locked(scn); LWLockRelease(&cluster_scn_state->lwlock); } - } else - scn = cluster_scn_advance(); + } pg_atomic_fetch_add_u64(&cluster_scn_state->commit_advance_count, 1); CLUSTER_INJECTION_POINT("cluster-scn-commit-post-advance"); @@ -910,6 +967,321 @@ cluster_scn_adg_pending_overflowed(void) return overflowed; } +/* + * ============================================================ + * spec-7.4 D1: durable_safe_scn frontier registry. + * ============================================================ + * + * Register at commit-SCN allocation (inside the reservation loop in + * cluster_scn_advance_for_commit, under lwlock), fill in the commit + * record LSN once XactLogCommitRecord returns, discharge on one of + * three paths: + * + * (a) sync commit -- the committing backend proved its own record + * durable via XLogFlush; it discharges by SCN after leaving + * the commit critical section. + * (b) async commit -- walwriter discharges every entry whose LSN + * is at or below the background-flush horizon. + * (c) abort -- allocation never became a commit; remove it so the + * frontier is not blocked forever. + * + * The published frontier is min(pending)-1 (time predecessor) while + * entries are pending, else the last allocated commit SCN. Both are + * safe: no own-origin commit SCN above the published value is ever + * claimed durable, and every commit SCN at or below it has been + * discharged as durable (aborted allocations are not commits and make + * no durability claim). + */ + +/* + * cluster_scn_durable_pending_register_locked -- caller holds lwlock + * LW_EXCLUSIVE. Registers a freshly allocated commit SCN with an + * unknown LSN. On overflow the frontier freezes stickily instead of + * risking an unsafe claim (LOG once on the transition; counter for + * monitoring). + */ +static void +cluster_scn_durable_pending_register_locked(SCN commit_scn) +{ + if (cluster_scn_state == NULL || !SCN_VALID(commit_scn)) + return; + + /* Track every commit allocation, frozen or not: the empty-registry + * frontier re-derivation depends on it. */ + if (!SCN_VALID(cluster_scn_state->durable_last_allocated) + || scn_total_cmp(commit_scn, cluster_scn_state->durable_last_allocated) > 0) + cluster_scn_state->durable_last_allocated = commit_scn; + + if (cluster_scn_state->durable_frozen) + return; + + if (cluster_scn_state->durable_pending_count >= CLUSTER_SCN_DURABLE_PENDING_MAX) { + cluster_scn_state->durable_frozen = true; + pg_atomic_fetch_add_u64(&cluster_scn_state->durable_overflow_count, 1); + ereport(LOG, (errmsg("cluster durable SCN frontier frozen: pending registry overflow " + "(more than %d in-flight commits)", + CLUSTER_SCN_DURABLE_PENDING_MAX), + errhint("Remote nodes fall back to fetch-reply SCN sampling until this " + "node restarts."))); + return; + } + + cluster_scn_state->durable_pending_scn[cluster_scn_state->durable_pending_count] = commit_scn; + cluster_scn_state->durable_pending_lsn[cluster_scn_state->durable_pending_count] + = InvalidXLogRecPtr; + cluster_scn_state->durable_pending_count++; + cluster_scn_backend_durable_pending_scn = commit_scn; +} + +/* + * cluster_scn_durable_pending_remove_locked -- caller holds lwlock + * LW_EXCLUSIVE. Swap-removes the entry for commit_scn (order is not + * meaningful; the frontier is derived by a min scan). Does NOT + * publish; callers decide whether removal was a durability proof. + */ +static bool +cluster_scn_durable_pending_remove_locked(SCN commit_scn) +{ + uint16 i; + + if (cluster_scn_state == NULL || !SCN_VALID(commit_scn)) + return false; + + for (i = 0; i < cluster_scn_state->durable_pending_count; i++) { + if (scn_total_cmp(cluster_scn_state->durable_pending_scn[i], commit_scn) == 0) { + uint16 last = (uint16)(cluster_scn_state->durable_pending_count - 1); + + cluster_scn_state->durable_pending_scn[i] + = cluster_scn_state->durable_pending_scn[last]; + cluster_scn_state->durable_pending_lsn[i] + = cluster_scn_state->durable_pending_lsn[last]; + cluster_scn_state->durable_pending_scn[last] = InvalidScn; + cluster_scn_state->durable_pending_lsn[last] = InvalidXLogRecPtr; + cluster_scn_state->durable_pending_count = last; + if (scn_total_cmp(cluster_scn_backend_durable_pending_scn, commit_scn) == 0) + cluster_scn_backend_durable_pending_scn = InvalidScn; + return true; + } + } + return false; +} + +/* + * cluster_scn_durable_frontier_publish_locked -- caller holds lwlock + * LW_EXCLUSIVE. Recomputes the contiguous durable frontier and + * publishes it monotonically. A recomputed value below the published + * one is a bug in the discharge protocol: refuse + count (L441 + * monotonic discipline), never regress the public claim. + */ +static void +cluster_scn_durable_frontier_publish_locked(void) +{ + SCN frontier; + SCN current; + + if (cluster_scn_state == NULL || cluster_scn_state->durable_frozen) + return; + + if (cluster_scn_state->durable_pending_count > 0) { + SCN min_pending = cluster_scn_state->durable_pending_scn[0]; + uint16 i; + + for (i = 1; i < cluster_scn_state->durable_pending_count; i++) { + if (scn_total_cmp(cluster_scn_state->durable_pending_scn[i], min_pending) < 0) + min_pending = cluster_scn_state->durable_pending_scn[i]; + } + frontier = cluster_scn_time_predecessor(min_pending); + } else + frontier = cluster_scn_state->durable_last_allocated; + + if (!SCN_VALID(frontier)) + return; + + current = (SCN)pg_atomic_read_u64(&cluster_scn_state->durable_safe_scn); + if (SCN_VALID(current) && scn_total_cmp(frontier, current) < 0) { + pg_atomic_fetch_add_u64(&cluster_scn_state->durable_regression_count, 1); + return; + } + pg_atomic_write_u64(&cluster_scn_state->durable_safe_scn, frontier); +} + +/* + * cluster_scn_durable_safe_scn -- lock-free read of the published + * contiguous durable frontier (InvalidScn until anything published). + */ +SCN +cluster_scn_durable_safe_scn(void) +{ + if (cluster_scn_state == NULL) + return InvalidScn; + return (SCN)pg_atomic_read_u64(&cluster_scn_state->durable_safe_scn); +} + +/* + * cluster_scn_durable_pending_fill_lsn -- record the commit-record LSN + * for a pending commit SCN once XactLogCommitRecord has returned. + * Until filled, the walwriter flush horizon can never discharge the + * entry (an unwritten record has no durability horizon). Pure shmem + * write under the lock; safe inside the commit critical section. + */ +bool +cluster_scn_durable_pending_fill_lsn(SCN commit_scn, XLogRecPtr commit_lsn) +{ + uint16 i; + bool found = false; + + if (cluster_scn_state == NULL || !SCN_VALID(commit_scn) || XLogRecPtrIsInvalid(commit_lsn)) + return false; + + LWLockAcquire(&cluster_scn_state->lwlock, LW_EXCLUSIVE); + for (i = 0; i < cluster_scn_state->durable_pending_count; i++) { + if (scn_total_cmp(cluster_scn_state->durable_pending_scn[i], commit_scn) == 0) { + cluster_scn_state->durable_pending_lsn[i] = commit_lsn; + found = true; + break; + } + } + LWLockRelease(&cluster_scn_state->lwlock); + return found; +} + +/* + * cluster_scn_durable_pending_discharge_scn -- sync-commit discharge: + * the caller proved this commit record durable (its own XLogFlush + * returned). Runs AFTER the commit critical section (mini-plan v1.1 + * ruling #3: no publish work inside the crit section). + */ +bool +cluster_scn_durable_pending_discharge_scn(SCN commit_scn) +{ + bool removed; + + if (cluster_scn_state == NULL || !SCN_VALID(commit_scn)) + return false; + + LWLockAcquire(&cluster_scn_state->lwlock, LW_EXCLUSIVE); + removed = cluster_scn_durable_pending_remove_locked(commit_scn); + if (removed) + cluster_scn_durable_frontier_publish_locked(); + LWLockRelease(&cluster_scn_state->lwlock); + return removed; +} + +/* + * cluster_scn_durable_pending_abort_self -- abort-path cleanup: this + * backend's in-flight allocation never became a commit; unblock the + * frontier. Mirrors cluster_scn_pending_commit_clear_my_pending. + * + * Guard: an entry with a FILLED lsn is a real commit (its commit + * record was inserted; abort cannot happen past that point without + * PANIC, and an async commit leaves its entry filled for the + * walwriter). Such an entry is only detached from this backend, never + * removed -- removing it would let the frontier cover an unflushed + * commit (the P0 this registry exists to prevent). + */ +bool +cluster_scn_durable_pending_abort_self(void) +{ + SCN pending = cluster_scn_backend_durable_pending_scn; + bool removed = false; + uint16 i; + + if (cluster_scn_state == NULL || !SCN_VALID(pending)) + return false; + + LWLockAcquire(&cluster_scn_state->lwlock, LW_EXCLUSIVE); + for (i = 0; i < cluster_scn_state->durable_pending_count; i++) { + if (scn_total_cmp(cluster_scn_state->durable_pending_scn[i], pending) == 0) { + if (XLogRecPtrIsInvalid(cluster_scn_state->durable_pending_lsn[i])) { + removed = cluster_scn_durable_pending_remove_locked(pending); + if (removed) + cluster_scn_durable_frontier_publish_locked(); + } + break; + } + } + LWLockRelease(&cluster_scn_state->lwlock); + cluster_scn_backend_durable_pending_scn = InvalidScn; + return removed; +} + +/* + * cluster_scn_durable_pending_discharge_upto -- walwriter discharge: + * every pending entry whose (known) commit-record LSN is at or below + * the background-flush horizon is durable. Entries with an unknown + * LSN stay pending regardless of the horizon. Returns the number of + * entries discharged. + */ +uint32 +cluster_scn_durable_pending_discharge_upto(XLogRecPtr flushed_lsn) +{ + uint32 discharged = 0; + uint16 i; + + if (cluster_scn_state == NULL || XLogRecPtrIsInvalid(flushed_lsn)) + return 0; + + LWLockAcquire(&cluster_scn_state->lwlock, LW_EXCLUSIVE); + i = 0; + while (i < cluster_scn_state->durable_pending_count) { + XLogRecPtr entry_lsn = cluster_scn_state->durable_pending_lsn[i]; + + if (!XLogRecPtrIsInvalid(entry_lsn) && entry_lsn <= flushed_lsn) { + /* swap-remove keeps index i pointing at an unvisited entry */ + (void)cluster_scn_durable_pending_remove_locked( + cluster_scn_state->durable_pending_scn[i]); + discharged++; + } else + i++; + } + if (discharged > 0) + cluster_scn_durable_frontier_publish_locked(); + LWLockRelease(&cluster_scn_state->lwlock); + return discharged; +} + +uint64 +cluster_scn_durable_pending_count(void) +{ + uint64 count; + + if (cluster_scn_state == NULL) + return 0; + LWLockAcquire(&cluster_scn_state->lwlock, LW_SHARED); + count = cluster_scn_state->durable_pending_count; + LWLockRelease(&cluster_scn_state->lwlock); + return count; +} + +bool +cluster_scn_durable_frontier_frozen(void) +{ + bool frozen; + + if (cluster_scn_state == NULL) + return false; + LWLockAcquire(&cluster_scn_state->lwlock, LW_SHARED); + frozen = cluster_scn_state->durable_frozen; + LWLockRelease(&cluster_scn_state->lwlock); + return frozen; +} + +uint64 +cluster_scn_durable_frontier_overflow_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->durable_overflow_count); +} + +uint64 +cluster_scn_durable_frontier_regression_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->durable_regression_count); +} + /* * cluster_scn_current -- read current encoded SCN without advancing. * @@ -1027,37 +1399,208 @@ cluster_scn_observe_bump_count(void) * ============================================================ */ +/* + * cluster_scn_boc_payload_encode / _decode -- spec-7.4 D1 BOC payload v1. + * + * Explicit little-endian, independent of host byte order (the envelope + * wire is LE per spec-2.3; the payload follows the same convention). + * v1 carries exactly one field: origin_durable_safe_scn (8 bytes). + */ +void +cluster_scn_boc_payload_encode(SCN scn, uint8 *buf) +{ + int i; + + for (i = 0; i < CLUSTER_SCN_BOC_PAYLOAD_V1_LEN; i++) + buf[i] = (uint8)((scn >> (8 * i)) & 0xFF); +} + +SCN +cluster_scn_boc_payload_decode(const uint8 *buf) +{ + SCN scn = 0; + int i; + + for (i = 0; i < CLUSTER_SCN_BOC_PAYLOAD_V1_LEN; i++) + scn |= ((SCN)buf[i]) << (8 * i); + return scn; +} + /* * cluster_scn_boc_broadcast_handler -- spec-2.9 D3 dispatch handler. * * Registered via cluster_lmon.c phase 1 (spec-2.9 D1) for msg_type = * PGRAC_IC_MSG_BOC_BROADCAST (= 3). * - * NO-OP body by design (spec-2.9 §3.0 I6 SCN-via-envelope-piggyback): + * Envelope-side Lamport piggyback is unchanged (spec-2.9 §3.0 I6): * envelope.scn (frozen at offset 20 per spec-2.3) is observed via * cluster_ic_envelope_verify -> cluster_ic_envelope_observe_scn * (spec-2.4 D5) BEFORE this handler fires. Handler MUST NOT call * cluster_scn_observe directly (spec-2.9 §3.0 I6 + T-scn-13c grep * invariant). * - * DEBUG2 log is the only side-effect (spec-2.9 §3.0 I5 payload-zero): - * BOC pulse traffic is otherwise invisible from SQL surface; this - * helps dev/test trace cross-instance pulse arrival. + * spec-7.4 D1 (spec-2.9 I5 v0.5 amend): payload_length in {0, 8}. + * 0 = pre-v1 sender or event publish off -- pulse-only frame, DEBUG2 + * trace only. 8 = payload v1 origin_durable_safe_scn (LE): validate + * length / decodability / SCN node bits == env.source_node_id, then + * update the per-(origin, epoch) cache. Every reject is a counted + * fail-closed drop; a lower SCN within the same epoch is a counted + * regression reject (reordered frame); an epoch change rebuilds the + * entry (R5). + * + * PGRAC modifications by SqlRush : + * What changed: spec-7.4 D1 replaced the payload-zero Assert with the + * versioned {0, 8} contract above. + * Why: the durable frontier rides the existing BOC pulse (zero new + * msg_type); mixed-version peers keep sending 0-byte frames, which + * remain valid pulse-only traffic. */ void -cluster_scn_boc_broadcast_handler(const ClusterICEnvelope *env, - const void *payload pg_attribute_unused()) +cluster_scn_boc_broadcast_handler(const ClusterICEnvelope *env, const void *payload) { - /* spec-2.9 §3.0 I5 payload-zero invariant: BOC_BROADCAST carries - * no payload. Receive-side Assert mirrors send-side payload_len=0 - * (cluster_scn_emit_broadcast_pulse, spec-2.9 D2). Field name on - * the envelope wire is payload_length (uint32, offset 28). */ + SCN scn; + uint32 origin; + uint32 seq; + bool same_epoch; + SCN cached; + Assert(env != NULL); - Assert(env->payload_length == 0); - ereport(DEBUG2, + if (env->payload_length == 0) { + /* Pre-v1 sender / publish off: pulse-only frame (I5 v0.4 + * behavior preserved bit-for-bit). */ + ereport( + DEBUG2, (errmsg("cluster_scn: BOC broadcast received from peer %u (env.scn=" UINT64_FORMAT ")", env->source_node_id, env->scn))); + return; + } + + if (cluster_scn_state == NULL) + return; /* shmem not up; nothing to update */ + + if (env->payload_length != CLUSTER_SCN_BOC_PAYLOAD_V1_LEN || payload == NULL) { + pg_atomic_fetch_add_u64(&cluster_scn_state->boc_payload_bad_length_count, 1); + ereport(DEBUG1, (errmsg("cluster_scn: dropped BOC payload with bad length %u from peer %u", + env->payload_length, env->source_node_id))); + return; + } + + scn = cluster_scn_boc_payload_decode(payload); + if (!SCN_VALID(scn)) { + /* Structurally 8 bytes but undecodable (InvalidScn): count it + * with the malformed-frame drops. */ + pg_atomic_fetch_add_u64(&cluster_scn_state->boc_payload_bad_length_count, 1); + ereport(DEBUG1, (errmsg("cluster_scn: dropped undecodable BOC payload from peer %u", + env->source_node_id))); + return; + } + + origin = env->source_node_id; + if (origin >= CLUSTER_MAX_NODES || (uint32)scn_node_id(scn) != origin) { + pg_atomic_fetch_add_u64(&cluster_scn_state->boc_payload_node_mismatch_count, 1); + ereport(DEBUG1, (errmsg("cluster_scn: dropped BOC payload with node mismatch (scn node %d, " + "envelope peer %u)", + (int)scn_node_id(scn), env->source_node_id))); + return; + } + + /* + * Per-(origin, epoch) cache update. Single writer (the LMON dispatch + * running this handler), so reading the current fields directly is + * safe; the seqlock protects concurrent readers only. + */ + same_epoch = (cluster_scn_state->remote_durable_epoch[origin] == env->epoch); + cached = cluster_scn_state->remote_durable_scn[origin]; + if (same_epoch && SCN_VALID(cached) && scn_total_cmp(scn, cached) < 0) { + pg_atomic_fetch_add_u64(&cluster_scn_state->boc_payload_regression_count, 1); + return; /* reordered older frame; keep the newer claim */ + } + + seq = pg_atomic_read_u32(&cluster_scn_state->remote_durable_seq[origin]); + pg_atomic_write_u32(&cluster_scn_state->remote_durable_seq[origin], seq + 1); + pg_write_barrier(); + cluster_scn_state->remote_durable_epoch[origin] = env->epoch; + cluster_scn_state->remote_durable_scn[origin] = scn; + pg_write_barrier(); + pg_atomic_write_u32(&cluster_scn_state->remote_durable_seq[origin], seq + 2); + pg_atomic_fetch_add_u64(&cluster_scn_state->boc_payload_accept_count, 1); + + ereport(DEBUG2, (errmsg("cluster_scn: BOC durable frontier from peer %u = " UINT64_FORMAT + " (epoch " UINT64_FORMAT ")", + env->source_node_id, scn, env->epoch))); +} + +/* + * cluster_scn_remote_durable_safe -- read the cached durable frontier + * claim for a remote origin. Returns false when the origin is out of + * range, nothing has been cached yet, or the bounded seqlock spin is + * exhausted -- callers fall back to the fetch-reply piggyback sampling + * (never block on this cache). + */ +bool +cluster_scn_remote_durable_safe(NodeId origin, uint64 *epoch_out, SCN *scn_out) +{ + int retries; + + if (cluster_scn_state == NULL || origin < 0 || origin >= CLUSTER_MAX_NODES) + return false; + + for (retries = 0; retries < 1000; retries++) { + uint32 s1 = pg_atomic_read_u32(&cluster_scn_state->remote_durable_seq[origin]); + uint64 epoch; + SCN scn; + + if (s1 & 1) + continue; /* write in progress */ + pg_read_barrier(); + epoch = cluster_scn_state->remote_durable_epoch[origin]; + scn = cluster_scn_state->remote_durable_scn[origin]; + pg_read_barrier(); + if (pg_atomic_read_u32(&cluster_scn_state->remote_durable_seq[origin]) != s1) + continue; /* torn read; retry */ + + if (!SCN_VALID(scn)) + return false; /* nothing cached for this origin yet */ + if (epoch_out != NULL) + *epoch_out = epoch; + if (scn_out != NULL) + *scn_out = scn; + return true; + } + return false; /* bounded spin exhausted; fail open to fallback path */ +} + +uint64 +cluster_scn_boc_payload_accept_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->boc_payload_accept_count); +} + +uint64 +cluster_scn_boc_payload_bad_length_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->boc_payload_bad_length_count); +} + +uint64 +cluster_scn_boc_payload_node_mismatch_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->boc_payload_node_mismatch_count); +} + +uint64 +cluster_scn_boc_payload_regression_count(void) +{ + if (cluster_scn_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_scn_state->boc_payload_regression_count); } /* @@ -1215,6 +1758,15 @@ cluster_scn_boc_tick(void) if (cluster_scn_state == NULL) return; + /* + * spec-7.4 D1: async-commit durable frontier discharge. Runs on + * every walwriter wakeup (NOT throttled by the sweep interval): the + * caller sits right behind XLogBackgroundFlush, so the flush horizon + * proves every filled pending entry at or below it durable. Cheap + * no-op when nothing is pending. + */ + (void)cluster_scn_durable_pending_discharge_upto(GetFlushRecPtr(NULL)); + now_ts = GetCurrentTimestamp(); /* Throttle: skip if elapsed < cluster.boc_sweep_interval_ms. @@ -1486,6 +2038,31 @@ cluster_scn_shmem_init(void) pg_atomic_init_u64(&cluster_scn_state->adg_pending_register_count, 0); pg_atomic_init_u64(&cluster_scn_state->adg_pending_clear_count, 0); pg_atomic_init_u64(&cluster_scn_state->adg_pending_overflow_count, 0); + /* spec-7.4 D1: durable_safe_scn frontier registry. */ + cluster_scn_state->durable_pending_count = 0; + cluster_scn_state->durable_frozen = false; + memset(cluster_scn_state->durable_pending_scn, 0, + sizeof(cluster_scn_state->durable_pending_scn)); + memset(cluster_scn_state->durable_pending_lsn, 0, + sizeof(cluster_scn_state->durable_pending_lsn)); + cluster_scn_state->durable_last_allocated = InvalidScn; + pg_atomic_init_u64(&cluster_scn_state->durable_safe_scn, InvalidScn); + pg_atomic_init_u64(&cluster_scn_state->durable_overflow_count, 0); + pg_atomic_init_u64(&cluster_scn_state->durable_regression_count, 0); + /* spec-7.4 D1: per-origin remote frontier cache + payload counters. */ + { + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + pg_atomic_init_u32(&cluster_scn_state->remote_durable_seq[i], 0); + cluster_scn_state->remote_durable_epoch[i] = 0; + cluster_scn_state->remote_durable_scn[i] = InvalidScn; + } + } + pg_atomic_init_u64(&cluster_scn_state->boc_payload_accept_count, 0); + pg_atomic_init_u64(&cluster_scn_state->boc_payload_bad_length_count, 0); + pg_atomic_init_u64(&cluster_scn_state->boc_payload_node_mismatch_count, 0); + pg_atomic_init_u64(&cluster_scn_state->boc_payload_regression_count, 0); } } diff --git a/src/include/cluster/cluster_scn.h b/src/include/cluster/cluster_scn.h index b7727672f6..1576d1ab23 100644 --- a/src/include/cluster/cluster_scn.h +++ b/src/include/cluster/cluster_scn.h @@ -297,6 +297,58 @@ extern SCN cluster_scn_adg_thread_safe_scn(void); extern uint64 cluster_scn_adg_pending_count(void); extern bool cluster_scn_adg_pending_overflowed(void); +/* + * spec-7.4 D1 durable_safe_scn frontier registry. + * + * The frontier is the CONTIGUOUS durable watermark this origin publishes on + * the BOC channel: every own-origin commit SCN <= the frontier is durable + * on WAL. A plain atomic-max of flushed commit SCNs is forbidden (P0): + * a stalled earlier allocation would be falsely claimed durable. + * + * Lifecycle: cluster_scn_advance_for_commit() registers the returned SCN + * with an unknown LSN; the commit path fills in the commit-record LSN once + * known; discharge happens on one of three paths (sync commit after its + * own XLogFlush, walwriter behind XLogBackgroundFlush for async commits, + * abort cleanup). Registry overflow freezes the frontier stickily (until + * restart) rather than publishing an unsafe claim; consumers fall back to + * the existing fetch-reply piggyback sampling. + */ +#define CLUSTER_SCN_DURABLE_PENDING_MAX 256 + +extern SCN cluster_scn_durable_safe_scn(void); +extern bool cluster_scn_durable_pending_fill_lsn(SCN commit_scn, XLogRecPtr commit_lsn); +extern bool cluster_scn_durable_pending_discharge_scn(SCN commit_scn); +extern bool cluster_scn_durable_pending_abort_self(void); +extern uint32 cluster_scn_durable_pending_discharge_upto(XLogRecPtr flushed_lsn); +extern uint64 cluster_scn_durable_pending_count(void); +extern bool cluster_scn_durable_frontier_frozen(void); +extern uint64 cluster_scn_durable_frontier_overflow_count(void); +extern uint64 cluster_scn_durable_frontier_regression_count(void); + +/* + * spec-7.4 D1: BOC payload v1 (wire) + per-origin remote frontier cache. + * + * Wire contract (spec-2.9 I5 v0.5 amend): BOC_BROADCAST payload_length is + * 0 (pre-v1 sender / event publish off) or 8 (v1 = origin_durable_safe_scn, + * explicit little-endian). Any other length is a counted fail-closed drop. + * The v2 WAL-LSN field (7.5 Smart Fusion) will be a separate appended + * field; SCN and WAL-LSN domains never share bytes (L457). + * + * Accepted values land in a per-(origin, epoch) seqlock cache (single + * writer: the LMON dispatch that runs the handler). Readers use + * cluster_scn_remote_durable_safe(); a bounded-spin failure returns false + * so consumers fall back to the fetch-reply piggyback sampling. + */ +#define CLUSTER_SCN_BOC_PAYLOAD_V1_LEN 8 + +extern void cluster_scn_boc_payload_encode(SCN scn, uint8 *buf); +extern SCN cluster_scn_boc_payload_decode(const uint8 *buf); +extern bool cluster_scn_remote_durable_safe(NodeId origin, uint64 *epoch_out, SCN *scn_out); +extern uint64 cluster_scn_boc_payload_accept_count(void); +extern uint64 cluster_scn_boc_payload_bad_length_count(void); +extern uint64 cluster_scn_boc_payload_node_mismatch_count(void); +extern uint64 cluster_scn_boc_payload_regression_count(void); + /* spec-1.16 stat accessors (LW_SHARED). */ extern uint64 cluster_scn_commit_advance_count(void); extern uint64 cluster_scn_abort_advance_count(void); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 094792a98f..c7a60fbed4 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -37,7 +37,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_conf \ test_cluster_ic_mock test_cluster_inject test_cluster_pgstat \ test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr \ - test_cluster_scn test_cluster_block_format test_cluster_itl_slot \ + test_cluster_scn test_cluster_scn_frontier test_cluster_block_format test_cluster_itl_slot \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ @@ -219,7 +219,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_lms_shard test_cluster_gcs_block_dedup test_cluster_gcs_block_shard,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_scn_frontier test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_lms_shard test_cluster_gcs_block_dedup test_cluster_gcs_block_shard,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region # + pg_atomic_*; the test stubs all of these and exercises only the @@ -1156,6 +1156,14 @@ test_cluster_scn: test_cluster_scn.c unit_test.h \ $(CC) $(CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_VERSION_O) $(CLUSTER_SCN_O) -o $@ +# spec-7.4 D1: test_cluster_scn_frontier links cluster_scn.o standalone +# (same stub model as test_cluster_scn) and exercises the durable +# pending registry real-behavior paths via the in-process shmem buffer. +test_cluster_scn_frontier: test_cluster_scn_frontier.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_SCN_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_SCN_O) -o $@ + # spec-6.4 D1: test_cluster_adg links the dependency-light ADG pure # primitives. SCN comparators are stubbed locally so this binary does not pull # cluster_scn.o's backend/shmem dependencies. diff --git a/src/test/cluster_unit/test_cluster_scn.c b/src/test/cluster_unit/test_cluster_scn.c index 556898e242..1616daadee 100644 --- a/src/test/cluster_unit/test_cluster_scn.c +++ b/src/test/cluster_unit/test_cluster_scn.c @@ -169,7 +169,8 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) * address in standalone unit tests. */ static union { uint64 force_align; - char data[8192]; /* generous; cluster_scn_shmem_size() << 8KB */ + char data[16384]; /* generous; spec-7.4 D1 grew the state past 8KB + * (durable frontier registry + remote cache) */ } scn_buf; static bool scn_initialized = false; @@ -343,6 +344,20 @@ cluster_conf_node_count(void) volatile uint32 CritSectionCount = 0; BackendType MyBackendType = B_LMON; +/* spec-7.4 D1 / L104: cluster_scn.o boc_tick now calls GetFlushRecPtr + * for the walwriter async-commit frontier discharge. Standalone unit + * binaries link cluster_scn.o only; stub returns 0 (InvalidXLogRecPtr) + * so discharge_upto is a guarded no-op unless a test drives the API + * directly with explicit horizons. */ +XLogRecPtr +GetFlushRecPtr(TimeLineID *insertTLI) +{ + if (insertTLI != NULL) + *insertTLI = 0; + return 0; +} + + UT_DEFINE_GLOBALS(); diff --git a/src/test/cluster_unit/test_cluster_scn_frontier.c b/src/test/cluster_unit/test_cluster_scn_frontier.c new file mode 100644 index 0000000000..64584936b4 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_scn_frontier.c @@ -0,0 +1,861 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_scn_frontier.c + * Real-behavior tests for the spec-7.4 D1 durable_safe_scn frontier + * registry in cluster_scn.c. + * + * The frontier is the CONTIGUOUS durable watermark an origin may + * publish on the BOC channel: every own-origin commit SCN <= the + * frontier is durable on WAL. These tests pin the pending-registry + * semantics (register at commit-SCN allocation, LSN fill-in at + * commit-record insert, three-way discharge: sync commit / walwriter + * flush horizon / abort) plus the sticky overflow freeze and the + * monotonic-publish guard. + * + * Linking model mirrors test_cluster_scn.c: cluster_scn.o standalone + * with PG-backend symbols stubbed; the ShmemInitStruct stub backs + * "pgrac cluster scn" with a static aligned buffer so shmem-state + * paths run for real in-process. Wire / event / TAP-level behavior + * (BOC payload v1, dirty wakeup, walwriter hook) is TAP-tested. + * + * Spec: spec-7.4-commit-scn-propagation-freshness.md D1 + * Wire contract: spec-2.9-scn-broadcast-service-activation.md (I5 v0.5) + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_scn_frontier.c + * + * NOTES + * This is a pgrac-original file (no derivation from PostgreSQL). + * Each test_*.c is a standalone executable; see unit_test.h. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include + +#include "access/xact.h" /* PGRAC: spec-1.18 xl_xact_scn struct */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ +#include "cluster/cluster_ic_envelope.h" /* spec-2.9 D4: ClusterICEnvelope + PGRAC_IC_MSG_BOC_BROADCAST */ +#include "cluster/cluster_ic_router.h" /* spec-2.9 D4: ClusterICFanoutResult */ +#include "cluster/cluster_scn.h" +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2 stub — profiling gate */ +#include "port/atomics.h" +#include "storage/lwlock.h" + +/* Drop PG's port.h printf -> pg_printf override; unit_test.h uses + * stdlib printf and we don't link libpgport in this test binary. */ +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + + +/* ---------- + * Stubs needed to link cluster_scn.o standalone. Runtime paths + * (advance/observe with LWLock + shmem + ereport) are not exercised + * here; only the pure-function cmp layer is invoked. + * ---------- + */ + +bool IsUnderPostmaster = false; + +/* spec-5.59 D2 stubs: cluster_scn.o now carries GUC-gated profiling probes + * (cluster_xnode_profile.h); the unit harness links neither cluster_guc.o + * nor cluster_xnode_profile.o, so define the two gate symbols inertly + * (probes early-return on enabled=false / Ctl=NULL). */ +bool cluster_xnode_profile_enabled = false; +ClusterXnodeProfileShared *ClusterXnodeProfileCtl = NULL; + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} +int +errmsg(const char *f pg_attribute_unused(), ...) +{ + return 0; +} +int +errmsg_internal(const char *f pg_attribute_unused(), ...) +{ + return 0; +} +int +errdetail(const char *f pg_attribute_unused(), ...) +{ + return 0; +} +int +errhint(const char *f pg_attribute_unused(), ...) +{ + return 0; +} +void +elog_start(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +void +elog_finish(int e pg_attribute_unused(), const char *f pg_attribute_unused(), ...) +{} + +/* shmem / lwlock / fmgr / pg_atomic stubs. + * + * spec-2.11 P1.2 修订: 原 stub 返回 NULL + *foundPtr=true → 任何对 + * cluster_scn_state 字段的 atomic 读写都会 SEGV(NULL pointer deref). + * T-scn-15c 真行为测试需要 atomic fetch_add 真生效,所以扩展 stub: + * + * (i) 维护一个 per-name static buffer cache(指针稳定),首次访问 + * *foundPtr=false 触发 caller init path(zero-fill atomic + * fields);后续访问 *foundPtr=true 保持复用 + * (ii) buffer 用 BSS static(zero-init by C runtime) + uint64 union + * member 强制至少 8-byte alignment — 满足 LWLock / + * pg_atomic_uint64 alignment 和 init 前置条件 + * (iii) 仅 "pgrac cluster scn" 名走真 buffer 路径;其他 region 名 + * retain 旧 NULL 行为(避免影响其他 spec stub 假设) + * + * T-scn-15c 用法:测试体前置调 cluster_scn_shmem_init() → 触发 + * ShmemInitStruct("pgrac cluster scn", ...) → 首次 *foundPtr=false + * → cluster_scn_shmem_init body 执行 atomic init zero loop → + * cluster_scn_state 真指向 valid buffer → atomic ops 真生效. + */ +void * +ShmemInitStruct(const char *name, Size size, bool *foundPtr) +{ + /* spec-2.11 P1.2: per-name persistent buffer cache for cluster_scn. + * Use a union instead of plain char[] so pg_atomic_uint64 and LWLock + * inside ClusterScnSharedState are not placed on a 1-byte-aligned + * address in standalone unit tests. */ + static union { + uint64 force_align; + char data[16384]; /* generous; covers spec-7.4 durable_pending growth */ + } scn_buf; + static bool scn_initialized = false; + + if (name != NULL && strcmp(name, "pgrac cluster scn") == 0) { + Assert(size <= sizeof(scn_buf.data)); /* catch shmem layout growth */ + *foundPtr = scn_initialized; + scn_initialized = true; /* subsequent calls see found=true */ + return scn_buf.data; + } + + /* All other names: retain spec-1.X stub behavior. */ + *foundPtr = true; + return NULL; +} +void +RequestAddinShmemSpace(Size s pg_attribute_unused()) +{} +void +LWLockInitialize(LWLock *l pg_attribute_unused(), int t pg_attribute_unused()) +{} +bool +LWLockAcquire(LWLock *l pg_attribute_unused(), LWLockMode m pg_attribute_unused()) +{ + return true; +} +void +LWLockRelease(LWLock *l pg_attribute_unused()) +{} + +/* + * spec-2.12 D6 / T-scn-16d: GetCurrentTimestamp stub now backed by + * a writable test_clock_us global. Default 0 preserves spec-1.X / + * 2.X test semantics (existing tests do not set the global); + * T-scn-16d sets test_clock_us before each cluster_scn_observe call + * to drive deterministic max_gap_ms transitions. + */ +TimestampTz test_clock_us = 0; + +TimestampTz +GetCurrentTimestamp(void) +{ + return test_clock_us; +} + +NodeId cluster_node_id = 0; /* GUC backing store mock */ +ClusterConf test_cluster_conf = { + .magic = PGRAC_CLUSTER_CONF_MAGIC, + .node_count = 2, +}; +ClusterConf *ClusterConfShmem = &test_cluster_conf; + +/* Spec-1.16 Hardening v1.0.1 (round 9 P1 finding 1): cluster_scn skip + * helper now references cluster_enabled. Pin to true so unit-test + * advance/observe paths aren't silently skipped (behavior tests live + * in TAP 066 L12). */ +bool cluster_enabled = true; +bool cluster_enable_adg = false; /* spec-7.4 frontier tests: keep the ADG + * registry quiet so durable_pending assertions are not entangled with + * adg_pending bookkeeping (ADG real-behavior tests live in + * test_cluster_scn.c). */ + +/* superuser stub for SQL UDF wrappers (never called in this binary). */ +bool +superuser(void) +{ + return true; +} + +/* injection stub */ +bool +cluster_injection_should_skip(const char *p pg_attribute_unused()) +{ + return false; +} +void +cluster_injection_check(const char *p pg_attribute_unused()) +{} + +/* fmgr stub for SQL UDFs (address-only) */ +struct FunctionCallInfoBaseData; + +/* injection extras (cluster_scn.c references run + armed_count) */ +pg_atomic_uint32 cluster_injection_armed_count; +void +cluster_injection_run(const char *p pg_attribute_unused()) +{} +bool +cluster_injection_should_skip_full(const char *p pg_attribute_unused()) +{ + return false; +} + +/* timestamp helper used by wraparound watermark logging */ +bool +TimestampDifferenceExceeds(TimestampTz a pg_attribute_unused(), TimestampTz b pg_attribute_unused(), + int ms pg_attribute_unused()) +{ + return false; +} + +/* spec-1.17 D2: TimestampDifference stub for boc_tick throttle. */ +void +TimestampDifference(TimestampTz a pg_attribute_unused(), TimestampTz b pg_attribute_unused(), + long *secs, int *usecs) +{ + *secs = 0; + *usecs = 0; +} + +/* spec-1.17 D4: cluster_boc_sweep_interval_ms GUC backing var. */ +int cluster_boc_sweep_interval_ms = 1; + +/* shmem region registry stub (advance/observe path NOT exercised) */ +void +cluster_shmem_register_region(const void *r pg_attribute_unused()) +{} + +/* + * spec-2.9 D4 / L104 standalone-test-stub-must-cover-cross-module-call: + * + * cluster_scn.c now references cluster_ic_send_envelope_fanout and + * cluster_cssd_get_alive_peer_count from its LMON-side BOC drain body + * (spec-2.9 D2 review fix). Both symbols live in sibling cluster_ic_router.o and + * cluster_cssd.o respectively; this standalone unit-test binary links + * cluster_scn.o only, so the cross-module refs need local stubs to + * satisfy the linker. + * + * The LMON drain path itself is never invoked from any T-scn-13 test + * (handler branch is exercised directly), so the stubs can be vacuous: + * all peers PEER_DOWN for fanout, 0 for alive-peer-count. + */ +void +cluster_ic_send_envelope_fanout(uint8 msg_type pg_attribute_unused(), + const void *payload pg_attribute_unused(), + uint32 payload_len pg_attribute_unused(), + ClusterICFanoutResult per_peer[]) +{ + if (per_peer != NULL) { + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) + per_peer[i] = CLUSTER_IC_FANOUT_PEER_DOWN; + } +} + +int +cluster_cssd_get_alive_peer_count(void) +{ + return 0; +} + +int +cluster_conf_node_count(void) +{ + return 2; +} + +/* + * spec-2.9 v0.4 D2/D3 cross-module symbol stubs (CI strict-linker fix): + * + * cluster_scn.c references CritSectionCount (Q6/I8 Assert in + * cluster_scn_emit_broadcast_pulse; inlined into cluster_scn_boc_tick + * by LTO/static visibility) and MyBackendType (Q1 Assert in + * cluster_scn_lmon_drain_boc_broadcast). Linux ld + macOS strict ld + * require these as defined symbols at link time; local macOS ld is + * more permissive and missed the gap during cluster_unit make check. + * + * Standalone test binary never invokes either path, so initial values + * are vacuous (CritSectionCount = 0 satisfies the walwriter-not-in-crit + * Assert if ever exercised; MyBackendType = B_LMON would satisfy the + * drain LMON-only Assert if ever exercised). + */ +#include "miscadmin.h" +volatile uint32 CritSectionCount = 0; +BackendType MyBackendType = B_LMON; + +/* spec-7.4 D1 / L104: cluster_scn.o boc_tick now calls GetFlushRecPtr + * for the walwriter async-commit frontier discharge. Standalone unit + * binaries link cluster_scn.o only; stub returns 0 (InvalidXLogRecPtr) + * so discharge_upto is a guarded no-op unless a test drives the API + * directly with explicit horizons. */ +XLogRecPtr +GetFlushRecPtr(TimeLineID *insertTLI) +{ + if (insertTLI != NULL) + *insertTLI = 0; + return 0; +} + + +UT_DEFINE_GLOBALS(); + + +/* ============================================================ + * Spec-7.4 D1 durable_safe_scn frontier registry tests + * ============================================================ + * + * The published watermark is a CONTIGUOUS durable frontier: every + * own-origin commit SCN <= frontier is durable on WAL. A plain + * atomic-max of flushed commit SCNs is forbidden (P0): with T1 + * allocating S then stalling while T2 allocates S+1 and flushes + * first, max would publish S+1 and falsely claim S durable. + * + * Tests share one in-process shmem buffer (ShmemInitStruct stub + * caches per-name); state is cumulative across tests, so every + * assertion below is written against SCNs allocated locally in the + * same test. The overflow test freezes the frontier STICKILY and + * therefore MUST run last. + * + * Spec: spec-7.4-commit-scn-propagation-freshness.md D1 + * Mini-plan: docs/stage7-74-d1-miniplan.md v1.1 §1.1 (pgrac repo) + */ + +UT_TEST(test_spec74_frontier_accessors_linkable) +{ + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_safe_scn); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_pending_fill_lsn); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_pending_discharge_scn); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_pending_abort_self); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_pending_discharge_upto); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_pending_count); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_frontier_frozen); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_frontier_overflow_count); + UT_ASSERT_NOT_NULL((void *)cluster_scn_durable_frontier_regression_count); +} + +UT_TEST(test_spec74_frontier_initial_state_invalid_unfrozen) +{ + /* Real bound check: the ShmemInitStruct stub buffer is 16KB and + * its Assert is compiled out in no-cassert unit builds -- keep an + * always-on guard against silent BSS overflow as the state struct + * grows. */ + UT_ASSERT(cluster_scn_shmem_size() <= 16384); + + cluster_scn_shmem_init(); + + UT_ASSERT_EQ(cluster_scn_durable_safe_scn(), InvalidScn); + UT_ASSERT_EQ(cluster_scn_durable_pending_count(), 0); + UT_ASSERT(!cluster_scn_durable_frontier_frozen()); + UT_ASSERT_EQ(cluster_scn_durable_frontier_overflow_count(), 0); + UT_ASSERT_EQ(cluster_scn_durable_frontier_regression_count(), 0); +} + +UT_TEST(test_spec74_frontier_commit_advance_registers_pending) +{ + uint64 pre; + SCN s; + SCN f; + + cluster_scn_shmem_init(); + pre = cluster_scn_durable_pending_count(); + + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + UT_ASSERT_EQ(cluster_scn_durable_pending_count(), pre + 1); + + /* An allocated-but-not-durable commit SCN must not be covered. */ + f = cluster_scn_durable_safe_scn(); + UT_ASSERT(!SCN_VALID(f) || scn_total_cmp(f, s) < 0); + + /* Clean up for the next test: this commit never flushes. */ + UT_ASSERT(cluster_scn_durable_pending_abort_self()); + UT_ASSERT_EQ(cluster_scn_durable_pending_count(), pre); +} + +UT_TEST(test_spec74_frontier_sync_discharge_advances_to_scn) +{ + SCN s; + + cluster_scn_shmem_init(); + + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)1000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s), 0); +} + +UT_TEST(test_spec74_frontier_out_of_order_flush_does_not_pass_stalled) +{ + SCN s1; + SCN s2; + SCN f; + + cluster_scn_shmem_init(); + + /* T1 allocates s1 and stalls BEFORE its commit record exists + * (lsn never filled). T2 allocates s2, inserts and flushes. */ + s1 = cluster_scn_advance_for_commit(); + s2 = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s1)); + UT_ASSERT(SCN_VALID(s2)); + UT_ASSERT(scn_total_cmp(s1, s2) < 0); + + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s2, (XLogRecPtr)2000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s2)); + + /* P0 heart: the frontier must NOT pass the stalled s1. */ + f = cluster_scn_durable_safe_scn(); + UT_ASSERT(SCN_VALID(f)); + UT_ASSERT(scn_total_cmp(f, s1) < 0); + + /* T1 resumes: record inserted at lsn 1500, walwriter flushes + * through 1500 -> both discharged -> frontier reaches s2. */ + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s1, (XLogRecPtr)1500)); + UT_ASSERT_EQ(cluster_scn_durable_pending_discharge_upto((XLogRecPtr)1500), 1); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s2), 0); +} + +UT_TEST(test_spec74_frontier_walwriter_upto_respects_lsn_order) +{ + SCN s; + SCN f_before; + + cluster_scn_shmem_init(); + + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + f_before = cluster_scn_durable_safe_scn(); + + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)5000)); + + /* Flush horizon below the entry's lsn: nothing discharges. */ + UT_ASSERT_EQ(cluster_scn_durable_pending_discharge_upto((XLogRecPtr)4999), 0); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), f_before), 0); + + /* Flush horizon reaches the entry: discharge + frontier = s. */ + UT_ASSERT_EQ(cluster_scn_durable_pending_discharge_upto((XLogRecPtr)5000), 1); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s), 0); +} + +UT_TEST(test_spec74_frontier_unfilled_lsn_never_discharged_by_upto) +{ + SCN s; + SCN f; + + cluster_scn_shmem_init(); + + /* Allocated but commit record not yet inserted (lsn unknown): + * even an infinite flush horizon must not discharge it. */ + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + + UT_ASSERT_EQ(cluster_scn_durable_pending_discharge_upto((XLogRecPtr)PG_UINT64_MAX), 0); + f = cluster_scn_durable_safe_scn(); + UT_ASSERT(!SCN_VALID(f) || scn_total_cmp(f, s) < 0); + + UT_ASSERT(cluster_scn_durable_pending_abort_self()); +} + +UT_TEST(test_spec74_frontier_abort_unblocks_frontier) +{ + SCN s1; + SCN s2; + SCN f; + + cluster_scn_shmem_init(); + + /* s1 aborts after allocation (commit record never written); a + * later durable commit s2 must not be held back by it. */ + s1 = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s1)); + f = cluster_scn_durable_safe_scn(); + UT_ASSERT(!SCN_VALID(f) || scn_total_cmp(f, s1) < 0); + + UT_ASSERT(cluster_scn_durable_pending_abort_self()); + + s2 = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s2, (XLogRecPtr)7000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s2)); + + /* Aborted s1 is not a commit: frontier covering s2 (> s1) makes + * no false durability claim about s1. */ + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s2), 0); +} + +UT_TEST(test_spec74_frontier_register_after_publish_does_not_regress) +{ + SCN s1; + SCN s2; + uint64 pre_regress; + + cluster_scn_shmem_init(); + pre_regress = cluster_scn_durable_frontier_regression_count(); + + s1 = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s1, (XLogRecPtr)8000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s1)); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s1), 0); + + /* A new pending registration recomputes the frontier to + * min(pending)-1 == s1: published value must hold, and the + * monotonic guard must not count a regression. */ + s2 = cluster_scn_advance_for_commit(); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s1), 0); + UT_ASSERT_EQ(cluster_scn_durable_frontier_regression_count(), pre_regress); + + UT_ASSERT(cluster_scn_durable_pending_abort_self()); +} + + +/* ============================================================ + * Spec-7.4 D1 BOC payload v1 codec + handler + remote cache tests + * ============================================================ + * + * Wire contract (spec-2.9 I5 v0.5 amend): payload_length in {0, 8}. + * 8B = origin_durable_safe_scn, explicit little-endian. Receive side + * validates length, decodability, and that the SCN's node bits match + * env.source_node_id; every reject is a counted fail-closed drop. + * Accepted values land in a per-(origin, epoch) cache; an epoch + * change rebuilds the entry, a lower SCN within the same epoch is a + * counted regression reject. + */ + +static ClusterICEnvelope +frontier_test_env(uint32 source, uint64 epoch, uint32 payload_length) +{ + ClusterICEnvelope env; + + memset(&env, 0, sizeof(env)); + env.magic = PGRAC_IC_ENVELOPE_MAGIC; + env.version = PGRAC_IC_ENVELOPE_VERSION_V1; + env.msg_type = PGRAC_IC_MSG_BOC_BROADCAST; + env.source_node_id = source; + env.epoch = epoch; + env.scn = 12345; + env.payload_length = payload_length; + return env; +} + +UT_TEST(test_spec74_payload_codec_le_roundtrip) +{ + uint8 buf[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN scn = (SCN)0x0102030405060708ULL; + + cluster_scn_boc_payload_encode(scn, buf); + UT_ASSERT_EQ(buf[0], 0x08); + UT_ASSERT_EQ(buf[1], 0x07); + UT_ASSERT_EQ(buf[2], 0x06); + UT_ASSERT_EQ(buf[3], 0x05); + UT_ASSERT_EQ(buf[4], 0x04); + UT_ASSERT_EQ(buf[5], 0x03); + UT_ASSERT_EQ(buf[6], 0x02); + UT_ASSERT_EQ(buf[7], 0x01); + UT_ASSERT_EQ(cluster_scn_boc_payload_decode(buf), scn); +} + +UT_TEST(test_spec74_handler_v1_payload_updates_cache) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN sent = scn_encode(3, 500); + SCN got = InvalidScn; + uint64 got_epoch = 0; + uint64 pre_accept; + + cluster_scn_shmem_init(); + pre_accept = cluster_scn_boc_payload_accept_count(); + + cluster_scn_boc_payload_encode(sent, payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_accept_count(), pre_accept + 1); + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, sent), 0); + UT_ASSERT_EQ(got_epoch, 7); +} + +UT_TEST(test_spec74_handler_zero_payload_keeps_cache) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, 0); + SCN got = InvalidScn; + uint64 got_epoch = 0; + + cluster_scn_shmem_init(); + + /* Old sender / publish off: 0-byte frame stays a no-op and the + * cache from the previous test survives. */ + cluster_scn_boc_broadcast_handler(&env, NULL); + + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, scn_encode(3, 500)), 0); +} + +UT_TEST(test_spec74_handler_bad_length_dropped) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, 4); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN got = InvalidScn; + uint64 got_epoch = 0; + uint64 pre_bad; + + cluster_scn_shmem_init(); + pre_bad = cluster_scn_boc_payload_bad_length_count(); + + cluster_scn_boc_payload_encode(scn_encode(3, 900), payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_bad_length_count(), pre_bad + 1); + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, scn_encode(3, 500)), 0); /* unchanged */ +} + +UT_TEST(test_spec74_handler_undecodable_payload_dropped) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + uint64 pre_bad; + + cluster_scn_shmem_init(); + pre_bad = cluster_scn_boc_payload_bad_length_count(); + + /* All-zero payload decodes to InvalidScn: malformed frame. */ + memset(payload, 0, sizeof(payload)); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_bad_length_count(), pre_bad + 1); +} + +UT_TEST(test_spec74_handler_node_mismatch_dropped) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN got = InvalidScn; + uint64 got_epoch = 0; + uint64 pre_mismatch; + + cluster_scn_shmem_init(); + pre_mismatch = cluster_scn_boc_payload_node_mismatch_count(); + + /* SCN claims node 5 but the envelope sender is node 3. */ + cluster_scn_boc_payload_encode(scn_encode(5, 900), payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_node_mismatch_count(), pre_mismatch + 1); + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, scn_encode(3, 500)), 0); /* unchanged */ +} + +UT_TEST(test_spec74_handler_regression_rejected_same_epoch) +{ + ClusterICEnvelope env = frontier_test_env(3, 7, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN got = InvalidScn; + uint64 got_epoch = 0; + uint64 pre_regress; + + cluster_scn_shmem_init(); + pre_regress = cluster_scn_boc_payload_regression_count(); + + /* Advance the cache to 600 first. */ + cluster_scn_boc_payload_encode(scn_encode(3, 600), payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + /* A reordered older frame (550) must not roll the cache back. */ + cluster_scn_boc_payload_encode(scn_encode(3, 550), payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_regression_count(), pre_regress + 1); + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, scn_encode(3, 600)), 0); +} + +UT_TEST(test_spec74_handler_epoch_change_rebuilds_cache) +{ + ClusterICEnvelope env = frontier_test_env(3, 8, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + SCN got = InvalidScn; + uint64 got_epoch = 0; + uint64 pre_regress; + + cluster_scn_shmem_init(); + pre_regress = cluster_scn_boc_payload_regression_count(); + + /* New epoch 8 with a LOWER scn than epoch 7's cached 600: the + * entry is rebuilt, not regression-rejected (R5 epoch semantics). */ + cluster_scn_boc_payload_encode(scn_encode(3, 550), payload); + cluster_scn_boc_broadcast_handler(&env, payload); + + UT_ASSERT_EQ(cluster_scn_boc_payload_regression_count(), pre_regress); + UT_ASSERT(cluster_scn_remote_durable_safe(3, &got_epoch, &got)); + UT_ASSERT_EQ(scn_total_cmp(got, scn_encode(3, 550)), 0); + UT_ASSERT_EQ(got_epoch, 8); +} + +UT_TEST(test_spec74_remote_accessor_unseen_origin_false) +{ + SCN got = InvalidScn; + uint64 got_epoch = 0; + + cluster_scn_shmem_init(); + + UT_ASSERT(!cluster_scn_remote_durable_safe(99, &got_epoch, &got)); + UT_ASSERT(!cluster_scn_remote_durable_safe(CLUSTER_MAX_NODES, &got_epoch, &got)); + UT_ASSERT(!cluster_scn_remote_durable_safe(-1, &got_epoch, &got)); +} + +UT_TEST(test_spec74_frontier_abort_self_keeps_filled_entry) +{ + SCN s; + uint64 pre_count; + + cluster_scn_shmem_init(); + pre_count = cluster_scn_durable_pending_count(); + + /* Async-commit shape: the commit record was inserted (lsn filled) + * but not yet flushed. A later abort of a DIFFERENT transaction on + * this backend must not discharge the still-pending real commit -- + * a filled lsn proves a commit record exists, so only the walwriter + * flush horizon may prove it durable. */ + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)4242)); + + UT_ASSERT(!cluster_scn_durable_pending_abort_self()); + UT_ASSERT_EQ(cluster_scn_durable_pending_count(), pre_count + 1); + + /* The walwriter horizon still discharges it normally. */ + UT_ASSERT_EQ(cluster_scn_durable_pending_discharge_upto((XLogRecPtr)4242), 1); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s), 0); +} + +/* + * MUST RUN LAST: overflow freezes the frontier stickily (until + * restart); every test after this one would see frozen state. + */ +UT_TEST(test_spec74_frontier_overflow_freezes_sticky) +{ + SCN last_safe; + SCN first_s = InvalidScn; + SCN s; + uint64 pre_overflow; + int i; + + cluster_scn_shmem_init(); + pre_overflow = cluster_scn_durable_frontier_overflow_count(); + last_safe = cluster_scn_durable_safe_scn(); + + /* Fill the registry to capacity, then one more. */ + for (i = 0; i <= CLUSTER_SCN_DURABLE_PENDING_MAX; i++) { + s = cluster_scn_advance_for_commit(); + UT_ASSERT(SCN_VALID(s)); + if (i == 0) + first_s = s; + } + + UT_ASSERT(cluster_scn_durable_frontier_frozen()); + UT_ASSERT(cluster_scn_durable_frontier_overflow_count() > pre_overflow); + + /* Frozen frontier never advances -- discharging a tracked entry + * still removes it, but the published value stays put (sticky + * until restart; consumers fall back to the existing fetch-reply + * piggyback sampling). */ + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(first_s, (XLogRecPtr)9000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(first_s)); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), last_safe), 0); + UT_ASSERT(cluster_scn_durable_frontier_frozen()); +} + +int +main(void) +{ + /* Spec-7.4 D1 durable_safe_scn frontier registry (9) -- + * T-scn-74-1a..1i. Cumulative shmem state; overflow test last. */ + UT_RUN(test_spec74_frontier_accessors_linkable); + UT_RUN(test_spec74_frontier_initial_state_invalid_unfrozen); + UT_RUN(test_spec74_frontier_commit_advance_registers_pending); + UT_RUN(test_spec74_frontier_sync_discharge_advances_to_scn); + UT_RUN(test_spec74_frontier_out_of_order_flush_does_not_pass_stalled); + UT_RUN(test_spec74_frontier_walwriter_upto_respects_lsn_order); + UT_RUN(test_spec74_frontier_unfilled_lsn_never_discharged_by_upto); + UT_RUN(test_spec74_frontier_abort_unblocks_frontier); + UT_RUN(test_spec74_frontier_register_after_publish_does_not_regress); + UT_RUN(test_spec74_payload_codec_le_roundtrip); + UT_RUN(test_spec74_handler_v1_payload_updates_cache); + UT_RUN(test_spec74_handler_zero_payload_keeps_cache); + UT_RUN(test_spec74_handler_bad_length_dropped); + UT_RUN(test_spec74_handler_undecodable_payload_dropped); + UT_RUN(test_spec74_handler_node_mismatch_dropped); + UT_RUN(test_spec74_handler_regression_rejected_same_epoch); + UT_RUN(test_spec74_handler_epoch_change_rebuilds_cache); + UT_RUN(test_spec74_remote_accessor_unseen_origin_false); + UT_RUN(test_spec74_frontier_abort_self_keeps_filled_entry); + UT_RUN(test_spec74_frontier_overflow_freezes_sticky); + + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From c58552a69a5a15465fb15f3375c647b0de9b328f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:29:25 +0800 Subject: [PATCH 3/7] feat(cluster): spec-7.4 D1 commit-event BOC publish (dirty + LMON wake + payload attach) Event half of the durable-frontier channel. Producer: any discharge that STRICTLY advances the published frontier signals after releasing the SCN lwlock -- publish-before-signal order is inherent -- via exchange-set of a dirty flag, waking LMON only on the 0->1 transition (natural batching: N commits in one window collapse to one drain). Consumer: the LMON BOC drain exchange-clears dirty BEFORE snapshotting the frontier so a racing publish re-arms the wakeup, then fans out one BOC frame carrying payload v1; either a sweep beat or a commit event justifies a fanout, and the zero-peer short-circuit now runs before either gate so neither is consumed with nobody to send to. New GUC cluster.boc_event_publish (bool, on, SIGHUP): off restores the pre-D1 wire bytes (0-length BOC frames) and stops event wakeups; the walwriter sweep cadence remains the fallback either way. Injection point cluster-boc-event-publish suppresses the producer signal to prove the sweep fallback. Unit: 7 new tests (27 total in test_cluster_scn_frontier) pin the wake-on-advance / dedup single-wake / no-advance-no-wake / off / injection-suppressed producer matrix and the drain payload attach (on = 8B LE frontier, off = 0B even with a stale dirty flag). Spec: spec-7.4-commit-scn-propagation-freshness.md Spec: spec-2.9-scn-broadcast-service-activation.md --- src/backend/cluster/cluster_guc.c | 15 ++ src/backend/cluster/cluster_scn.c | 124 ++++++++-- src/include/cluster/cluster_guc.h | 4 + src/include/cluster/cluster_scn.h | 1 + src/test/cluster_unit/test_cluster_scn.c | 10 + .../cluster_unit/test_cluster_scn_frontier.c | 215 +++++++++++++++++- 6 files changed, 345 insertions(+), 24 deletions(-) diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 643ec959d6..d164cb1125 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -554,6 +554,9 @@ bool cluster_undo_retention_horizon_enabled = true; * check_GUC_init Assert (boot_val 与 C-var 初值不一致会触发 guc.c:4820 TRAP). */ int cluster_boc_sweep_interval_ms = 100; +/* PGRAC: spec-7.4 D1-2 — event-driven BOC durable-frontier publish. */ +bool cluster_boc_event_publish = true; + /* PGRAC: spec-2.12 D1 — SCN cross-instance propagation lag bound. * * Configuration bound (NOT enforcement action) — used by TAP 102 as @@ -3667,6 +3670,18 @@ cluster_init_guc(void) * Range 1..1000 保持. */ &cluster_boc_sweep_interval_ms, 100, 1, 1000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + /* PGRAC: spec-7.4 D1-2 — commit-driven BOC publish events. off = + * sweep-only cadence with 0-length BOC frames (pre-D1 wire bytes) + * and no LMON event wakeups. */ + DefineCustomBoolVariable( + "cluster.boc_event_publish", + gettext_noop("Publish the durable SCN frontier on commit events."), + gettext_noop("When on, a commit that advances the durable SCN frontier wakes LMON to " + "fan out a BOC frame carrying the frontier immediately; the periodic " + "walwriter sweep remains the fallback cadence. When off, BOC frames " + "carry no payload and only the sweep cadence runs."), + &cluster_boc_event_publish, true, PGC_SIGHUP, 0, NULL, NULL, NULL); + /* PGRAC: spec-2.12 D1 — SCN cross-instance propagation lag bound. * Configuration only (no enforcement action — TAP 102 uses this as * hard threshold; in-process metric is local staleness proxy). */ diff --git a/src/backend/cluster/cluster_scn.c b/src/backend/cluster/cluster_scn.c index f4241616ce..f47fc69729 100644 --- a/src/backend/cluster/cluster_scn.c +++ b/src/backend/cluster/cluster_scn.c @@ -54,6 +54,7 @@ #include "cluster/cluster_ic_envelope.h" /* ClusterICEnvelope (spec-2.9 D3) */ #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope_fanout + PGRAC_IC_MSG_BOC_BROADCAST */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup (spec-7.4 D1-2) */ #include "cluster/cluster_shmem.h" #include "cluster/cluster_xnode_profile.h" /* PGRAC: spec-5.59 D2 profiling */ #include "access/xlog.h" /* GetFlushRecPtr (spec-7.4 D1 walwriter discharge) */ @@ -205,6 +206,17 @@ typedef struct ClusterScnSharedState { pg_atomic_uint64 boc_payload_bad_length_count; pg_atomic_uint64 boc_payload_node_mismatch_count; pg_atomic_uint64 boc_payload_regression_count; + /* + * spec-7.4 D1-2: commit-event publish protocol. Producer (any + * discharge path that ADVANCES the published frontier): publish + * under the lwlock -> exchange-set dirty -> wake LMON only on the + * 0->1 transition (publish-before-signal, L387; the exchange makes + * concurrent producers race-free on the single wakeup). Consumer + * (LMON drain): exchange-clear dirty BEFORE snapshotting the + * frontier, so a publish racing past the snapshot re-arms a wakeup + * instead of being lost. + */ + pg_atomic_uint32 boc_event_dirty; } ClusterScnSharedState; /* spec-2.12 D2: defensive — raw-bits storage of TimestampTz in @@ -226,7 +238,8 @@ static bool cluster_scn_pending_commit_register_locked(SCN commit_scn); static bool cluster_scn_pending_commit_remove_locked(SCN commit_scn); static void cluster_scn_durable_pending_register_locked(SCN commit_scn); static bool cluster_scn_durable_pending_remove_locked(SCN commit_scn); -static void cluster_scn_durable_frontier_publish_locked(void); +static bool cluster_scn_durable_frontier_publish_locked(void); +static void cluster_scn_boc_event_signal(void); /* @@ -1071,16 +1084,18 @@ cluster_scn_durable_pending_remove_locked(SCN commit_scn) * LW_EXCLUSIVE. Recomputes the contiguous durable frontier and * publishes it monotonically. A recomputed value below the published * one is a bug in the discharge protocol: refuse + count (L441 - * monotonic discipline), never regress the public claim. + * monotonic discipline), never regress the public claim. Returns true + * only when the published value STRICTLY advanced (drives the D1-2 + * event signal; republishing an equal claim is not an event). */ -static void +static bool cluster_scn_durable_frontier_publish_locked(void) { SCN frontier; SCN current; if (cluster_scn_state == NULL || cluster_scn_state->durable_frozen) - return; + return false; if (cluster_scn_state->durable_pending_count > 0) { SCN min_pending = cluster_scn_state->durable_pending_scn[0]; @@ -1095,14 +1110,55 @@ cluster_scn_durable_frontier_publish_locked(void) frontier = cluster_scn_state->durable_last_allocated; if (!SCN_VALID(frontier)) - return; + return false; current = (SCN)pg_atomic_read_u64(&cluster_scn_state->durable_safe_scn); - if (SCN_VALID(current) && scn_total_cmp(frontier, current) < 0) { - pg_atomic_fetch_add_u64(&cluster_scn_state->durable_regression_count, 1); - return; + if (SCN_VALID(current)) { + int cmp = scn_total_cmp(frontier, current); + + if (cmp < 0) { + pg_atomic_fetch_add_u64(&cluster_scn_state->durable_regression_count, 1); + return false; + } + if (cmp == 0) + return false; /* republish of the same claim; no event */ } pg_atomic_write_u64(&cluster_scn_state->durable_safe_scn, frontier); + return true; /* strictly advanced */ +} + +/* + * cluster_scn_boc_event_signal -- producer half of the D1-2 event + * protocol. Call AFTER releasing the SCN lwlock, only when the + * publish strictly advanced the frontier. Publish-before-signal + * (L387) is inherent: the frontier write happened under the lock + * before this runs. The injection point suppresses the event to + * prove the sweep cadence remains a sufficient fallback (L2). + */ +static void +cluster_scn_boc_event_signal(void) +{ + if (!cluster_boc_event_publish) + return; + if (cluster_scn_state == NULL) + return; + if (cluster_injection_should_skip("cluster-boc-event-publish")) + return; + if (pg_atomic_exchange_u32(&cluster_scn_state->boc_event_dirty, 1) == 0) + cluster_lmon_marker_complete_wakeup(); +} + +/* + * cluster_scn_boc_event_consume -- consumer half: LMON drain clears + * the dirty flag BEFORE snapshotting the frontier, so a publish that + * races in after the snapshot re-arms a wakeup (no lost update). + */ +bool +cluster_scn_boc_event_consume(void) +{ + if (cluster_scn_state == NULL) + return false; + return pg_atomic_exchange_u32(&cluster_scn_state->boc_event_dirty, 0) != 0; } /* @@ -1155,6 +1211,7 @@ bool cluster_scn_durable_pending_discharge_scn(SCN commit_scn) { bool removed; + bool advanced = false; if (cluster_scn_state == NULL || !SCN_VALID(commit_scn)) return false; @@ -1162,8 +1219,10 @@ cluster_scn_durable_pending_discharge_scn(SCN commit_scn) LWLockAcquire(&cluster_scn_state->lwlock, LW_EXCLUSIVE); removed = cluster_scn_durable_pending_remove_locked(commit_scn); if (removed) - cluster_scn_durable_frontier_publish_locked(); + advanced = cluster_scn_durable_frontier_publish_locked(); LWLockRelease(&cluster_scn_state->lwlock); + if (advanced) + cluster_scn_boc_event_signal(); return removed; } @@ -1184,6 +1243,7 @@ cluster_scn_durable_pending_abort_self(void) { SCN pending = cluster_scn_backend_durable_pending_scn; bool removed = false; + bool advanced = false; uint16 i; if (cluster_scn_state == NULL || !SCN_VALID(pending)) @@ -1195,13 +1255,15 @@ cluster_scn_durable_pending_abort_self(void) if (XLogRecPtrIsInvalid(cluster_scn_state->durable_pending_lsn[i])) { removed = cluster_scn_durable_pending_remove_locked(pending); if (removed) - cluster_scn_durable_frontier_publish_locked(); + advanced = cluster_scn_durable_frontier_publish_locked(); } break; } } LWLockRelease(&cluster_scn_state->lwlock); cluster_scn_backend_durable_pending_scn = InvalidScn; + if (advanced) + cluster_scn_boc_event_signal(); return removed; } @@ -1216,6 +1278,7 @@ uint32 cluster_scn_durable_pending_discharge_upto(XLogRecPtr flushed_lsn) { uint32 discharged = 0; + bool advanced = false; uint16 i; if (cluster_scn_state == NULL || XLogRecPtrIsInvalid(flushed_lsn)) @@ -1235,8 +1298,10 @@ cluster_scn_durable_pending_discharge_upto(XLogRecPtr flushed_lsn) i++; } if (discharged > 0) - cluster_scn_durable_frontier_publish_locked(); + advanced = cluster_scn_durable_frontier_publish_locked(); LWLockRelease(&cluster_scn_state->lwlock); + if (advanced) + cluster_scn_boc_event_signal(); return discharged; } @@ -1654,6 +1719,11 @@ cluster_scn_lmon_drain_boc_broadcast(void) static uint64 last_drained_sweep_count = 0; ClusterICFanoutResult per_peer[CLUSTER_MAX_NODES]; uint64 sweep_count; + bool event_pending; + SCN frontier; + uint8 payload[CLUSTER_SCN_BOC_PAYLOAD_V1_LEN]; + const void *send_payload = NULL; + uint32 send_len = 0; int peer; int done = 0; int would_block = 0; @@ -1667,20 +1737,37 @@ cluster_scn_lmon_drain_boc_broadcast(void) Assert(MyBackendType == B_LMON); - sweep_count = pg_atomic_read_u64(&cluster_scn_state->boc_sweep_count); - if (sweep_count == last_drained_sweep_count) - return; - /* v0.3 Q7 / I9: 0 peer means no router/fanout work and no log spam. - * Do not mark the sweep drained here; if a peer becomes alive before - * the next walwriter sweep, LMON may still emit the latest SCN. */ + * Checked BEFORE consuming the event flag so neither the sweep nor a + * pending commit event is marked drained; if a peer becomes alive + * later, LMON still emits the latest state. (spec-7.4 D1-2 moved + * this check ahead of the sweep gate for exactly that reason.) */ if (cluster_cssd_get_alive_peer_count() == 0) return; + /* spec-7.4 D1-2: exchange-clear the event flag BEFORE snapshotting + * (a publish racing past the snapshot re-arms a wakeup). Either a + * new sweep beat or a commit event justifies a fanout. */ + event_pending = cluster_scn_boc_event_consume(); + sweep_count = pg_atomic_read_u64(&cluster_scn_state->boc_sweep_count); + if (sweep_count == last_drained_sweep_count && !event_pending) + return; + /* PGRAC: spec-5.59 D2 profiling */ cluster_xp_begin(&xps, CLXP_C_SCN_BOC_BROADCAST); - cluster_ic_send_envelope_fanout(PGRAC_IC_MSG_BOC_BROADCAST, NULL, 0, per_peer); + /* spec-7.4 D1-2: attach payload v1 (the published durable frontier) + * when event publish is on and a frontier exists; otherwise keep + * the pre-D1 0-length pulse (byte equivalence when the GUC is off, + * and old receivers treat 0-length as pulse-only either way). */ + frontier = cluster_scn_durable_safe_scn(); + if (cluster_boc_event_publish && SCN_VALID(frontier)) { + cluster_scn_boc_payload_encode(frontier, payload); + send_payload = payload; + send_len = CLUSTER_SCN_BOC_PAYLOAD_V1_LEN; + } + + cluster_ic_send_envelope_fanout(PGRAC_IC_MSG_BOC_BROADCAST, send_payload, send_len, per_peer); last_drained_sweep_count = sweep_count; for (peer = 0; peer < CLUSTER_MAX_NODES; peer++) { @@ -2063,6 +2150,7 @@ cluster_scn_shmem_init(void) pg_atomic_init_u64(&cluster_scn_state->boc_payload_bad_length_count, 0); pg_atomic_init_u64(&cluster_scn_state->boc_payload_node_mismatch_count, 0); pg_atomic_init_u64(&cluster_scn_state->boc_payload_regression_count, 0); + pg_atomic_init_u32(&cluster_scn_state->boc_event_dirty, 0); } } diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index 2e2dc478e6..a2fc3e8e08 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -594,6 +594,10 @@ extern int cluster_interconnect_rdma_max_send_wr; * rate caps actual sweep frequency (Min(WalWriterDelay, this)). */ extern int cluster_boc_sweep_interval_ms; +/* spec-7.4 D1-2: event-driven BOC durable-frontier publish (dirty + + * LMON wakeup + payload v1 attach). off = pre-D1 sweep-only channel, + * byte-identical 0-length BOC frames and no event wakeups. */ +extern bool cluster_boc_event_publish; /* diff --git a/src/include/cluster/cluster_scn.h b/src/include/cluster/cluster_scn.h index 1576d1ab23..9b7e012a1f 100644 --- a/src/include/cluster/cluster_scn.h +++ b/src/include/cluster/cluster_scn.h @@ -343,6 +343,7 @@ extern uint64 cluster_scn_durable_frontier_regression_count(void); extern void cluster_scn_boc_payload_encode(SCN scn, uint8 *buf); extern SCN cluster_scn_boc_payload_decode(const uint8 *buf); +extern bool cluster_scn_boc_event_consume(void); extern bool cluster_scn_remote_durable_safe(NodeId origin, uint64 *epoch_out, SCN *scn_out); extern uint64 cluster_scn_boc_payload_accept_count(void); extern uint64 cluster_scn_boc_payload_bad_length_count(void); diff --git a/src/test/cluster_unit/test_cluster_scn.c b/src/test/cluster_unit/test_cluster_scn.c index 1616daadee..38b02fdc39 100644 --- a/src/test/cluster_unit/test_cluster_scn.c +++ b/src/test/cluster_unit/test_cluster_scn.c @@ -357,6 +357,16 @@ GetFlushRecPtr(TimeLineID *insertTLI) return 0; } +/* spec-7.4 D1-2 / L104: cluster_scn.o now references the event-publish + * GUC and the LMON wakeup helper (cluster_lmon.o not linked here). + * Event-protocol behavior tests live in test_cluster_scn_frontier.c; + * these stubs only satisfy the linker (wakeup vacuous, GUC on). */ +bool cluster_boc_event_publish = true; + +void +cluster_lmon_marker_complete_wakeup(void) +{} + UT_DEFINE_GLOBALS(); diff --git a/src/test/cluster_unit/test_cluster_scn_frontier.c b/src/test/cluster_unit/test_cluster_scn_frontier.c index 64584936b4..e1f448622a 100644 --- a/src/test/cluster_unit/test_cluster_scn_frontier.c +++ b/src/test/cluster_unit/test_cluster_scn_frontier.c @@ -238,10 +238,14 @@ superuser(void) } /* injection stub */ +/* spec-7.4 D1-2: configurable suppress-injection stub -- tests set + * test_injection_skip_point to a point name to suppress it. */ +const char *test_injection_skip_point = NULL; + bool -cluster_injection_should_skip(const char *p pg_attribute_unused()) +cluster_injection_should_skip(const char *p) { - return false; + return test_injection_skip_point != NULL && strcmp(p, test_injection_skip_point) == 0; } void cluster_injection_check(const char *p pg_attribute_unused()) @@ -300,12 +304,23 @@ cluster_shmem_register_region(const void *r pg_attribute_unused()) * (handler branch is exercised directly), so the stubs can be vacuous: * all peers PEER_DOWN for fanout, 0 for alive-peer-count. */ +/* spec-7.4 D1-2: capturing fanout stub -- drain tests assert on the + * last frame's msg_type / payload bytes. */ +int test_fanout_calls = 0; +uint8 test_fanout_last_msg_type = 0; +uint32 test_fanout_last_len = 0; +uint8 test_fanout_last_payload[64]; + void -cluster_ic_send_envelope_fanout(uint8 msg_type pg_attribute_unused(), - const void *payload pg_attribute_unused(), - uint32 payload_len pg_attribute_unused(), +cluster_ic_send_envelope_fanout(uint8 msg_type, const void *payload, uint32 payload_len, ClusterICFanoutResult per_peer[]) { + test_fanout_calls++; + test_fanout_last_msg_type = msg_type; + test_fanout_last_len = payload_len; + memset(test_fanout_last_payload, 0, sizeof(test_fanout_last_payload)); + if (payload != NULL && payload_len > 0 && payload_len <= sizeof(test_fanout_last_payload)) + memcpy(test_fanout_last_payload, payload, payload_len); if (per_peer != NULL) { int i; @@ -314,10 +329,14 @@ cluster_ic_send_envelope_fanout(uint8 msg_type pg_attribute_unused(), } } +/* spec-7.4 D1-2: configurable so drain tests can pass the zero-peer + * short-circuit (I9). */ +int test_alive_peer_count = 0; + int cluster_cssd_get_alive_peer_count(void) { - return 0; + return test_alive_peer_count; } int @@ -358,6 +377,18 @@ GetFlushRecPtr(TimeLineID *insertTLI) return 0; } +/* spec-7.4 D1-2: LMON wakeup stub counts SetLatch-equivalent signals. */ +int test_lmon_wakeup_count = 0; + +void +cluster_lmon_marker_complete_wakeup(void) +{ + test_lmon_wakeup_count++; +} + +/* spec-7.4 D1-2 GUC backing var (cluster_guc.o not linked). */ +bool cluster_boc_event_publish = true; + UT_DEFINE_GLOBALS(); @@ -793,6 +824,171 @@ UT_TEST(test_spec74_frontier_abort_self_keeps_filled_entry) UT_ASSERT_EQ(scn_total_cmp(cluster_scn_durable_safe_scn(), s), 0); } + +/* ============================================================ + * Spec-7.4 D1-2 event protocol tests (dirty + wake + drain payload) + * ============================================================ + * + * Producer protocol (mini-plan v1.1 ruling #4): publish (under the + * SCN lwlock) -> exchange-set dirty -> SetLatch(LMON) only on the + * 0->1 transition. Consumer (LMON drain): exchange-clear dirty + * BEFORE snapshotting, so a racing publish re-arms the wakeup. + * cluster.boc_event_publish=off restores payload=0 frames AND no + * wakeups (byte equivalence with the pre-D1 sweep-only channel). + */ + +UT_TEST(test_spec74_event_dirty_wake_on_frontier_advance) +{ + SCN s; + + cluster_scn_shmem_init(); + (void)cluster_scn_boc_event_consume(); /* drain stale dirty */ + test_lmon_wakeup_count = 0; + + s = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)11000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + + UT_ASSERT_EQ(test_lmon_wakeup_count, 1); + UT_ASSERT(cluster_scn_boc_event_consume()); + UT_ASSERT(!cluster_scn_boc_event_consume()); +} + +UT_TEST(test_spec74_event_dirty_dedup_second_advance_no_wake) +{ + SCN s1; + SCN s2; + + cluster_scn_shmem_init(); + (void)cluster_scn_boc_event_consume(); + test_lmon_wakeup_count = 0; + + s1 = cluster_scn_advance_for_commit(); + s2 = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s1, (XLogRecPtr)12000)); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s2, (XLogRecPtr)12001)); + + /* First advance wakes; the second finds dirty already set and + * must not signal again (natural batching). */ + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s1)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s2)); + + UT_ASSERT_EQ(test_lmon_wakeup_count, 1); + UT_ASSERT(cluster_scn_boc_event_consume()); +} + +UT_TEST(test_spec74_event_no_frontier_advance_no_wake) +{ + SCN s1; + SCN s2; + + cluster_scn_shmem_init(); + (void)cluster_scn_boc_event_consume(); + test_lmon_wakeup_count = 0; + + s1 = cluster_scn_advance_for_commit(); + s2 = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s1, (XLogRecPtr)13000)); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s2, (XLogRecPtr)13001)); + + /* Discharging the LATER scn first leaves min(pending)=s1: the + * recomputed frontier equals the already-published value, so no + * event fires. */ + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s2)); + UT_ASSERT_EQ(test_lmon_wakeup_count, 0); + UT_ASSERT(!cluster_scn_boc_event_consume()); + + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s1)); + UT_ASSERT_EQ(test_lmon_wakeup_count, 1); + UT_ASSERT(cluster_scn_boc_event_consume()); +} + +UT_TEST(test_spec74_event_publish_off_no_dirty_no_wake) +{ + SCN s; + + cluster_scn_shmem_init(); + (void)cluster_scn_boc_event_consume(); + test_lmon_wakeup_count = 0; + + cluster_boc_event_publish = false; + s = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)14000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + cluster_boc_event_publish = true; + + UT_ASSERT_EQ(test_lmon_wakeup_count, 0); + UT_ASSERT(!cluster_scn_boc_event_consume()); +} + +UT_TEST(test_spec74_event_injection_suppresses_signal) +{ + SCN s; + + cluster_scn_shmem_init(); + (void)cluster_scn_boc_event_consume(); + test_lmon_wakeup_count = 0; + + test_injection_skip_point = "cluster-boc-event-publish"; + s = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)15000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + test_injection_skip_point = NULL; + + /* Suppressed event; the sweep cadence remains the fallback (L2). */ + UT_ASSERT_EQ(test_lmon_wakeup_count, 0); + UT_ASSERT(!cluster_scn_boc_event_consume()); +} + +UT_TEST(test_spec74_drain_attaches_payload_v1_when_on) +{ + SCN s; + + cluster_scn_shmem_init(); + test_alive_peer_count = 1; + + /* Arm an event, then drain as LMON would. */ + s = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)16000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + + test_fanout_calls = 0; + cluster_scn_lmon_drain_boc_broadcast(); + + UT_ASSERT_EQ(test_fanout_calls, 1); + UT_ASSERT_EQ(test_fanout_last_msg_type, PGRAC_IC_MSG_BOC_BROADCAST); + UT_ASSERT_EQ(test_fanout_last_len, CLUSTER_SCN_BOC_PAYLOAD_V1_LEN); + UT_ASSERT_EQ(scn_total_cmp(cluster_scn_boc_payload_decode(test_fanout_last_payload), + cluster_scn_durable_safe_scn()), + 0); + test_alive_peer_count = 0; +} + +UT_TEST(test_spec74_drain_zero_len_when_off_even_if_dirty) +{ + SCN s; + + cluster_scn_shmem_init(); + test_alive_peer_count = 1; + + /* Dirty was armed while the GUC was on; a SIGHUP flips it off + * before the drain runs. The stale event drains as a 0-byte + * pulse: off means payload=0 on the wire, unconditionally. */ + s = cluster_scn_advance_for_commit(); + UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)17000)); + UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + + cluster_boc_event_publish = false; + test_fanout_calls = 0; + cluster_scn_lmon_drain_boc_broadcast(); + cluster_boc_event_publish = true; + + UT_ASSERT_EQ(test_fanout_calls, 1); + UT_ASSERT_EQ(test_fanout_last_msg_type, PGRAC_IC_MSG_BOC_BROADCAST); + UT_ASSERT_EQ(test_fanout_last_len, 0); + test_alive_peer_count = 0; +} + /* * MUST RUN LAST: overflow freezes the frontier stickily (until * restart); every test after this one would see frozen state. @@ -854,6 +1050,13 @@ main(void) UT_RUN(test_spec74_handler_epoch_change_rebuilds_cache); UT_RUN(test_spec74_remote_accessor_unseen_origin_false); UT_RUN(test_spec74_frontier_abort_self_keeps_filled_entry); + UT_RUN(test_spec74_event_dirty_wake_on_frontier_advance); + UT_RUN(test_spec74_event_dirty_dedup_second_advance_no_wake); + UT_RUN(test_spec74_event_no_frontier_advance_no_wake); + UT_RUN(test_spec74_event_publish_off_no_dirty_no_wake); + UT_RUN(test_spec74_event_injection_suppresses_signal); + UT_RUN(test_spec74_drain_attaches_payload_v1_when_on); + UT_RUN(test_spec74_drain_zero_len_when_off_even_if_dirty); UT_RUN(test_spec74_frontier_overflow_freezes_sticky); UT_DONE(); From 1e3e283e69161bd98bb9bbe4dec39f8d8f123ebc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:37:18 +0800 Subject: [PATCH 4/7] feat(cluster): spec-7.4 D1 durable frontier + BOC payload observability rows pg_cluster_debug scn section gains the durable-frontier registry stats (safe scn as full 64-bit hex, pending count, frozen flag, overflow + regression counters), the four BOC payload v1 receive-side counters (accept / bad-length / node-mismatch / regression drops), and one sparse row pair (frontier hex + epoch) per remote origin with a cached claim. These back the TAP legs and keep every new counter SQL-visible. Spec: spec-7.4-commit-scn-propagation-freshness.md --- src/backend/cluster/cluster_debug.c | 41 +++++++++++++++++ src/test/cluster_unit/test_cluster_debug.c | 53 ++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 1e8cd28be9..a5badb6dc5 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1056,6 +1056,47 @@ dump_scn(ReturnSetInfo *rsinfo) emit_row(rsinfo, "scn", "scn_observed_max_observe_gap_ms", fmt_int64((int64)cluster_scn_observed_max_observe_gap_ms())); } + + /* PGRAC: spec-7.4 D1 — durable frontier registry + BOC payload + * v1 stats (producer + receive-side reject counters), plus one + * sparse row pair per remote origin with a cached frontier + * claim. scn_durable_safe_scn is the full 64-bit encoded SCN + * (hex, same convention as scn_current_encoded). */ + emit_row(rsinfo, "scn", "scn_durable_safe_scn", + fmt_uint64_hex((uint64)cluster_scn_durable_safe_scn())); + emit_row(rsinfo, "scn", "scn_durable_pending_count", + fmt_int64((int64)cluster_scn_durable_pending_count())); + emit_row(rsinfo, "scn", "scn_durable_frontier_frozen", + fmt_int32(cluster_scn_durable_frontier_frozen() ? 1 : 0)); + emit_row(rsinfo, "scn", "scn_durable_frontier_overflow_count", + fmt_int64((int64)cluster_scn_durable_frontier_overflow_count())); + emit_row(rsinfo, "scn", "scn_durable_frontier_regression_count", + fmt_int64((int64)cluster_scn_durable_frontier_regression_count())); + emit_row(rsinfo, "scn", "scn_boc_payload_accept_count", + fmt_int64((int64)cluster_scn_boc_payload_accept_count())); + emit_row(rsinfo, "scn", "scn_boc_payload_bad_length_count", + fmt_int64((int64)cluster_scn_boc_payload_bad_length_count())); + emit_row(rsinfo, "scn", "scn_boc_payload_node_mismatch_count", + fmt_int64((int64)cluster_scn_boc_payload_node_mismatch_count())); + emit_row(rsinfo, "scn", "scn_boc_payload_regression_count", + fmt_int64((int64)cluster_scn_boc_payload_regression_count())); + + { + NodeId origin; + + for (origin = 0; origin < CLUSTER_MAX_NODES; origin++) { + uint64 repoch = 0; + SCN rscn = InvalidScn; + char keybuf[64]; + + if (!cluster_scn_remote_durable_safe(origin, &repoch, &rscn)) + continue; + snprintf(keybuf, sizeof(keybuf), "scn_remote_durable_node%d", (int)origin); + emit_row(rsinfo, "scn", pstrdup(keybuf), fmt_uint64_hex((uint64)rscn)); + snprintf(keybuf, sizeof(keybuf), "scn_remote_durable_epoch_node%d", (int)origin); + emit_row(rsinfo, "scn", pstrdup(keybuf), fmt_int64((int64)repoch)); + } + } } } diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index d41a5a04c3..ac2b50e65d 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -3236,6 +3236,59 @@ cluster_scn_last_observe_at(void) return 0; } +/* Spec-7.4 D1 durable frontier / BOC payload accessor stubs. */ +SCN +cluster_scn_durable_safe_scn(void) +{ + return 0; +} +uint64 +cluster_scn_durable_pending_count(void) +{ + return 0; +} +bool +cluster_scn_durable_frontier_frozen(void) +{ + return false; +} +uint64 +cluster_scn_durable_frontier_overflow_count(void) +{ + return 0; +} +uint64 +cluster_scn_durable_frontier_regression_count(void) +{ + return 0; +} +uint64 +cluster_scn_boc_payload_accept_count(void) +{ + return 0; +} +uint64 +cluster_scn_boc_payload_bad_length_count(void) +{ + return 0; +} +uint64 +cluster_scn_boc_payload_node_mismatch_count(void) +{ + return 0; +} +uint64 +cluster_scn_boc_payload_regression_count(void) +{ + return 0; +} +bool +cluster_scn_remote_durable_safe(NodeId origin pg_attribute_unused(), + uint64 *epoch_out pg_attribute_unused(), + SCN *scn_out pg_attribute_unused()) +{ + return false; +} uint64 cluster_scn_observed_max_observe_gap_ms(void) { From 13a4e04d5c60437cd1d0e33eab5ef7036a96ba82 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 13:39:29 +0800 Subject: [PATCH 5/7] test(cluster): spec-7.4 D1 -- t/068 L11 scn dump key baseline 19 -> 28 The durable-frontier / BOC payload observability rows added 9 keys to the scn dump section; sparse per-origin rows never appear single-node so the count stays deterministic. Spec: spec-7.4-commit-scn-propagation-freshness.md --- src/test/cluster_tap/t/068_wal_xl_scn.pl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/cluster_tap/t/068_wal_xl_scn.pl b/src/test/cluster_tap/t/068_wal_xl_scn.pl index fb4c43a5e4..5f895cf6d1 100644 --- a/src/test/cluster_tap/t/068_wal_xl_scn.pl +++ b/src/test/cluster_tap/t/068_wal_xl_scn.pl @@ -326,11 +326,16 @@ sub waldump_range # scn keys now 19 after spec-2.12 added 3 convergence boundary keys # (scn_last_observe_at + scn_seconds_since_last_observe + # scn_observed_max_observe_gap_ms). +# scn keys 19 -> 28 after spec-7.4 D1 added 9 durable-frontier / BOC +# payload v1 rows (safe scn + pending count + frozen + overflow + +# regression + 4 payload receive counters). The sparse per-origin +# scn_remote_durable_* rows never appear single-node (no cached +# origins), so the count stays deterministic here. my $scn_key_count = $primary->safe_psql('postgres', q{ SELECT count(*) FROM pg_cluster_state WHERE category='scn' }); -is($scn_key_count, '19', - 'L11 pg_cluster_state has 19 scn keys after spec-2.12 3 convergence boundary metrics'); +is($scn_key_count, '28', + 'L11 pg_cluster_state has 28 scn keys after spec-7.4 D1 durable frontier rows'); # ---------- From a308468727735d434c41b33849575b5327a2f50f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 15:49:07 +0800 Subject: [PATCH 6/7] test(cluster): spec-7.4 D1 -- t/387 durable-frontier 2-node legs L1-L8 + boc-event-publish injection point Adds t/387: node0 commits, node1 observes node0's published contiguous durable frontier through the BOC payload v1 channel. Legs L1 staleness (event path), L2 sweep fallback, L3 commit-unperturbed under a publish fault, L4 batching, L5 off byte-equivalence, L6 out-of-order (frontier must not pass a stalled commit), L7 idle async walwriter discharge, L8 seqlock reader liveness across an LMON kill/crash-restart. The commit-event publish path referenced cluster-boc-event-publish but the point was never in the registry, so suppression was a no-op. Register it and switch the probe from should_skip (one-shot SKIP dispatch, never armed here) to an armed-state peek (cluster_cr_injection_armed): the probe sits on the post-commit-durable path, so an armed ERROR/CRASH dispatch must never fire, and L2 needs sustained suppression while armed, not a single skip. Registry grows 164 -> 165; sweep the count assertions in t/015 (+ name list), t/017/018/020/021/022/023/024/030 accordingly. Unit stub updated for the armed-peek + armed_count gate. Spec: spec-7.4-commit-scn-propagation-freshness.md (D1) --- src/backend/cluster/cluster_inject.c | 13 + src/backend/cluster/cluster_scn.c | 11 +- src/test/cluster_tap/t/015_inject.pl | 8 +- src/test/cluster_tap/t/017_debug.pl | 8 +- src/test/cluster_tap/t/018_shared_fs.pl | 4 +- src/test/cluster_tap/t/020_shmem_registry.pl | 4 +- src/test/cluster_tap/t/021_block_format.pl | 4 +- src/test/cluster_tap/t/022_itl_slot.pl | 4 +- .../cluster_tap/t/023_buffer_descriptor.pl | 4 +- src/test/cluster_tap/t/024_pcm_lock.pl | 4 +- src/test/cluster_tap/t/030_acceptance.pl | 6 +- .../t/387_cluster_7_4_durable_frontier.pl | 545 ++++++++++++++++++ .../cluster_unit/test_cluster_scn_frontier.c | 20 +- 13 files changed, 608 insertions(+), 27 deletions(-) create mode 100644 src/test/cluster_tap/t/387_cluster_7_4_durable_frontier.pl diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index a2875d638f..c214ec88c3 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -739,6 +739,19 @@ static ClusterInjectPoint cluster_injection_points[] = { * exercise the fail-closed leg + guardrail counter). */ { .name = "cluster-mxid-halfspace-hard-limit" }, + + /* + * spec-7.4 D1-2 — durable-frontier commit-event publish. + * cluster-boc-event-publish: armed-state peek (any fault type) + * suppresses the commit-side BOC event signal (no dirty flag, + * no LMON wakeup), degrading propagation to the walwriter + * sweep cadence. Drives the t/387 L2 sweep-fallback leg and + * the L3 commit-unperturbed leg. Armed-peek style (no + * CLUSTER_INJECTION_POINT dispatch) because the probe sits on + * the post-commit path where an armed ERROR/CRASH dispatch + * could fire after the commit record is already durable. + */ + { .name = "cluster-boc-event-publish" }, }; #define CLUSTER_INJECTION_COUNT lengthof(cluster_injection_points) diff --git a/src/backend/cluster/cluster_scn.c b/src/backend/cluster/cluster_scn.c index f47fc69729..0cb68d0ebf 100644 --- a/src/backend/cluster/cluster_scn.c +++ b/src/backend/cluster/cluster_scn.c @@ -1142,7 +1142,16 @@ cluster_scn_boc_event_signal(void) return; if (cluster_scn_state == NULL) return; - if (cluster_injection_should_skip("cluster-boc-event-publish")) + + /* + * Armed-state peek, not CLUSTER_INJECTION_POINT dispatch: this + * runs on the post-commit path (commit record already durable), so + * an armed ERROR/SLEEP/CRASH dispatch must never fire here. The + * armed_count fast gate keeps the probe free when no injection is + * armed anywhere in this process. + */ + if (cluster_injection_armed_count > 0 + && cluster_cr_injection_armed("cluster-boc-event-publish", NULL)) return; if (pg_atomic_exchange_u32(&cluster_scn_state->boc_event_dirty, 1) == 0) cluster_lmon_marker_complete_wakeup(); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 0979ef1884..3ce025ce5b 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'pg_stat_cluster_injections returns 164 rows (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', + 'pg_stat_cluster_injections returns 165 rows (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '164 injection point names match the full registry (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '165 injection point names match the full registry (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index eaaf1d5170..352da505f1 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '164', - 'all 164 injection points have a .fault_type entry under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', + 'all 165 injection points have a .fault_type entry under inject category (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '164', - 'all 164 injection points have a .hits entry under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', + 'all 165 injection points have a .hits entry under inject category (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index ee4dfb7794..28d3ad1cc1 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L9 total injection registry size is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', + 'L9 total injection registry size is 165 (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index 653707ac01..3d4028d2aa 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L15 total injection registry size is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '165', + 'L15 total injection registry size is 165 (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index e2ed11b217..bad5e01059 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L11 pg_stat_cluster_injections is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '165', + 'L11 pg_stat_cluster_injections is 165 (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 1d40fb4013..deaee74190 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L12a pg_stat_cluster_injections is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '165', + 'L12a pg_stat_cluster_injections is 165 (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index 2f4d78c328..997bbe75e6 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L11 pg_stat_cluster_injections is 164 (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '165', + 'L11 pg_stat_cluster_injections is 165 (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 4181f4b730..c15f8e4b2f 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -163,8 +163,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', - 'L6a pg_stat_cluster_injections has 164 entries (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', + 'L6a pg_stat_cluster_injections has 165 entries (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 7fa47b5e54..b88b4b4364 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '164', 'M1 164 injection points (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '165', 'M1 165 injection points (spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '328', - 'M5 inject category has 164×2 = 328 sub-keys (.fault_type + .hits; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '330', + 'M5 inject category has 165×2 = 330 sub-keys (.fault_type + .hits; spec-7.4 D1-2 +1 cluster-boc-event-publish; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/387_cluster_7_4_durable_frontier.pl b/src/test/cluster_tap/t/387_cluster_7_4_durable_frontier.pl new file mode 100644 index 0000000000..2c71045468 --- /dev/null +++ b/src/test/cluster_tap/t/387_cluster_7_4_durable_frontier.pl @@ -0,0 +1,545 @@ +#------------------------------------------------------------------------- +# spec-7.4 D1 durable frontier + BOC payload v1 -- 2-node legs L1-L8. +# +# node0 commits; node1 observes node0's published contiguous durable +# frontier (origin_durable_safe_scn) through the BOC payload v1 channel +# and caches it per (origin, epoch). Legs: +# +# L1 staleness, event path on: node0 commit -> node1 cached-frontier +# advance. Hard: every round converges; p50 clearly below the +# sweep-only cadence. (The strict local-machine numbers -- p99 +# <= 10ms and >= 5x vs sweep-only -- are enforced by the census +# runner report, not by this CI leg; set PGRAC_74_STRICT=1 for +# the strict local gate.) +# L2 sweep fallback: arm cluster-boc-event-publish (suppresses the +# commit-event signal); staleness degrades to the sweep + LMON +# cadence but still converges. Proves the 100ms sweep remains a +# sufficient fallback and that the injection probe engages. +# L3 commit-unperturbed (behavioural half): with the event publish +# suppressed AND LMON stalled via injection, commits neither +# fail nor wait. (The <= 3% commit-p99 red line is measured by +# the census runner on a no-cassert build -- see the id-ledger +# note for t/387.) +# L4 batching: a concurrent commit burst produces sublinear BOC +# fanout (dirty-flag dedup coalesces commits per drain). +# L5 off byte-equivalence: cluster.boc_event_publish=off reverts to +# 0-length BOC pulses -- no payload accepts, remote cache holds +# its stale value, sweep pulses still flow (observe bumps), and +# the local frontier keeps advancing; flipping back on resumes. +# L6 out-of-order hard leg: T1 stalls between commit-SCN allocation +# and its commit record (sleep injection at +# cluster-scn-commit-post-advance); T2 (later SCN) commits fully +# and durably; the published frontier must NOT advance past the +# stalled T1 and must advance once T1 completes. +# L7 idle-async hard leg: a single synchronous_commit=off commit +# with no follow-up traffic is eventually discharged by the +# walwriter flush horizon and published (locally and remotely). +# L8 reader-liveness hard leg: SIGKILL node1's LMON (the seqlock +# writer for the remote-frontier cache) -> crash-restart; scn +# state reads complete (no permanent reader spin) and the cache +# repopulates after the peers reconnect. +# +# Author: SqlRush +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/387_cluster_7_4_durable_frontier.pl +#------------------------------------------------------------------------- + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Time::HiRes qw(usleep time); +use IPC::Run (); +use Test::More; + +my $STRICT = $ENV{PGRAC_74_STRICT} ? 1 : 0; + +sub state_str +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) ? $v : ''; +} + +sub state_int +{ + my ($node, $cat, $key) = @_; + my $v = state_str($node, $cat, $key); + return $v ne '' ? $v + 0 : 0; +} + +# fmt_uint64_hex emits fixed-width uppercase 0x%016X, so plain string +# comparison orders same-origin frontier claims correctly. +sub hex_gt { return $_[0] gt $_[1]; } + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +sub write_retry +{ + my ($node, $sql) = @_; + for my $i (1 .. 12) + { + my $ok = eval { $node->safe_psql('postgres', $sql); 1 }; + return 1 if $ok; + usleep(500_000); + } + return 0; +} + +# Median / p99 over a sorted copy (ms values). +sub pct +{ + my ($vals, $p) = @_; + my @s = sort { $a <=> $b } @$vals; + my $idx = int($p * (scalar(@s) - 1) + 0.5); + return $s[$idx]; +} + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'frontier74', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'jit = off', + # L8 kills LMON: the node must crash-restart (shmem rebuild), + # not shut down (the TAP harness default is off). + 'restart_after_crash = on', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 5', + 'cluster.online_join = on', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.join_convergence_timeout_ms = 30000', + ]); +$pair->start_pair; +usleep(2_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'node0 sees node1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'node1 sees node0'); +my ($node0, $node1) = ($pair->node0, $pair->node1); + +is($node0->safe_psql('postgres', 'SHOW cluster.boc_event_publish'), + 'on', 'cluster.boc_event_publish defaults to on'); + +ok(write_retry($node0, 'CREATE TABLE t387 (id int, v int)'), 'seed table created'); + +# Persistent sessions: commit driver on node0, poll reader on node1. +# (Existing sessions pick up SIGHUP re-arms at their next command, so +# these survive the per-leg injection / GUC flips below.) +my $bg0 = $node0->background_psql('postgres'); +my $bg1 = $node1->background_psql('postgres'); + +# The default BackgroundPsql timer is cumulative for the session; this +# test issues hundreds of poll queries, so restart it per query. +$bg0->set_query_timer_restart; +$bg1->set_query_timer_restart; + +my $REMOTE_SQL = q{SELECT COALESCE((SELECT value FROM pg_cluster_state + WHERE category='scn' AND key='scn_remote_durable_node0'), '')}; +my $FRONTIER_SQL = q{SELECT COALESCE((SELECT value FROM pg_cluster_state + WHERE category='scn' AND key='scn_durable_safe_scn'), '')}; + +# One measured staleness round: node0 commit -> node1 cached frontier +# advances past its pre-round value. Returns latency in ms, or -1 on +# a convergence timeout. +my $round_seq = 0; +sub staleness_round +{ + my ($timeout_s) = @_; + my $before = $bg1->query($REMOTE_SQL); + $round_seq++; + $bg0->query("INSERT INTO t387 VALUES ($round_seq, 0)"); + my $t0 = time(); + while (time() - $t0 < $timeout_s) + { + my $now = $bg1->query($REMOTE_SQL); + return (time() - $t0) * 1000.0 + if $now ne '' && ($before eq '' || hex_gt($now, $before)); + usleep(1_000); + } + return -1; +} + +# ---------- +# L1: event-path staleness. +# ---------- + +# Warm-up: the very first commit must populate node1's cache at all. +my $warm = staleness_round(10); +cmp_ok($warm, '>=', 0, 'L1 warm-up: remote durable-frontier cache populated'); + +my @event_ms; +for my $r (1 .. 25) +{ + my $ms = staleness_round(5); + cmp_ok($ms, '>=', 0, "L1 round $r converged (<= 5s)"); + push @event_ms, $ms; +} +my $event_p50 = pct(\@event_ms, 0.50); +my $event_p99 = pct(\@event_ms, 0.99); +diag(sprintf('L1 event-path staleness ms: p50=%.2f p99=%.2f min=%.2f max=%.2f', + $event_p50, $event_p99, (sort { $a <=> $b } @event_ms)[0], + (sort { $a <=> $b } @event_ms)[-1])); +cmp_ok($event_p50, '<=', 150, + 'L1 event-path p50 clearly below the sweep-only cadence'); +if ($STRICT) +{ + cmp_ok($event_p99, '<=', 10, 'L1 STRICT: event-path p99 <= 10ms (local machine)'); +} + +# ---------- +# L2: suppressed event -> sweep cadence remains a sufficient fallback. +# ---------- + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-boc-event-publish'}), + '1', + 'L2 event-publish suppression injection point is registered'); + +# Injection arm state and hit counters are process-local, so read the +# hits through bg0: that session's backend is the one committing (and +# therefore probing) below. +my $HITS_SQL = q{SELECT hits FROM pg_stat_cluster_injections + WHERE name = 'cluster-boc-event-publish'}; +my $hits0 = ($bg0->query($HITS_SQL) || 0) + 0; +set_injection($node0, 'cluster-boc-event-publish'); + +my @sweep_ms; +for my $r (1 .. 8) +{ + # Bound: 100ms sweep beat + <= 1000ms LMON loop pickup + fanout and + # CI margin. + my $ms = staleness_round(3.5); + cmp_ok($ms, '>=', 0, "L2 suppressed round $r still converged via sweep (<= 3.5s)"); + push @sweep_ms, $ms; +} +my $sweep_p50 = pct(\@sweep_ms, 0.50); +diag(sprintf('L2 sweep-fallback staleness ms: p50=%.2f max=%.2f', + $sweep_p50, (sort { $a <=> $b } @sweep_ms)[-1])); + +my $hits1 = ($bg0->query($HITS_SQL) || 0) + 0; +cmp_ok($hits1, '>', $hits0, 'L2 suppression probe engaged (hits advanced)'); +# A truly suppressed round can only converge on a sweep beat, so its +# median cannot sit at event-path (sub-sweep-interval) latencies. This +# guards against a silently non-functional suppression probe. +cmp_ok($sweep_p50, '>', 20, + 'L2 suppressed staleness sits above event-path latencies'); +cmp_ok($sweep_p50, '>', $event_p50, + 'L2 suppressed staleness is measurably slower than the event path'); +if ($STRICT) +{ + cmp_ok($sweep_p50, '>=', 5 * $event_p50, + 'L2 STRICT: event path >= 5x faster than sweep-only (local machine)'); +} + +set_injection($node0, ''); +my $recover_ms = staleness_round(3.5); +cmp_ok($recover_ms, '>=', 0, 'L2 event path recovers after disarm'); + +# ---------- +# L3 (behavioural half): a publish-side fault never perturbs commits. +# +# The fault is the event-publish suppression itself (the D1-added +# commit-side step made to fail). Deliberately NOT an LMON stall: LMON +# owns the whole IC transport, so stalling it perturbs pre-existing +# cross-node commit dependencies (GCS replies etc.) that have nothing +# to do with D1 -- the first RED run measured exactly that (~6s commit +# walls tracking the injected LMON sleep). The <= 3% commit-p99 red +# line vs the D0 baseline is enforced by the census runner. +# ---------- + +sub timed_commits +{ + my ($label, $lo) = @_; + my @ms; + my $failures = 0; + for my $r (1 .. 12) + { + my $t0 = time(); + my $ok = eval { $bg0->query('INSERT INTO t387 VALUES (' . ($lo + $r) . ', 3)'); 1 }; + push @ms, (time() - $t0) * 1000.0; + $failures++ unless $ok; + } + diag(sprintf('L3 %s commit ms: p50=%.2f max=%.2f', + $label, pct(\@ms, 0.50), (sort { $a <=> $b } @ms)[-1])); + return ($failures, pct(\@ms, 0.50), (sort { $a <=> $b } @ms)[-1]); +} + +my ($base_fail, $base_p50, $base_max) = timed_commits('baseline', 1000); +is($base_fail, 0, 'L3 baseline commits all succeeded'); + +my $f_pre_l3 = $bg0->query($FRONTIER_SQL); +set_injection($node0, 'cluster-boc-event-publish'); +my ($sup_fail, $sup_p50, $sup_max) = timed_commits('suppressed-publish', 1100); + +is($sup_fail, 0, 'L3 all commits succeeded under a suppressed event publish'); +cmp_ok($sup_max, '<=', 1000, 'L3 no commit stalled under the publish fault'); +cmp_ok($sup_p50, '<=', (25 > 5 * $base_p50 ? 25 : 5 * $base_p50), + 'L3 commit latency did not degrade under the publish fault'); + +my $f_post_l3 = $bg0->query($FRONTIER_SQL); +ok($f_post_l3 ne '' && ($f_pre_l3 eq '' || hex_gt($f_post_l3, $f_pre_l3)), + 'L3 local durable frontier still advanced (discharge is backend-side)'); + +set_injection($node0, ''); +my $l3_recover = staleness_round(5); +cmp_ok($l3_recover, '>=', 0, 'L3 propagation recovers after disarm'); + +# ---------- +# L4: batching -- a publish window merges many commits into few BOCs. +# +# Phase A (events on) records the fanout-per-commit ratio for the +# measure report: at TAP-achievable local TPS each commit event can +# legitimately wake one drain, so no hard sublinearity gate here (the +# high-TPS sublinear number is the census runner's). Phase B proves +# the merge semantic deterministically: with the event signal +# suppressed, a 40-commit burst drains on sweep beats only -- few +# fanouts, and the final broadcast carries the merged (latest) +# frontier that covers the whole burst. +# ---------- + +my $fanout0 = state_int($node0, 'scn', 'scn_boc_broadcast_fanout_count'); +my $commit0 = state_int($node0, 'scn', 'scn_commit_advance_count'); +$bg0->query("INSERT INTO t387 VALUES (2000 + $_, 4)") for 1 .. 40; +my $fanout_a = state_int($node0, 'scn', 'scn_boc_broadcast_fanout_count') - $fanout0; +my $commit_a = state_int($node0, 'scn', 'scn_commit_advance_count') - $commit0; +diag("L4 phase A (events on): commits=$commit_a fanouts=$fanout_a"); +cmp_ok($commit_a, '>=', 40, 'L4 phase A committed 40+ times'); +cmp_ok($fanout_a, '>=', 1, 'L4 phase A produced fanouts'); +cmp_ok($fanout_a, '<=', $commit_a + 10, + 'L4 phase A no wakeup amplification (at most ~one fanout per commit)'); + +set_injection($node0, 'cluster-boc-event-publish'); +my $fanout_b0 = state_int($node0, 'scn', 'scn_boc_broadcast_fanout_count'); +my $commit_b0 = state_int($node0, 'scn', 'scn_commit_advance_count'); +$bg0->query("INSERT INTO t387 VALUES (2100 + $_, 4)") for 1 .. 40; +my $f_after_burst = $bg0->query($FRONTIER_SQL); + +# Sweep-cadence convergence: the remote cache must reach the local +# frontier claim that covers the WHOLE burst -- one merged payload. +my $t_l4 = time(); +my $l4_converged = 0; +while (time() - $t_l4 < 3.5) +{ + my $r = $bg1->query($REMOTE_SQL); + if ($r ne '' && !hex_gt($f_after_burst, $r)) + { + $l4_converged = 1; + last; + } + usleep(2_000); +} +my $fanout_b = state_int($node0, 'scn', 'scn_boc_broadcast_fanout_count') - $fanout_b0; +my $commit_b = state_int($node0, 'scn', 'scn_commit_advance_count') - $commit_b0; +set_injection($node0, ''); +diag("L4 phase B (suppressed): commits=$commit_b fanouts=$fanout_b"); +cmp_ok($commit_b, '>=', 40, 'L4 phase B committed 40+ times'); +ok($l4_converged, 'L4 phase B remote cache converged to the merged frontier'); +cmp_ok($fanout_b, '<=', $commit_b / 2, + 'L4 phase B few broadcasts carried the whole burst (merge is real)'); + +# ---------- +# L5: off byte-equivalence contract. +# ---------- + +$node0->safe_psql('postgres', 'ALTER SYSTEM SET cluster.boc_event_publish = off'); +$node0->safe_psql('postgres', 'SELECT pg_reload_conf()'); +# LMON re-reads the GUC on its own loop (up to ~1s), so a straggler +# payload-carrying fanout can land right after the reload; settle +# before taking the off-window baselines. +usleep(2_500_000); + +my $remote_pre_l5 = $bg1->query($REMOTE_SQL); +my $accept0 = state_int($node1, 'scn', 'scn_boc_payload_accept_count'); +my $badlen0 = state_int($node1, 'scn', 'scn_boc_payload_bad_length_count'); +my $obs0 = state_int($node1, 'scn', 'scn_observe_bump_count'); +my $f_pre_l5 = $bg0->query($FRONTIER_SQL); + +$bg0->query("INSERT INTO t387 VALUES (3000 + $_, 5)") for 1 .. 10; +usleep(2_500_000); + +my $f_post_l5 = $bg0->query($FRONTIER_SQL); +ok($f_post_l5 ne '' && hex_gt($f_post_l5, $f_pre_l5), + 'L5 local frontier keeps advancing while publish is off'); +is(state_int($node1, 'scn', 'scn_boc_payload_accept_count'), + $accept0, 'L5 no payload accepted remotely while off (0-length pulses)'); +is(state_int($node1, 'scn', 'scn_boc_payload_bad_length_count'), + $badlen0, 'L5 0-length pulses are legal, never counted bad-length'); +is($bg1->query($REMOTE_SQL), $remote_pre_l5, + 'L5 remote cached frontier held its stale value while off'); +cmp_ok(state_int($node1, 'scn', 'scn_observe_bump_count'), '>', $obs0, + 'L5 sweep pulses still flowed while off (observe bumps advanced)'); + +$node0->safe_psql('postgres', 'ALTER SYSTEM SET cluster.boc_event_publish = on'); +$node0->safe_psql('postgres', 'SELECT pg_reload_conf()'); +usleep(700_000); +my $l5_resume = staleness_round(5); +cmp_ok($l5_resume, '>=', 0, 'L5 payload propagation resumes after flipping back on'); +cmp_ok(state_int($node1, 'scn', 'scn_boc_payload_accept_count'), '>', $accept0, + 'L5 remote payload accepts resumed'); + +# ---------- +# L6: out-of-order flush must not let the frontier pass a stalled commit. +# ---------- + +ok(write_retry($node0, 'CREATE TABLE t387_l6 (id int, v int)'), 'L6 side table created'); + +my $regress0 = state_int($node0, 'scn', 'scn_durable_frontier_regression_count'); +my $c_pre_l6 = state_int($node0, 'scn', 'scn_commit_advance_count'); + +# T1 arms the stall in ITS OWN backend (injection arm state is +# process-local), so T2 and every other process stay untouched -- the +# GUC + SIGHUP arming route raced its own disarm here and put T2 to +# sleep too. T1 also writes a private table so T2 shares no page/ITL +# with the stalled transaction. The 8s stall covers the window +# assertions below with margin. +my $connstr = $node0->connstr('postgres'); +my ($t1_out, $t1_err) = ('', ''); +my $t1 = IPC::Run::start( + ['psql', '-XAtq', '-d', $connstr, '-c', + q{SELECT cluster_inject_fault('cluster-scn-commit-post-advance', 'sleep', 8000000); } + . 'INSERT INTO t387_l6 VALUES (9001, 6)'], + '>', \$t1_out, '2>', \$t1_err); + +my $saw_pending = 0; +my $t_l6 = time(); +while (time() - $t_l6 < 3) +{ + if (state_int($node0, 'scn', 'scn_durable_pending_count') == 1) + { + $saw_pending = 1; + last; + } + usleep(100_000); +} +ok($saw_pending, 'L6 T1 registered its pending commit SCN and stalled'); + +# T2: a fresh backend (never armed -- the stall arm is local to T1's +# process) commits fully and durably while T1 is still stalled. +my $t2_t0 = time(); +$node0->safe_psql('postgres', 'INSERT INTO t387 VALUES (9002, 6)'); +my $t2_ms = (time() - $t2_t0) * 1000.0; +diag(sprintf('L6 T2 commit wall: %.1f ms', $t2_ms)); +cmp_ok($t2_ms, '<=', 3000, + 'L6 T2 committed promptly (no coupling to the stalled T1)'); + +is(state_int($node0, 'scn', 'scn_commit_advance_count') - $c_pre_l6, + 2, 'L6 exactly T1+T2 allocated commit SCNs in the window'); +is(state_int($node0, 'scn', 'scn_durable_pending_count'), + 1, 'L6 T2 discharged; only the stalled T1 remains pending'); + +my $f_window_a = $node0->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='scn' AND key='scn_durable_safe_scn'}); +usleep(300_000); +my $f_window_b = $node0->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='scn' AND key='scn_durable_safe_scn'}); +is($f_window_b, $f_window_a, + 'L6 frontier held (not merely slow) while T1 stayed pending'); + +# finish() returns true iff every child exited 0 (psql -q suppresses +# the INSERT tag on stdout, so the exit status is the success signal). +ok($t1->finish, 'L6 T1 commit completed after its stall'); +diag("L6 T1 stderr: $t1_err") if $t1_err ne ''; + +my $f_final = ''; +my $t_l6b = time(); +my $drained = 0; +while (time() - $t_l6b < 12) +{ + if (state_int($node0, 'scn', 'scn_durable_pending_count') == 0) + { + $drained = 1; + $f_final = $node0->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state WHERE category='scn' AND key='scn_durable_safe_scn'}); + last; + } + usleep(100_000); +} +ok($drained, 'L6 pending drained once T1 completed'); +ok($f_final ne '' && hex_gt($f_final, $f_window_a), + 'L6 frontier advanced only after the stalled T1 completed'); +is(state_int($node0, 'scn', 'scn_durable_frontier_regression_count'), + $regress0, 'L6 no frontier regression was ever attempted'); + +# ---------- +# L7: idle async commit is eventually discharged by the walwriter. +# ---------- + +my $f_pre_l7 = $bg0->query($FRONTIER_SQL); +my $remote_pre_l7 = $bg1->query($REMOTE_SQL); +$node0->safe_psql('postgres', + 'SET synchronous_commit = off; INSERT INTO t387 VALUES (9100, 7)'); + +my $t_l7 = time(); +my ($l7_local, $l7_remote) = (0, 0); +while (time() - $t_l7 < 5 && (!$l7_local || !$l7_remote)) +{ + $l7_local = 1 + if !$l7_local && hex_gt($bg0->query($FRONTIER_SQL), $f_pre_l7); + $l7_remote = 1 + if !$l7_remote && hex_gt($bg1->query($REMOTE_SQL), $remote_pre_l7); + usleep(50_000); +} +ok($l7_local, + 'L7 idle async commit discharged by the walwriter flush horizon (local frontier)'); +ok($l7_remote, 'L7 walwriter-side event published the async frontier remotely'); + +# ---------- +# L8: seqlock reader liveness across LMON kill / crash-restart. +# ---------- + +$bg0->quit; +$bg1->quit; + +my $lmon_pid = $node1->safe_psql('postgres', + q{SELECT pid FROM pg_stat_activity WHERE backend_type = 'lmon' LIMIT 1}); +ok($lmon_pid =~ /^\d+$/, 'L8 found node1 LMON pid'); +kill 'KILL', $lmon_pid; + +# Aux-process death crash-restarts node1 (shmem rebuilt). +$node1->poll_query_until('postgres', 'SELECT true') + or BAIL_OUT('node1 did not come back after LMON kill'); + +my ($rc, $out, $err) = $node1->psql( + 'postgres', + q{SELECT count(*) FROM pg_cluster_state WHERE category = 'scn'}, + timeout => 15); +is($rc, 0, 'L8 scn state read completed after crash-restart (no reader spin)'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 60), 'L8 node0 re-sees node1'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 60), 'L8 node1 re-sees node0'); + +$node0->safe_psql('postgres', 'INSERT INTO t387 VALUES (9200, 8)'); +my $t_l8 = time(); +my $repop = 0; +while (time() - $t_l8 < 15) +{ + my $v = state_str($node1, 'scn', 'scn_remote_durable_node0'); + if ($v ne '') + { + $repop = 1; + last; + } + # Keep the channel warm: sweeps republish, but a fresh commit event + # is the fast path. + $node0->safe_psql('postgres', 'INSERT INTO t387 VALUES (9201, 8)'); + usleep(500_000); +} +ok($repop, 'L8 remote durable-frontier cache repopulated after restart'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_scn_frontier.c b/src/test/cluster_unit/test_cluster_scn_frontier.c index e1f448622a..0fc57ef2e5 100644 --- a/src/test/cluster_unit/test_cluster_scn_frontier.c +++ b/src/test/cluster_unit/test_cluster_scn_frontier.c @@ -239,7 +239,10 @@ superuser(void) /* injection stub */ /* spec-7.4 D1-2: configurable suppress-injection stub -- tests set - * test_injection_skip_point to a point name to suppress it. */ + * test_injection_skip_point to a point name to make the armed-state + * peek (cluster_cr_injection_armed) report it armed. The production + * probe also requires cluster_injection_armed_count > 0 (fast gate), + * so tests raise the gate for the suppression window. */ const char *test_injection_skip_point = NULL; bool @@ -247,6 +250,14 @@ cluster_injection_should_skip(const char *p) { return test_injection_skip_point != NULL && strcmp(p, test_injection_skip_point) == 0; } + +bool +cluster_cr_injection_armed(const char *p, uint64 *out_param) +{ + if (out_param != NULL) + *out_param = 0; + return test_injection_skip_point != NULL && strcmp(p, test_injection_skip_point) == 0; +} void cluster_injection_check(const char *p pg_attribute_unused()) {} @@ -254,8 +265,9 @@ cluster_injection_check(const char *p pg_attribute_unused()) /* fmgr stub for SQL UDFs (address-only) */ struct FunctionCallInfoBaseData; -/* injection extras (cluster_scn.c references run + armed_count) */ -pg_atomic_uint32 cluster_injection_armed_count; +/* injection extras (cluster_scn.c references run + armed_count; the + * gate is a plain int in cluster_inject.h, mirror that here). */ +int cluster_injection_armed_count; void cluster_injection_run(const char *p pg_attribute_unused()) {} @@ -930,9 +942,11 @@ UT_TEST(test_spec74_event_injection_suppresses_signal) test_lmon_wakeup_count = 0; test_injection_skip_point = "cluster-boc-event-publish"; + cluster_injection_armed_count = 1; s = cluster_scn_advance_for_commit(); UT_ASSERT(cluster_scn_durable_pending_fill_lsn(s, (XLogRecPtr)15000)); UT_ASSERT(cluster_scn_durable_pending_discharge_scn(s)); + cluster_injection_armed_count = 0; test_injection_skip_point = NULL; /* Suppressed event; the sweep cadence remains the fallback (L2). */ From 7d89e3965baaea3519bb521a53fe24e5f43b2e43 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 11 Jul 2026 16:05:02 +0800 Subject: [PATCH 7/7] fix(cluster): spec-7.4 D1 -- add cluster_cr_injection_armed stub to test_cluster_scn link test_cluster_scn links cluster_scn.o standalone. The D1 boc-event-publish probe switched cluster_scn.c from cluster_injection_should_skip to the armed-state peek cluster_cr_injection_armed, so every binary linking cluster_scn.o now needs that symbol; only test_cluster_scn_frontier had the stub. A local incremental build reused a stale test_cluster_scn.o and masked the missing symbol -- a clean CI build (and a clean local rebuild, all 169 binaries) exposed the link failure. Add the stub and correct cluster_injection_armed_count to int to match cluster_inject.h. Spec: spec-7.4-commit-scn-propagation-freshness.md (D1) --- src/test/cluster_unit/test_cluster_scn.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/test/cluster_unit/test_cluster_scn.c b/src/test/cluster_unit/test_cluster_scn.c index 38b02fdc39..c9bcdc6a7d 100644 --- a/src/test/cluster_unit/test_cluster_scn.c +++ b/src/test/cluster_unit/test_cluster_scn.c @@ -242,6 +242,15 @@ cluster_injection_should_skip(const char *p pg_attribute_unused()) { return false; } +/* spec-7.4 D1-2: cluster_scn.c's boc-event-publish probe uses the + * armed-state peek; this binary never arms, so always report unarmed. */ +bool +cluster_cr_injection_armed(const char *p pg_attribute_unused(), uint64 *out_param) +{ + if (out_param != NULL) + *out_param = 0; + return false; +} void cluster_injection_check(const char *p pg_attribute_unused()) {} @@ -249,8 +258,9 @@ cluster_injection_check(const char *p pg_attribute_unused()) /* fmgr stub for SQL UDFs (address-only) */ struct FunctionCallInfoBaseData; -/* injection extras (cluster_scn.c references run + armed_count) */ -pg_atomic_uint32 cluster_injection_armed_count; +/* injection extras (cluster_scn.c references run + armed_count; the gate + * is a plain int in cluster_inject.h, mirror that here). */ +int cluster_injection_armed_count; void cluster_injection_run(const char *p pg_attribute_unused()) {}