From b58c69e54a0e7ce2d899922aaed44c1bbf3042b7 Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 9 Sep 2026 13:43:08 -0400 Subject: [PATCH 01/13] bootstrap before verifying and indexing blocks --- common/api.go | 25 +++++ instance.go | 128 ++++++++++++++++------ instance_helpers_test.go | 52 +++++++-- instance_test.go | 77 ++++++++++++- nonvalidator/epochs.go | 8 +- nonvalidator/epochs_test.go | 16 +-- nonvalidator/non_validator.go | 111 ++++++++++++++++--- nonvalidator/non_validator_test.go | 167 +++++++++++++++++++++++++++++ util.go | 6 ++ 9 files changed, 510 insertions(+), 80 deletions(-) diff --git a/common/api.go b/common/api.go index a3f5b4c2..78dab28f 100644 --- a/common/api.go +++ b/common/api.go @@ -203,6 +203,31 @@ func (nws Nodes) Contains(nodeID NodeID) bool { return false } +// Equal returns whether both hold the same nodes, ignoring order. +func (nws Nodes) Equal(other Nodes) bool { + if len(nws) != len(other) { + return false + } + + nwsClone := slices.Clone(nws) + otherClone := slices.Clone(other) + SortNodes(nwsClone) + SortNodes(otherClone) + + for i := range nwsClone { + if !bytes.Equal(nwsClone[i].Id, otherClone[i].Id) { + return false + } + if !bytes.Equal(nwsClone[i].PK, otherClone[i].PK) { + return false + } + if nwsClone[i].Weight != otherClone[i].Weight { + return false + } + } + return true +} + // Node is a struct that pairs a node ID with its weight and public key. type Node struct { Id NodeID diff --git a/instance.go b/instance.go index 5888e099..9f580614 100644 --- a/instance.go +++ b/instance.go @@ -74,6 +74,11 @@ type Instance struct { epochOrNV timeAdvancer epochChanges chan epochChange stopCh chan struct{} + + // bootstrapped represents whether the instance has completed bootstrapping. + // This is false on Start, and also set to false when our notices it's validator + // has fallen behind by many epochs. + bootstrapped bool } func NewInstance(config Config) *Instance { @@ -113,18 +118,64 @@ func (i *Instance) Start(ctx context.Context) error { context.AfterFunc(ctx, i.Stop) - nodes, epochNum, err := getLastAcceptedEpochAndValidatorSet(&i.Config) + if err := i.bootstrap(); err != nil { + return err + } + + go i.tick() + go i.listenForEpochChanges() + + return nil +} + +func (i *Instance) bootstrap() error { + latestValidatorSet, err := getLatestPlatformChainValidatorSet(i.Config.PlatformChain) if err != nil { - return fmt.Errorf("error determining latest epoch and validator set: %w", err) + return err } - if err := i.startAtEpoch(nodes); err != nil { - return fmt.Errorf("error starting instance at epoch %d: %w", epochNum, err) + latestIndexedEpochValidators, _, err := getLastAcceptedEpochAndValidatorSet(&i.Config) + if err != nil { + return err } - go i.tick() - go i.listenForEpochChanges() + // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. + // Note: this may not be the latest epoch, but our futureEpochCollector will eventually notice we are behind and transition properly. + if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { + i.bootstrapped = true + return i.startValidator(latestIndexedEpochValidators) + } + + // Start as non-validator if our last indexed validator set does not equal, the latest p-chain validator set + // Note: the epoch may be transitioning, so the latest p-chain validator set actually points to a future epoch. + // The non-validator should finish bootstrapping and convert our non-validator to a validator in this case. + return i.startNonValidator() +} + +func (i *Instance) onBootstrapFinish(highestKnownEpoch uint64, highestKnownValidators common.Nodes) error { + i.bootstrapped = true + i.Config.Logger.Debug( + "Node finished bootstrapping", + zap.Stringers("Highest Validators", + highestKnownValidators.NodeIDs()), + zap.Uint64("Highest Epoch", highestKnownEpoch), + ) + + _, lastAcceptedEpoch, err := getLastAcceptedEpochAndValidatorSet(&i.Config) + if err != nil { + return err + } + // the latest epoch contains our node, we should asynchronously notify the listener + // which will convert our node to a validator for this epoch. + if highestKnownValidators.Contains(i.Config.ID) && lastAcceptedEpoch == highestKnownEpoch { + i.Config.Logger.Debug("Our node completed bootstrapping and it is a validator") + i.notifyEpochChange(highestKnownEpoch, highestKnownValidators) + return nil + } + + // we have completed bootstrapping, but our node is still a non-validator + i.Config.Logger.Debug("Our node completed bootstrapping but it is not a validator") return nil } @@ -168,13 +219,12 @@ func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { return nonvalidator.Config{}, err } - height := i.Config.PlatformChain.GetCurrentHeight() - mappings, err := i.Config.PlatformChain.GetValidatorSet(height) + latestValidatorSet, err := getLatestPlatformChainValidatorSet(i.Config.PlatformChain) if err != nil { return nonvalidator.Config{}, err } - comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, mappings.Nodes()) + comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, latestValidatorSet.Nodes()) // Plant an artificial MSM. A non-validator never verifies the state machine transition, // it only verifies the inner block (see common.OnlyVMVerifyOpt), so this MSM is only @@ -203,6 +253,8 @@ func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, TransitionToValidator: i.notifyEpochChange, + OnFinishBootstrapping: i.onBootstrapFinish, + Bootstrapped: i.bootstrapped, } return config, nil } @@ -330,36 +382,42 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } } + if i.nv != nil { + return i.nv.HandleMessage(msg, from) + } + if i.e != nil { - switch { - case msg.AuxiliaryInfo != nil: - if msg.AuxiliaryInfo.Epoch != i.e.Epoch { - i.Config.Logger.Debug( - "Received an auxiliary info from an old epoch", - zap.Uint64("Aux Info Epoch", msg.AuxiliaryInfo.Epoch), - zap.Uint64("Our Epoch", i.e.Epoch), - zap.Stringer("From", from)) - return nil - } - i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) - case msg.EpochTransitionApproval != nil: - if !from.Equals(msg.EpochTransitionApproval.NodeID[:]) { - i.Config.Logger.Debug("Dropping approval not sent by its signer", - zap.Stringer("from", from), - zap.Stringer("signer", common.NodeID(msg.EpochTransitionApproval.NodeID[:]))) - return nil - } - // TODO: pass in time.Now() rather than uint64 - i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().UnixMilli())) - return nil - } - return i.e.HandleMessage(msg, from) + return i.handleMessageForEpoch(msg, from) } - if i.nv != nil { - return i.nv.HandleMessage(msg, from) + return errors.New("we are not running as a validator or not validator") +} + +func (i *Instance) handleMessageForEpoch(msg *common.Message, from common.NodeID) error { + + switch { + case msg.AuxiliaryInfo != nil: + if msg.AuxiliaryInfo.Epoch != i.e.Epoch { + i.Config.Logger.Debug( + "Received an auxiliary info from an old epoch", + zap.Uint64("Aux Info Epoch", msg.AuxiliaryInfo.Epoch), + zap.Uint64("Our Epoch", i.e.Epoch), + zap.Stringer("From", from)) + return nil + } + i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + case msg.EpochTransitionApproval != nil: + if !from.Equals(msg.EpochTransitionApproval.NodeID[:]) { + i.Config.Logger.Debug("Dropping approval not sent by its signer", + zap.Stringer("from", from), + zap.Stringer("signer", common.NodeID(msg.EpochTransitionApproval.NodeID[:]))) + return nil + } + // TODO: pass in time.Now() rather than uint64 + i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().UnixMilli())) + return nil } - return nil + return i.e.HandleMessage(msg, from) } func (i *Instance) wireReplicationResponse(msg *common.Message) error { diff --git a/instance_helpers_test.go b/instance_helpers_test.go index e67d443a..4a44f9c0 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -59,6 +59,8 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte var ( genesisPChainHeight uint64 = 0 genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} + // epochBlockTime fixes the timestamp of the epoch-defining block + epochBlockTime = genesisBlock.TS.Add(time.Millisecond) ) var paramConfig = ParameterConfig{ @@ -246,6 +248,33 @@ func newTestStorageWithGenesis(t *testing.T) *testStorage { return s } +// newChainStorage builds and indexes the minimum chain a node can start from: genesis plus +// epoch 1's defining block, which carries the descriptor naming the epoch's validator set. +// It returns the storage and the epoch-defining block at its tip. +func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*testStorage, metadata.StateMachineBlock) { + storage := newTestStorageWithGenesis(t) + genesis, _, err := storage.GetBlock(0) + require.NoError(t, err) + + epochBlock := metadata.StateMachineBlock{ + InnerBlock: &testInnerBlock{Height_: 1, TS: epochBlockTime, Payload: []byte("epoch")}, + Metadata: metadata.StateMachineMetadata{ + Timestamp: uint64(epochBlockTime.UnixMilli()), + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 1, Seq: 1, Prev: common.Digest(genesis.Digest())}, + SimplexEpochInfo: metadata.SimplexEpochInfo{ + BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: validators}, + }, + }, + }, + } + + block := &ParsedBlock{StateMachineBlock: epochBlock.Clone()} + finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(validators)}, block, validators.NodeIDs()) + require.NoError(t, storage.Index(context.Background(), block, finalization)) + return storage, epochBlock +} + func (m *testStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { block, fin, err := m.Retrieve(seq) if err != nil { @@ -601,20 +630,21 @@ func (n *node) restart() *node { return newNode } +// role reports whether the instance currently runs a validator epoch rather than a +// non-validator, and whether it has finished bootstrapping. +func (n *node) role() (isValidator bool, bootstrapped bool) { + n.inst.lock.Lock() + defer n.inst.lock.Unlock() + + return n.inst.e != nil, n.inst.bootstrapped +} + // sync syncs a node by waiting for the commit of the latest sequence. func (n *node) sync() *node { n.storage.WaitForBlockCommit(n.net.seq - 1) return n } -// role reports whether the node is running a validator rather than a non-validator. -func (n *node) role() (isValidator bool) { - n.inst.lock.Lock() - defer n.inst.lock.Unlock() - - return n.inst.e != nil -} - const firstEverEpoch uint64 = 1 type network struct { @@ -754,8 +784,10 @@ func (n *network) waitUntilValidatorsReady() { continue } - require.Eventually(n.t, node.role, time.Minute, time.Millisecond, - "node %x never started running a validator", node.id) + require.Eventually(n.t, func() bool { + isValidator, _ := node.role() + return isValidator + }, time.Minute, time.Millisecond, "node %x never started running a validator", node.id) } } diff --git a/instance_test.go b/instance_test.go index 736172b0..e8697f24 100644 --- a/instance_test.go +++ b/instance_test.go @@ -89,7 +89,8 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { } } -// TestNonValidatorSyncs that a non-validator syncs the chain when added to the network. +// TestNonValidatorSyncs asserts a node outside the validator set syncs the chain when added +// to the network, and stays a non-validator once it has bootstrapped. func TestNonValidatorSyncs(t *testing.T) { validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} @@ -101,8 +102,14 @@ func TestNonValidatorSyncs(t *testing.T) { network.acceptNewBlock() nonValidator := newNodeMapping(2) - network.addNode(nonValidator.NodeID[:]) + node := network.addNode(nonValidator.NodeID[:]) network.acceptNewBlock() + node.sync() + + // ensure we bootstrap and are not a validator + isValidator, bootstrapped := node.role() + require.True(t, bootstrapped) + require.False(t, isValidator) } // TestNonValidatorBecomesValidator tests that an upcoming validator becomes a validator @@ -385,6 +392,19 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { parent, _, err := nonValidatorNode.storage.GetBlock(1) require.NoError(t, err) + // The non-validator drops every message until it bootstraps. One peer reporting the block + // sealing epoch 1 meets the threshold of F(1)+1. + sealing := &ParsedBlock{StateMachineBlock: parent.Clone()} + sealingFinalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: 1}, sealing, []common.NodeID{validator.NodeID[:]}) + require.NoError(t, nonValidatorNode.inst.HandleMessage(&common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + LatestSeq: &common.QuorumRound{Block: sealing, Finalization: &sealingFinalization}, + }, + }, validator.NodeID[:])) + + _, bootstrapped := nonValidatorNode.role() + require.True(t, bootstrapped) + // A block whose only defect is its state machine transition: its timestamp precedes its // parent's. invalid := metadata.StateMachineBlock{ @@ -556,3 +576,56 @@ func TestValidatorSetsMetadataFromSnowman(t *testing.T) { require.Equal(t, uint64(1), block.BlockHeader().Round) require.Equal(t, numNonSimplexBlocks, block.BlockHeader().Seq) } + +// TestBootstrap_ValidatorInLatestEpoch asserts a node whose indexed epoch names exactly the +// latest P-chain validator set skips bootstrapping and starts as a validator of that epoch. +func TestBootstrap_ValidatorInLatestEpoch(t *testing.T) { + v1 := newNodeMapping(1) + v2 := newNodeMapping(2) + genesisValidatorSet := metadata.NodeBLSMappings{v1, v2} + + pChain := newTestPChain(genesisValidatorSet) + storage, _ := newChainStorage(t, genesisValidatorSet) + node := newNetwork(t, pChain).addNodeWithConfig(v1.NodeID[:], nodeConfig{storage: storage}) + + isValidator, bootstrapped := node.role() + require.True(t, bootstrapped, "a node already at the latest validator set has nothing to bootstrap") + require.True(t, isValidator) +} + +// TestBootstrap_ValidatorDuringTransition asserts a validator whose epoch is mid-transition, +// so its indexed set disagrees with the latest P-chain set, starts as a non-validator and +// converts back to a validator once bootstrapping confirms its indexed epoch. +func TestBootstrap_ValidatorDuringTransition(t *testing.T) { + ourNodeMapping := newNodeMapping(1) + v2 := newNodeMapping(2) + futureValidator := newNodeMapping(3) + genesisValidatorSet := metadata.NodeBLSMappings{ourNodeMapping, v2} + + pChain := newTestPChain(genesisValidatorSet) + // The P-chain moved on to a set that contains ourNode, but no sealing block for it has + // been indexed, so our indexed set and the latest set disagree. + pChain.setValidatorSetAt(10, metadata.NodeBLSMappings{ourNodeMapping, v2, futureValidator}) + pChain.advanceHeight(10) + + storage, sealing := newChainStorage(t, genesisValidatorSet) + node := newNetwork(t, pChain).addNodeWithConfig(ourNodeMapping.NodeID[:], nodeConfig{storage: storage}) + + isValidator, bootstrapped := node.role() + require.False(t, bootstrapped) + require.False(t, isValidator, "a node whose indexed set is not the latest must bootstrap first") // even though we are a validator + + // One peer reporting the latest sealing block meets the threshold of F(3)+1. + block := &ParsedBlock{StateMachineBlock: sealing.Clone()} + finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(genesisValidatorSet)}, block, genesisValidatorSet.NodeIDs()) + require.NoError(t, node.inst.HandleMessage(&common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + LatestSeq: &common.QuorumRound{Block: block, Finalization: &finalization}, + }, + }, v2.NodeID[:])) + + require.Eventually(t, func() bool { + isValidator, bootstrapped := node.role() + return isValidator && bootstrapped + }, 10*time.Second, 10*time.Millisecond, "the node never converted back to a validator of its indexed epoch") +} diff --git a/nonvalidator/epochs.go b/nonvalidator/epochs.go index 102a4ff3..612a7563 100644 --- a/nonvalidator/epochs.go +++ b/nonvalidator/epochs.go @@ -135,12 +135,10 @@ func (e epochs) canValidate(block common.Block) bool { // latestValidatorSetRetriever is an allows the epoch replicator to get the latest validator set. // This is used to calculate the threshold of votes needed to validate an epoch. -type latestValidatorSetRetriever interface { - Validators() common.Nodes -} +type latestValidatorSetRetriever func() common.Nodes // epochDigestCounter counts sealing block responses from validators for each epoch. -// It uses latestValidatorSetRetriever to determine when the required response threshold +// It uses LatestValidatorSetRetriever to determine when the required response threshold // has been reached. type epochDigestCounter struct { logger common.Logger @@ -174,7 +172,7 @@ func (e *epochDigestCounter) collectedSealingBlockInfo(sealingBlockInfo *common. return false } - validators := e.latestValidatorSetRetriever.Validators() + validators := e.latestValidatorSetRetriever() if !validators.Contains(from) { e.logger.Debug("Received a quorum round from a node that is not a validator", zap.Stringer("from", from)) diff --git a/nonvalidator/epochs_test.go b/nonvalidator/epochs_test.go index dc4a3ba9..9ca2c300 100644 --- a/nonvalidator/epochs_test.go +++ b/nonvalidator/epochs_test.go @@ -251,14 +251,6 @@ func newSealingQuorumRound(epoch uint64, numValidators int) *common.QuorumRound } } -type testValidatorSetRetriever struct { - nodes common.Nodes -} - -func (v *testValidatorSetRetriever) Validators() common.Nodes { - return v.nodes -} - // TestCollectedQuorumRound feeds an epochReplicator a sealing-block quorum round // for an unknown epoch and asserts collectedQuorumRound only confirms the epoch // once a threshold of distinct validators have voted for the same digest. @@ -283,8 +275,8 @@ func TestCollectedQuorumRound(t *testing.T) { // votes required to confirm the epoch. threshold := common.F(len(voters)) + 1 require.GreaterOrEqual(t, len(voters), threshold, "need at least threshold validators to vote with") - e := newEpochReplicator(testutil.MakeLogger(t, 1), &testValidatorSetRetriever{ - nodes: voters, + e := newEpochReplicator(testutil.MakeLogger(t, 1), func() common.Nodes { + return voters }) // Each distinct vote below the threshold leaves the epoch unconfirmed. @@ -305,8 +297,8 @@ func TestCollectedSealingBlockInfoOneResponsePerValidator(t *testing.T) { qr := newSealingQuorumRound(1, 4) info := qr.Block.SealingBlockInfo() validators := info.ValidatorSet - e := newEpochReplicator(testutil.MakeLogger(t, 1), &testValidatorSetRetriever{ - nodes: validators, + e := newEpochReplicator(testutil.MakeLogger(t, 1), func() common.Nodes { + return validators }) seq5 := newSealingTestBlock(5, 1, common.Digest{}, info).BlockHeader() diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index 0d36505d..26f748a7 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -6,6 +6,7 @@ package nonvalidator import ( "bytes" "context" + "errors" "fmt" "math/rand/v2" "sync" @@ -58,6 +59,12 @@ type Config struct { // TransitionToValidator is called when our non-validator indexes the highest known epoch // and it is in the validator set TransitionToValidator func(epoch uint64, validators common.Nodes) + + OnFinishBootstrapping func(epoch uint64, validators common.Nodes) error + + // a non-validator is considered bootstrapped when it has received a threshold of votes + // from the latest validator set. Until then, it cannot verify or index any blocks. + Bootstrapped bool } type NonValidator struct { @@ -117,7 +124,7 @@ func NewNonValidator(config Config) (*NonValidator, error) { epochs: epochs, verifier: common.NewBlockVerificationScheduler(config.Logger, simplex.DefaultProcessingBlocks, scheduler), lock: lock, - highestEpochCollector: newEpochReplicator(config.Logger, config.Comm), + highestEpochCollector: newEpochReplicator(config.Logger, config.Comm.Validators), oneTimeVerifier: simplex.NewOneTimeVerifier(config.Logger), sequenceReplicator: replicator, }, nil @@ -154,6 +161,10 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er return n.haltedError } + if !n.Bootstrapped { + return n.handleBootstrap(msg, from) + } + switch { case msg.BlockMessage != nil && msg.BlockMessage.Block != nil: return n.handleBlock(msg.BlockMessage.Block, from) @@ -165,7 +176,68 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er n.Logger.Debug("Received unexpected message", zap.Any("Message", msg), zap.Stringer("from", from)) return nil } +} + +// handleBootstrap handles messages received before we are bootstrapped. +// Only replication responses are processed, all other messages are dropped. +func (n *NonValidator) handleBootstrap(msg *common.Message, from common.NodeID) error { + resp := msg.ReplicationResponse + if resp == nil { + n.Logger.Debug("Dropping message received while bootstrapping, we only accept replication responses", zap.Any("Message", msg), zap.Stringer("From", from)) + return nil + } + + for _, qr := range resp.Data { + if err := n.maybeBootstrapFromQuorumRound(&qr, from); err != nil { + n.Logger.Debug("Failed processing quorum round while bootstrapping", zap.Stringer("QR", &qr), zap.Error(err)) + } + } + + if err := n.maybeBootstrapFromQuorumRound(resp.LatestSeq, from); err != nil { + n.Logger.Debug("Failed processing latest seq while bootstrapping", zap.Stringer("QR", resp.LatestSeq), zap.Error(err)) + } + + if !n.Bootstrapped { + return nil + } + + // Begin processing the quorum rounds stored if bootstrapping has finished. + return n.processReplicationState() +} + +// maybeBootstrapFromQuorumRound records the sealing block info of qr with the highest epoch collector. +// Once a threshold of nodes report the same sealing block, we consider ourselves bootstrapped, +// validate the epoch it seals, and store the quorum round for replication. +func (n *NonValidator) maybeBootstrapFromQuorumRound(qr *common.QuorumRound, from common.NodeID) error { + if err := verifyQuorumRound(qr); err != nil { + return err + } + + // We can only bootstrap from a sealing block, request the one sealing this block's epoch. + if qr.Block.SealingBlockInfo() == nil { + n.sendRequest(qr.Block.BlockHeader().Epoch, from) + return nil + } + + if !n.highestEpochCollector.collectedSealingBlockInfo(qr.Block.SealingBlockInfo(), qr.Block.BlockHeader(), from) { + return nil + } + + n.Logger.Info("Bootstrapped, received a threshold of sealing block info for an epoch", zap.Stringer("Info", qr.Block.SealingBlockInfo())) + n.Bootstrapped = true + + n.maybeValidateNextEpoch(qr.Block) + // We are storing a quorum round with a finalization we have not yet verified. + // We do this to tell the replicator a valid sequence exists and to begin replication if necessary. + // We will check the validity when we process this round. + n.sequenceReplicator.StoreQuorumRound(qr) + + if n.OnFinishBootstrapping == nil { + n.Logger.Debug("OnFinishBootstrapping not set for the non-validator") + return nil + } + return n.OnFinishBootstrapping(qr.Block.BlockHeader().Seq, qr.Block.SealingBlockInfo().ValidatorSet) } // handleBlock handles a block message. BlockMessages are sent when the leader proposes a block for its round. @@ -329,6 +401,10 @@ func (n *NonValidator) removeOldSequencesAndEpochs(lastCommittedSeq, minEpochToK // handleFinalization process a finalization message. If its for a future epoch, it will forward the finalization // to the replication handler. func (n *NonValidator) handleFinalization(finalization *common.Finalization, from common.NodeID) error { + if !n.Bootstrapped { + return nil + } + bh := finalization.Finalization.BlockHeader n.Logger.Debug("Received a finalization", zap.Uint64("Seq", bh.Seq), zap.Stringer("From", from)) @@ -501,25 +577,11 @@ func (n *NonValidator) processReplicationState() error { // epochs when qr has a sealing block, either by checking that we have received a threshold, or by backwards hash chain validation. // Returns an error if the qr could not be processed. func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.NodeID) error { - if qr == nil { - return nil - } - - if err := qr.VerifyQCConsistentWithBlock(); err != nil { + if err := verifyQuorumRound(qr); err != nil { return err } block := qr.Block - finalization := qr.Finalization - - // Non validators only process quorum rounds with finalizations - if finalization == nil { - return nil - } - - if block == nil { - return fmt.Errorf("received a quorum round with a finalization but no block") - } if n.isAccepted(block.BlockHeader().Seq) { return fmt.Errorf("processing quorum round for a block we already indexed") @@ -542,6 +604,23 @@ func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.No return nil } +// verifyQuorumRound verifies a qr can be processed by the non-validator. +func verifyQuorumRound(qr *common.QuorumRound) error { + if qr == nil { + return errors.New("nil quorum round") + } + + if err := qr.VerifyQCConsistentWithBlock(); err != nil { + return err + } + + if qr.Block == nil || qr.Finalization == nil { + return errors.New("ignoring quorum round without a block and finalization") + } + + return nil +} + // storeQuorumRound updates replication state, and stores qr if its within MaxSequenceWindow. func (n *NonValidator) storeQuorumRound(qr *common.QuorumRound) { seq := qr.Block.BlockHeader().Seq diff --git a/nonvalidator/non_validator_test.go b/nonvalidator/non_validator_test.go index 0ccf79a0..68dbae28 100644 --- a/nonvalidator/non_validator_test.go +++ b/nonvalidator/non_validator_test.go @@ -322,6 +322,7 @@ func TestHandleMessages(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -404,6 +405,7 @@ func TestNonValidator_StopsGracefully(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -437,6 +439,7 @@ func TestHandleMessages_DuplicateBlock(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -588,6 +591,7 @@ func TestNonValidator_CallsTransition(t *testing.T) { defer lock.Unlock() calls = append(calls, transitionCall{epoch: epoch, validators: validators}) }, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -752,6 +756,7 @@ func TestNonValidator_Bootstrap(t *testing.T) { MaxSequenceWindow: tt.maxSequenceWindow, ID: myNodeID, StartTime: time.Now(), + Bootstrapped: false, }, ) require.NoError(t, err) @@ -793,6 +798,7 @@ func TestNonValidator_ReplicationRequests(t *testing.T) { MaxSequenceWindow: maxSeqWindow, ID: myNodeID, StartTime: startTime, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -874,6 +880,7 @@ func TestNonValidator_VerifiesFinalizationDuringReplication(t *testing.T) { MaxSequenceWindow: 5, // significantly lower the max round window ID: testNodes.NodeIDs()[0], StartTime: startTime, + Bootstrapped: true, }, ) @@ -1012,6 +1019,7 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) { MaxSequenceWindow: 10, ID: validators.NodeIDs()[0], StartTime: time.Now(), + Bootstrapped: true, }, ) require.NoError(t, err) @@ -1046,6 +1054,163 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) { } } +// TestNonValidator_BootstrapGatesMessages asserts that blocks and finalizations are dropped +// until a threshold of replication responses vouch for the same sealing block, +// after which the stored round is committed and messages are processed normally. +func TestNonValidator_BootstrapGatesMessages(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + var bootstrappedHighestValidators common.Nodes + var bootstrappedHighestEpoch uint64 + + nv, err := NewNonValidator( + Config{ + Storage: tc, + Comm: testutil.NewNoopComm(testNodes.NodeIDs()), + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: common.NodeID{100}, + OnFinishBootstrapping: func(epoch uint64, validators common.Nodes) error { + bootstrappedHighestEpoch = epoch + bootstrappedHighestValidators = validators + return nil + }, + }, + ) + require.NoError(t, err) + defer nv.Stop() + + b3 := tc.appendSealing(testNodes) + f3 := tc.newFinalization(b3) + + block := blockMsg(t, b3, testNodes) + require.NoError(t, nv.HandleMessage(block.msg, block.from)) + fin := finalizationMsg(t, b3, testNodes) + require.NoError(t, nv.HandleMessage(fin.msg, fin.from)) + + require.Never(t, + func() bool { return tc.NumBlocks() > 3 }, + 2*time.Second, 50*time.Millisecond, + "indexed a block before bootstrapping", + ) + + qrMsg := &common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + Data: []common.QuorumRound{{Block: b3, Finalization: &f3}}, + }, + } + threshold := common.F(len(testNodes)) + 1 + for i := 0; i < threshold; i++ { + require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[i])) + } + + // bootstrapping commits the collected sealing block + tc.WaitForBlockCommit(3) + + // messages flow normally after bootstrapping + b4 := tc.appendBlock() + block = blockMsg(t, b4, testNodes) + require.NoError(t, nv.HandleMessage(block.msg, block.from)) + fin = finalizationMsg(t, b4, testNodes) + require.NoError(t, nv.HandleMessage(fin.msg, fin.from)) + tc.WaitForBlockCommit(4) + + require.Equal(t, b3.BlockHeader().Seq, bootstrappedHighestEpoch) + require.Equal(t, testNodes, bootstrappedHighestValidators) +} + +// TestNonValidator_BootstrapRequestsSealingBlock asserts that a replication response +// carrying a non-sealing block while bootstrapping triggers a replication request +// for the block's epoch, whose seq is the sealing block that opened it. +func TestNonValidator_BootstrapRequestsSealingBlock(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + myNodeID := common.NodeID{100} + msgQueue := &messageQueue{} + nv, err := NewNonValidator( + Config{ + Storage: tc, + Comm: &routerComm{ + nodes: testNodes, + t: t, + ID: myNodeID, + messageQueue: msgQueue, + }, + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: myNodeID, + }, + ) + require.NoError(t, err) + defer nv.Stop() + + b3 := tc.appendBlock() + f3 := tc.newFinalization(b3) + sender := testNodes.NodeIDs()[2] + + require.NoError(t, nv.HandleMessage(&common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + Data: []common.QuorumRound{{Block: b3, Finalization: &f3}}, + }, + }, sender)) + + msg, ok := msgQueue.popResponse() + require.True(t, ok) + require.NotNil(t, msg.msg.ReplicationRequest) + require.Equal(t, []uint64{b3.BlockHeader().Epoch}, msg.msg.ReplicationRequest.Seqs) + require.Equal(t, sender, msg.to) + + // the non-sealing block must not bootstrap the node + require.False(t, nv.Bootstrapped) + _, ok = msgQueue.popResponse() + require.False(t, ok) +} + +// TestNonValidator_BootstrapLatestKnownEpoch asserts a node caught up to the network +// bootstraps from responses vouching for the sealing block of the latest epoch it +// already has indexed, without re-indexing it. +func TestNonValidator_BootstrapLatestKnownEpoch(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + nv, err := NewNonValidator( + Config{ + Storage: tc, + Comm: testutil.NewNoopComm(testNodes.NodeIDs()), + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: common.NodeID{100}, + }, + ) + require.NoError(t, err) + defer nv.Stop() + + // the sealing block of epoch 1, indexed at seq 1 + sealing, fin, err := tc.Retrieve(1) + require.NoError(t, err) + + qrMsg := &common.Message{ + ReplicationResponse: &common.ReplicationResponse{ + LatestSeq: &common.QuorumRound{Block: sealing.(common.Block), Finalization: &fin}, + }, + } + + threshold := common.F(len(testNodes)) + 1 + for i := 0; i < threshold; i++ { + require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[i])) + } + + require.True(t, nv.Bootstrapped) + require.Equal(t, uint64(3), tc.NumBlocks()) + + // messages flow normally after bootstrapping + b3 := tc.appendBlock() + block := blockMsg(t, b3, testNodes) + require.NoError(t, nv.HandleMessage(block.msg, block.from)) + fin3 := finalizationMsg(t, b3, testNodes) + require.NoError(t, nv.HandleMessage(fin3.msg, fin3.from)) + tc.WaitForBlockCommit(3) +} + func advanceUntil(nv *NonValidator, epochs *testEpochs, msgQueue *messageQueue, seq uint64) { startTime := nv.StartTime for { @@ -1131,6 +1296,7 @@ func TestNonValidatorAcceptsProposalFromUnsortedValidatorSet(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: common.NodeID{100}, + Bootstrapped: true, }, ) require.NoError(t, err) @@ -1168,6 +1334,7 @@ func TestNonValidatorRejectsQuorumRoundWithMismatchedHeader(t *testing.T) { Comm: testutil.NewNoopComm(testNodes.NodeIDs()), Logger: logger, SignatureAggregatorCreator: tc.signatureAggregatorCreator, + Bootstrapped: true, MaxSequenceWindow: 10, ID: common.NodeID{16}, StartTime: time.Now(), diff --git a/util.go b/util.go index 977dabb5..d83d9a72 100644 --- a/util.go +++ b/util.go @@ -109,3 +109,9 @@ func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.Node } return validatorSet } + +func getLatestPlatformChainValidatorSet(platformChain PlatformChain) (metadata.NodeBLSMappings, error) { + height := platformChain.GetCurrentHeight() + mappings, err := platformChain.GetValidatorSet(height) + return mappings, err +} From 2873d6b9f5d584107e8c39588612f22eb18d34f7 Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 9 Sep 2026 21:19:26 -0400 Subject: [PATCH 02/13] bootstrapping reworked --- common/timeout_handler.go | 8 + instance.go | 49 +----- instance_helpers_test.go | 5 +- nonvalidator/epochs.go | 2 + nonvalidator/non_validator.go | 249 ++++++++++++++++++----------- nonvalidator/non_validator_test.go | 207 +++++++++++++++++++----- 6 files changed, 345 insertions(+), 175 deletions(-) diff --git a/common/timeout_handler.go b/common/timeout_handler.go index 3a635d3e..e7d69ce8 100644 --- a/common/timeout_handler.go +++ b/common/timeout_handler.go @@ -133,6 +133,14 @@ func (t *TimeoutHandler[T]) RemoveTask(ID T) { delete(t.tasks, ID) } +// HasTasks reports whether any task is still outstanding. +func (t *TimeoutHandler[T]) HasTasks() bool { + t.lock.Lock() + defer t.lock.Unlock() + + return len(t.tasks) > 0 +} + func (t *TimeoutHandler[T]) RemoveOldTasks(shouldRemove func(id T, _ struct{}) bool) { t.lock.Lock() defer t.lock.Unlock() diff --git a/instance.go b/instance.go index 9f580614..959e1e36 100644 --- a/instance.go +++ b/instance.go @@ -74,11 +74,6 @@ type Instance struct { epochOrNV timeAdvancer epochChanges chan epochChange stopCh chan struct{} - - // bootstrapped represents whether the instance has completed bootstrapping. - // This is false on Start, and also set to false when our notices it's validator - // has fallen behind by many epochs. - bootstrapped bool } func NewInstance(config Config) *Instance { @@ -142,41 +137,13 @@ func (i *Instance) bootstrap() error { // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. // Note: this may not be the latest epoch, but our futureEpochCollector will eventually notice we are behind and transition properly. if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { - i.bootstrapped = true return i.startValidator(latestIndexedEpochValidators) } // Start as non-validator if our last indexed validator set does not equal, the latest p-chain validator set // Note: the epoch may be transitioning, so the latest p-chain validator set actually points to a future epoch. // The non-validator should finish bootstrapping and convert our non-validator to a validator in this case. - return i.startNonValidator() -} - -func (i *Instance) onBootstrapFinish(highestKnownEpoch uint64, highestKnownValidators common.Nodes) error { - i.bootstrapped = true - i.Config.Logger.Debug( - "Node finished bootstrapping", - zap.Stringers("Highest Validators", - highestKnownValidators.NodeIDs()), - zap.Uint64("Highest Epoch", highestKnownEpoch), - ) - - _, lastAcceptedEpoch, err := getLastAcceptedEpochAndValidatorSet(&i.Config) - if err != nil { - return err - } - - // the latest epoch contains our node, we should asynchronously notify the listener - // which will convert our node to a validator for this epoch. - if highestKnownValidators.Contains(i.Config.ID) && lastAcceptedEpoch == highestKnownEpoch { - i.Config.Logger.Debug("Our node completed bootstrapping and it is a validator") - i.notifyEpochChange(highestKnownEpoch, highestKnownValidators) - return nil - } - - // we have completed bootstrapping, but our node is still a non-validator - i.Config.Logger.Debug("Our node completed bootstrapping but it is not a validator") - return nil + return i.startNonValidator(false) } func (i *Instance) startValidator(validators common.Nodes) error { @@ -197,8 +164,10 @@ func (i *Instance) startValidator(validators common.Nodes) error { return epoch.Start() } -func (i *Instance) startNonValidator() error { - config, err := i.createNonValidatorConfig() +// startNonValidator runs a non-validator. bootstrapped is true when we already hold the +// newest sealing block, such as when a validator leaves the validator set. +func (i *Instance) startNonValidator(bootstrapped bool) error { + config, err := i.createNonValidatorConfig(bootstrapped) if err != nil { return err } @@ -213,7 +182,7 @@ func (i *Instance) startNonValidator() error { return nil } -func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { +func (i *Instance) createNonValidatorConfig(bootstrapped bool) (nonvalidator.Config, error) { source, err := simplex.NewRandomSource() if err != nil { return nonvalidator.Config{}, err @@ -253,8 +222,7 @@ func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, TransitionToValidator: i.notifyEpochChange, - OnFinishBootstrapping: i.onBootstrapFinish, - Bootstrapped: i.bootstrapped, + Bootstrapped: bootstrapped, } return config, nil } @@ -394,7 +362,6 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } func (i *Instance) handleMessageForEpoch(msg *common.Message, from common.NodeID) error { - switch { case msg.AuxiliaryInfo != nil: if msg.AuxiliaryInfo.Epoch != i.e.Epoch { @@ -652,7 +619,7 @@ func (i *Instance) startAtEpoch(validators common.Nodes) error { return i.startValidator(validators) } - return i.startNonValidator() + return i.startNonValidator(true) } type epochConfig struct { diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 4a44f9c0..3eae7d93 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -636,7 +636,10 @@ func (n *node) role() (isValidator bool, bootstrapped bool) { n.inst.lock.Lock() defer n.inst.lock.Unlock() - return n.inst.e != nil, n.inst.bootstrapped + if n.inst.e != nil { + return true, true + } + return false, n.inst.nv != nil && n.inst.nv.IsBootstrapped() } // sync syncs a node by waiting for the commit of the latest sequence. diff --git a/nonvalidator/epochs.go b/nonvalidator/epochs.go index 612a7563..d73892c8 100644 --- a/nonvalidator/epochs.go +++ b/nonvalidator/epochs.go @@ -111,6 +111,8 @@ func (e epochs) removeOldEpochs(minEpochToKeep uint64) { } } +// canValidate returns true if `block` is valid sealing block in the chain. It can +// be valid if `block` is a backwards pointer to any sealing block already validated (backwards hash chain validation). func (e epochs) canValidate(block common.Block) bool { if block.SealingBlockInfo() == nil { return false diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index 26f748a7..11574b75 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -60,10 +60,8 @@ type Config struct { // and it is in the validator set TransitionToValidator func(epoch uint64, validators common.Nodes) - OnFinishBootstrapping func(epoch uint64, validators common.Nodes) error - - // a non-validator is considered bootstrapped when it has received a threshold of votes - // from the latest validator set. Until then, it cannot verify or index any blocks. + // Bootstrapped is set once every epoch from our tip up to the one a threshold of the latest + // validator set reported has been validated. Until then only replication responses are handled. Bootstrapped bool } @@ -92,6 +90,11 @@ type NonValidator struct { epochs epochs verifier *common.BlockDependencyManager + + // sealingBlockTimeouts re-requests the sealing blocks between the highest validated epoch and + // our tip that have not been validated yet. It holds exactly the missing ones, so no tasks + // means every sealing block down to our tip is validated. + sealingBlockTimeouts *common.TimeoutHandler[uint64] } // NewNonValidator creates a NonValidator with the given `config`. @@ -116,7 +119,7 @@ func NewNonValidator(config Config) (*NonValidator, error) { replicator := simplex.NewReplicationState(config.Logger, config.Comm, config.ID, config.MaxSequenceWindow, true, config.StartTime, lock, randomSource) - return &NonValidator{ + nv := &NonValidator{ Config: config, incompleteSequences: make(map[uint64]*finalizedSeq), ctx: ctx, @@ -127,7 +130,13 @@ func NewNonValidator(config Config) (*NonValidator, error) { highestEpochCollector: newEpochReplicator(config.Logger, config.Comm.Validators), oneTimeVerifier: simplex.NewOneTimeVerifier(config.Logger), sequenceReplicator: replicator, - }, nil + } + nv.sealingBlockTimeouts = common.NewTimeoutHandler(config.Logger, "sealing block replication", config.StartTime, simplex.DefaultReplicationRequestTimeout, nv.requestMissingSealingBlocks) + if !config.Bootstrapped { + nv.sealingBlockTimeouts.AddTask(startBroadcastTask) + } + + return nv, nil } func (n *NonValidator) Start() { @@ -139,11 +148,22 @@ func (n *NonValidator) Stop() { n.Logger.Info("Shutting down non-validator", zap.Stringer("ID", n.ID)) n.cancelCtx() n.sequenceReplicator.Close() + n.sealingBlockTimeouts.Close() n.verifier.Close() } func (n *NonValidator) AdvanceTime(t time.Time) { n.sequenceReplicator.AdvanceTime(t) + n.sealingBlockTimeouts.Tick(t) +} + +// IsBootstrapped reports whether bootstrapping has finished. +// Bootstrapping finishes when every sealing block down to our tip is validated. +func (n *NonValidator) IsBootstrapped() bool { + n.lock.Lock() + defer n.lock.Unlock() + + return n.Bootstrapped } func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) error { @@ -161,8 +181,9 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er return n.haltedError } - if !n.Bootstrapped { - return n.handleBootstrap(msg, from) + if !n.Bootstrapped && msg.ReplicationResponse == nil { + n.Logger.Debug("Dropping message received while bootstrapping, we only accept replication responses", zap.Any("Message", msg), zap.Stringer("From", from)) + return nil } switch { @@ -178,66 +199,93 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er } } -// handleBootstrap handles messages received before we are bootstrapped. -// Only replication responses are processed, all other messages are dropped. -func (n *NonValidator) handleBootstrap(msg *common.Message, from common.NodeID) error { - resp := msg.ReplicationResponse - if resp == nil { - n.Logger.Debug("Dropping message received while bootstrapping, we only accept replication responses", zap.Any("Message", msg), zap.Stringer("From", from)) +// processBootstrapQuorumRound handles quorum rounds until bootstrapping finishes. +// Once a threshold validates an epoch, only sealing blocks are validated +// and stored(in a backwards manner). +func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from common.NodeID) error { + block := qr.Block + bh := block.BlockHeader() + sealingInfo := block.SealingBlockInfo() + + if sealingInfo == nil { + n.sendRequest(bh.Epoch, from) return nil } - for _, qr := range resp.Data { - if err := n.maybeBootstrapFromQuorumRound(&qr, from); err != nil { - n.Logger.Debug("Failed processing quorum round while bootstrapping", zap.Stringer("QR", &qr), zap.Error(err)) + switch { + case n.epochs.canValidate(block): + // The sealing block in the backwards hash chain + n.validateSealingBlock(qr, from) + case n.highestEpochCollector.collectedSealingBlockInfo(sealingInfo, bh, from): + n.Logger.Info("A threshold of validators reported a sealing block", zap.Uint64("Seq", bh.Seq), zap.Stringer("Info", sealingInfo)) + n.sealingBlockTimeouts.RemoveTask(startBroadcastTask) + if !n.isIndexed(bh.Seq) { + n.validateSealingBlock(qr, from) } - } - - if err := n.maybeBootstrapFromQuorumRound(resp.LatestSeq, from); err != nil { - n.Logger.Debug("Failed processing latest seq while bootstrapping", zap.Stringer("QR", resp.LatestSeq), zap.Error(err)) - } - - if !n.Bootstrapped { + default: return nil } - // Begin processing the quorum rounds stored if bootstrapping has finished. - return n.processReplicationState() + // No sealing block is missing, so every epoch from our tip to the highest is validated. + if !n.sealingBlockTimeouts.HasTasks() { + n.finishBootstrap() + // If the highest epoch is already indexed, nothing more gets indexed to trigger the transition. + highestEpoch, validators := n.epochs.highestEpoch() + n.maybeTransitionToValidator(highestEpoch, validators) + } + return nil } -// maybeBootstrapFromQuorumRound records the sealing block info of qr with the highest epoch collector. -// Once a threshold of nodes report the same sealing block, we consider ourselves bootstrapped, -// validate the epoch it seals, and store the quorum round for replication. -func (n *NonValidator) maybeBootstrapFromQuorumRound(qr *common.QuorumRound, from common.NodeID) error { - if err := verifyQuorumRound(qr); err != nil { - return err - } +// validateSealingBlock validates the epoch a sealing block opens and stores its quorum round. +// The finalization has not been verified yet. Storing tells the replicator a valid sequence exists +// and its validity is checked when the round is processed. +func (n *NonValidator) validateSealingBlock(qr *common.QuorumRound, from common.NodeID) { + n.maybeValidateNextEpoch(qr.Block, from) + n.sequenceReplicator.StoreQuorumRound(qr) +} - // We can only bootstrap from a sealing block, request the one sealing this block's epoch. - if qr.Block.SealingBlockInfo() == nil { - n.sendRequest(qr.Block.BlockHeader().Epoch, from) - return nil - } +// finishBootstrap marks bootstrapping done. Every epoch from our tip to the highest one a +// threshold of validators reported is validated, so replication and live messages can be handled. +func (n *NonValidator) finishBootstrap() { + n.Bootstrapped = true + highestEpoch, _ := n.epochs.highestEpoch() + n.Logger.Info("Finished bootstrapping", zap.Uint64("Highest Epoch", highestEpoch)) +} - if !n.highestEpochCollector.collectedSealingBlockInfo(qr.Block.SealingBlockInfo(), qr.Block.BlockHeader(), from) { - return nil +// maybeTransitionToValidator calls TransitionToValidator when epoch is the highest validated epoch, +// its sealing block is indexed and its validator set contains us. +func (n *NonValidator) maybeTransitionToValidator(epoch uint64, validators common.Nodes) { + highestEpoch, highestValidatorSet := n.epochs.highestEpoch() + if highestEpoch != epoch || !n.isIndexed(epoch) || !highestValidatorSet.Contains(n.ID) || n.TransitionToValidator == nil { + return } + n.TransitionToValidator(epoch, validators) +} - n.Logger.Info("Bootstrapped, received a threshold of sealing block info for an epoch", zap.Stringer("Info", qr.Block.SealingBlockInfo())) - n.Bootstrapped = true +// startBroadcastTask is the sealingBlockTimeouts task that repeats the start broadcast until a +// threshold of responses validates an epoch above our tip. Seq 0 is genesis, never a sealing block we request. +const startBroadcastTask uint64 = 0 - n.maybeValidateNextEpoch(qr.Block) - // We are storing a quorum round with a finalization we have not yet verified. - // We do this to tell the replicator a valid sequence exists and to begin replication if necessary. - // We will check the validity when we process this round. - n.sequenceReplicator.StoreQuorumRound(qr) +// requestMissingSealingBlocks re-requests sealing blocks of the hash chain that timed out +// from every validator. Runs on the timeout handler's goroutine. +func (n *NonValidator) requestMissingSealingBlocks(seqs []uint64) { + n.lock.Lock() + defer n.lock.Unlock() - if n.OnFinishBootstrapping == nil { - n.Logger.Debug("OnFinishBootstrapping not set for the non-validator") - return nil + if n.ctx.Err() != nil { + return } - return n.OnFinishBootstrapping(qr.Block.BlockHeader().Seq, qr.Block.SealingBlockInfo().ValidatorSet) + for _, seq := range seqs { + if seq == startBroadcastTask { + n.broadcastLatestEpoch() + continue + } + n.Logger.Debug("Re-requesting a sealing block", zap.Uint64("Seq", seq)) + n.Comm.Broadcast(&common.Message{ + ReplicationRequest: &common.ReplicationRequest{Seqs: []uint64{seq}}, + }) + } } // handleBlock handles a block message. BlockMessages are sent when the leader proposes a block for its round. @@ -264,7 +312,7 @@ func (n *NonValidator) handleBlock(block common.Block, from common.NodeID) error } // If we have already verified the block discard it - if n.isAccepted(bh.Seq) { + if n.isIndexed(bh.Seq) { n.Logger.Debug("Already accepted a block from this round") return nil } @@ -297,11 +345,11 @@ func (n *NonValidator) handleBlock(block common.Block, from common.NodeID) error incomplete.block = block - n.maybeValidateNextEpoch(block) + n.maybeValidateNextEpoch(block, from) return n.scheduleNewFinalizedBlockTask(block, incomplete.finalization) } -func (n *NonValidator) isAccepted(seq uint64) bool { +func (n *NonValidator) isIndexed(seq uint64) bool { return n.nextSeqToCommit() > seq } @@ -342,18 +390,9 @@ func (n *NonValidator) newFinalizedBlockTask(block common.Block, finalization *c return md.Digest } - // If we are indexing a sealing block, we may need to transition to become a validator + // Indexing a sealing block may make us a validator of the epoch it opens. if block.SealingBlockInfo() != nil { - highestEpoch, highestValidatorSet := n.epochs.highestEpoch() - - // We should only transition to become a validator, if the sealing block is creating the highest - // epoch we have validated. Since we are fetching from the epochs map, we know this epoch has been validated - // either by a threshold of responses, or backwards hash chain validation. - if highestValidatorSet.Contains(n.ID) && highestEpoch == md.Seq { - if n.TransitionToValidator != nil { - n.TransitionToValidator(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet) - } - } + n.maybeTransitionToValidator(md.Seq, block.SealingBlockInfo().ValidatorSet) } n.Logger.Info("Verified and Indexed Block", zap.Uint64("Block Seq", md.Seq), zap.Stringer("Block Digest", md.Digest)) @@ -371,8 +410,12 @@ func (n *NonValidator) newFinalizedBlockTask(block common.Block, finalization *c } } -func (n *NonValidator) maybeValidateNextEpoch(block common.Block) { - nextEpoch := block.BlockHeader().Seq +// maybeValidateNextEpoch validates the epoch block opens when block is a sealing block. While +// bootstrapping it also requests the sealing block that opened block's own epoch, following the +// hash chain back until every sealing block down to an epoch we have indexed is validated. +func (n *NonValidator) maybeValidateNextEpoch(block common.Block, from common.NodeID) { + bh := block.BlockHeader() + nextEpoch := bh.Seq sealingInfo := block.SealingBlockInfo() if sealingInfo == nil { return @@ -385,6 +428,22 @@ func (n *NonValidator) maybeValidateNextEpoch(block common.Block) { n.Logger.Info("We have a valid sealing block, messages for that epoch can be processed.", zap.Uint64("Epoch", nextEpoch)) n.epochs[nextEpoch] = newEpochMetadata(nextEpoch, sealingInfo, n.SignatureAggregatorCreator) + + if n.Bootstrapped { + return + } + + n.sealingBlockTimeouts.RemoveTask(nextEpoch) + + // The first simplex block opens its own epoch, so there is no earlier sealing block. + prevSealingSeq := bh.Epoch + _, known := n.epochs[prevSealingSeq] + if prevSealingSeq == nextEpoch || n.isIndexed(prevSealingSeq) || known { + return + } + + n.sendRequest(prevSealingSeq, from) + n.sealingBlockTimeouts.AddTask(prevSealingSeq) } func (n *NonValidator) removeOldSequencesAndEpochs(lastCommittedSeq, minEpochToKeep uint64) { @@ -401,15 +460,11 @@ func (n *NonValidator) removeOldSequencesAndEpochs(lastCommittedSeq, minEpochToK // handleFinalization process a finalization message. If its for a future epoch, it will forward the finalization // to the replication handler. func (n *NonValidator) handleFinalization(finalization *common.Finalization, from common.NodeID) error { - if !n.Bootstrapped { - return nil - } - bh := finalization.Finalization.BlockHeader n.Logger.Debug("Received a finalization", zap.Uint64("Seq", bh.Seq), zap.Stringer("From", from)) - if n.isAccepted(bh.Seq) { + if n.isIndexed(bh.Seq) { n.Logger.Debug("Received a stale finalization", zap.Uint64("Seq", bh.Seq), zap.Stringer("From", from)) return nil } @@ -505,7 +560,7 @@ func (n *NonValidator) handleFinalization(finalization *common.Finalization, fro return nil } - n.maybeValidateNextEpoch(incomplete.block) + n.maybeValidateNextEpoch(incomplete.block, from) return n.scheduleNewFinalizedBlockTask(incomplete.block, incomplete.finalization) } @@ -520,7 +575,7 @@ func (n *NonValidator) scheduleNewFinalizedBlockTask(block common.Block, finaliz finalizedBlockTask := n.newFinalizedBlockTask(n.oneTimeVerifier.Wrap(block), finalization) var prev *common.Digest - if bh.Seq > 0 && !n.isAccepted(bh.Seq-1) { + if bh.Seq > 0 && !n.isIndexed(bh.Seq-1) { prev = &bh.Prev } return n.verifier.ScheduleTaskWithDependencies(finalizedBlockTask, bh.Seq, prev, []uint64{}) @@ -543,6 +598,10 @@ func (n *NonValidator) handleReplicationResponse(resp *common.ReplicationRespons } func (n *NonValidator) processReplicationState() error { + if !n.Bootstrapped { + return nil + } + nextSeqToCommit := n.nextSeqToCommit() n.sequenceReplicator.MaybeAdvanceState(nextSeqToCommit, 0, 0) @@ -581,9 +640,14 @@ func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.No return err } + // Runs before rejecting indexed blocks, an indexed sealing block is still a vote while bootstrapping. + if !n.Bootstrapped { + return n.processBootstrapQuorumRound(qr, from) + } + block := qr.Block - if n.isAccepted(block.BlockHeader().Seq) { + if n.isIndexed(block.BlockHeader().Seq) { return fmt.Errorf("processing quorum round for a block we already indexed") } @@ -599,7 +663,7 @@ func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.No } // This block could be a sealing block, validate the next epoch if so. - n.maybeValidateNextEpoch(block) + n.maybeValidateNextEpoch(block, from) n.storeQuorumRound(qr) return nil } @@ -651,42 +715,37 @@ func (n *NonValidator) storeQuorumRound(qr *common.QuorumRound) { func (n *NonValidator) handleQrFromUnknownEpoch(qr *common.QuorumRound, from common.NodeID) { block := qr.Block + bh := block.BlockHeader() n.Logger.Debug("Received a QR from an Epoch that we have not validated", - zap.Uint64("Epoch", block.BlockHeader().Epoch), - zap.Uint64("Block Seq", block.BlockHeader().Seq), - zap.Stringer("Block digest", block.BlockHeader().Digest)) - n.sendRequest(block.BlockHeader().Epoch, from) + zap.Uint64("Epoch", bh.Epoch), + zap.Uint64("Block Seq", bh.Seq), + zap.Stringer("Block digest", bh.Digest)) + + n.sendRequest(bh.Epoch, from) // This block is in an epoch that we do not have. Therefore, we cannot verify its finalization. // However, if it is a sealing block we may be able to validate the epoch if its part of the sealing block hash-chain. if n.epochs.canValidate(block) { n.Logger.Debug("We can validate an epoch block as we have validated the one after it.", zap.Stringer("Info", block.SealingBlockInfo())) - n.maybeValidateNextEpoch(block) - n.storeQuorumRound(qr) + n.validateSealingBlock(qr, from) return } - if n.highestEpochCollector.collectedSealingBlockInfo(qr.Block.SealingBlockInfo(), qr.Block.BlockHeader(), from) { - n.Logger.Debug("We can validate an epoch because we have received a threshold of messages of it.", zap.Stringer("Info", block.SealingBlockInfo())) - n.maybeValidateNextEpoch(block) - // We are storing a quorum round with a finalization we have not yet verified. - // We do this to tell the replicator a valid sequence exists and to begin replication if necessary. - // We will check the validity when we process this round. - n.storeQuorumRound(qr) + if n.highestEpochCollector.collectedSealingBlockInfo(block.SealingBlockInfo(), bh, from) { + n.Logger.Debug("We can validate an epoch because we have received a threshold of messages of it.", zap.Stringer("Info", block.SealingBlockInfo())) + n.validateSealingBlock(qr, from) } } -// TODO: add a re-broadcast timeout task until we have validated an epoch. func (n *NonValidator) broadcastLatestEpoch() { highestEpoch, _ := n.epochs.highestEpoch() + request := &common.ReplicationRequest{ + LatestFinalizedSeq: highestEpoch, + } // Sending a LatestFinalizedSeq of 0 gets ignored by validators. if highestEpoch == 0 { - highestEpoch = 1 - } - - request := &common.ReplicationRequest{ - LatestFinalizedSeq: highestEpoch, + request.LatestFinalizedSeq = 1 } n.Comm.Broadcast(&common.Message{ diff --git a/nonvalidator/non_validator_test.go b/nonvalidator/non_validator_test.go index 68dbae28..3de15306 100644 --- a/nonvalidator/non_validator_test.go +++ b/nonvalidator/non_validator_test.go @@ -372,6 +372,7 @@ func TestNonValidatorDropsTelockQuorumRound(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, + Bootstrapped: true, }) require.NoError(t, err) defer nv.Stop() @@ -650,6 +651,7 @@ func TestNonValidator_RequestHighestEpochOnStart(t *testing.T) { require.True(t, ok) require.NotNil(t, msg.msg.ReplicationRequest) require.Equal(t, uint64(1), msg.msg.ReplicationRequest.LatestFinalizedSeq) + require.Empty(t, msg.msg.ReplicationRequest.Seqs) } // TestNonValidator_Bootstrap ensures a non-validator can replicate sequences given different states of the chain. @@ -765,6 +767,7 @@ func TestNonValidator_Bootstrap(t *testing.T) { defer nv.Stop() advanceUntil(nv, epochs, msgQueue, tt.lastSeq) + require.Eventually(t, nv.IsBootstrapped, 5*time.Second, 10*time.Millisecond) }) } } @@ -1055,12 +1058,10 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) { } // TestNonValidator_BootstrapGatesMessages asserts that blocks and finalizations are dropped -// until a threshold of replication responses vouch for the same sealing block, -// after which the stored round is committed and messages are processed normally. +// until a threshold of replication responses vote for the same sealing block, after which +// the stored round is committed and messages are processed normally. func TestNonValidator_BootstrapGatesMessages(t *testing.T) { tc := newSeededChain(t, testNodes, 2) - var bootstrappedHighestValidators common.Nodes - var bootstrappedHighestEpoch uint64 nv, err := NewNonValidator( Config{ @@ -1070,11 +1071,6 @@ func TestNonValidator_BootstrapGatesMessages(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: common.NodeID{100}, - OnFinishBootstrapping: func(epoch uint64, validators common.Nodes) error { - bootstrappedHighestEpoch = epoch - bootstrappedHighestValidators = validators - return nil - }, }, ) require.NoError(t, err) @@ -1100,11 +1096,14 @@ func TestNonValidator_BootstrapGatesMessages(t *testing.T) { }, } threshold := common.F(len(testNodes)) + 1 - for i := 0; i < threshold; i++ { + for i := 0; i < threshold-1; i++ { require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[i])) } + require.False(t, nv.IsBootstrapped(), "bootstrapped below the threshold") - // bootstrapping commits the collected sealing block + // the sealing block's epoch was opened by the indexed epoch 1, so the threshold vote finishes bootstrapping + require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[threshold-1])) + require.True(t, nv.IsBootstrapped()) tc.WaitForBlockCommit(3) // messages flow normally after bootstrapping @@ -1114,55 +1113,186 @@ func TestNonValidator_BootstrapGatesMessages(t *testing.T) { fin = finalizationMsg(t, b4, testNodes) require.NoError(t, nv.HandleMessage(fin.msg, fin.from)) tc.WaitForBlockCommit(4) +} - require.Equal(t, b3.BlockHeader().Seq, bootstrappedHighestEpoch) - require.Equal(t, testNodes, bootstrappedHighestValidators) +// sealingResponse wraps the block and finalization indexed at seq on tc in a replication response. +func sealingResponse(tc *testChain, seq uint64) *common.Message { + block, fin, err := tc.Retrieve(seq) + require.NoError(tc.t, err) + return &common.Message{ReplicationResponse: &common.ReplicationResponse{ + Data: []common.QuorumRound{{Block: block.(common.Block), Finalization: &fin}}, + }} +} + +// popRequestedSeqs drains the message queue and returns every seq requested. +func popRequestedSeqs(t *testing.T, msgQueue *messageQueue) []uint64 { + seqs := []uint64{} + for msg, ok := msgQueue.popResponse(); ok; msg, ok = msgQueue.popResponse() { + require.NotNil(t, msg.msg.ReplicationRequest, "unexpected message %v", msg.msg) + seqs = append(seqs, msg.msg.ReplicationRequest.Seqs...) + } + return seqs } -// TestNonValidator_BootstrapRequestsSealingBlock asserts that a replication response -// carrying a non-sealing block while bootstrapping triggers a replication request -// for the block's epoch, whose seq is the sealing block that opened it. -func TestNonValidator_BootstrapRequestsSealingBlock(t *testing.T) { +// TestNonValidator_BootstrapWalksHashChain asserts a non-validator several epochs behind requests +// sealing blocks one hop back at a time once a threshold reports the highest one, and finishes +// bootstrapping when the chain reaches an epoch it has indexed. +func TestNonValidator_BootstrapWalksHashChain(t *testing.T) { tc := newSeededChain(t, testNodes, 2) + tc.indexEpochs(5, 10, 20) myNodeID := common.NodeID{100} msgQueue := &messageQueue{} nv, err := NewNonValidator( Config{ - Storage: tc, - Comm: &routerComm{ - nodes: testNodes, - t: t, - ID: myNodeID, - messageQueue: msgQueue, - }, + Storage: tc.CloneUntil(3), + Comm: &routerComm{nodes: tc.nodes(), t: t, ID: myNodeID, messageQueue: msgQueue}, Logger: testutil.MakeLogger(t, 1), SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: myNodeID, + StartTime: time.Now(), }, ) require.NoError(t, err) defer nv.Stop() - b3 := tc.appendBlock() - f3 := tc.newFinalization(b3) - sender := testNodes.NodeIDs()[2] + threshold := common.F(len(tc.nodes())) + 1 + for i := 0; i < threshold; i++ { + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 20), tc.nodes().NodeIDs()[i])) + } + require.False(t, nv.IsBootstrapped()) - require.NoError(t, nv.HandleMessage(&common.Message{ - ReplicationResponse: &common.ReplicationResponse{ - Data: []common.QuorumRound{{Block: b3, Finalization: &f3}}, + // each sealing block validates by hash and requests the one that opened its epoch + for _, seq := range []uint64{10, 5} { + require.Equal(t, []uint64{seq}, popRequestedSeqs(t, msgQueue)) + require.False(t, nv.IsBootstrapped()) + require.NoError(t, nv.HandleMessage(sealingResponse(tc, seq), tc.nodes().NodeIDs()[0])) + } + + // seq 5 was opened by the indexed epoch 1, so every sealing block is validated + require.True(t, nv.IsBootstrapped()) + require.NotEmpty(t, popRequestedSeqs(t, msgQueue), "replication of the sequences behind the tip never started") +} + +// TestNonValidator_BootstrapIgnoresSealingBlockOffChain asserts that while following the hash chain +// a sealing block further down it is dropped until the epoch pointing back to it has been validated. +func TestNonValidator_BootstrapIgnoresSealingBlockOffChain(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + tc.indexEpochs(5, 10, 20) + myNodeID := common.NodeID{100} + msgQueue := &messageQueue{} + nv, err := NewNonValidator( + Config{ + Storage: tc.CloneUntil(3), + Comm: &routerComm{nodes: tc.nodes(), t: t, ID: myNodeID, messageQueue: msgQueue}, + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: myNodeID, + StartTime: time.Now(), }, - }, sender)) + ) + require.NoError(t, err) + defer nv.Stop() - msg, ok := msgQueue.popResponse() - require.True(t, ok) - require.NotNil(t, msg.msg.ReplicationRequest) - require.Equal(t, []uint64{b3.BlockHeader().Epoch}, msg.msg.ReplicationRequest.Seqs) - require.Equal(t, sender, msg.to) + threshold := common.F(len(tc.nodes())) + 1 + for i := 0; i < threshold; i++ { + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 20), tc.nodes().NodeIDs()[i])) + } + require.Equal(t, []uint64{10}, popRequestedSeqs(t, msgQueue)) + + // seq 5 opened the epoch seq 10 was produced in, but seq 10 has not been validated yet + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 5), tc.nodes().NodeIDs()[0])) + require.Empty(t, popRequestedSeqs(t, msgQueue)) + require.False(t, nv.IsBootstrapped()) + + // once seq 10 validates, seq 5 is still requested, so it was dropped, and is accepted on resend + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 10), tc.nodes().NodeIDs()[0])) + require.Equal(t, []uint64{5}, popRequestedSeqs(t, msgQueue)) + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 5), tc.nodes().NodeIDs()[0])) + require.True(t, nv.IsBootstrapped()) +} - // the non-sealing block must not bootstrap the node +// TestNonValidator_BootstrapRetriesSealingBlock asserts an unanswered request for a sealing block +// of the hash chain is re-requested once the replication timeout passes. +func TestNonValidatorBootstrapRetriesSealingBlock(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + tc.indexEpochs(5, 10, 20) + myNodeID := common.NodeID{100} + msgQueue := &messageQueue{} + nv, err := NewNonValidator( + Config{ + Storage: tc.CloneUntil(3), + Comm: &routerComm{nodes: tc.nodes(), t: t, ID: myNodeID, messageQueue: msgQueue}, + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: myNodeID, + StartTime: time.Now(), + }, + ) + require.NoError(t, err) + defer nv.Stop() + + threshold := common.F(len(tc.nodes())) + 1 + for i := 0; i < threshold; i++ { + require.NoError(t, nv.HandleMessage(sealingResponse(tc, 20), tc.nodes().NodeIDs()[i])) + } + require.Equal(t, []uint64{10}, popRequestedSeqs(t, msgQueue)) + + nv.AdvanceTime(nv.StartTime.Add(simplex.DefaultReplicationRequestTimeout)) + require.Eventually(t, func() bool { + msg, ok := msgQueue.popResponse() + return ok && slices.Equal(msg.msg.ReplicationRequest.Seqs, []uint64{10}) + }, 5*time.Second, 10*time.Millisecond, "the sealing block was never re-requested") +} + +// TestNonValidator_BootstrapRequestsSealingBlock asserts that a replication response carrying a +// non-sealing block while bootstrapping triggers a request to its sender for the sealing block that +// opened its epoch, whether that epoch is ours or unknown. +func TestNonValidatorBootstrapRequestsSealingBlock(t *testing.T) { + tc := newSeededChain(t, testNodes, 2) + myNodeID := common.NodeID{100} + msgQueue := &messageQueue{} + nv, err := NewNonValidator( + Config{ + Storage: tc.CloneUntil(3), + Comm: &routerComm{nodes: testNodes, t: t, ID: myNodeID, messageQueue: msgQueue}, + Logger: testutil.MakeLogger(t, 1), + SignatureAggregatorCreator: tc.signatureAggregatorCreator, + MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + ID: myNodeID, + }, + ) + require.NoError(t, err) + defer nv.Stop() + + inOurEpoch := tc.appendBlock() + require.NoError(t, tc.Index(context.Background(), inOurEpoch, tc.newFinalization(inOurEpoch))) + sealing := tc.appendSealing(testNodes) + require.NoError(t, tc.Index(context.Background(), sealing, tc.newFinalization(sealing))) + inUnknownEpoch := tc.appendBlock() + require.NoError(t, tc.Index(context.Background(), inUnknownEpoch, tc.newFinalization(inUnknownEpoch))) + sender := testNodes.NodeIDs()[2] + + for _, tt := range []struct { + seq uint64 + sealingSeq uint64 + }{ + {seq: inOurEpoch.BlockHeader().Seq, sealingSeq: 1}, + {seq: inUnknownEpoch.BlockHeader().Seq, sealingSeq: sealing.BlockHeader().Seq}, + } { + require.NoError(t, nv.HandleMessage(sealingResponse(tc, tt.seq), sender)) + msg, ok := msgQueue.popResponse() + require.True(t, ok) + require.NotNil(t, msg.msg.ReplicationRequest) + require.Equal(t, []uint64{tt.sealingSeq}, msg.msg.ReplicationRequest.Seqs) + require.Equal(t, sender, msg.to) + } + + // non-sealing blocks never bootstrap the node require.False(t, nv.Bootstrapped) - _, ok = msgQueue.popResponse() + _, ok := msgQueue.popResponse() require.False(t, ok) } @@ -1392,6 +1522,7 @@ func TestNonValidatorDropsQuorumRoundPastSequenceWindow(t *testing.T) { MaxSequenceWindow: maxSequenceWindow, ID: common.NodeID{16}, StartTime: time.Now(), + Bootstrapped: true, }, ) require.NoError(t, err) From 32159a9e2a926672b17cf9320a394d6218f9c46a Mon Sep 17 00:00:00 2001 From: samliok Date: Wed, 9 Sep 2026 21:20:41 -0400 Subject: [PATCH 03/13] lint --- nonvalidator/epochs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nonvalidator/epochs.go b/nonvalidator/epochs.go index d73892c8..b4f615d0 100644 --- a/nonvalidator/epochs.go +++ b/nonvalidator/epochs.go @@ -140,7 +140,7 @@ func (e epochs) canValidate(block common.Block) bool { type latestValidatorSetRetriever func() common.Nodes // epochDigestCounter counts sealing block responses from validators for each epoch. -// It uses LatestValidatorSetRetriever to determine when the required response threshold +// It uses latestValidatorSetRetriever to determine when the required response threshold // has been reached. type epochDigestCounter struct { logger common.Logger From 5b06b79c1593a27b0b7114dc5d27695f57aa9a0b Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 10 Sep 2026 10:12:57 -0400 Subject: [PATCH 04/13] reduce diff --- instance.go | 61 +++++++++++++++++++++++++--------------------------- util.go | 20 ++++++++--------- util_test.go | 8 +------ 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/instance.go b/instance.go index 959e1e36..1016948d 100644 --- a/instance.go +++ b/instance.go @@ -129,13 +129,13 @@ func (i *Instance) bootstrap() error { return err } - latestIndexedEpochValidators, _, err := getLastAcceptedEpochAndValidatorSet(&i.Config) + latestIndexedEpochValidators, err := getLastAcceptedValidatorSet(&i.Config) if err != nil { return err } // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. - // Note: this may not be the latest epoch, but our futureEpochCollector will eventually notice we are behind and transition properly. + // Note: this may not be the latest epoch, but a future PR will eventually notice we are behind and transition properly. if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { return i.startValidator(latestIndexedEpochValidators) } @@ -350,41 +350,38 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } } - if i.nv != nil { - return i.nv.HandleMessage(msg, from) - } - if i.e != nil { - return i.handleMessageForEpoch(msg, from) - } - - return errors.New("we are not running as a validator or not validator") -} - -func (i *Instance) handleMessageForEpoch(msg *common.Message, from common.NodeID) error { - switch { - case msg.AuxiliaryInfo != nil: - if msg.AuxiliaryInfo.Epoch != i.e.Epoch { - i.Config.Logger.Debug( - "Received an auxiliary info from an old epoch", - zap.Uint64("Aux Info Epoch", msg.AuxiliaryInfo.Epoch), - zap.Uint64("Our Epoch", i.e.Epoch), - zap.Stringer("From", from)) + switch { + case msg.AuxiliaryInfo != nil: + if msg.AuxiliaryInfo.Epoch != i.e.Epoch { + i.Config.Logger.Debug( + "Received an auxiliary info from an old epoch", + zap.Uint64("Aux Info Epoch", msg.AuxiliaryInfo.Epoch), + zap.Uint64("Our Epoch", i.e.Epoch), + zap.Stringer("From", from)) + return nil + } + i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + case msg.EpochTransitionApproval != nil: + if !from.Equals(msg.EpochTransitionApproval.NodeID[:]) { + i.Config.Logger.Debug("Dropping approval not sent by its signer", + zap.Stringer("from", from), + zap.Stringer("signer", common.NodeID(msg.EpochTransitionApproval.NodeID[:]))) + return nil + } + // TODO: pass in time.Now() rather than uint64 + i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().UnixMilli())) return nil } - i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) - case msg.EpochTransitionApproval != nil: - if !from.Equals(msg.EpochTransitionApproval.NodeID[:]) { - i.Config.Logger.Debug("Dropping approval not sent by its signer", - zap.Stringer("from", from), - zap.Stringer("signer", common.NodeID(msg.EpochTransitionApproval.NodeID[:]))) - return nil + + if i.nv != nil { + return i.nv.HandleMessage(msg, from) } - // TODO: pass in time.Now() rather than uint64 - i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().UnixMilli())) - return nil + + return i.e.HandleMessage(msg, from) } - return i.e.HandleMessage(msg, from) + + return errors.New("we are not running as a validator or not validator") } func (i *Instance) wireReplicationResponse(msg *common.Message) error { diff --git a/util.go b/util.go index d83d9a72..eca3a84d 100644 --- a/util.go +++ b/util.go @@ -32,15 +32,15 @@ func LastBlock(storage Storage) (metadata.StateMachineBlock, uint64, error) { return lastBlock, numBlocks, nil } -// getLastAcceptedEpochAndValidatorSet determines the epoch the instance should start at based on -// the last block in storage. If the ledger only contains non-Simplex blocks, the -// epoch is the first Simplex height. If the last block is a sealing block, the -// epoch it seals has ended, so the next epoch is returned. Otherwise, the epoch -// of the last block is returned. -func getLastAcceptedEpochAndValidatorSet(config *Config) (common.Nodes, uint64, error) { +// getLastAcceptedValidatorSet returns the validator set of the epoch the +// instance should start at, based on the last block in storage. If the ledger only +// contains non-Simplex blocks, the genesis validator set is returned. If the last +// block is a sealing block, the set it seals in is returned. Otherwise, the set is +// loaded from the sealing block of the last block's epoch. +func getLastAcceptedValidatorSet(config *Config) (common.Nodes, error) { lastBlock, numBlocks, err := LastBlock(config.Storage) if err != nil { - return nil, 0, fmt.Errorf("error retrieving last block: %w", err) + return nil, fmt.Errorf("error retrieving last block: %w", err) } lastNonSimplexHeight := config.LastNonSimplexInnerBlock.Height() @@ -70,17 +70,17 @@ func getLastAcceptedEpochAndValidatorSet(config *Config) (common.Nodes, uint64, sealingBlockSeq := parsedLastBlock.BlockHeader().Epoch sealingBlock, _, err := config.Storage.GetBlock(sealingBlockSeq) if err != nil { - return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) + return nil, fmt.Errorf("error retrieving sealing block from storage: %w", err) } if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { - return nil, 0, fmt.Errorf("%w at seq %d", errNonSealingBlock, sealingBlockSeq) + return nil, fmt.Errorf("%w at seq %d", errNonSealingBlock, sealingBlockSeq) } validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) nodes = validatorSetToNodes(validatorSet) config.Logger.Debug("Determined epoch and validator set from sealing block in storage", zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) } - return nodes, epochNum, nil + return nodes, nil } func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { diff --git a/util_test.go b/util_test.go index 6d63f086..6661a5f9 100644 --- a/util_test.go +++ b/util_test.go @@ -124,14 +124,12 @@ func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { tests := []struct { name string blocks []metadata.StateMachineBlock - expectedEpoch uint64 expectedNodes common.Nodes expectedErr error }{ { name: "only non-Simplex blocks starts at first Simplex height with genesis set", blocks: []metadata.StateMachineBlock{nonSimplexBlock(0)}, - expectedEpoch: 1, expectedNodes: vdrSet.Nodes(), }, { @@ -141,7 +139,6 @@ func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { nonSimplexBlock(1), nonSimplexBlock(2), }, - expectedEpoch: 3, expectedNodes: vdrSet.Nodes(), }, { @@ -150,7 +147,6 @@ func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { simplexBlock(1, 1), sealingBlock(1, 2, vdrSet), }, - expectedEpoch: 2, expectedNodes: vdrSet.Nodes(), }, { @@ -161,7 +157,6 @@ func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { sealingBlock(1, 2, vdrSet), simplexBlock(2, 3), }, - expectedEpoch: 2, expectedNodes: vdrSet.Nodes(), }, { @@ -185,13 +180,12 @@ func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { storage := &stubStorage{blocks: tt.blocks} config := epochTestConfig(t, storage, vdrSet) - nodes, epoch, err := getLastAcceptedEpochAndValidatorSet(config) + nodes, err := getLastAcceptedValidatorSet(config) if tt.expectedErr != nil { require.ErrorIs(t, err, tt.expectedErr) return } require.NoError(t, err) - require.Equal(t, tt.expectedEpoch, epoch) require.Equal(t, tt.expectedNodes, nodes) }) } From 5c7afc862ab6fbede8a1229862be7e3e3a2ab918 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 10 Sep 2026 10:14:16 -0400 Subject: [PATCH 05/13] reduce more diff --- instance.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/instance.go b/instance.go index 1016948d..f5497c58 100644 --- a/instance.go +++ b/instance.go @@ -374,13 +374,13 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error return nil } - if i.nv != nil { - return i.nv.HandleMessage(msg, from) - } - return i.e.HandleMessage(msg, from) } + if i.nv != nil { + return i.nv.HandleMessage(msg, from) + } + return errors.New("we are not running as a validator or not validator") } From b4f07f1e6ebc081072e7bc49bfc9bcbac1cf8b34 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 10 Sep 2026 14:43:54 -0400 Subject: [PATCH 06/13] add offline test --- instance.go | 3 ++ instance_helpers_test.go | 40 ++++++++++++++++++--- instance_test.go | 78 ++++++++++++++++++++-------------------- 3 files changed, 78 insertions(+), 43 deletions(-) diff --git a/instance.go b/instance.go index f5497c58..a66f29f3 100644 --- a/instance.go +++ b/instance.go @@ -124,6 +124,7 @@ func (i *Instance) Start(ctx context.Context) error { } func (i *Instance) bootstrap() error { + i.Config.Logger.Debug("Node started bootstrapping") latestValidatorSet, err := getLatestPlatformChainValidatorSet(i.Config.PlatformChain) if err != nil { return err @@ -137,12 +138,14 @@ func (i *Instance) bootstrap() error { // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. // Note: this may not be the latest epoch, but a future PR will eventually notice we are behind and transition properly. if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { + i.Config.Logger.Debug("Node finished bootstrapping, its latest epoch is up to date with the Platform Chain") return i.startValidator(latestIndexedEpochValidators) } // Start as non-validator if our last indexed validator set does not equal, the latest p-chain validator set // Note: the epoch may be transitioning, so the latest p-chain validator set actually points to a future epoch. // The non-validator should finish bootstrapping and convert our non-validator to a validator in this case. + i.Config.Logger.Debug("Node starting bootstrapping as a non-validator") return i.startNonValidator(false) } diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 3eae7d93..bf054feb 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -59,8 +59,7 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte var ( genesisPChainHeight uint64 = 0 genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} - // epochBlockTime fixes the timestamp of the epoch-defining block - epochBlockTime = genesisBlock.TS.Add(time.Millisecond) + epochBlockTime = genesisBlock.TS.Add(time.Millisecond) ) var paramConfig = ParameterConfig{ @@ -413,6 +412,10 @@ func (i *instanceComm) enqueue(m inflightMessage) { } func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { + if c.n.isOffline(c.id) || c.n.isOffline(destination) { + return + } + for _, n := range c.n.nodesSnapshot() { if !bytes.Equal(n.id, destination) { continue @@ -429,9 +432,13 @@ func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { } func (c *instanceComm) Broadcast(msg *common.Message) { + if c.n.isOffline(c.id) { + return + } + // every node in the network but ourselves, each with its own re-parsed copy for _, n := range c.n.nodesSnapshot() { - if bytes.Equal(n.id, c.id) { + if bytes.Equal(n.id, c.id) || c.n.isOffline(n.id) { continue } @@ -661,9 +668,11 @@ type network struct { // pending holds the block the network has been asked to build, claimable by any leader. pending *pendingBlockSignal - // lock guards nodes, which comm goroutines read while addNode appends. + // lock guards nodes and offline, which comm goroutines read while tests mutate them. lock sync.Mutex nodes []node + // offline nodes stay in the network but neither send nor receive messages. + offline map[string]struct{} } func (n *network) nodesSnapshot() []node { @@ -672,6 +681,25 @@ func (n *network) nodesSnapshot() []node { return append([]node(nil), n.nodes...) } +func (n *network) setOffline(id common.NodeID) { + n.lock.Lock() + defer n.lock.Unlock() + n.offline[string(id)] = struct{}{} +} + +func (n *network) setOnline(id common.NodeID) { + n.lock.Lock() + defer n.lock.Unlock() + delete(n.offline, string(id)) +} + +func (n *network) isOffline(id common.NodeID) bool { + n.lock.Lock() + defer n.lock.Unlock() + _, offline := n.offline[string(id)] + return offline +} + func newNetwork(t *testing.T, pChain *testPlatformChain) *network { genesisNodes := pChain.GenesisValidatorSet().Nodes() common.SortNodes(genesisNodes) @@ -680,6 +708,7 @@ func newNetwork(t *testing.T, pChain *testPlatformChain) *network { t: t, pChain: pChain, pending: newPendingBlockSignal(), + offline: make(map[string]struct{}), epochValidatorSet: genesisNodes, // Genesis at seq 0. Then first simplex block is built automatically @@ -863,6 +892,9 @@ func (n *network) waitUntilSealingBlock(expectedValidatorSet common.Nodes) commo for { var block common.VerifiedBlock for _, node := range n.nodes { + if n.isOffline(node.id) { + continue + } committedBlock := node.storage.WaitForBlockCommit(n.seq) if block == nil { block = committedBlock diff --git a/instance_test.go b/instance_test.go index e8697f24..8003e935 100644 --- a/instance_test.go +++ b/instance_test.go @@ -24,7 +24,11 @@ func TestValidatorIndexes(t *testing.T) { pChain := newTestPChain(genesisSet) network := newNetwork(t, pChain) - network.addNode(validator.NodeID[:]).sync() + node := network.addNode(validator.NodeID[:]).sync() + + isValidator, bootstrapped := node.role() + require.True(t, bootstrapped, "a node already at the latest validator set has nothing to bootstrap") + require.True(t, isValidator) network.acceptNewBlock() } @@ -577,55 +581,51 @@ func TestValidatorSetsMetadataFromSnowman(t *testing.T) { require.Equal(t, numNonSimplexBlocks, block.BlockHeader().Seq) } -// TestBootstrap_ValidatorInLatestEpoch asserts a node whose indexed epoch names exactly the -// latest P-chain validator set skips bootstrapping and starts as a validator of that epoch. -func TestBootstrap_ValidatorInLatestEpoch(t *testing.T) { - v1 := newNodeMapping(1) +// TestBootstrapValidatorDuringTransition asserts a validator bootstraps when +// its latest epoch is mid-transition. i.e. the latest pchain validator set disagrees +// with the validators latest index validator set. +func TestBootstrapValidatorDuringTransition(t *testing.T) { + ourNodeMapping := newNodeMapping(1) v2 := newNodeMapping(2) - genesisValidatorSet := metadata.NodeBLSMappings{v1, v2} + v3 := newNodeMapping(3) + v4 := newNodeMapping(4) + futureValidator := newNodeMapping(5) + genesisValidatorSet := metadata.NodeBLSMappings{ourNodeMapping, v2, v3, v4} + futureValidatorSet := metadata.NodeBLSMappings{ourNodeMapping, v2, v3, v4, futureValidator} + // The P-chain moved on to a set that contains ourNode, but no sealing block for it has + // been indexed, so our indexed set and the latest set disagree. pChain := newTestPChain(genesisValidatorSet) - storage, _ := newChainStorage(t, genesisValidatorSet) - node := newNetwork(t, pChain).addNodeWithConfig(v1.NodeID[:], nodeConfig{storage: storage}) - isValidator, bootstrapped := node.role() - require.True(t, bootstrapped, "a node already at the latest validator set has nothing to bootstrap") - require.True(t, isValidator) -} + network := newNetwork(t, pChain) + network.addNode(futureValidator.NodeID[:]) + network.addNode(v2.NodeID[:]) + network.addNode(v3.NodeID[:]) + network.addNode(v4.NodeID[:]) + network.sync() -// TestBootstrap_ValidatorDuringTransition asserts a validator whose epoch is mid-transition, -// so its indexed set disagrees with the latest P-chain set, starts as a non-validator and -// converts back to a validator once bootstrapping confirms its indexed epoch. -func TestBootstrap_ValidatorDuringTransition(t *testing.T) { - ourNodeMapping := newNodeMapping(1) - v2 := newNodeMapping(2) - futureValidator := newNodeMapping(3) - genesisValidatorSet := metadata.NodeBLSMappings{ourNodeMapping, v2} + // all validators are offline + network.setOffline(v4.NodeID[:]) + network.setOffline(v3.NodeID[:]) + network.setOffline(v2.NodeID[:]) - pChain := newTestPChain(genesisValidatorSet) - // The P-chain moved on to a set that contains ourNode, but no sealing block for it has - // been indexed, so our indexed set and the latest set disagree. - pChain.setValidatorSetAt(10, metadata.NodeBLSMappings{ourNodeMapping, v2, futureValidator}) + // the future validator set is different than the current validator set ourNodeIsIn + pChain.setValidatorSetAt(10, futureValidatorSet) pChain.advanceHeight(10) - storage, sealing := newChainStorage(t, genesisValidatorSet) - node := newNetwork(t, pChain).addNodeWithConfig(ourNodeMapping.NodeID[:], nodeConfig{storage: storage}) - + // the node joins, but because the pchain validator set is different than our epoch we will sync as a non-validator + node := network.addNode(ourNodeMapping.NodeID[:]) isValidator, bootstrapped := node.role() require.False(t, bootstrapped) require.False(t, isValidator, "a node whose indexed set is not the latest must bootstrap first") // even though we are a validator - // One peer reporting the latest sealing block meets the threshold of F(3)+1. - block := &ParsedBlock{StateMachineBlock: sealing.Clone()} - finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(genesisValidatorSet)}, block, genesisValidatorSet.NodeIDs()) - require.NoError(t, node.inst.HandleMessage(&common.Message{ - ReplicationResponse: &common.ReplicationResponse{ - LatestSeq: &common.QuorumRound{Block: block, Finalization: &finalization}, - }, - }, v2.NodeID[:])) + // bring 2 node back online. The threshold for non-validators to complete bootstrapping is 2 votes, + // but to make a quorum is 3. This means the pchain transition will not occur, however the node should now sync as a validator + network.setOnline(v2.NodeID[:]) + network.setOnline(v3.NodeID[:]) + node.sync() - require.Eventually(t, func() bool { - isValidator, bootstrapped := node.role() - return isValidator && bootstrapped - }, 10*time.Second, 10*time.Millisecond, "the node never converted back to a validator of its indexed epoch") + // The only way for the epoch transition to finish is if ourNode becomes a validator + // and produces an approval & participates in the finalization. + network.waitUntilSealingBlock(futureValidatorSet.Nodes()) } From bc4270a82ac60d8ef457ec6100e39ddd1abfbc19 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 10 Sep 2026 14:44:55 -0400 Subject: [PATCH 07/13] lint --- instance_helpers_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/instance_helpers_test.go b/instance_helpers_test.go index bf054feb..2f14262d 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -59,7 +59,6 @@ func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte var ( genesisPChainHeight uint64 = 0 genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} - epochBlockTime = genesisBlock.TS.Add(time.Millisecond) ) var paramConfig = ParameterConfig{ @@ -247,33 +246,6 @@ func newTestStorageWithGenesis(t *testing.T) *testStorage { return s } -// newChainStorage builds and indexes the minimum chain a node can start from: genesis plus -// epoch 1's defining block, which carries the descriptor naming the epoch's validator set. -// It returns the storage and the epoch-defining block at its tip. -func newChainStorage(t *testing.T, validators metadata.NodeBLSMappings) (*testStorage, metadata.StateMachineBlock) { - storage := newTestStorageWithGenesis(t) - genesis, _, err := storage.GetBlock(0) - require.NoError(t, err) - - epochBlock := metadata.StateMachineBlock{ - InnerBlock: &testInnerBlock{Height_: 1, TS: epochBlockTime, Payload: []byte("epoch")}, - Metadata: metadata.StateMachineMetadata{ - Timestamp: uint64(epochBlockTime.UnixMilli()), - SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: 1, Round: 1, Seq: 1, Prev: common.Digest(genesis.Digest())}, - SimplexEpochInfo: metadata.SimplexEpochInfo{ - BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ - AggregatedMembership: metadata.AggregatedMembership{Members: validators}, - }, - }, - }, - } - - block := &ParsedBlock{StateMachineBlock: epochBlock.Clone()} - finalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: len(validators)}, block, validators.NodeIDs()) - require.NoError(t, storage.Index(context.Background(), block, finalization)) - return storage, epochBlock -} - func (m *testStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { block, fin, err := m.Retrieve(seq) if err != nil { From 8a741ee3ce7c0c9f2b5504b387574b163e9aa82a Mon Sep 17 00:00:00 2001 From: samliok Date: Mon, 14 Sep 2026 16:16:09 -0400 Subject: [PATCH 08/13] comments --- common/global_test.go | 46 ++++++++++++++++++++++++++++++ common/timeout_handler.go | 3 +- instance.go | 8 +++--- instance_test.go | 4 +-- nonvalidator/non_validator.go | 2 +- nonvalidator/non_validator_test.go | 5 ++-- 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/common/global_test.go b/common/global_test.go index 004401db..a6cf9532 100644 --- a/common/global_test.go +++ b/common/global_test.go @@ -24,3 +24,49 @@ func TestNodeIDs(t *testing.T) { } } } + +// TestNodesEqual checks Equal ignores order but compares +// length, Id, PK and Weight of every node. +func TestNodesEqual(t *testing.T) { + a := Node{Id: NodeID{1}, Weight: 10, PK: PublicKeyBytes{0xa}} + b := Node{Id: NodeID{2}, Weight: 20, PK: PublicKeyBytes{0xb}} + c := Node{Id: NodeID{3}, Weight: 30, PK: PublicKeyBytes{0xc}} + + testCases := []struct { + name string + nws Nodes + other Nodes + equal bool + }{ + {name: "both nil", equal: true}, + {name: "nil and empty", other: Nodes{}, equal: true}, + {name: "same order", nws: Nodes{a, b, c}, other: Nodes{a, b, c}, equal: true}, + {name: "different order", nws: Nodes{a, b, c}, other: Nodes{c, a, b}, equal: true}, + {name: "different length", nws: Nodes{a, b}, other: Nodes{a, b, c}, equal: false}, + {name: "empty and non empty", nws: Nodes{}, other: Nodes{a}, equal: false}, + {name: "different id", nws: Nodes{a, b}, other: Nodes{a, {Id: NodeID{9}, Weight: b.Weight, PK: b.PK}}, equal: false}, + {name: "different weight", nws: Nodes{a, b}, other: Nodes{a, {Id: b.Id, Weight: 99, PK: b.PK}}, equal: false}, + {name: "different pk", nws: Nodes{a, b}, other: Nodes{a, {Id: b.Id, Weight: b.Weight, PK: PublicKeyBytes{0xff}}}, equal: false}, + {name: "duplicate vs distinct", nws: Nodes{a, a}, other: Nodes{a, b}, equal: false}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + require.Equal(t, testCase.equal, testCase.nws.Equal(testCase.other)) + require.Equal(t, testCase.equal, testCase.other.Equal(testCase.nws)) + }) + } +} + +// TestNodesEqualDoesNotMutate checks Equal sorts clones and +// leaves the receiver and argument in their original order. +func TestNodesEqualDoesNotMutate(t *testing.T) { + a := Node{Id: NodeID{1}, Weight: 10, PK: PublicKeyBytes{0xa}} + b := Node{Id: NodeID{2}, Weight: 20, PK: PublicKeyBytes{0xb}} + + nws := Nodes{b, a} + other := Nodes{a, b} + require.True(t, nws.Equal(other)) + require.Equal(t, Nodes{b, a}, nws) + require.Equal(t, Nodes{a, b}, other) +} diff --git a/common/timeout_handler.go b/common/timeout_handler.go index e7d69ce8..18dcfb6c 100644 --- a/common/timeout_handler.go +++ b/common/timeout_handler.go @@ -133,8 +133,7 @@ func (t *TimeoutHandler[T]) RemoveTask(ID T) { delete(t.tasks, ID) } -// HasTasks reports whether any task is still outstanding. -func (t *TimeoutHandler[T]) HasTasks() bool { +func (t *TimeoutHandler[T]) Empty() bool { t.lock.Lock() defer t.lock.Unlock() diff --git a/instance.go b/instance.go index a66f29f3..8c8b4810 100644 --- a/instance.go +++ b/instance.go @@ -113,7 +113,7 @@ func (i *Instance) Start(ctx context.Context) error { context.AfterFunc(ctx, i.Stop) - if err := i.bootstrap(); err != nil { + if err := i.maybeBootstrap(); err != nil { return err } @@ -123,8 +123,8 @@ func (i *Instance) Start(ctx context.Context) error { return nil } -func (i *Instance) bootstrap() error { - i.Config.Logger.Debug("Node started bootstrapping") +func (i *Instance) maybeBootstrap() error { + i.Config.Logger.Debug("Checking if bootstrapping is required") latestValidatorSet, err := getLatestPlatformChainValidatorSet(i.Config.PlatformChain) if err != nil { return err @@ -138,7 +138,7 @@ func (i *Instance) bootstrap() error { // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. // Note: this may not be the latest epoch, but a future PR will eventually notice we are behind and transition properly. if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { - i.Config.Logger.Debug("Node finished bootstrapping, its latest epoch is up to date with the Platform Chain") + i.Config.Logger.Debug("Node skipping bootstrapping because its latest epoch is up to date with the Platform Chain") return i.startValidator(latestIndexedEpochValidators) } diff --git a/instance_test.go b/instance_test.go index 8003e935..bef3fb13 100644 --- a/instance_test.go +++ b/instance_test.go @@ -396,8 +396,8 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { parent, _, err := nonValidatorNode.storage.GetBlock(1) require.NoError(t, err) - // The non-validator drops every message until it bootstraps. One peer reporting the block - // sealing epoch 1 meets the threshold of F(1)+1. + // The non-validator requires a threshold of F(1)+1 responses for the first sealing block. + // Otherwise it will block non-replication messages. sealing := &ParsedBlock{StateMachineBlock: parent.Clone()} sealingFinalization, _ := testutil.NewFinalizationRecord(t, &testutil.TestSignatureAggregator{N: 1}, sealing, []common.NodeID{validator.NodeID[:]}) require.NoError(t, nonValidatorNode.inst.HandleMessage(&common.Message{ diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index 11574b75..915c53cd 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -227,7 +227,7 @@ func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from } // No sealing block is missing, so every epoch from our tip to the highest is validated. - if !n.sealingBlockTimeouts.HasTasks() { + if !n.sealingBlockTimeouts.Empty() { n.finishBootstrap() // If the highest epoch is already indexed, nothing more gets indexed to trigger the transition. highestEpoch, validators := n.epochs.highestEpoch() diff --git a/nonvalidator/non_validator_test.go b/nonvalidator/non_validator_test.go index 3de15306..1e0273ff 100644 --- a/nonvalidator/non_validator_test.go +++ b/nonvalidator/non_validator_test.go @@ -1296,10 +1296,10 @@ func TestNonValidatorBootstrapRequestsSealingBlock(t *testing.T) { require.False(t, ok) } -// TestNonValidator_BootstrapLatestKnownEpoch asserts a node caught up to the network +// TestNonValidatorBootstrapLatestKnownEpoch asserts a node caught up to the network // bootstraps from responses vouching for the sealing block of the latest epoch it // already has indexed, without re-indexing it. -func TestNonValidator_BootstrapLatestKnownEpoch(t *testing.T) { +func TestNonValidatorBootstrapLatestKnownEpoch(t *testing.T) { tc := newSeededChain(t, testNodes, 2) nv, err := NewNonValidator( Config{ @@ -1312,6 +1312,7 @@ func TestNonValidator_BootstrapLatestKnownEpoch(t *testing.T) { }, ) require.NoError(t, err) + require.False(t, nv.Bootstrapped) defer nv.Stop() // the sealing block of epoch 1, indexed at seq 1 From 002a90a8489a9d2c53c2cc1933fa506425b92160 Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 18 Sep 2026 12:17:59 -0400 Subject: [PATCH 09/13] bootstrap --- nonvalidator/non_validator.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index 915c53cd..9ea18f30 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -132,9 +132,6 @@ func NewNonValidator(config Config) (*NonValidator, error) { sequenceReplicator: replicator, } nv.sealingBlockTimeouts = common.NewTimeoutHandler(config.Logger, "sealing block replication", config.StartTime, simplex.DefaultReplicationRequestTimeout, nv.requestMissingSealingBlocks) - if !config.Bootstrapped { - nv.sealingBlockTimeouts.AddTask(startBroadcastTask) - } return nv, nil } @@ -142,6 +139,9 @@ func NewNonValidator(config Config) (*NonValidator, error) { func (n *NonValidator) Start() { n.Logger.Info("Starting non-validator", zap.Stringer("ID", n.ID)) n.broadcastLatestEpoch() + if !n.Bootstrapped { + n.sealingBlockTimeouts.AddTask(initialBootstrapTask) + } } func (n *NonValidator) Stop() { @@ -215,10 +215,12 @@ func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from switch { case n.epochs.canValidate(block): // The sealing block in the backwards hash chain - n.validateSealingBlock(qr, from) + if !n.isIndexed(bh.Seq) { + n.validateSealingBlock(qr, from) + } case n.highestEpochCollector.collectedSealingBlockInfo(sealingInfo, bh, from): n.Logger.Info("A threshold of validators reported a sealing block", zap.Uint64("Seq", bh.Seq), zap.Stringer("Info", sealingInfo)) - n.sealingBlockTimeouts.RemoveTask(startBroadcastTask) + n.sealingBlockTimeouts.RemoveTask(initialBootstrapTask) if !n.isIndexed(bh.Seq) { n.validateSealingBlock(qr, from) } @@ -262,9 +264,9 @@ func (n *NonValidator) maybeTransitionToValidator(epoch uint64, validators commo n.TransitionToValidator(epoch, validators) } -// startBroadcastTask is the sealingBlockTimeouts task that repeats the start broadcast until a -// threshold of responses validates an epoch above our tip. Seq 0 is genesis, never a sealing block we request. -const startBroadcastTask uint64 = 0 +// initialBootstrapTask is the sealingBlockTimeouts task that continuously asks for sealing blocks until a +// threshold of responses validates an epoch. We use Seq 1, since requests with Seq 0 are dropped. +const initialBootstrapTask uint64 = 1 // requestMissingSealingBlocks re-requests sealing blocks of the hash chain that timed out // from every validator. Runs on the timeout handler's goroutine. @@ -277,10 +279,6 @@ func (n *NonValidator) requestMissingSealingBlocks(seqs []uint64) { } for _, seq := range seqs { - if seq == startBroadcastTask { - n.broadcastLatestEpoch() - continue - } n.Logger.Debug("Re-requesting a sealing block", zap.Uint64("Seq", seq)) n.Comm.Broadcast(&common.Message{ ReplicationRequest: &common.ReplicationRequest{Seqs: []uint64{seq}}, From 2803568fd10a32772a241c0478bbea35672b7f68 Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 18 Sep 2026 12:35:23 -0400 Subject: [PATCH 10/13] one source of truth for storage --- nonvalidator/non_validator.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index 9ea18f30..c1b49907 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -243,7 +243,7 @@ func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from // and its validity is checked when the round is processed. func (n *NonValidator) validateSealingBlock(qr *common.QuorumRound, from common.NodeID) { n.maybeValidateNextEpoch(qr.Block, from) - n.sequenceReplicator.StoreQuorumRound(qr) + n.storeQuorumRound(qr) } // finishBootstrap marks bootstrapping done. Every epoch from our tip to the highest one a @@ -684,10 +684,17 @@ func verifyQuorumRound(qr *common.QuorumRound) error { } // storeQuorumRound updates replication state, and stores qr if its within MaxSequenceWindow. +// Sealing blocks are always stored, the replicator needs them to know a valid sequence exists. func (n *NonValidator) storeQuorumRound(qr *common.QuorumRound) { seq := qr.Block.BlockHeader().Seq nextSeqToCommit := n.nextSeqToCommit() + // Store sealing blocks regardless of MaxSequenceWindow, since . + if qr.Block.SealingBlockInfo() != nil { + n.sequenceReplicator.StoreQuorumRound(qr) + return + } + if seq > n.MaxSequenceWindow+nextSeqToCommit { n.Logger.Debug("Received a quorum round from a sequence too far ahead", zap.Uint64("Next Seq To Commit", nextSeqToCommit), zap.Uint64("Block Sequence", seq)) n.sequenceReplicator.ReceivedFutureFinalization(qr.Finalization, nextSeqToCommit) From bfbc3ef63f342cd07f3fd83ac8ef499510169bdf Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 18 Sep 2026 13:00:12 -0400 Subject: [PATCH 11/13] add comments and is empty --- common/timeout_handler.go | 2 +- nonvalidator/non_validator.go | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/common/timeout_handler.go b/common/timeout_handler.go index 18dcfb6c..29c3fb49 100644 --- a/common/timeout_handler.go +++ b/common/timeout_handler.go @@ -137,7 +137,7 @@ func (t *TimeoutHandler[T]) Empty() bool { t.lock.Lock() defer t.lock.Unlock() - return len(t.tasks) > 0 + return len(t.tasks) == 0 } func (t *TimeoutHandler[T]) RemoveOldTasks(shouldRemove func(id T, _ struct{}) bool) { diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index c1b49907..bf303859 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -202,40 +202,39 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er // processBootstrapQuorumRound handles quorum rounds until bootstrapping finishes. // Once a threshold validates an epoch, only sealing blocks are validated // and stored(in a backwards manner). -func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from common.NodeID) error { +func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from common.NodeID) { block := qr.Block bh := block.BlockHeader() sealingInfo := block.SealingBlockInfo() if sealingInfo == nil { n.sendRequest(bh.Epoch, from) - return nil + return } switch { case n.epochs.canValidate(block): - // The sealing block in the backwards hash chain + // The sealing block is in the backwards hash chain, try to validate it if !n.isIndexed(bh.Seq) { n.validateSealingBlock(qr, from) } case n.highestEpochCollector.collectedSealingBlockInfo(sealingInfo, bh, from): + // The sealing block is from an unknown epoch and we have collected enough votes to validate it n.Logger.Info("A threshold of validators reported a sealing block", zap.Uint64("Seq", bh.Seq), zap.Stringer("Info", sealingInfo)) n.sealingBlockTimeouts.RemoveTask(initialBootstrapTask) + if !n.isIndexed(bh.Seq) { n.validateSealingBlock(qr, from) } default: - return nil + // The sealing block cannot be validated, or there is not enough votes for it. + return } // No sealing block is missing, so every epoch from our tip to the highest is validated. - if !n.sealingBlockTimeouts.Empty() { + if n.sealingBlockTimeouts.Empty() { n.finishBootstrap() - // If the highest epoch is already indexed, nothing more gets indexed to trigger the transition. - highestEpoch, validators := n.epochs.highestEpoch() - n.maybeTransitionToValidator(highestEpoch, validators) } - return nil } // validateSealingBlock validates the epoch a sealing block opens and stores its quorum round. @@ -250,8 +249,9 @@ func (n *NonValidator) validateSealingBlock(qr *common.QuorumRound, from common. // threshold of validators reported is validated, so replication and live messages can be handled. func (n *NonValidator) finishBootstrap() { n.Bootstrapped = true - highestEpoch, _ := n.epochs.highestEpoch() + highestEpoch, validators := n.epochs.highestEpoch() n.Logger.Info("Finished bootstrapping", zap.Uint64("Highest Epoch", highestEpoch)) + n.maybeTransitionToValidator(highestEpoch, validators) } // maybeTransitionToValidator calls TransitionToValidator when epoch is the highest validated epoch, @@ -640,7 +640,8 @@ func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.No // Runs before rejecting indexed blocks, an indexed sealing block is still a vote while bootstrapping. if !n.Bootstrapped { - return n.processBootstrapQuorumRound(qr, from) + n.processBootstrapQuorumRound(qr, from) + return nil } block := qr.Block From 64e89d4fa8539c07a6a7d998bfc67fab7038317e Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 18 Sep 2026 13:21:56 -0400 Subject: [PATCH 12/13] bootstrap rename to epochs replicated --- instance.go | 24 ++++---- instance_helpers_test.go | 6 +- instance_test.go | 28 +++++----- nonvalidator/non_validator.go | 52 ++++++++--------- nonvalidator/non_validator_test.go | 90 +++++++++++++++--------------- 5 files changed, 100 insertions(+), 100 deletions(-) diff --git a/instance.go b/instance.go index 8c8b4810..c38d0adb 100644 --- a/instance.go +++ b/instance.go @@ -113,7 +113,7 @@ func (i *Instance) Start(ctx context.Context) error { context.AfterFunc(ctx, i.Stop) - if err := i.maybeBootstrap(); err != nil { + if err := i.maybeReplicateEpochs(); err != nil { return err } @@ -123,8 +123,8 @@ func (i *Instance) Start(ctx context.Context) error { return nil } -func (i *Instance) maybeBootstrap() error { - i.Config.Logger.Debug("Checking if bootstrapping is required") +func (i *Instance) maybeReplicateEpochs() error { + i.Config.Logger.Debug("Checking if epoch replication is required") latestValidatorSet, err := getLatestPlatformChainValidatorSet(i.Config.PlatformChain) if err != nil { return err @@ -135,17 +135,17 @@ func (i *Instance) maybeBootstrap() error { return err } - // We have indexed the latest validator set, therefore we can skip bootstrapping and start as a validator. + // We have indexed the latest validator set, therefore we can skip epoch replication and start as a validator. // Note: this may not be the latest epoch, but a future PR will eventually notice we are behind and transition properly. if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) { - i.Config.Logger.Debug("Node skipping bootstrapping because its latest epoch is up to date with the Platform Chain") + i.Config.Logger.Debug("Node skipping epoch replication because its latest epoch is up to date with the Platform Chain") return i.startValidator(latestIndexedEpochValidators) } // Start as non-validator if our last indexed validator set does not equal, the latest p-chain validator set // Note: the epoch may be transitioning, so the latest p-chain validator set actually points to a future epoch. - // The non-validator should finish bootstrapping and convert our non-validator to a validator in this case. - i.Config.Logger.Debug("Node starting bootstrapping as a non-validator") + // The non-validator should finish replicating epochs and convert our non-validator to a validator in this case. + i.Config.Logger.Debug("Node starting epoch replication as a non-validator") return i.startNonValidator(false) } @@ -167,10 +167,10 @@ func (i *Instance) startValidator(validators common.Nodes) error { return epoch.Start() } -// startNonValidator runs a non-validator. bootstrapped is true when we already hold the +// startNonValidator runs a non-validator. epochsReplicated is true when we already hold the // newest sealing block, such as when a validator leaves the validator set. -func (i *Instance) startNonValidator(bootstrapped bool) error { - config, err := i.createNonValidatorConfig(bootstrapped) +func (i *Instance) startNonValidator(epochsReplicated bool) error { + config, err := i.createNonValidatorConfig(epochsReplicated) if err != nil { return err } @@ -185,7 +185,7 @@ func (i *Instance) startNonValidator(bootstrapped bool) error { return nil } -func (i *Instance) createNonValidatorConfig(bootstrapped bool) (nonvalidator.Config, error) { +func (i *Instance) createNonValidatorConfig(epochsReplicated bool) (nonvalidator.Config, error) { source, err := simplex.NewRandomSource() if err != nil { return nonvalidator.Config{}, err @@ -225,7 +225,7 @@ func (i *Instance) createNonValidatorConfig(bootstrapped bool) (nonvalidator.Con SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, TransitionToValidator: i.notifyEpochChange, - Bootstrapped: bootstrapped, + EpochsReplicated: epochsReplicated, } return config, nil } diff --git a/instance_helpers_test.go b/instance_helpers_test.go index 2f14262d..df529ef3 100644 --- a/instance_helpers_test.go +++ b/instance_helpers_test.go @@ -610,15 +610,15 @@ func (n *node) restart() *node { } // role reports whether the instance currently runs a validator epoch rather than a -// non-validator, and whether it has finished bootstrapping. -func (n *node) role() (isValidator bool, bootstrapped bool) { +// non-validator, and whether it has finished replicating epochs. +func (n *node) role() (isValidator bool, epochsReplicated bool) { n.inst.lock.Lock() defer n.inst.lock.Unlock() if n.inst.e != nil { return true, true } - return false, n.inst.nv != nil && n.inst.nv.IsBootstrapped() + return false, n.inst.nv != nil && n.inst.nv.HasReplicatedEpochs() } // sync syncs a node by waiting for the commit of the latest sequence. diff --git a/instance_test.go b/instance_test.go index bef3fb13..93de2291 100644 --- a/instance_test.go +++ b/instance_test.go @@ -26,8 +26,8 @@ func TestValidatorIndexes(t *testing.T) { network := newNetwork(t, pChain) node := network.addNode(validator.NodeID[:]).sync() - isValidator, bootstrapped := node.role() - require.True(t, bootstrapped, "a node already at the latest validator set has nothing to bootstrap") + isValidator, epochsReplicated := node.role() + require.True(t, epochsReplicated, "a node already at the latest validator set has no epochs to replicate") require.True(t, isValidator) network.acceptNewBlock() @@ -94,7 +94,7 @@ func TestEpochInvokesMSMWaitForPendingBlock(t *testing.T) { } // TestNonValidatorSyncs asserts a node outside the validator set syncs the chain when added -// to the network, and stays a non-validator once it has bootstrapped. +// to the network, and stays a non-validator once it has epochsReplicated. func TestNonValidatorSyncs(t *testing.T) { validator := newNodeMapping(1) genesisSet := []metadata.NodeBLSMapping{validator} @@ -110,9 +110,9 @@ func TestNonValidatorSyncs(t *testing.T) { network.acceptNewBlock() node.sync() - // ensure we bootstrap and are not a validator - isValidator, bootstrapped := node.role() - require.True(t, bootstrapped) + // ensure we replicated epochs and are not a validator + isValidator, epochsReplicated := node.role() + require.True(t, epochsReplicated) require.False(t, isValidator) } @@ -406,8 +406,8 @@ func TestNonValidatorSkipsMSMVerification(t *testing.T) { }, }, validator.NodeID[:])) - _, bootstrapped := nonValidatorNode.role() - require.True(t, bootstrapped) + _, epochsReplicated := nonValidatorNode.role() + require.True(t, epochsReplicated) // A block whose only defect is its state machine transition: its timestamp precedes its // parent's. @@ -581,10 +581,10 @@ func TestValidatorSetsMetadataFromSnowman(t *testing.T) { require.Equal(t, numNonSimplexBlocks, block.BlockHeader().Seq) } -// TestBootstrapValidatorDuringTransition asserts a validator bootstraps when +// TestValidatorReplicatesEpochsDuringTransition asserts a validator replicates epochs when // its latest epoch is mid-transition. i.e. the latest pchain validator set disagrees // with the validators latest index validator set. -func TestBootstrapValidatorDuringTransition(t *testing.T) { +func TestValidatorReplicatesEpochsDuringTransition(t *testing.T) { ourNodeMapping := newNodeMapping(1) v2 := newNodeMapping(2) v3 := newNodeMapping(3) @@ -615,11 +615,11 @@ func TestBootstrapValidatorDuringTransition(t *testing.T) { // the node joins, but because the pchain validator set is different than our epoch we will sync as a non-validator node := network.addNode(ourNodeMapping.NodeID[:]) - isValidator, bootstrapped := node.role() - require.False(t, bootstrapped) - require.False(t, isValidator, "a node whose indexed set is not the latest must bootstrap first") // even though we are a validator + isValidator, epochsReplicated := node.role() + require.False(t, epochsReplicated) + require.False(t, isValidator, "a node whose indexed set is not the latest must replicate epochs first") // even though we are a validator - // bring 2 node back online. The threshold for non-validators to complete bootstrapping is 2 votes, + // bring 2 node back online. The threshold for non-validators to complete epoch replication is 2 votes, // but to make a quorum is 3. This means the pchain transition will not occur, however the node should now sync as a validator network.setOnline(v2.NodeID[:]) network.setOnline(v3.NodeID[:]) diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index bf303859..c8deea10 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -60,9 +60,9 @@ type Config struct { // and it is in the validator set TransitionToValidator func(epoch uint64, validators common.Nodes) - // Bootstrapped is set once every epoch from our tip up to the one a threshold of the latest + // EpochsReplicated is set once every epoch from our tip up to the one a threshold of the latest // validator set reported has been validated. Until then only replication responses are handled. - Bootstrapped bool + EpochsReplicated bool } type NonValidator struct { @@ -139,8 +139,8 @@ func NewNonValidator(config Config) (*NonValidator, error) { func (n *NonValidator) Start() { n.Logger.Info("Starting non-validator", zap.Stringer("ID", n.ID)) n.broadcastLatestEpoch() - if !n.Bootstrapped { - n.sealingBlockTimeouts.AddTask(initialBootstrapTask) + if !n.EpochsReplicated { + n.sealingBlockTimeouts.AddTask(initialEpochReplicationTask) } } @@ -157,13 +157,13 @@ func (n *NonValidator) AdvanceTime(t time.Time) { n.sealingBlockTimeouts.Tick(t) } -// IsBootstrapped reports whether bootstrapping has finished. -// Bootstrapping finishes when every sealing block down to our tip is validated. -func (n *NonValidator) IsBootstrapped() bool { +// HasReplicatedEpochs reports whether epoch replication has finished. +// Epoch replication finishes when every sealing block down to our tip is validated. +func (n *NonValidator) HasReplicatedEpochs() bool { n.lock.Lock() defer n.lock.Unlock() - return n.Bootstrapped + return n.EpochsReplicated } func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) error { @@ -181,8 +181,8 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er return n.haltedError } - if !n.Bootstrapped && msg.ReplicationResponse == nil { - n.Logger.Debug("Dropping message received while bootstrapping, we only accept replication responses", zap.Any("Message", msg), zap.Stringer("From", from)) + if !n.EpochsReplicated && msg.ReplicationResponse == nil { + n.Logger.Debug("Dropping message received while replicating epochs, we only accept replication responses", zap.Any("Message", msg), zap.Stringer("From", from)) return nil } @@ -199,10 +199,10 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er } } -// processBootstrapQuorumRound handles quorum rounds until bootstrapping finishes. +// processEpochReplicationQuorumRound handles quorum rounds until epoch replication finishes. // Once a threshold validates an epoch, only sealing blocks are validated // and stored(in a backwards manner). -func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from common.NodeID) { +func (n *NonValidator) processEpochReplicationQuorumRound(qr *common.QuorumRound, from common.NodeID) { block := qr.Block bh := block.BlockHeader() sealingInfo := block.SealingBlockInfo() @@ -221,7 +221,7 @@ func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from case n.highestEpochCollector.collectedSealingBlockInfo(sealingInfo, bh, from): // The sealing block is from an unknown epoch and we have collected enough votes to validate it n.Logger.Info("A threshold of validators reported a sealing block", zap.Uint64("Seq", bh.Seq), zap.Stringer("Info", sealingInfo)) - n.sealingBlockTimeouts.RemoveTask(initialBootstrapTask) + n.sealingBlockTimeouts.RemoveTask(initialEpochReplicationTask) if !n.isIndexed(bh.Seq) { n.validateSealingBlock(qr, from) @@ -233,7 +233,7 @@ func (n *NonValidator) processBootstrapQuorumRound(qr *common.QuorumRound, from // No sealing block is missing, so every epoch from our tip to the highest is validated. if n.sealingBlockTimeouts.Empty() { - n.finishBootstrap() + n.finishEpochReplication() } } @@ -245,12 +245,12 @@ func (n *NonValidator) validateSealingBlock(qr *common.QuorumRound, from common. n.storeQuorumRound(qr) } -// finishBootstrap marks bootstrapping done. Every epoch from our tip to the highest one a +// finishEpochReplication marks epoch replication done. Every epoch from our tip to the highest one a // threshold of validators reported is validated, so replication and live messages can be handled. -func (n *NonValidator) finishBootstrap() { - n.Bootstrapped = true +func (n *NonValidator) finishEpochReplication() { + n.EpochsReplicated = true highestEpoch, validators := n.epochs.highestEpoch() - n.Logger.Info("Finished bootstrapping", zap.Uint64("Highest Epoch", highestEpoch)) + n.Logger.Info("Finished replicating epochs", zap.Uint64("Highest Epoch", highestEpoch)) n.maybeTransitionToValidator(highestEpoch, validators) } @@ -264,9 +264,9 @@ func (n *NonValidator) maybeTransitionToValidator(epoch uint64, validators commo n.TransitionToValidator(epoch, validators) } -// initialBootstrapTask is the sealingBlockTimeouts task that continuously asks for sealing blocks until a +// initialEpochReplicationTask is the sealingBlockTimeouts task that continuously asks for sealing blocks until a // threshold of responses validates an epoch. We use Seq 1, since requests with Seq 0 are dropped. -const initialBootstrapTask uint64 = 1 +const initialEpochReplicationTask uint64 = 1 // requestMissingSealingBlocks re-requests sealing blocks of the hash chain that timed out // from every validator. Runs on the timeout handler's goroutine. @@ -409,7 +409,7 @@ func (n *NonValidator) newFinalizedBlockTask(block common.Block, finalization *c } // maybeValidateNextEpoch validates the epoch block opens when block is a sealing block. While -// bootstrapping it also requests the sealing block that opened block's own epoch, following the +// replicating epochs it also requests the sealing block that opened block's own epoch, following the // hash chain back until every sealing block down to an epoch we have indexed is validated. func (n *NonValidator) maybeValidateNextEpoch(block common.Block, from common.NodeID) { bh := block.BlockHeader() @@ -427,7 +427,7 @@ func (n *NonValidator) maybeValidateNextEpoch(block common.Block, from common.No n.Logger.Info("We have a valid sealing block, messages for that epoch can be processed.", zap.Uint64("Epoch", nextEpoch)) n.epochs[nextEpoch] = newEpochMetadata(nextEpoch, sealingInfo, n.SignatureAggregatorCreator) - if n.Bootstrapped { + if n.EpochsReplicated { return } @@ -596,7 +596,7 @@ func (n *NonValidator) handleReplicationResponse(resp *common.ReplicationRespons } func (n *NonValidator) processReplicationState() error { - if !n.Bootstrapped { + if !n.EpochsReplicated { return nil } @@ -638,9 +638,9 @@ func (n *NonValidator) processQuorumRound(qr *common.QuorumRound, from common.No return err } - // Runs before rejecting indexed blocks, an indexed sealing block is still a vote while bootstrapping. - if !n.Bootstrapped { - n.processBootstrapQuorumRound(qr, from) + // Runs before rejecting indexed blocks, an indexed sealing block is still a vote while replicating epochs. + if !n.EpochsReplicated { + n.processEpochReplicationQuorumRound(qr, from) return nil } diff --git a/nonvalidator/non_validator_test.go b/nonvalidator/non_validator_test.go index 1e0273ff..1a2acfc9 100644 --- a/nonvalidator/non_validator_test.go +++ b/nonvalidator/non_validator_test.go @@ -322,7 +322,7 @@ func TestHandleMessages(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -372,7 +372,7 @@ func TestNonValidatorDropsTelockQuorumRound(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, - Bootstrapped: true, + EpochsReplicated: true, }) require.NoError(t, err) defer nv.Stop() @@ -406,7 +406,7 @@ func TestNonValidator_StopsGracefully(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -440,7 +440,7 @@ func TestHandleMessages_DuplicateBlock(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: testNodes[0].Id, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -592,7 +592,7 @@ func TestNonValidator_CallsTransition(t *testing.T) { defer lock.Unlock() calls = append(calls, transitionCall{epoch: epoch, validators: validators}) }, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -654,8 +654,8 @@ func TestNonValidator_RequestHighestEpochOnStart(t *testing.T) { require.Empty(t, msg.msg.ReplicationRequest.Seqs) } -// TestNonValidator_Bootstrap ensures a non-validator can replicate sequences given different states of the chain. -func TestNonValidator_Bootstrap(t *testing.T) { +// TestNonValidator_EpochReplication ensures a non-validator can replicate sequences given different states of the chain. +func TestNonValidator_EpochReplication(t *testing.T) { tests := []struct { name string // setup builds the full network chain the non-validator replicates from. @@ -758,7 +758,7 @@ func TestNonValidator_Bootstrap(t *testing.T) { MaxSequenceWindow: tt.maxSequenceWindow, ID: myNodeID, StartTime: time.Now(), - Bootstrapped: false, + EpochsReplicated: false, }, ) require.NoError(t, err) @@ -767,7 +767,7 @@ func TestNonValidator_Bootstrap(t *testing.T) { defer nv.Stop() advanceUntil(nv, epochs, msgQueue, tt.lastSeq) - require.Eventually(t, nv.IsBootstrapped, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, nv.HasReplicatedEpochs, 5*time.Second, 10*time.Millisecond) }) } } @@ -801,7 +801,7 @@ func TestNonValidator_ReplicationRequests(t *testing.T) { MaxSequenceWindow: maxSeqWindow, ID: myNodeID, StartTime: startTime, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -883,7 +883,7 @@ func TestNonValidator_VerifiesFinalizationDuringReplication(t *testing.T) { MaxSequenceWindow: 5, // significantly lower the max round window ID: testNodes.NodeIDs()[0], StartTime: startTime, - Bootstrapped: true, + EpochsReplicated: true, }, ) @@ -1022,7 +1022,7 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) { MaxSequenceWindow: 10, ID: validators.NodeIDs()[0], StartTime: time.Now(), - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -1057,10 +1057,10 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) { } } -// TestNonValidator_BootstrapGatesMessages asserts that blocks and finalizations are dropped +// TestNonValidator_EpochReplicationGatesMessages asserts that blocks and finalizations are dropped // until a threshold of replication responses vote for the same sealing block, after which // the stored round is committed and messages are processed normally. -func TestNonValidator_BootstrapGatesMessages(t *testing.T) { +func TestNonValidator_EpochReplicationGatesMessages(t *testing.T) { tc := newSeededChain(t, testNodes, 2) nv, err := NewNonValidator( @@ -1087,7 +1087,7 @@ func TestNonValidator_BootstrapGatesMessages(t *testing.T) { require.Never(t, func() bool { return tc.NumBlocks() > 3 }, 2*time.Second, 50*time.Millisecond, - "indexed a block before bootstrapping", + "indexed a block before epochs were replicated", ) qrMsg := &common.Message{ @@ -1099,14 +1099,14 @@ func TestNonValidator_BootstrapGatesMessages(t *testing.T) { for i := 0; i < threshold-1; i++ { require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[i])) } - require.False(t, nv.IsBootstrapped(), "bootstrapped below the threshold") + require.False(t, nv.HasReplicatedEpochs(), "epochsReplicated below the threshold") - // the sealing block's epoch was opened by the indexed epoch 1, so the threshold vote finishes bootstrapping + // the sealing block's epoch was opened by the indexed epoch 1, so the threshold vote finishes epoch replication require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[threshold-1])) - require.True(t, nv.IsBootstrapped()) + require.True(t, nv.HasReplicatedEpochs()) tc.WaitForBlockCommit(3) - // messages flow normally after bootstrapping + // messages flow normally after epoch replication b4 := tc.appendBlock() block = blockMsg(t, b4, testNodes) require.NoError(t, nv.HandleMessage(block.msg, block.from)) @@ -1134,10 +1134,10 @@ func popRequestedSeqs(t *testing.T, msgQueue *messageQueue) []uint64 { return seqs } -// TestNonValidator_BootstrapWalksHashChain asserts a non-validator several epochs behind requests +// TestNonValidator_EpochReplicationWalksHashChain asserts a non-validator several epochs behind requests // sealing blocks one hop back at a time once a threshold reports the highest one, and finishes -// bootstrapping when the chain reaches an epoch it has indexed. -func TestNonValidator_BootstrapWalksHashChain(t *testing.T) { +// epoch replication when the chain reaches an epoch it has indexed. +func TestNonValidator_EpochReplicationWalksHashChain(t *testing.T) { tc := newSeededChain(t, testNodes, 2) tc.indexEpochs(5, 10, 20) myNodeID := common.NodeID{100} @@ -1160,23 +1160,23 @@ func TestNonValidator_BootstrapWalksHashChain(t *testing.T) { for i := 0; i < threshold; i++ { require.NoError(t, nv.HandleMessage(sealingResponse(tc, 20), tc.nodes().NodeIDs()[i])) } - require.False(t, nv.IsBootstrapped()) + require.False(t, nv.HasReplicatedEpochs()) // each sealing block validates by hash and requests the one that opened its epoch for _, seq := range []uint64{10, 5} { require.Equal(t, []uint64{seq}, popRequestedSeqs(t, msgQueue)) - require.False(t, nv.IsBootstrapped()) + require.False(t, nv.HasReplicatedEpochs()) require.NoError(t, nv.HandleMessage(sealingResponse(tc, seq), tc.nodes().NodeIDs()[0])) } // seq 5 was opened by the indexed epoch 1, so every sealing block is validated - require.True(t, nv.IsBootstrapped()) + require.True(t, nv.HasReplicatedEpochs()) require.NotEmpty(t, popRequestedSeqs(t, msgQueue), "replication of the sequences behind the tip never started") } -// TestNonValidator_BootstrapIgnoresSealingBlockOffChain asserts that while following the hash chain +// TestNonValidator_EpochReplicationIgnoresSealingBlockOffChain asserts that while following the hash chain // a sealing block further down it is dropped until the epoch pointing back to it has been validated. -func TestNonValidator_BootstrapIgnoresSealingBlockOffChain(t *testing.T) { +func TestNonValidator_EpochReplicationIgnoresSealingBlockOffChain(t *testing.T) { tc := newSeededChain(t, testNodes, 2) tc.indexEpochs(5, 10, 20) myNodeID := common.NodeID{100} @@ -1204,18 +1204,18 @@ func TestNonValidator_BootstrapIgnoresSealingBlockOffChain(t *testing.T) { // seq 5 opened the epoch seq 10 was produced in, but seq 10 has not been validated yet require.NoError(t, nv.HandleMessage(sealingResponse(tc, 5), tc.nodes().NodeIDs()[0])) require.Empty(t, popRequestedSeqs(t, msgQueue)) - require.False(t, nv.IsBootstrapped()) + require.False(t, nv.HasReplicatedEpochs()) // once seq 10 validates, seq 5 is still requested, so it was dropped, and is accepted on resend require.NoError(t, nv.HandleMessage(sealingResponse(tc, 10), tc.nodes().NodeIDs()[0])) require.Equal(t, []uint64{5}, popRequestedSeqs(t, msgQueue)) require.NoError(t, nv.HandleMessage(sealingResponse(tc, 5), tc.nodes().NodeIDs()[0])) - require.True(t, nv.IsBootstrapped()) + require.True(t, nv.HasReplicatedEpochs()) } -// TestNonValidator_BootstrapRetriesSealingBlock asserts an unanswered request for a sealing block +// TestNonValidator_EpochReplicationRetriesSealingBlock asserts an unanswered request for a sealing block // of the hash chain is re-requested once the replication timeout passes. -func TestNonValidatorBootstrapRetriesSealingBlock(t *testing.T) { +func TestNonValidator_EpochReplicationRetriesSealingBlock(t *testing.T) { tc := newSeededChain(t, testNodes, 2) tc.indexEpochs(5, 10, 20) myNodeID := common.NodeID{100} @@ -1247,10 +1247,10 @@ func TestNonValidatorBootstrapRetriesSealingBlock(t *testing.T) { }, 5*time.Second, 10*time.Millisecond, "the sealing block was never re-requested") } -// TestNonValidator_BootstrapRequestsSealingBlock asserts that a replication response carrying a -// non-sealing block while bootstrapping triggers a request to its sender for the sealing block that +// TestNonValidator_EpochReplicationRequestsSealingBlock asserts that a replication response carrying a +// non-sealing block while replicating epochs triggers a request to its sender for the sealing block that // opened its epoch, whether that epoch is ours or unknown. -func TestNonValidatorBootstrapRequestsSealingBlock(t *testing.T) { +func TestNonValidator_EpochReplicationRequestsSealingBlock(t *testing.T) { tc := newSeededChain(t, testNodes, 2) myNodeID := common.NodeID{100} msgQueue := &messageQueue{} @@ -1290,16 +1290,16 @@ func TestNonValidatorBootstrapRequestsSealingBlock(t *testing.T) { require.Equal(t, sender, msg.to) } - // non-sealing blocks never bootstrap the node - require.False(t, nv.Bootstrapped) + // non-sealing blocks never complete epoch replication + require.False(t, nv.EpochsReplicated) _, ok := msgQueue.popResponse() require.False(t, ok) } -// TestNonValidatorBootstrapLatestKnownEpoch asserts a node caught up to the network -// bootstraps from responses vouching for the sealing block of the latest epoch it +// TestNonValidator_EpochReplicationLatestKnownEpoch asserts a node caught up to the network +// replicates epochs from responses vouching for the sealing block of the latest epoch it // already has indexed, without re-indexing it. -func TestNonValidatorBootstrapLatestKnownEpoch(t *testing.T) { +func TestNonValidator_EpochReplicationLatestKnownEpoch(t *testing.T) { tc := newSeededChain(t, testNodes, 2) nv, err := NewNonValidator( Config{ @@ -1312,7 +1312,7 @@ func TestNonValidatorBootstrapLatestKnownEpoch(t *testing.T) { }, ) require.NoError(t, err) - require.False(t, nv.Bootstrapped) + require.False(t, nv.EpochsReplicated) defer nv.Stop() // the sealing block of epoch 1, indexed at seq 1 @@ -1330,10 +1330,10 @@ func TestNonValidatorBootstrapLatestKnownEpoch(t *testing.T) { require.NoError(t, nv.HandleMessage(qrMsg, testNodes.NodeIDs()[i])) } - require.True(t, nv.Bootstrapped) + require.True(t, nv.EpochsReplicated) require.Equal(t, uint64(3), tc.NumBlocks()) - // messages flow normally after bootstrapping + // messages flow normally after epoch replication b3 := tc.appendBlock() block := blockMsg(t, b3, testNodes) require.NoError(t, nv.HandleMessage(block.msg, block.from)) @@ -1427,7 +1427,7 @@ func TestNonValidatorAcceptsProposalFromUnsortedValidatorSet(t *testing.T) { SignatureAggregatorCreator: tc.signatureAggregatorCreator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, ID: common.NodeID{100}, - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) @@ -1465,7 +1465,7 @@ func TestNonValidatorRejectsQuorumRoundWithMismatchedHeader(t *testing.T) { Comm: testutil.NewNoopComm(testNodes.NodeIDs()), Logger: logger, SignatureAggregatorCreator: tc.signatureAggregatorCreator, - Bootstrapped: true, + EpochsReplicated: true, MaxSequenceWindow: 10, ID: common.NodeID{16}, StartTime: time.Now(), @@ -1523,7 +1523,7 @@ func TestNonValidatorDropsQuorumRoundPastSequenceWindow(t *testing.T) { MaxSequenceWindow: maxSequenceWindow, ID: common.NodeID{16}, StartTime: time.Now(), - Bootstrapped: true, + EpochsReplicated: true, }, ) require.NoError(t, err) From a9d458fdd99e4b734283ba01ca3bef2469eef359 Mon Sep 17 00:00:00 2001 From: samliok Date: Mon, 21 Sep 2026 12:57:23 -0500 Subject: [PATCH 13/13] update comment and reording --- nonvalidator/epochs.go | 5 +++-- nonvalidator/non_validator.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nonvalidator/epochs.go b/nonvalidator/epochs.go index b4f615d0..371d5b0e 100644 --- a/nonvalidator/epochs.go +++ b/nonvalidator/epochs.go @@ -111,8 +111,9 @@ func (e epochs) removeOldEpochs(minEpochToKeep uint64) { } } -// canValidate returns true if `block` is valid sealing block in the chain. It can -// be valid if `block` is a backwards pointer to any sealing block already validated (backwards hash chain validation). +// canValidate returns true if `block` is valid sealing block in the chain. +// block is valid if it's a sealing block that isn't in `e` +// and for which there exists a sealing block in e that has a backwards hash pointer to block. func (e epochs) canValidate(block common.Block) bool { if block.SealingBlockInfo() == nil { return false diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index c8deea10..94e43bbd 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -187,12 +187,12 @@ func (n *NonValidator) HandleMessage(msg *common.Message, from common.NodeID) er } switch { + case msg.ReplicationResponse != nil: + return n.handleReplicationResponse(msg.ReplicationResponse, from) case msg.BlockMessage != nil && msg.BlockMessage.Block != nil: return n.handleBlock(msg.BlockMessage.Block, from) case msg.Finalization != nil: return n.handleFinalization(msg.Finalization, from) - case msg.ReplicationResponse != nil: - return n.handleReplicationResponse(msg.ReplicationResponse, from) default: n.Logger.Debug("Received unexpected message", zap.Any("Message", msg), zap.Stringer("from", from)) return nil