From 681e8511ed1383f568808c86d5e42228a93ca177 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Fri, 1 Aug 2025 22:56:24 +0000 Subject: [PATCH 1/3] TQ: Add support for alarms in the protocol An alarm represents a protocol invariant violation. It's unclear exactly what should be done about these other than recording them and allowing them to be reported upstack, which is what is done in this PR. An argument could be made for "freezing" the state machine such that trust quorum nodes stop working and the only thing they can do is report alarm status. However, that would block the trust quorum from operating at all, and it's unclear if this should cause an outage on that node. I'm also somewhat hesitant to put the alarms into the persistent state as that would prevent unlock in the case of a sled/rack reboot. On the flip side of just recording is the possible danger resulting from operating with an invariant violation. This could potentially be risky, and since we shouldn't ever see these maybe pausing for a support call is the right thing. TBD, once more work is done on the protocol. --- trust-quorum/gfss/src/shamir.rs | 12 ++++++- trust-quorum/src/alarm.rs | 42 +++++++++++++++++++++++++ trust-quorum/src/compute_key_share.rs | 12 +++---- trust-quorum/src/configuration.rs | 4 ++- trust-quorum/src/lib.rs | 2 ++ trust-quorum/src/node.rs | 45 +++++++++++++++------------ trust-quorum/src/node_ctx.rs | 20 +++++++++++- trust-quorum/tests/cluster.rs | 15 +++++++++ 8 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 trust-quorum/src/alarm.rs diff --git a/trust-quorum/gfss/src/shamir.rs b/trust-quorum/gfss/src/shamir.rs index 41f09a3b7cd..c315b8dfd1a 100644 --- a/trust-quorum/gfss/src/shamir.rs +++ b/trust-quorum/gfss/src/shamir.rs @@ -23,7 +23,17 @@ pub enum SplitError { TooFewTotalShares { n: u8, k: u8 }, } -#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] +#[derive( + Debug, + Clone, + thiserror::Error, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, +)] pub enum CombineError { #[error("must be at least 2 shares to combine")] TooFewShares, diff --git a/trust-quorum/src/alarm.rs b/trust-quorum/src/alarm.rs new file mode 100644 index 00000000000..488a8b4ea1e --- /dev/null +++ b/trust-quorum/src/alarm.rs @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Mechanism for reporting protocol invariant violations + +use serde::{Deserialize, Serialize}; + +use crate::{Configuration, Epoch}; + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] +pub enum Alarm { + /// Different configurations found for the same epoch + /// + /// Reason: Nexus creates configurations and stores them in CRDB before + /// sending them to a coordinator of its choosing. Nexus will not send the + /// same reconfiguration request to different coordinators. If it does those + /// coordinators will generate different key shares. However, since Nexus + /// will not tell different nodes to coordinate the same configuration, this + /// state should be impossible to reach. + MismatchedConfigurations { config1: Configuration, config2: Configuration }, + + /// We received a `CommitAdvance` while coordinating for the same epoch + /// + /// Reason: `CommitAdvance` is a reply for a key share request in an + /// old epoch that we don't have the latest committed coordination. + /// However we are actually the coordinator for the configuration in the + /// `CommitAdvance`. While it's possible that another node could learn + /// of the commit from Nexus before the coordinator, the coordinator will + /// never ask for a key share for that epoch. Therefore this state should be + /// impossible to reach. + CommitAdvanceForCoordinatingEpoch { config: Configuration }, + + /// The `keyShareComputer` could not compute this node's share + /// + /// Reason: A threshold of valid key shares were received based on the the + /// share digests in the Configuration. However, computation of the share + /// still failed. This should be impossible. + ShareComputationFailed { epoch: Epoch, err: gfss::shamir::CombineError }, +} diff --git a/trust-quorum/src/compute_key_share.rs b/trust-quorum/src/compute_key_share.rs index 2bee03abbea..4aae82e39c6 100644 --- a/trust-quorum/src/compute_key_share.rs +++ b/trust-quorum/src/compute_key_share.rs @@ -9,7 +9,9 @@ //! other nodes so that it can compute its own key share. use crate::crypto::Sha3_256Digest; -use crate::{Configuration, Epoch, NodeHandlerCtx, PeerMsgKind, PlatformId}; +use crate::{ + Alarm, Configuration, Epoch, NodeHandlerCtx, PeerMsgKind, PlatformId, +}; use gfss::gf256::Gf256; use gfss::shamir::{self, Share}; use slog::{Logger, error, o, warn}; @@ -137,11 +139,9 @@ impl KeyShareComputer { }); true } - Err(e) => { - // TODO: put the node into into an `Alarm` state similar to - // https://github.com/oxidecomputer/omicron/pull/8062 once we - // have alarms? - error!(self.log, "Failed to compute share: {}", e); + Err(err) => { + error!(self.log, "Failed to compute share: {}", err); + ctx.raise_alarm(Alarm::ShareComputationFailed { epoch, err }); false } } diff --git a/trust-quorum/src/configuration.rs b/trust-quorum/src/configuration.rs index 23b4f6f3c8d..a6057c62ed1 100644 --- a/trust-quorum/src/configuration.rs +++ b/trust-quorum/src/configuration.rs @@ -30,7 +30,9 @@ pub enum ConfigurationError { /// The configuration for a given epoch. /// /// Only valid for non-lrtq configurations -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] pub struct Configuration { /// Unique Id of the rack pub rack_id: RackUuid, diff --git a/trust-quorum/src/lib.rs b/trust-quorum/src/lib.rs index 39418714296..8bb8d8de5d3 100644 --- a/trust-quorum/src/lib.rs +++ b/trust-quorum/src/lib.rs @@ -23,7 +23,9 @@ mod persistent_state; mod validators; pub use configuration::Configuration; pub use coordinator_state::{CoordinatorOperation, CoordinatorState}; +mod alarm; +pub use alarm::Alarm; pub use crypto::RackSecret; pub use messages::*; pub use node::Node; diff --git a/trust-quorum/src/node.rs b/trust-quorum/src/node.rs index 72d38a5b674..1f6d615af89 100644 --- a/trust-quorum/src/node.rs +++ b/trust-quorum/src/node.rs @@ -20,7 +20,7 @@ use crate::validators::{ MismatchedRackIdError, ReconfigurationError, ValidatedReconfigureMsg, }; use crate::{ - Configuration, CoordinatorState, Epoch, NodeHandlerCtx, PlatformId, + Alarm, Configuration, CoordinatorState, Epoch, NodeHandlerCtx, PlatformId, messages::*, }; use gfss::shamir::Share; @@ -311,25 +311,28 @@ impl Node { } // Do we have the configuration in our persistent state? If not save it. - ctx.update_persistent_state(|ps| { - if let Err(e) = ps.configs.insert_unique(config.clone()) { - let existing = - e.duplicates().first().expect("duplicate exists"); - if *existing != &config { - error!( - self.log, - "Received a configuration mismatch"; - "from" => %from, - "existing_config" => #?existing, - "received_config" => #?config - ); - // TODO: Alarm - } - false - } else { - true + if let Some(existing) = + ctx.persistent_state().configuration(config.epoch) + { + if existing != &config { + error!( + self.log, + "Received a configuration mismatch"; + "from" => %from, + "existing_config" => #?existing, + "received_config" => #?config + ); + ctx.raise_alarm(Alarm::MismatchedConfigurations { + config1: (*existing).clone(), + config2: config.clone(), + }); } - }); + } else { + ctx.update_persistent_state(|ps| { + ps.configs.insert_unique(config.clone()).expect("new config"); + true + }); + } // Are we coordinating for an older epoch? If so, cancel. if let Some(cs) = &self.coordinator_state { @@ -350,7 +353,9 @@ impl Node { "from" => %from, "epoch" => %config.epoch ); - // TODO: Alarm + ctx.raise_alarm(Alarm::CommitAdvanceForCoordinatingEpoch { + config, + }); return; } else { info!( diff --git a/trust-quorum/src/node_ctx.rs b/trust-quorum/src/node_ctx.rs index 30cfe15b174..e3a4f7fed32 100644 --- a/trust-quorum/src/node_ctx.rs +++ b/trust-quorum/src/node_ctx.rs @@ -4,7 +4,9 @@ //! Parameter to Node API calls that allows interaction with the system at large -use crate::{Envelope, PeerMsg, PeerMsgKind, PersistentState, PlatformId}; +use crate::{ + Alarm, Envelope, PeerMsg, PeerMsgKind, PersistentState, PlatformId, +}; use std::collections::BTreeSet; /// An API shared by [`NodeCallerCtx`] and [`NodeHandlerCtx`] @@ -12,6 +14,7 @@ pub trait NodeCommonCtx { fn platform_id(&self) -> &PlatformId; fn persistent_state(&self) -> &PersistentState; fn connected(&self) -> &BTreeSet; + fn alarms(&self) -> &BTreeSet; } /// An API for an [`NodeCtx`] usable from a [`crate::Node`] @@ -54,6 +57,9 @@ pub trait NodeHandlerCtx: NodeCommonCtx { /// Remove a peer from the connected set fn remove_connection(&mut self, id: &PlatformId); + + /// Record (in-memory) that an alarm has occurred + fn raise_alarm(&mut self, alarm: Alarm); } /// Common parameter to [`crate::Node`] methods @@ -79,6 +85,9 @@ pub struct NodeCtx { /// Connected peer nodes connected: BTreeSet, + + /// Any alarms that have occurred + alarms: BTreeSet, } impl NodeCtx { @@ -89,6 +98,7 @@ impl NodeCtx { persistent_state_changed: false, outgoing: Vec::new(), connected: BTreeSet::new(), + alarms: BTreeSet::new(), } } } @@ -105,6 +115,10 @@ impl NodeCommonCtx for NodeCtx { fn connected(&self) -> &BTreeSet { &self.connected } + + fn alarms(&self) -> &BTreeSet { + &self.alarms + } } impl NodeHandlerCtx for NodeCtx { @@ -138,6 +152,10 @@ impl NodeHandlerCtx for NodeCtx { fn remove_connection(&mut self, id: &PlatformId) { self.connected.remove(id); } + + fn raise_alarm(&mut self, alarm: Alarm) { + self.alarms.insert(alarm); + } } impl NodeCallerCtx for NodeCtx { diff --git a/trust-quorum/tests/cluster.rs b/trust-quorum/tests/cluster.rs index 9dec6000640..9bc7da94c65 100644 --- a/trust-quorum/tests/cluster.rs +++ b/trust-quorum/tests/cluster.rs @@ -834,6 +834,7 @@ impl TestState { self.invariant_nodes_have_prepared_if_coordinator_has_acks()?; self.invariant_nodes_have_committed_if_nexus_has_acks()?; self.invariant_nodes_not_coordinating_and_computing_key_share_simultaneously()?; + self.invariant_no_alarms()?; Ok(()) } @@ -953,6 +954,20 @@ impl TestState { Ok(()) } + + // Ensure there has been no alarm at any node + fn invariant_no_alarms(&self) -> Result<(), TestCaseError> { + for (id, (_, ctx)) in &self.sut.nodes { + let alarms = ctx.alarms(); + prop_assert!( + alarms.is_empty(), + "Alarms found for {}: {:#?}", + id, + alarms + ); + } + Ok(()) + } } /// Broken out of `TestState` to alleviate borrow checker woes From ad388eb59393960945f796b0a6dd286c6e0d6549 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Sat, 2 Aug 2025 23:39:24 +0000 Subject: [PATCH 2/3] Remove Bogus Alarm It's not actually an error to receive a `CommitAdvance` while coordinating for the same epoch. The `GetShare` from the coordinator could have been delayed in the network` and the node that received it already committed before the coordinator knew it was done preparing. In essence, the following would happen: 1. The coordinator would send GetShare requests for the prior epoch 2. Enough nodes would reply so that the coordinator would start sending prepares. 3. Enough nodes would ack prepares to commit 4. Nexus would poll and send commits. Other nodes would get those commits, but not the coordinator 5. A node that hadn't yet received the `GetShare` would get a `CommitAdvance` or see the `Commit` from nexus and get it's configuration and recompute it's own share and commit. It may have been a prior coordinator with delayed deliveries to other nodes of `GetShare` messages. 6. The node that just committed finally receives the `GetShare` and sends back a `CommitAdvance` to the coordinator This is all valid, and was similar to a proptest counterexample --- trust-quorum/src/alarm.rs | 19 ++++++------------- trust-quorum/src/node.rs | 6 ++---- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/trust-quorum/src/alarm.rs b/trust-quorum/src/alarm.rs index 488a8b4ea1e..63a0c321c0d 100644 --- a/trust-quorum/src/alarm.rs +++ b/trust-quorum/src/alarm.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; -use crate::{Configuration, Epoch}; +use crate::{Configuration, Epoch, PlatformId}; #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, @@ -20,18 +20,11 @@ pub enum Alarm { /// coordinators will generate different key shares. However, since Nexus /// will not tell different nodes to coordinate the same configuration, this /// state should be impossible to reach. - MismatchedConfigurations { config1: Configuration, config2: Configuration }, - - /// We received a `CommitAdvance` while coordinating for the same epoch - /// - /// Reason: `CommitAdvance` is a reply for a key share request in an - /// old epoch that we don't have the latest committed coordination. - /// However we are actually the coordinator for the configuration in the - /// `CommitAdvance`. While it's possible that another node could learn - /// of the commit from Nexus before the coordinator, the coordinator will - /// never ask for a key share for that epoch. Therefore this state should be - /// impossible to reach. - CommitAdvanceForCoordinatingEpoch { config: Configuration }, + MismatchedConfigurations { + config1: Configuration, + config2: Configuration, + from: PlatformId, + }, /// The `keyShareComputer` could not compute this node's share /// diff --git a/trust-quorum/src/node.rs b/trust-quorum/src/node.rs index 1f6d615af89..a91f48ca3d5 100644 --- a/trust-quorum/src/node.rs +++ b/trust-quorum/src/node.rs @@ -325,6 +325,7 @@ impl Node { ctx.raise_alarm(Alarm::MismatchedConfigurations { config1: (*existing).clone(), config2: config.clone(), + from: from.clone(), }); } } else { @@ -347,15 +348,12 @@ impl Node { ); self.coordinator_state = None; } else if coordinating_epoch == config.epoch { - error!( + info!( self.log, "Received CommitAdvance while coordinating for same epoch!"; "from" => %from, "epoch" => %config.epoch ); - ctx.raise_alarm(Alarm::CommitAdvanceForCoordinatingEpoch { - config, - }); return; } else { info!( From 9465f5a89dbff985aaf904dee740a2a74e4c71f0 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Sun, 3 Aug 2025 03:46:21 +0000 Subject: [PATCH 3/3] Fix CommitAdvance behavior for expunged nodes --- trust-quorum/src/alarm.rs | 1 + trust-quorum/src/compute_key_share.rs | 24 ++++++++++---- trust-quorum/src/node.rs | 31 ++++++++++++++++++- .../tests/cluster.proptest-regressions | 7 +++++ 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 trust-quorum/tests/cluster.proptest-regressions diff --git a/trust-quorum/src/alarm.rs b/trust-quorum/src/alarm.rs index 63a0c321c0d..0580c024d34 100644 --- a/trust-quorum/src/alarm.rs +++ b/trust-quorum/src/alarm.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::{Configuration, Epoch, PlatformId}; +#[allow(clippy::large_enum_variant)] #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, )] diff --git a/trust-quorum/src/compute_key_share.rs b/trust-quorum/src/compute_key_share.rs index 4aae82e39c6..8cc780f752e 100644 --- a/trust-quorum/src/compute_key_share.rs +++ b/trust-quorum/src/compute_key_share.rs @@ -103,6 +103,7 @@ impl KeyShareComputer { "epoch" => %epoch, "from" => %from ); + return false; } // A valid share was received. Is it new? @@ -118,12 +119,23 @@ impl KeyShareComputer { // What index are we in the configuration? This is our "x-coordinate" // for our key share calculation. We always start indexing from 1, since // 0 is the rack secret. - let index = self - .config - .members - .keys() - .position(|id| id == ctx.platform_id()) - .expect("node exists"); + let index = + self.config.members.keys().position(|id| id == ctx.platform_id()); + + let Some(index) = index else { + let msg = concat!( + "Failed to get index for ourselves in current configuration. ", + "We are not a member, and must have been expunged." + ); + error!( + self.log, + "{msg}"; + "platform_id" => %ctx.platform_id(), + "config" => ?self.config + ); + return false; + }; + let x_coordinate = Gf256::new(u8::try_from(index + 1).expect("index fits in u8")); diff --git a/trust-quorum/src/node.rs b/trust-quorum/src/node.rs index a91f48ca3d5..a6613f9062f 100644 --- a/trust-quorum/src/node.rs +++ b/trust-quorum/src/node.rs @@ -308,6 +308,7 @@ impl Node { "epoch" => %config.epoch ); ctx.update_persistent_state(|ps| ps.commits.insert(config.epoch)); + return; } // Do we have the configuration in our persistent state? If not save it. @@ -347,6 +348,7 @@ impl Node { "received_epoch" => %config.epoch ); self.coordinator_state = None; + // Intentionally fall through } else if coordinating_epoch == config.epoch { info!( self.log, @@ -402,7 +404,8 @@ impl Node { } } - // We either were collectiong shares for an old epoch or haven't started yet. + // We either were collectiong shares for an old epoch or haven't started + // yet. self.key_share_computer = Some(KeyShareComputer::new(&self.log, ctx, config)); } @@ -417,6 +420,18 @@ impl Node { ctx.persistent_state().latest_committed_configuration() { if latest_committed_config.epoch > epoch { + if !latest_committed_config.members.contains_key(&from) { + info!( + self.log, + "Received a GetShare message from expunged node"; + "from" => %from, + "latest_committed_epoch" => + %latest_committed_config.epoch, + "requested_epoch" => %epoch + ); + // TODO: Send an expunged message + return; + } info!( self.log, concat!( @@ -435,6 +450,20 @@ impl Node { } } + // Do we have the configuration? Is the requesting peer a member? + if let Some(config) = ctx.persistent_state().configuration(epoch) { + if !config.members.contains_key(&from) { + info!( + self.log, + "Received a GetShare message from expunged node"; + "from" => %from, + "epoch" => %epoch + ); + // TODO: Send an expunged message + return; + } + } + // If we have the share for the requested epoch, we always return it. We // know that it is at least as new as the last committed epoch. We might // not have learned about the configuration being committed yet, but diff --git a/trust-quorum/tests/cluster.proptest-regressions b/trust-quorum/tests/cluster.proptest-regressions new file mode 100644 index 00000000000..bb7577eca65 --- /dev/null +++ b/trust-quorum/tests/cluster.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc aa229dbfa63db6ed21d624e6c474f3591be97204fc051fe2b7b6d1af78409532 # shrinks to input = _TestTrustQuorumProtocolArgs { input: TestInput { initial_config: GeneratedConfiguration { members: {2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 21}, threshold: Index(0) }, initial_down_nodes: {}, actions: [DeliverEnvelopes([Index(14493870343628933413), Index(0), Index(2635249153387079100), Index(7659274142974149207), Index(10538101999560721780), Index(9052169730812637246), Index(4315275857016173272), Index(1740015673248333392), Index(13987162340212539423)]), DeliverEnvelopes([Index(9041798398182823156), Index(13868876044032355650)]), DeliverEnvelopes([Index(3011348768397116341), Index(8660180482978775225), Index(9182847551331361735), Index(16121357191531813301), Index(3788564878545978377), Index(17393195961508650460), Index(3346167418243659948), Index(3573581286569374339), Index(13250013466448051467), Index(4820013139542767201), Index(8039736122411843390), Index(17013067520142601277), Index(16122004323767661676), Index(17569103901503114628), Index(1932949711347229750), Index(4319513761709523737), Index(4584565250890182062), Index(1279075436995347802), Index(17062300948886540969)]), PollPrepareAcks, Commit([Index(17389235045933586108), Index(8560515657089253499), Index(10855801671237462872), Index(13083943140650726535), Index(6596360928255073579), Index(12912936435579810120), Index(13555346217578137976), Index(1669092046242131079)]), DeliverNexusReplies(6), Reconfigure { num_added_nodes: 3, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(1703742760083928913), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 0, removed_nodes: [], threshold: Index(5259489828371974487), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 0, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(14446131236589207074), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(2073236698708178332)]), DeliverEnvelopes([Index(3453196791926166099), Index(568387599084033335), Index(3409840665184802609), Index(12286030226091071672), Index(7286708483759199339), Index(2055658592347030945), Index(1173213978658236675), Index(8649978366863725550)]), Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(9996579049375553061), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(17841859598948908161), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 1, removed_nodes: [], threshold: Index(8689652795643676030), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(13933299526419321128), Index(5225873483252317482), Index(5244472839316863196), Index(15065206300363858421), Index(1331991639281083654), Index(9969299029824142306), Index(10418763353833126365)]), Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(17441105865747581233), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9629417235951452701), Index(17699594351241716570), Index(6446358179225870872), Index(10477494232845674894), Index(226357282448375442), Index(17511469032238010586), Index(5868458999694820988), Index(9008597573607186863), Index(14889469521169823547), Index(16216562580497901897), Index(10976472819558126038), Index(8297592251652111615)]), DeliverEnvelopes([Index(6225179597717370182), Index(3456377812674314165), Index(4870110218414913904), Index(12933525926916895630), Index(5251263308628043457), Index(14960131063828769882), Index(954293712144926568), Index(15613441814815191176), Index(7681056590698508795), Index(3497512649074481205)]), DeliverEnvelopes([Index(5458890086829541581), Index(4300690563898456475), Index(469725690344733484), Index(3627582562296635119), Index(13485721932604488035), Index(838877244787605597), Index(17616669406954758185), Index(10912008557925580957), Index(2064843088888151834), Index(13592137105530054033), Index(464925174084213053), Index(12477077518224200179), Index(7062021569203679845)]), DeliverEnvelopes([Index(15629509651933908728), Index(7593448817552421286), Index(6069697258859901799), Index(7447724974852314797), Index(6976391078682036797), Index(15954111707567866812), Index(9739269646592130920), Index(7840844524173616953), Index(17251031726274733868), Index(3236370306236870045), Index(10198745931145551826), Index(11526777721718675946), Index(7855592439162647423), Index(17024415010144483949), Index(4229155328323504098), Index(5472111026499192469)]), DeliverEnvelopes([Index(4411786472941543389), Index(2352452285251361371), Index(7822068502251264399), Index(10423488690252370246), Index(6800625973591759468), Index(14891503574606637490), Index(4929072166198014740), Index(15521754316309705239), Index(8420972048384534331), Index(14271222805435196129), Index(15731413232976718648), Index(7620446100066998584), Index(16071224481319847817), Index(3144698519065113997), Index(2809650099793229888), Index(5232005717265660268), Index(7349929989771864370), Index(5385953262323237136), Index(5995048832340625664)]), DeliverEnvelopes([Index(17995955460911334433), Index(4708641667603374303), Index(7732664601632819026), Index(17423877575181254121), Index(13237365779440508178), Index(16219013419076737978)]), DeliverEnvelopes([Index(8304946145816664660), Index(862057291908146042), Index(15725440490586181656), Index(16995826471578986413), Index(2215118704959889028), Index(5868186664753940774), Index(8206076788005089132), Index(5701603538374868810), Index(7882633833049924931), Index(14920895499854492495), Index(9744657067126826847), Index(17217199825680194648)]), DeliverEnvelopes([Index(8037762189481305143), Index(7161454571813702078), Index(16624697899890776024), Index(6776753215496123550), Index(3552663706596513637), Index(13168137737732814449), Index(17773039112791415335), Index(5357529117839942678), Index(11360548594432633510), Index(17152936725964058392), Index(3735622760167456433), Index(3729434446292991499), Index(8239146570926600189), Index(17213753850553118615), Index(3586505624037292033), Index(5676170410763085337), Index(7854336347370863376), Index(3296193835517592123)]), DeliverEnvelopes([Index(7450515981053560543), Index(7281049720524507073), Index(11805621530410984780), Index(14212481354680952709), Index(11677201894475474096), Index(15579047847775346789), Index(8933612206363673270), Index(14589715969816048496), Index(9146028479816005688), Index(15678520830606285653), Index(13646715682213052692), Index(16542748546042997802), Index(3018629145641526806), Index(3279754668523137850), Index(9190080933222987746)]), DeliverEnvelopes([Index(4442754352407199000), Index(6726639451905550261), Index(276325053358083205), Index(8309639208446119622), Index(17071310980996705540), Index(13681920258353201127), Index(6824753331681591316), Index(9564449636752909493), Index(11781421339321328484), Index(17060651525797945236), Index(6256326521108731728), Index(882364767508101600), Index(18155289916698303405), Index(3604801106996280475), Index(18030768360505361161), Index(10828708533993573006), Index(10140811484372325170)]), DeliverEnvelopes([Index(14675779565613765883), Index(7875237876508366946), Index(1080598541644522468), Index(9795208880339871790), Index(16788173384975729322), Index(7862287353069942075), Index(18287036425108952119), Index(438554228774108801), Index(16936278258143324901), Index(17297858047970493225), Index(16280794788037943206)]), DeliverEnvelopes([Index(14318968306187897375), Index(15260180391589155604), Index(15065168303490293135), Index(16283608916485249870), Index(13275033248234956364), Index(2830029696975378334), Index(10826126602511348625), Index(3503489386636778134), Index(4143173347019462961), Index(14194382779242307605), Index(6222922790114912969), Index(4040121466287304927), Index(9113628229529655615), Index(5415084019849945730), Index(1626553538397648371), Index(15501687776809538824)]), DeliverEnvelopes([Index(5723406985380791968), Index(1155733174009260305), Index(7032746625126254072), Index(4743835759260085268), Index(7266303314157497289), Index(9385508013014315790), Index(17143596343958976419), Index(12019102422355865994), Index(17164688997776033771), Index(5601248314369099348), Index(8909509441657159134), Index(434646333244277371), Index(2059402500670808177), Index(10728320671205025035), Index(10839709495944012941), Index(13074991387519599432), Index(16684428541064080728), Index(1516391557911492208)]), PollPrepareAcks, Commit([Index(10338703738028056944), Index(18153730223737876722), Index(7878439710336920970), Index(16934827903768804991)]), DeliverNexusReplies(7), DeliverEnvelopes([Index(12043925503899430601), Index(13756482947028862137), Index(16512953710007648247), Index(17789811749467591337), Index(10320232143017481771), Index(1459556606276719356), Index(845781064116242142), Index(1956519909910251783), Index(3164835970372175816), Index(14120105518532170713), Index(10524629999594340195), Index(2646948465709084646), Index(15782024841931074195), Index(4857968548620380892), Index(10321553606781845802), Index(2234980294207553151), Index(14471054211349921145)]), DeliverEnvelopes([Index(3767742994858916851), Index(5436520998904493278), Index(10088257053026246598), Index(1498263247328214760), Index(14106058185302211861), Index(6061739775064231445), Index(2023011050214169455), Index(7250794870201811289), Index(17600053360168057888), Index(1740642731188709568), Index(2663072436983654497), Index(809252200146375679), Index(1869788575371162759), Index(4781549348184833023), Index(3523664822500907563)]), Commit([Index(14652573856526224379), Index(5781738428227077912), Index(12854715630867365273), Index(1911123227362573689)]), DeliverEnvelopes([Index(5483103026978400083), Index(14279025710451447544), Index(4937116848351103172), Index(16927987341169693780), Index(9896521330207311715)]), DeliverEnvelopes([Index(16187414459998690745), Index(65365848735262627), Index(16029309594081455593), Index(8738644556875519057), Index(8210105897167225670), Index(15476129597523288716), Index(5183379879833502204), Index(11717624468492776210), Index(15229030068062309892), Index(11851728872298706575), Index(10702820350636537827), Index(2687836824532707096), Index(2996236953690420395)]), Commit([Index(10707814609213298068), Index(12699776013897419563)]), Commit([Index(10157815118387593249), Index(5123540499752415756), Index(15808738534837068446)]), DeliverEnvelopes([Index(1541961047234018357), Index(1525243190754021203), Index(12707227425511543885), Index(1363732177719220534), Index(6262127490926569596), Index(15087875812043424072), Index(7757884893278869289), Index(11575998173089334269), Index(7651957407489583015), Index(384152082221813754), Index(13402749590934510602), Index(9025488436292943022)]), Commit([Index(2522165368668848218), Index(13296047016915538553), Index(15622931071258404786), Index(3114595942384622948), Index(12412617090084599671), Index(11664985786937734966)]), PollPrepareAcks, Commit([Index(13180524526118109937), Index(17193218842005402908), Index(17790813705747893301), Index(739262928517866423)]), PollPrepareAcks, PollPrepareAcks, DeliverNexusReplies(3), DeliverEnvelopes([Index(652695315058809065), Index(727847806737003567), Index(5404780742271511029), Index(16694185347801040155), Index(4098481543012673009), Index(7791187420263002075)]), Reconfigure { num_added_nodes: 1, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(199832759506365010), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9168374719730985799), Index(10794102811332111465), Index(17028470677432611953)]), DeliverNexusReplies(3), Reconfigure { num_added_nodes: 1, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(18233548140847603442), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, PollPrepareAcks, DeliverNexusReplies(7), DeliverNexusReplies(6), PollPrepareAcks, Reconfigure { num_added_nodes: 4, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(6273134768375120567), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverNexusReplies(1), DeliverNexusReplies(8), Reconfigure { num_added_nodes: 4, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(15266709888734590000), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9098454520105484266), Index(5401214547661529702), Index(729613142155688864), Index(18165312717926349510), Index(13596594098232187840), Index(17864609912832060176), Index(6576389962946327472), Index(6748861532334916574), Index(6270337184347778455), Index(4621189873065888654), Index(1329701404467837166), Index(15253391592315247523), Index(7548879285352433967), Index(1519898158722201861), Index(9221399647181265582), Index(14775495632976400799)]), DeliverNexusReplies(7), DeliverNexusReplies(7), DeliverEnvelopes([Index(6844824367736706288), Index(15644179546212865649), Index(4684448243389276339), Index(7196508453975880576)]), PollPrepareAcks, PollPrepareAcks, DeliverNexusReplies(8), DeliverEnvelopes([Index(5940056868932490196), Index(7905305070747045242), Index(18010824367591471345)]), PollPrepareAcks, DeliverEnvelopes([Index(11313344465270829281), Index(14770881005951866888), Index(14791521919982503413), Index(13986963003913738122), Index(15146722185811966483), Index(5670599236968951456), Index(17258904022024285117), Index(9970983405711455637), Index(1037749372107185535), Index(14590287308375860226), Index(14401361711946635290), Index(11544751816992154044)]), DeliverNexusReplies(2), DeliverNexusReplies(5), PollPrepareAcks, Commit([Index(4768387033456628201), Index(4474917877175302218), Index(16571851457975840247), Index(15255821624041417534), Index(16757765128366937769), Index(2630722755438967257), Index(9285796362852384853), Index(14015858312178579018)]), DeliverEnvelopes([Index(15871463961649635696), Index(18163198373939775702), Index(2600226515950816111)]), DeliverEnvelopes([Index(3137454651926879610), Index(4284065310722833297), Index(16055853410816331071), Index(15166506491941155629), Index(12590336857066018677)]), Commit([Index(5641127885216290645), Index(11167110485586573589), Index(1014364951816464696), Index(12696711847665095026), Index(17978099696654320596)]), DeliverNexusReplies(8), Commit([Index(4029921677534804644), Index(15185718783597090882), Index(11314695466132516141), Index(5987253657617228904), Index(12519437426641924797), Index(6263639313484088578), Index(1682708770096528814), Index(17514314903812047682), Index(11457587421741508124)]), Commit([Index(9774639841375717942), Index(10038533130474007843), Index(17060633271273876314), Index(7000629506295319285)]), DeliverEnvelopes([Index(15061341259133073864), Index(5027422415066345386), Index(9379255978562524739), Index(6150067533282387651), Index(213696060201653414), Index(16059755076226721578), Index(16824493282550752097), Index(15722153504680899440), Index(5759912639700558277), Index(4924300060842152804), Index(9327426990216358558), Index(17734827167724690660), Index(1357167295908663567), Index(13405304061122455105)]), Commit([Index(17426666830188603729), Index(12810851899054823276), Index(5888927766144618223), Index(18098665668806060067), Index(8508378395275761614), Index(1650853648920734436), Index(5663558801920224835)]), DeliverEnvelopes([Index(15295993199711016882), Index(3859998607405058373), Index(16246966108341368602), Index(2383124365549873480), Index(16408321263194864709), Index(14444156841149526789), Index(6495747943129340003), Index(17772779185757377240), Index(14604635448037405545), Index(4847143012976409235), Index(1145991671159544728), Index(6507514046328470924), Index(13464350535917834973), Index(14483501505596509725), Index(4511687336665279669), Index(9620101944261959773), Index(484390403697139876), Index(9857456322741422009), Index(3341222595136110266)]), PollPrepareAcks, DeliverNexusReplies(5), PollPrepareAcks, PollPrepareAcks, DeliverEnvelopes([Index(213757553854772440), Index(3794994835931277050), Index(1230995856688641619), Index(615222509626393634), Index(11235657579233697266), Index(380444266263585305), Index(17306335717947557462), Index(8806641371157136569), Index(544411057042609898), Index(14217584839742171129), Index(17179942378194078060), Index(18111746867895443277), Index(4705170374898011720), Index(3617427108480506863), Index(4184498722027448066), Index(17681194391468303007), Index(12378580499825910547), Index(1587437098921461308), Index(5646864724861626575)]), Commit([Index(2895265222191031484), Index(17950221019931177667), Index(9263853737651325899), Index(3842185010035964272), Index(12827291054333377474)]), Commit([Index(15759430573635913179), Index(9798702836106401259), Index(5201468168437920159), Index(2657315818353075169), Index(8828036642617900104), Index(16884723312357945697), Index(14437294211944068336)]), DeliverNexusReplies(5), DeliverEnvelopes([Index(7768905154491618039), Index(3438302915553662917), Index(14091186660758608634)]), Commit([Index(17866948588368032403), Index(11604953794048654451)]), DeliverNexusReplies(2), DeliverNexusReplies(7), Commit([Index(13712612038145219747), Index(12085627743433809088), Index(8699014604701166458), Index(2994093657804205227), Index(16225639718416622872), Index(6900536451203328895), Index(11227963432608776486), Index(13049675711087060279), Index(5214683319822193770)]), DeliverNexusReplies(4), PollPrepareAcks, DeliverNexusReplies(2), DeliverEnvelopes([Index(6044528710470823771), Index(7296331177112754699), Index(9457525726669633929), Index(5079497433348808932), Index(17348870535556670336), Index(512738896472721452), Index(12256764748176290216), Index(7604437520746076905), Index(6769996048460626115), Index(16864410288957109112), Index(3088590075318437038), Index(2095012960284709716)]), Commit([Index(15776963438323035828), Index(16592172527267608874), Index(4574510404731010758), Index(9801172050530399304), Index(12591565060149844827), Index(17758031771693043328), Index(2356648783237251688), Index(2403981669904249702), Index(4639941478079293934)])] } }