From 6b9cfd500a039ed4eae68a757c93bd06241d8a34 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Mon, 17 Mar 2025 13:41:42 -0400
Subject: [PATCH 01/20] rebase

---
 .gitignore                 |   4 +-
 Cargo.toml                 |   3 +-
 chain/src/bin/setup.rs     | 222 ++++++++++++++++++++++++++++++++++++-
 chain/src/bin/validator.rs |  27 +++--
 chain/src/lib.rs           |   2 +
 scripts/build.sh           |   6 +
 scripts/run.sh             |   5 +
 7 files changed, 256 insertions(+), 13 deletions(-)
 create mode 100755 scripts/build.sh
 create mode 100644 scripts/run.sh

diff --git a/.gitignore b/.gitignore
index 18dfef8f..1bfca715 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,6 @@
 /target
 *.vscode
 /chain/assets
-/types/pkg
\ No newline at end of file
+/types/pkg
+
+targets
\ No newline at end of file
diff --git a/Cargo.toml b/Cargo.toml
index a257cb96..07f04bdb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,7 +12,8 @@ alto-client = { version = "0.0.6", path = "client" }
 alto-types = { version = "0.0.6", path = "types" }
 commonware-consensus = { version = "0.0.43" }
 commonware-cryptography = { version = "0.0.43" }
-commonware-deployer = { version = "0.0.43" }
+# commonware-deployer = { version = "0.0.43" }
+commonware-deployer = { path = "..//cm-monorop/deployer" }
 commonware-macros = { version = "0.0.43" }
 commonware-p2p = { version = "0.0.43" }
 commonware-resolver = { version = "0.0.43" }
diff --git a/chain/src/bin/setup.rs b/chain/src/bin/setup.rs
index 08c7b403..df234629 100644
--- a/chain/src/bin/setup.rs
+++ b/chain/src/bin/setup.rs
@@ -8,10 +8,10 @@ use commonware_cryptography::{
     ed25519::PublicKey,
     Ed25519, Scheme,
 };
-use commonware_deployer::ec2;
+use commonware_deployer::ec2::{self};
 use commonware_utils::{from_hex_formatted, hex, quorum};
 use rand::{rngs::OsRng, seq::IteratorRandom};
-use std::{collections::BTreeMap, fs, ops::AddAssign};
+use std::{collections::BTreeMap, fs, path::Path, ops::AddAssign};
 use tracing::{error, info};
 use uuid::Uuid;
 
@@ -19,6 +19,7 @@ const BINARY_NAME: &str = "validator";
 const PORT: u16 = 4545;
 const STORAGE_CLASS: &str = "gp3";
 const DASHBOARD_FILE: &str = "dashboard.json";
+const MONITORING_PORT: u16 = 9000;
 
 fn main() {
     // Initialize logger
@@ -110,6 +111,65 @@ fn main() {
                         .value_parser(value_parser!(String)),
                 ),
         )
+        .subcommand(
+            Command::new("generate-local")
+                .about("Generate configuration files for an alto chain locally")
+                .arg(
+                    Arg::new("storage_dir")
+                        .long("storage-dir")
+                        .required(false)
+                        .default_value("/tmp/alto")
+                        .value_parser(value_parser!(String))
+                )
+                .arg(
+                    Arg::new("peers")
+                        .long("peers")
+                        .required(true)
+                        .value_parser(value_parser!(usize)),
+                )
+                .arg(
+                    Arg::new("bootstrappers")
+                        .long("bootstrappers")
+                        .required(true)
+                        .value_parser(value_parser!(usize)),
+                )
+                .arg(
+                    Arg::new("worker_threads")
+                        .long("worker-threads")
+                        .required(true)
+                        .value_parser(value_parser!(usize)),
+                )
+                .arg(
+                    Arg::new("log_level")
+                        .long("log-level")
+                        .required(true)
+                        .value_parser(value_parser!(String)),
+                )
+                .arg(
+                    Arg::new("message_backlog")
+                        .long("message-backlog")
+                        .required(true)
+                        .value_parser(value_parser!(usize)),
+                )
+                .arg(
+                    Arg::new("mailbox_size")
+                        .long("mailbox-size")
+                        .required(true)
+                        .value_parser(value_parser!(usize)),
+                )
+                .arg(
+                    Arg::new("dashboard")
+                        .long("dashboard")
+                        .required(true)
+                        .value_parser(value_parser!(String)),
+                )
+                .arg(
+                    Arg::new("output")
+                        .long("output")
+                        .required(true)
+                        .value_parser(value_parser!(String)),
+                ),
+        )
         .subcommand(
             Command::new("indexer")
                 .about("Add indexer support for an alto chain.")
@@ -155,6 +215,7 @@ fn main() {
     // Handle subcommands
     match matches.subcommand() {
         Some(("generate", sub_matches)) => generate(sub_matches),
+        Some(("generate-local", sub_matches)) => generate_local(sub_matches),
         Some(("indexer", sub_matches)) => indexer(sub_matches),
         Some(("explorer", sub_matches)) => explorer(sub_matches),
         _ => {
@@ -257,6 +318,8 @@ fn generate(sub_matches: &ArgMatches) {
             worker_threads,
             log_level: log_level.clone(),
 
+            metrics_port: 9090,
+
             allowed_peers: allowed_peers.clone(),
             bootstrappers: bootstrappers.clone(),
 
@@ -318,6 +381,161 @@ fn generate(sub_matches: &ArgMatches) {
     info!(path = "config.yaml", "wrote configuration file");
 }
 
+fn generate_local(sub_matches: &ArgMatches) {
+    // Extract arguments
+    let storage_dir = sub_matches.get_one::<String>("storage_dir").unwrap().clone();
+    let peers = *sub_matches.get_one::<usize>("peers").unwrap();
+    let bootstrappers = *sub_matches.get_one::<usize>("bootstrappers").unwrap();
+
+    let worker_threads = *sub_matches.get_one::<usize>("worker_threads").unwrap();
+    let log_level = sub_matches.get_one::<String>("log_level").unwrap().clone();
+    let message_backlog = *sub_matches.get_one::<usize>("message_backlog").unwrap();
+    let mailbox_size = *sub_matches.get_one::<usize>("mailbox_size").unwrap();
+    let dashboard = sub_matches.get_one::<String>("dashboard").unwrap().clone();
+    let output = sub_matches.get_one::<String>("output").unwrap().clone();
+
+    // Construct output path
+    let raw_current_dir = std::env::current_dir().unwrap();
+    let current_dir = raw_current_dir.to_str().unwrap();
+    let output = format!("{}/{}", current_dir, output);
+
+    // Check if output directory exists
+    if fs::metadata(&output).is_ok() {
+        error!("output directory already exists: {}", output);
+        std::process::exit(1);
+    }
+
+    // Generate UUID
+    let tag = Uuid::new_v4().to_string();
+    info!(tag, "generated deployment tag");
+
+    // Generate peers
+    assert!(
+        bootstrappers <= peers,
+        "bootstrappers must be less than or equal to peers"
+    );
+    let mut peer_schemes = (0..peers)
+        .map(|_| Ed25519::new(&mut OsRng))
+        .collect::<Vec<_>>();
+    peer_schemes.sort_by_key(|scheme| scheme.public_key());
+    let allowed_peers: Vec<String> = peer_schemes
+        .iter()
+        .map(|scheme| scheme.public_key().to_string())
+        .collect();
+    let bootstrappers = allowed_peers
+        .iter()
+        .choose_multiple(&mut OsRng, bootstrappers)
+        .into_iter()
+        .cloned()
+        .collect::<Vec<_>>();
+
+    // Generate consensus key
+    let peers_u32 = peers as u32;
+    let threshold = quorum(peers_u32).expect("unable to derive quorum");
+    let (identity, shares) = ops::generate_shares(&mut OsRng, None, peers_u32, threshold);
+    info!(
+        identity = hex(&poly::public(&identity).serialize()),
+        "generated network key"
+    );
+
+    // Generate instance configurations
+    let mut peer_configs = Vec::new();
+    let mut instance_configs = Vec::new();
+    let mut peers = Vec::new();
+
+    for (index, scheme) in peer_schemes.iter().enumerate() {
+        // Create peer config
+        let name = format!("validator{}", index);
+        let peer_config_file = format!("{}.yaml", name);
+
+        let peer_directory = Path::new(&storage_dir).join(format!("validator{}", index));
+        let peer_config = Config { private_key: scheme.private_key().to_string(),
+            share: hex(&shares[index].serialize()),
+            identity: hex(&identity.serialize()),
+
+            port: PORT + index as u16,
+            directory: peer_directory.to_string_lossy().into_owned(),
+            worker_threads,
+            log_level: log_level.clone(),
+
+            metrics_port: 9090 + index as u16,
+
+            allowed_peers: allowed_peers.clone(),
+            bootstrappers: bootstrappers.clone(),
+
+            message_backlog,
+            mailbox_size,
+
+            indexer: None,
+        };
+        peer_configs.push((peer_config_file.clone(), peer_config));
+
+        // Create instance config
+        let region = "local".to_string();
+        let instance = ec2::InstanceConfig {
+            name: scheme.public_key().to_string(),
+            region,
+            instance_type: "local".to_string(),
+            storage_size: 10,
+            storage_class: "local".to_string(),
+            binary: BINARY_NAME.to_string(),
+            config: peer_config_file,
+        };
+        instance_configs.push(instance);
+
+        // Create peer config
+        let peer = ec2::Peer {
+            name: scheme.public_key().to_string(),
+            region: "local".to_string(),
+            ip: "127.0.0.1".parse().expect("invalid IP address"),
+            port: PORT + index as u16,
+        };
+        peers.push(peer);
+    }
+
+    // Generate root config file
+    let config = ec2::Config {
+        tag,
+        instances: instance_configs,
+        monitoring: ec2::MonitoringConfig {
+            instance_type: "local".to_string(),
+            storage_size: 10,
+            storage_class: "local".to_string(),
+            dashboard: "dashboard.json".to_string(),
+        },
+        ports: vec![ec2::PortConfig {
+            protocol: "tcp".to_string(),
+            port: MONITORING_PORT,
+            cidr: "0.0.0.0/0".to_string(),
+        }],
+    };
+
+    // Write configuration files
+    fs::create_dir_all(&output).unwrap();
+    fs::copy(
+        format!("{}/{}", current_dir, dashboard),
+        format!("{}/dashboard.json", output),
+    )
+    .unwrap();
+    for (peer_config_file, peer_config) in peer_configs {
+        let path = format!("{}/{}", output, peer_config_file);
+        let file = fs::File::create(&path).unwrap();
+        serde_yaml::to_writer(file, &peer_config).unwrap();
+        info!(path = peer_config_file, "wrote peer configuration file");
+    }
+
+    let path = format!("{}/config.yaml", output);
+    let file = fs::File::create(&path).unwrap();
+    serde_yaml::to_writer(file, &config).unwrap();
+    info!(path = "config.yaml", "wrote configuration file");
+
+    let peers_path = format!("{}/peers.yaml", output);
+    let file = fs::File::create(peers_path).unwrap();
+    serde_yaml::to_writer(file, &ec2::Peers{peers}).unwrap();
+    info!(path = "peers.yaml", "wrote peers configuration file");
+}
+
+
 fn indexer(sub_matches: &ArgMatches) {
     // Extract arguments
     let count = *sub_matches.get_one::<usize>("count").unwrap();
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index 62ce4548..ba53fa32 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -17,7 +17,7 @@ use commonware_runtime::{tokio, Clock, Metrics, Network, Runner, Spawner};
 use commonware_utils::{from_hex_formatted, hex, quorum};
 use futures::future::try_join_all;
 use governor::Quota;
-use prometheus_client::metrics::gauge::Gauge;
+use prometheus_client::metrics::{self, gauge::Gauge};
 use std::{
     collections::HashMap,
     net::{IpAddr, Ipv4Addr, SocketAddr},
@@ -62,6 +62,11 @@ fn parse_log_level(level: &str) -> Option<Level> {
 }
 
 fn main() {
+    struct PeerAddr {
+        ip: IpAddr,
+        port: u16
+    }
+
     // Parse arguments
     let matches = Command::new("validator")
         .about("Validator for an alto chain.")
@@ -73,13 +78,16 @@ fn main() {
     let peer_file = matches.get_one::<String>("peers").unwrap();
     let peers_file = std::fs::read_to_string(peer_file).expect("Could not read peers file");
     let peers: Peers = serde_yaml::from_str(&peers_file).expect("Could not parse peers file");
-    let peers: HashMap<PublicKey, IpAddr> = peers
+    let peers: HashMap<PublicKey, PeerAddr> = peers
         .peers
         .into_iter()
         .map(|peer| {
             let key = from_hex_formatted(&peer.name).expect("Could not parse peer key");
             let key = PublicKey::try_from(key).expect("Peer key is invalid");
-            (key, peer.ip)
+            (key, PeerAddr {
+                ip: peer.ip,
+                port: peer.port
+            })
         })
         .collect();
     info!(peers = peers.len(), "loaded peers");
@@ -99,7 +107,8 @@ fn main() {
     let identity = poly::Public::deserialize(&identity, threshold).expect("Identity is invalid");
     let identity_public = *poly::public(&identity);
     let public_key = signer.public_key();
-    let ip = peers.get(&public_key).expect("Could not find self in IPs");
+    let metrics_port = config.metrics_port;
+    let ip = peers.get(&public_key).expect("Could not find self in IPs").ip;
     info!(
         ?public_key,
         identity = hex(&identity_public.serialize()),
@@ -123,8 +132,8 @@ fn main() {
     for bootstrapper in &config.bootstrappers {
         let key = from_hex_formatted(bootstrapper).expect("Could not parse bootstrapper key");
         let key = PublicKey::try_from(key).expect("Bootstrapper key is invalid");
-        let ip = peers.get(&key).expect("Could not find bootstrapper in IPs");
-        let bootstrapper_socket = format!("{}:{}", ip, config.port);
+        let peer_addr = peers.get(&key).expect("Could not find bootstrapper in IPs");
+        let bootstrapper_socket = format!("{}:{}", peer_addr.ip, peer_addr.port);
         let bootstrapper_socket = SocketAddr::from_str(&bootstrapper_socket)
             .expect("Could not parse bootstrapper socket");
         bootstrappers.push((key, bootstrapper_socket));
@@ -144,7 +153,7 @@ fn main() {
         signer.clone(),
         P2P_NAMESPACE,
         SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), config.port),
-        SocketAddr::new(*ip, config.port),
+        SocketAddr::new(ip, config.port),
         bootstrappers,
         MAX_MESSAGE_SIZE,
     );
@@ -278,8 +287,8 @@ fn main() {
         });
 
         // Serve metrics
-        let metrics = context.with_label("metrics").spawn(|context| async move {
-            let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), METRICS_PORT);
+        let metrics = context.with_label("metrics").spawn(move |context| async move {
+            let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), metrics_port);
             let listener = context
                 .bind(addr)
                 .await
diff --git a/chain/src/lib.rs b/chain/src/lib.rs
index 4e5bc2dc..9b10089c 100644
--- a/chain/src/lib.rs
+++ b/chain/src/lib.rs
@@ -69,6 +69,8 @@ pub struct Config {
     pub worker_threads: usize,
     pub log_level: String,
 
+    pub metrics_port: u16,
+
     pub allowed_peers: Vec<String>,
     pub bootstrappers: Vec<String>,
 
diff --git a/scripts/build.sh b/scripts/build.sh
new file mode 100755
index 00000000..216f62a3
--- /dev/null
+++ b/scripts/build.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+CWD=$(pwd)
+
+cd chain && cargo build --bin validator --target x86_64-unknown-linux-gnu --target-dir ../targets 
+cp ../targets/x86_64-unknown-linux-gnu/debug/validator $CWD
\ No newline at end of file
diff --git a/scripts/run.sh b/scripts/run.sh
new file mode 100644
index 00000000..20ca6e18
--- /dev/null
+++ b/scripts/run.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd ./chain
+
+NUM_VALIDATORS=${NUM_VALIDATORS:-5}
\ No newline at end of file

From 1f7ea0a3bf17d81f20b8643a3acc2a4228ad9277 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Fri, 14 Mar 2025 13:52:34 -0400
Subject: [PATCH 02/20] script to launch & stop network

---
 scripts/run.sh              |  5 -----
 scripts/start_validators.sh | 29 +++++++++++++++++++++++++++++
 scripts/stop_validators.sh  | 19 +++++++++++++++++++
 3 files changed, 48 insertions(+), 5 deletions(-)
 delete mode 100644 scripts/run.sh
 create mode 100755 scripts/start_validators.sh
 create mode 100755 scripts/stop_validators.sh

diff --git a/scripts/run.sh b/scripts/run.sh
deleted file mode 100644
index 20ca6e18..00000000
--- a/scripts/run.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-cd ./chain
-
-NUM_VALIDATORS=${NUM_VALIDATORS:-5}
\ No newline at end of file
diff --git a/scripts/start_validators.sh b/scripts/start_validators.sh
new file mode 100755
index 00000000..9bf88029
--- /dev/null
+++ b/scripts/start_validators.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+cd ./chain
+
+NUM_VALIDATORS=${NUM_VALIDATORS:-5}
+LOG_DIR=/tmp/alto-logs
+PID_FILE=${LOG_DIR}/validators.pid
+FLAG_FILE=${LOG_DIR}/network_running.flag
+
+# Check if a network is already running
+if [ -f ${FLAG_FILE} ]; then
+    echo "Existing network detected (flag file ${FLAG_FILE} exists). Aborting launch."
+    exit 1
+fi
+
+# Create log directory if it doesn't exist and clear any old PID file
+mkdir -p ${LOG_DIR}
+[ -f ${PID_FILE} ] && rm ${PID_FILE}
+
+# Create the flag file to indicate that the network is running
+touch ${FLAG_FILE}
+
+for i in $(seq 0 $(($NUM_VALIDATORS - 1))); do
+    echo "Launching validator $i..."
+    cargo run --bin validator -- --peers assets/peers.yaml --config assets/validator${i}.yaml > ${LOG_DIR}/validator${i}.log 2>&1 &
+    echo $! >> ${PID_FILE}
+done
+
+echo "Validators launched. PIDs stored in ${PID_FILE}."
diff --git a/scripts/stop_validators.sh b/scripts/stop_validators.sh
new file mode 100755
index 00000000..68dfa408
--- /dev/null
+++ b/scripts/stop_validators.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+LOG_DIR=/tmp/alto-logs
+PID_FILE=${LOG_DIR}/validators.pid
+FLAG_FILE=${LOG_DIR}/network_running.flag
+
+if [ ! -f ${PID_FILE} ]; then
+    echo "No validators are running (PID file not found)."
+    exit 1
+fi
+
+while read pid; do
+    echo "Stopping validator process with PID ${pid}..."
+    kill ${pid} 2>/dev/null
+done < ${PID_FILE}
+
+rm ${PID_FILE}
+rm -f ${FLAG_FILE}
+echo "All validators have been stopped and the network flag has been removed."

From 4031776507a1396afef4b7051cb9f73a22945663 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Mon, 17 Mar 2025 17:10:06 -0400
Subject: [PATCH 03/20] broadcast ckp

---
 Cargo.lock                              |  25 +++-
 Cargo.toml                              |   1 +
 chain/Cargo.toml                        |   1 +
 chain/src/actors/mempool/actor.rs       |  51 ++++++++
 chain/src/actors/mempool/collector.rs   | 154 ++++++++++++++++++++++++
 chain/src/actors/mempool/coordinator.rs |  80 ++++++++++++
 chain/src/actors/mempool/ingress.rs     |  44 +++++++
 chain/src/actors/mempool/mod.rs         |   4 +
 chain/src/actors/mod.rs                 |   1 +
 chain/src/bin/validator.rs              |  30 ++++-
 10 files changed, 385 insertions(+), 6 deletions(-)
 create mode 100644 chain/src/actors/mempool/actor.rs
 create mode 100644 chain/src/actors/mempool/collector.rs
 create mode 100644 chain/src/actors/mempool/coordinator.rs
 create mode 100644 chain/src/actors/mempool/ingress.rs
 create mode 100644 chain/src/actors/mempool/mod.rs

diff --git a/Cargo.lock b/Cargo.lock
index af3e7950..fb8ce354 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -45,6 +45,7 @@ dependencies = [
  "axum",
  "bytes",
  "clap",
+ "commonware-broadcast",
  "commonware-consensus",
  "commonware-cryptography",
  "commonware-deployer",
@@ -805,6 +806,28 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
+[[package]]
+name = "commonware-broadcast"
+version = "0.0.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2845a125a1996d82604970a3d72c4fd68e314432ff33023dafbde3a0d788ca2"
+dependencies = [
+ "bytes",
+ "commonware-cryptography",
+ "commonware-macros",
+ "commonware-p2p",
+ "commonware-runtime",
+ "commonware-storage",
+ "commonware-utils",
+ "futures",
+ "prometheus-client",
+ "prost",
+ "prost-build",
+ "rand",
+ "thiserror 2.0.12",
+ "tracing",
+]
+
 [[package]]
 name = "commonware-consensus"
 version = "0.0.43"
@@ -853,8 +876,6 @@ dependencies = [
 [[package]]
 name = "commonware-deployer"
 version = "0.0.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f986a14b443a53c96ad1831c9826e35dbb0aeccaad34a289fe4000d61d0a9c6"
 dependencies = [
  "aws-config",
  "aws-sdk-ec2",
diff --git a/Cargo.toml b/Cargo.toml
index 07f04bdb..a700ed5a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ resolver = "2"
 [workspace.dependencies]
 alto-client = { version = "0.0.6", path = "client" }
 alto-types = { version = "0.0.6", path = "types" }
+commonware-broadcast = { version = "0.0.43" }
 commonware-consensus = { version = "0.0.43" }
 commonware-cryptography = { version = "0.0.43" }
 # commonware-deployer = { version = "0.0.43" }
diff --git a/chain/Cargo.toml b/chain/Cargo.toml
index 20823756..127d3fde 100644
--- a/chain/Cargo.toml
+++ b/chain/Cargo.toml
@@ -13,6 +13,7 @@ documentation = "https://docs.rs/alto-chain"
 [dependencies]
 alto-types = { workspace = true }
 alto-client = { workspace = true }
+commonware-broadcast = { workspace = true }
 commonware-consensus = { workspace = true }
 commonware-cryptography = { workspace = true }
 commonware-deployer = { workspace = true }
diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
new file mode 100644
index 00000000..24b52073
--- /dev/null
+++ b/chain/src/actors/mempool/actor.rs
@@ -0,0 +1,51 @@
+use super:: ingress::{Mailbox, Message};
+use commonware_broadcast::{linked::Context, Application as A, Broadcaster};
+use commonware_cryptography::Digest;
+use commonware_utils::Array;
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt, StreamExt,
+};
+use tracing::error;
+
+
+pub struct Actor<D: Digest, P: Array> {
+    mailbox: mpsc::Receiver<Message<D, P>>,
+    // TODO: add a mempool structure here
+}
+
+impl<D: Digest, P: Array> Actor<D, P> {
+    pub fn new() -> (Self, Mailbox<D, P>) {
+        let (sender, receiver) = mpsc::channel(1024);
+        (Actor { mailbox: receiver }, Mailbox::new(sender))
+    }
+
+    pub async fn run(mut self, mut engine: impl Broadcaster<Digest = D>) {
+        while let Some(msg) = self.mailbox.next().await {
+            match msg {
+                Message::Broadcast(payload) => {
+                    let receiver = engine.broadcast(payload).await;
+                    let result = receiver.await;
+                    match result {
+                        Ok(true) => {}
+                        Ok(false) => {
+                            error!("broadcast returned false")
+                        }
+                        Err(_) => {
+                            error!("broadcast dropped")
+                        }
+                    }
+                }
+                Message::Verify(_context, _payload, sender) => {
+                    // TODO: add handler here to process batch received
+                    let result = sender.send(true);
+                    if result.is_err() {
+                        error!("verify dropped");
+                    }
+                }
+            }
+        }
+    }
+
+    // TODO: implement handler for data received
+}
diff --git a/chain/src/actors/mempool/collector.rs b/chain/src/actors/mempool/collector.rs
new file mode 100644
index 00000000..3c947958
--- /dev/null
+++ b/chain/src/actors/mempool/collector.rs
@@ -0,0 +1,154 @@
+use commonware_broadcast::{linked::Prover, Collector as Z, Proof, };
+use commonware_cryptography::{bls12381::primitives::group, Digest, Scheme};
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt, StreamExt,
+};
+use std::{
+    cmp::max,
+    collections::{BTreeMap, HashMap},
+};
+use tracing::error;
+
+enum Message<C: Scheme, D: Digest> {
+    Acknowledged(Proof, D),
+    GetTip(C::PublicKey, oneshot::Sender<Option<u64>>),
+    GetContiguousTip(C::PublicKey, oneshot::Sender<Option<u64>>),
+    Get(C::PublicKey, u64, oneshot::Sender<Option<D>>),
+}
+
+pub struct Collector<C: Scheme, D: Digest> {
+    mailbox: mpsc::Receiver<Message<C, D>>,
+
+    // Application namespace
+    namespace: Vec<u8>,
+
+    // Public key of the group
+    public: group::Public,
+
+    // All known digests
+    digests: HashMap<C::PublicKey, BTreeMap<u64, D>>,
+
+    // Highest contiguous known height for each sequencer
+    contiguous: HashMap<C::PublicKey, u64>,
+
+    // Highest known height for each sequencer
+    highest: HashMap<C::PublicKey, u64>,
+}
+
+impl<C: Scheme, D: Digest> Collector<C, D> {
+    pub fn new(namespace: &[u8], public: group::Public) -> (Self, Mailbox<C, D>) {
+        let (sender, receiver) = mpsc::channel(1024);
+        (
+            Collector {
+                mailbox: receiver,
+                namespace: namespace.to_vec(),
+                public,
+                digests: HashMap::new(),
+                contiguous: HashMap::new(),
+                highest: HashMap::new(),
+            },
+            Mailbox { sender },
+        )
+    }
+
+    pub async fn run(mut self) {
+        while let Some(msg) = self.mailbox.next().await {
+            match msg {
+                Message::Acknowledged(proof, payload) => {
+                    // Check proof.
+                    // The prover checks the validity of the threshold signature when deserializing
+                    let prover = Prover::<C, D>::new(&self.namespace, self.public);
+                    let context = match prover.deserialize_threshold(proof) {
+                        Some((context, _payload, _epoch, _threshold)) => context,
+                        None => {
+                            error!("invalid proof");
+                            continue;
+                        }
+                    };
+
+                    // Update the collector
+                    let digests = self.digests.entry(context.sequencer.clone()).or_default();
+                    digests.insert(context.height, payload);
+
+                    // Update the highest height
+                    let highest = self.highest.get(&context.sequencer).copied().unwrap_or(0);
+                    self.highest
+                        .insert(context.sequencer.clone(), max(highest, context.height));
+
+                    // Update the highest contiguous height
+                    let highest = self.contiguous.get(&context.sequencer);
+                    if (highest.is_none() && context.height == 0)
+                        || (highest.is_some() && context.height == highest.unwrap() + 1)
+                    {
+                        let mut contiguous = context.height;
+                        while digests.contains_key(&(contiguous + 1)) {
+                            contiguous += 1;
+                        }
+                        self.contiguous.insert(context.sequencer, contiguous);
+                    }
+                }
+                Message::GetTip(sequencer, sender) => {
+                    let hi = self.highest.get(&sequencer).copied();
+                    sender.send(hi).unwrap();
+                }
+                Message::GetContiguousTip(sequencer, sender) => {
+                    let contiguous = self.contiguous.get(&sequencer).copied();
+                    sender.send(contiguous).unwrap();
+                }
+                Message::Get(sequencer, height, sender) => {
+                    let digest = self
+                        .digests
+                        .get(&sequencer)
+                        .and_then(|map| map.get(&height))
+                        .cloned();
+                    sender.send(digest).unwrap();
+                }
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct Mailbox<C: Scheme, D: Digest> {
+    sender: mpsc::Sender<Message<C, D>>,
+}
+
+impl<C: Scheme, D: Digest> Z for Mailbox<C, D> {
+    type Digest = D;
+    async fn acknowledged(&mut self, proof: Proof, payload: Self::Digest) {
+        self.sender
+            .send(Message::Acknowledged(proof, payload))
+            .await
+            .expect("Failed to send acknowledged");
+    }
+}
+
+impl<C: Scheme, D: Digest> Mailbox<C, D> {
+    pub async fn get_tip(&mut self, sequencer: C::PublicKey) -> Option<u64> {
+        let (sender, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::GetTip(sequencer, sender))
+            .await
+            .unwrap();
+        receiver.await.unwrap()
+    }
+
+    pub async fn get_contiguous_tip(&mut self, sequencer: C::PublicKey) -> Option<u64> {
+        let (sender, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::GetContiguousTip(sequencer, sender))
+            .await
+            .unwrap();
+        receiver.await.unwrap()
+    }
+
+    pub async fn get(&mut self, sequencer: C::PublicKey, height: u64) -> Option<D> {
+        let (sender, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::Get(sequencer, height, sender))
+            .await
+            .unwrap();
+        receiver.await.unwrap()
+    }
+}
diff --git a/chain/src/actors/mempool/coordinator.rs b/chain/src/actors/mempool/coordinator.rs
new file mode 100644
index 00000000..3db14817
--- /dev/null
+++ b/chain/src/actors/mempool/coordinator.rs
@@ -0,0 +1,80 @@
+use commonware_broadcast::{linked::Epoch, Coordinator as S, ThresholdCoordinator as T};
+use commonware_cryptography::bls12381::primitives::{
+    group::{Public, Share},
+    poly::Poly,
+};
+use commonware_utils::Array;
+use std::collections::HashMap;
+
+// TODO: The implementation is copied from commonware-broadcast::linked::mocks, should be updated to track the 
+/// Implementation of `commonware-consensus::Coordinator`. 
+#[derive(Clone)]
+pub struct Coordinator<P: Array> {
+    view: u64,
+    identity: Poly<Public>,
+    signers: Vec<P>,
+    signers_map: HashMap<P, u32>,
+    share: Share,
+}
+
+impl<P: Array> Coordinator<P> {
+    pub fn new(identity: Poly<Public>, mut signers: Vec<P>, share: Share) -> Self {
+        // Setup signers
+        signers.sort();
+        let mut signers_map = HashMap::new();
+        for (index, validator) in signers.iter().enumerate() {
+            signers_map.insert(validator.clone(), index as u32);
+        }
+
+        // Return coordinator
+        Self {
+            view: 0,
+            identity,
+            signers,
+            signers_map,
+            share,
+        }
+    }
+
+    pub fn set_view(&mut self, view: u64) {
+        self.view = view;
+    }
+}
+
+impl<P: Array> S for Coordinator<P> {
+    type Index = Epoch;
+    type PublicKey = P;
+
+    fn index(&self) -> Self::Index {
+        self.view
+    }
+
+    fn signers(&self, _: Self::Index) -> Option<&Vec<Self::PublicKey>> {
+        Some(&self.signers)
+    }
+
+    fn is_signer(&self, _: Self::Index, candidate: &Self::PublicKey) -> Option<u32> {
+        self.signers_map.get(candidate).cloned()
+    }
+
+    fn sequencers(&self, _: Self::Index) -> Option<&Vec<Self::PublicKey>> {
+        Some(&self.signers)
+    }
+
+    fn is_sequencer(&self, _: Self::Index, candidate: &Self::PublicKey) -> Option<u32> {
+        self.signers_map.get(candidate).cloned()
+    }
+}
+
+impl<P: Array> T for Coordinator<P> {
+    type Identity = Poly<Public>;
+    type Share = Share;
+
+    fn identity(&self, _: Self::Index) -> Option<&Self::Identity> {
+        Some(&self.identity)
+    }
+
+    fn share(&self, _: Self::Index) -> Option<&Self::Share> {
+        Some(&self.share)
+    }
+}
diff --git a/chain/src/actors/mempool/ingress.rs b/chain/src/actors/mempool/ingress.rs
new file mode 100644
index 00000000..03827fe3
--- /dev/null
+++ b/chain/src/actors/mempool/ingress.rs
@@ -0,0 +1,44 @@
+use commonware_utils::Array;
+use commonware_cryptography::Digest;
+use commonware_broadcast::{linked::Context, Application as A, Broadcaster};
+use futures::{ channel::{mpsc, oneshot}, SinkExt};
+
+pub enum Message<D: Digest, P: Array> {
+    Broadcast(D),
+    Verify(Context<P>, D, oneshot::Sender<bool>),
+}
+
+#[derive(Clone)]
+pub struct Mailbox<D: Digest, P: Array> {
+    sender: mpsc::Sender<Message<D, P>>,
+}
+
+impl<D: Digest, P: Array> Mailbox<D, P> {
+    pub(super) fn new(sender: mpsc::Sender<Message<D, P>>) -> Self {
+        Self {
+            sender
+        }
+    }
+
+    pub async fn broadcast(&mut self, payload: D) {
+        let _ = self.sender.send(Message::Broadcast(payload)).await;
+    }
+}
+
+impl<D: Digest, P: Array> A for Mailbox<D, P> {
+    type Context = Context<P>;
+    type Digest = D;
+
+    async fn verify(
+        &mut self,
+        context: Self::Context,
+        payload: Self::Digest,
+    ) -> oneshot::Receiver<bool> {
+        let (sender, receiver) = oneshot::channel();
+        let _ = self
+            .sender
+            .send(Message::Verify(context, payload, sender))
+            .await;
+        receiver
+    }
+}
\ No newline at end of file
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
new file mode 100644
index 00000000..ca5e17a4
--- /dev/null
+++ b/chain/src/actors/mempool/mod.rs
@@ -0,0 +1,4 @@
+pub mod actor;
+pub mod ingress;
+pub mod coordinator;
+pub mod collector;
\ No newline at end of file
diff --git a/chain/src/actors/mod.rs b/chain/src/actors/mod.rs
index 9e920fb4..3bac1080 100644
--- a/chain/src/actors/mod.rs
+++ b/chain/src/actors/mod.rs
@@ -1,2 +1,3 @@
 pub mod application;
 pub mod syncer;
+pub mod mempool;
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index ba53fa32..4c399d8f 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -1,15 +1,14 @@
-use alto_chain::{engine, Config};
+use alto_chain::{actors::mempool, engine, Config};
 use alto_client::Client;
 use alto_types::P2P_NAMESPACE;
 use axum::{routing::get, serve, Extension, Router};
 use clap::{Arg, Command};
+use commonware_broadcast::linked;
 use commonware_cryptography::{
     bls12381::primitives::{
         group::{self, Element},
         poly,
-    },
-    ed25519::{PrivateKey, PublicKey},
-    Ed25519, Scheme,
+    }, ed25519::{PrivateKey, PublicKey}, sha256, Bls12381, Ed25519, Scheme
 };
 use commonware_deployer::ec2::Peers;
 use commonware_p2p::authenticated;
@@ -208,6 +207,28 @@ fn main() {
             indexer = Some(Client::new(&uri, identity_public.into()));
         }
 
+        // Create mempool/broadcast/Proof of Availability engine
+        let mempool_namespace = b"mempool";
+        let (mempool_application, mempool_mailbox) = mempool::actor::Actor::<sha256::Digest, PublicKey>::new();
+        let broadcast_coordinator = mempool::coordinator::Coordinator::new(identity.clone(), peer_keys.clone(), share);
+        let (broadcast_collector, collector_mailbox) = mempool::collector::Collector::<Ed25519, sha256::Digest>::new(mempool_namespace, identity_public);
+        let broadcast_engine = linked::Engine::new(context.with_label("broadcast_engine"), linked::Config { 
+            crypto:  signer, 
+            coordinator: broadcast_coordinator, 
+            application: mempool_mailbox, 
+            collector: collector_mailbox, 
+            mailbox_size: 1024, 
+            verify_concurrent: 1024,
+            namespace: mempool_namespace.to_vec(), 
+            refresh_epoch_timeout: Duration::from_millis(100), 
+            rebroadcast_timeout: Duration::from_secs(5),
+            epoch_bounds: (1,1), 
+            height_bound: 2, 
+            journal_name_prefix: format!("broadcast-linked-seq/{}/", public_key),
+            journal_heights_per_section: 10, 
+            journal_replay_concurrency: 1 
+        });
+
         // Create engine
         let config = engine::Config {
             partition_prefix: "engine".to_string(),
@@ -229,6 +250,7 @@ fn main() {
             fetch_rate_per_peer: resolver_limit,
             indexer,
         };
+
         let engine = engine::Engine::new(context.with_label("engine"), config).await;
 
         // Start engine

From 431d09601535f1434abbd027b0e2b1a0a6453101 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Mon, 17 Mar 2025 17:32:05 -0400
Subject: [PATCH 04/20] broadcaster

---
 chain/src/bin/validator.rs | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index 4c399d8f..83a629b3 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -36,6 +36,9 @@ const VOTER_CHANNEL: u32 = 0;
 const RESOLVER_CHANNEL: u32 = 1;
 const BROADCASTER_CHANNEL: u32 = 2;
 const BACKFILLER_CHANNEL: u32 = 3;
+const MEMPOOL_CHANNEL: u32 = 4;
+const MEMPOOL_ACK_CHANNEL: u32 = 5;
+
 
 const LEADER_TIMEOUT: Duration = Duration::from_secs(1);
 const NOTARIZATION_TIMEOUT: Duration = Duration::from_secs(2);
@@ -198,6 +201,24 @@ fn main() {
             Some(3),
         );
 
+        // Register mempool broadcast channel
+        let mempool_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
+        let mempool_broadcaster = network.register(
+            MEMPOOL_CHANNEL,
+            mempool_limit,
+            config.message_backlog,
+            Some(3),
+        );
+
+        let mempool_ack_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
+        let mempool_ack_broadcaster = network.register(
+            MEMPOOL_ACK_CHANNEL,
+            mempool_ack_limit,
+            config.message_backlog,
+            Some(3),
+        );
+        
+
         // Create network
         let p2p = network.start();
 
@@ -211,9 +232,9 @@ fn main() {
         let mempool_namespace = b"mempool";
         let (mempool_application, mempool_mailbox) = mempool::actor::Actor::<sha256::Digest, PublicKey>::new();
         let broadcast_coordinator = mempool::coordinator::Coordinator::new(identity.clone(), peer_keys.clone(), share);
-        let (broadcast_collector, collector_mailbox) = mempool::collector::Collector::<Ed25519, sha256::Digest>::new(mempool_namespace, identity_public);
-        let broadcast_engine = linked::Engine::new(context.with_label("broadcast_engine"), linked::Config { 
-            crypto:  signer, 
+        let (_, collector_mailbox) = mempool::collector::Collector::<Ed25519, sha256::Digest>::new(mempool_namespace, identity_public);
+        let (broadcast_engine, broadcast_mailbox) = linked::Engine::new(context.with_label("broadcast_engine"), linked::Config { 
+            crypto:  signer.clone(), 
             coordinator: broadcast_coordinator, 
             application: mempool_mailbox, 
             collector: collector_mailbox, 
@@ -228,6 +249,8 @@ fn main() {
             journal_heights_per_section: 10, 
             journal_replay_concurrency: 1 
         });
+        context.with_label("mempool").spawn(|_| mempool_application.run(broadcast_mailbox));
+        let broadcast_engine = broadcast_engine.start(mempool_broadcaster, mempool_ack_broadcaster);
 
         // Create engine
         let config = engine::Config {
@@ -327,7 +350,7 @@ fn main() {
         });
 
         // Wait for any task to error
-        if let Err(e) = try_join_all(vec![p2p, engine, system, metrics]).await {
+        if let Err(e) = try_join_all(vec![p2p, engine, broadcast_engine, system, metrics]).await {
             error!(?e, "task failed");
         }
     });

From 1ad9f3fc7e3504bead2af549899b906408fc5884 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Tue, 18 Mar 2025 11:11:29 -0400
Subject: [PATCH 05/20] unit test template

---
 chain/src/actors/mempool/mod.rs | 281 +++++++++++++++++++++++++++++++-
 1 file changed, 280 insertions(+), 1 deletion(-)

diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index ca5e17a4..0fb32c7c 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -1,4 +1,283 @@
 pub mod actor;
 pub mod ingress;
 pub mod coordinator;
-pub mod collector;
\ No newline at end of file
+pub mod collector;
+
+#[cfg(test)]
+mod tests {
+    use std::{collections::{BTreeMap, HashMap}, hash::Hash, sync::{Arc, Mutex}, time::Duration};
+    use bytes::Bytes;
+    use commonware_broadcast::linked::{Config, Engine};
+    use tracing::debug;
+
+    use commonware_cryptography::{bls12381::{self, dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
+    use commonware_macros::test_traced;
+    use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network};
+    use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
+    use futures::{channel::oneshot, future::join_all};
+
+    use super::collector;
+
+    type Registrations<P> = HashMap<P, ((Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>))>;
+
+    #[allow(dead_code)]
+    enum Action {
+        Link(Link),
+        Update(Link),
+        Unlink,
+    }
+
+    async fn register_validators(
+        oracle: &mut Oracle<PublicKey>,
+        validators: &[PublicKey],
+    ) -> HashMap<PublicKey, (
+        (Sender<PublicKey>, Receiver<PublicKey>),
+        (Sender<PublicKey>, Receiver<PublicKey>),
+    )> { 
+        let mut registrations = HashMap::new();        
+        for validator in validators.iter() {
+            let (chunk_sender, chunk_receiver) = oracle.register(validator.clone(), 4).await.unwrap();
+            let (ack_sender, ack_receiver) = oracle.register(validator.clone(), 5).await.unwrap();
+            registrations.insert(validator.clone(), (
+                (chunk_sender, chunk_receiver),
+                (ack_sender, ack_receiver),
+            ));
+        }
+        registrations
+    }
+
+    async fn link_validators(
+        oracle: &mut Oracle<PublicKey>,
+        validators: &[PublicKey],
+        link: Link,
+        restrict_to: Option<fn(usize, usize, usize) -> bool>,
+    ) {
+        for (i1, v1) in validators.iter().enumerate() {
+            for (i2, v2) in validators.iter().enumerate() {
+                // Ignore self
+                if v2 == v1 {
+                    continue;
+                }
+
+                // Restrict to certain connections
+                if let Some(f) = restrict_to {
+                    if !f(validators.len(), i1, i2) {
+                        continue;
+                    }
+                }
+
+                // Add link
+                oracle
+                    .add_link(v1.clone(), v2.clone(), link.clone())
+                    .await
+                    .unwrap();
+            }
+        }
+    }
+
+    async fn await_collectors(
+        context: Context,
+        collectors: &BTreeMap<PublicKey, collector::Mailbox<Ed25519, sha256::Digest>>,
+        threshold: u64,
+    ) {
+        let mut receivers = Vec::new();
+        for (sequencer, mailbox) in collectors.iter() {
+            // Create a oneshot channel to signal when the collector has reached the threshold.
+            let (tx, rx) = oneshot::channel();
+            receivers.push(rx);
+
+            // Spawn a watcher for the collector.
+            context.with_label("collector_watcher").spawn({
+                let sequencer = sequencer.clone();
+                let mut mailbox = mailbox.clone();
+                move |context| async move {
+                    loop {
+                        let tip = mailbox.get_tip(sequencer.clone()).await.unwrap_or(0);
+                        debug!(tip, ?sequencer, "collector");
+                        if tip >= threshold {
+                            let _ = tx.send(sequencer.clone());
+                            break;
+                        }
+                        context.sleep(Duration::from_millis(100)).await;
+                    }
+                }
+            });
+        }
+
+        // Wait for all oneshot receivers to complete.
+        let results = join_all(receivers).await;
+        assert_eq!(results.len(), collectors.len());
+    }
+
+        async fn initialize_simulation(
+        context: Context,
+        num_validators: u32,
+        shares_vec: &mut [Share],
+    ) -> (
+        Oracle<PublicKey>,
+        Vec<(PublicKey, Ed25519, Share)>,
+        Vec<PublicKey>,
+        Registrations<PublicKey>,
+    ) {
+        let (network, mut oracle) = Network::new(
+            context.with_label("network"),
+            commonware_p2p::simulated::Config {
+                max_size: 1024 * 1024,
+            },
+        );
+        network.start();
+
+        let mut schemes = (0..num_validators)
+            .map(|i| Ed25519::from_seed(i as u64))
+            .collect::<Vec<_>>();
+        schemes.sort_by_key(|s| s.public_key());
+        let validators: Vec<(PublicKey, Ed25519, Share)> = schemes
+            .iter()
+            .enumerate()
+            .map(|(i, scheme)| (scheme.public_key(), scheme.clone(), shares_vec[i]))
+            .collect();
+        let pks = validators
+            .iter()
+            .map(|(pk, _, _)| pk.clone())
+            .collect::<Vec<_>>();
+
+        let registrations = register_validators(&mut oracle, &pks).await;
+        let link = Link {
+            latency: 10.0,
+            jitter: 1.0,
+            success_rate: 1.0,
+        };
+        link_validators(&mut oracle, &pks, link, None).await;
+        (oracle, validators, pks, registrations)
+    }
+
+    fn spawn_proposer(
+        context: Context,
+        mailboxes: Arc<
+            Mutex<BTreeMap<PublicKey, super::ingress::Mailbox<sha256::Digest, PublicKey>>>,
+        >,
+        invalid_when: fn(u64) -> bool,
+    ) {
+        context
+            .clone()
+            .with_label("invalid signature proposer")
+            .spawn(move |context| async move {
+                let mut iter = 0;
+                loop {
+                    iter += 1;
+                    let mailbox_vec: Vec<super::ingress::Mailbox<sha256::Digest, PublicKey>> = {
+                        let guard = mailboxes.lock().unwrap();
+                        guard.values().cloned().collect()
+                    };
+                    for mut mailbox in mailbox_vec {
+                        let payload = Bytes::from(format!("hello world, iter {}", iter));
+                        let mut hasher = sha256::Sha256::default();
+                        hasher.update(&payload);
+
+                        // Inject an invalid digest by updating with the payload again.
+                        if invalid_when(iter) {
+                            hasher.update(&payload);
+                        }
+
+                        let digest = hasher.finalize();
+                        mailbox.broadcast(digest).await;
+                    }
+                    context.sleep(Duration::from_millis(250)).await;
+                }
+            });
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn spawn_validator_engines(
+        context: Context,
+        identity: poly::Public,
+        pks: &[PublicKey],
+        validators: &[(PublicKey, Ed25519, Share)],
+        registrations: &mut Registrations<PublicKey>,
+        mailboxes: &mut BTreeMap<PublicKey, super::ingress::Mailbox<sha256::Digest, PublicKey>>,
+        collectors: &mut BTreeMap<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>,
+        refresh_epoch_timeout: Duration,
+        rebroadcast_timeout: Duration,
+    ) {
+        let namespace = b"my testing namespace";
+        for (validator, scheme, share) in validators.iter() {
+            let context = context.with_label(&validator.to_string());
+            let mut coordinator = super::coordinator::Coordinator::<PublicKey>::new(
+                identity.clone(),
+                pks.to_vec(),
+                *share,
+            );
+            coordinator.set_view(111);
+
+            let (app, app_mailbox) =
+                super::actor::Actor::<sha256::Digest, PublicKey>::new();
+            mailboxes.insert(validator.clone(), app_mailbox.clone());
+
+            let (collector, collector_mailbox) =
+                super::collector::Collector::<Ed25519, sha256::Digest>::new(
+                    namespace,
+                    *poly::public(&identity),
+                );
+            context.with_label("collector").spawn(|_| collector.run());
+            collectors.insert(validator.clone(), collector_mailbox);
+
+            let (engine, mailbox) = Engine::new(
+                context.with_label("engine"),
+                Config {
+                    crypto: scheme.clone(),
+                    application: app_mailbox.clone(),
+                    collector: collectors.get(validator).unwrap().clone(),
+                    coordinator,
+                    mailbox_size: 1024,
+                    verify_concurrent: 1024,
+                    namespace: namespace.to_vec(),
+                    epoch_bounds: (1, 1),
+                    height_bound: 2,
+                    refresh_epoch_timeout,
+                    rebroadcast_timeout,
+                    journal_heights_per_section: 10,
+                    journal_replay_concurrency: 1,
+                    journal_name_prefix: format!("broadcast-linked-seq/{}/", validator),
+                },
+            );
+
+            context.with_label("app").spawn(|_| app.run(mailbox));
+            let ((a1, a2), (b1, b2)) = registrations.remove(validator).unwrap();
+            engine.start((a1, a2), (b1, b2));
+        }
+    }
+
+    #[test_traced]
+    fn test_all_online() {
+        let num_validators: u32 = 4;
+        let quorum: u32 = 3;
+        let (runner, mut context, _) = Executor::timed(Duration::from_secs(30));
+        let (identity, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
+        shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
+
+        runner.start(async move {
+            let (_oracle, validators, pks, mut registrations) = initialize_simulation(
+                context.with_label("simulation"), 
+                num_validators, 
+                &mut shares_vec).await;
+            let mailboxes = Arc::new(Mutex::new(BTreeMap::<
+                PublicKey,
+                super::ingress::Mailbox<sha256::Digest, PublicKey>,
+            >::new()));
+            let mut collectors = BTreeMap::<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>::new();
+            spawn_validator_engines(
+                context.with_label("validator"), 
+                identity.clone(), 
+                &pks, 
+                &validators, 
+                &mut registrations, 
+                &mut mailboxes.lock().unwrap(), 
+                &mut collectors, 
+                Duration::from_millis(100), 
+                Duration::from_secs(5)
+            );
+            spawn_proposer(context.with_label("proposer"), mailboxes.clone(), |_| false);
+            await_collectors(context.with_label("collector"), &collectors, 100).await;
+        });
+    }
+}
\ No newline at end of file

From 6fc323c8671fb0fce0b7125154fdd042e813b449 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Wed, 19 Mar 2025 17:00:48 -0400
Subject: [PATCH 06/20] mempool p2p test ckp

---
 Cargo.lock                          |   1 +
 Cargo.toml                          |   1 +
 chain/Cargo.toml                    |   1 +
 chain/src/actors/mempool/actor.rs   |   1 +
 chain/src/actors/mempool/ingress.rs |   6 +-
 chain/src/actors/mempool/mempool.rs | 283 ++++++++++++++++++++++++++++
 chain/src/actors/mempool/mod.rs     |  93 ++++++++-
 7 files changed, 379 insertions(+), 7 deletions(-)
 create mode 100644 chain/src/actors/mempool/mempool.rs

diff --git a/Cargo.lock b/Cargo.lock
index fb8ce354..b8bfe071 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -64,6 +64,7 @@ dependencies = [
  "serde_yaml",
  "sysinfo",
  "thiserror 2.0.12",
+ "tokio",
  "tracing",
  "tracing-subscriber",
  "uuid",
diff --git a/Cargo.toml b/Cargo.toml
index a700ed5a..a5fe08db 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,6 +34,7 @@ tracing-subscriber = "0.3.19"
 governor = "0.6.3"
 prometheus-client = "0.22.3"
 clap = "4.5.18"
+tokio = "1.43.0"
 
 [profile.bench]
 # Because we enable overflow checks in "release," we should benchmark with them.
diff --git a/chain/Cargo.toml b/chain/Cargo.toml
index 127d3fde..1acf082a 100644
--- a/chain/Cargo.toml
+++ b/chain/Cargo.toml
@@ -32,6 +32,7 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["fmt", "json"] }
 governor = { workspace = true }
 prometheus-client = { workspace = true }
+tokio = { workspace = true , features = ["time"] }
 clap = { workspace = true }
 sysinfo = "0.33.1"
 axum = "0.8.1"
diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 24b52073..54cc9fcb 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -21,6 +21,7 @@ impl<D: Digest, P: Array> Actor<D, P> {
     }
 
     pub async fn run(mut self, mut engine: impl Broadcaster<Digest = D>) {
+        // it passes msgs in the mailbox of the actor to the engine mailbox
         while let Some(msg) = self.mailbox.next().await {
             match msg {
                 Message::Broadcast(payload) => {
diff --git a/chain/src/actors/mempool/ingress.rs b/chain/src/actors/mempool/ingress.rs
index 03827fe3..ee4584b7 100644
--- a/chain/src/actors/mempool/ingress.rs
+++ b/chain/src/actors/mempool/ingress.rs
@@ -1,8 +1,12 @@
-use commonware_utils::Array;
+use commonware_utils::{Array, SizedSerialize};
 use commonware_cryptography::Digest;
 use commonware_broadcast::{linked::Context, Application as A, Broadcaster};
 use futures::{ channel::{mpsc, oneshot}, SinkExt};
 
+pub struct Payload {
+    data: Vec<u8>
+}
+
 pub enum Message<D: Digest, P: Array> {
     Broadcast(D),
     Verify(Context<P>, D, oneshot::Sender<bool>),
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
new file mode 100644
index 00000000..6f8cd179
--- /dev/null
+++ b/chain/src/actors/mempool/mempool.rs
@@ -0,0 +1,283 @@
+use core::num;
+use std::{collections::HashMap, time::{Duration, SystemTime}};
+
+use bytes::{Buf, BufMut, Bytes};
+use commonware_cryptography::{ed25519::PublicKey, sha256::Digest, Hasher, Sha256};
+use commonware_p2p::{Receiver, Recipients, Sender};
+use commonware_runtime::{Clock, Handle, Spawner};
+use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
+use commonware_macros::select;
+use tokio::{io::AsyncReadExt, time};
+use tracing::{debug, warn};
+
+#[derive(Clone)]
+pub struct Batch  {
+    pub txs: Vec<RawTransaction>,
+
+    pub digest: Digest,
+    // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
+    pub accepted: bool, 
+    pub timestamp: SystemTime,
+}
+
+impl Batch {
+    fn compute_digest(txs: &Vec<RawTransaction>) -> Digest {
+        let mut hasher = Sha256::new();
+        for tx in txs.into_iter() {
+            hasher.update(tx.raw.as_ref());
+        }
+
+        hasher.finalize()
+    }
+
+    pub fn new(txs: Vec<RawTransaction>, timestamp: SystemTime) -> Self {
+        let digest = Self::compute_digest(&txs);
+
+        Self {
+            txs,
+            digest,
+            accepted: false,
+            timestamp 
+        }
+    }
+
+    pub fn serialize(&self) -> Vec<u8> {
+        let mut bytes = Vec::new();
+        bytes.put_u64(self.txs.len() as u64);
+        for tx in self.txs.iter() {
+            bytes.put_u64(tx.size());
+            bytes.extend_from_slice(&tx.raw);
+        }
+        bytes
+    }
+
+    pub fn deserialize(mut bytes: &[u8]) -> Option<Self> {
+        use bytes::Buf;
+        // We expect at least 8 bytes for the number of transactions.
+        if bytes.remaining() < 8 {
+            return None;
+        }
+        let tx_count = bytes.get_u64();
+        let mut txs = Vec::with_capacity(tx_count as usize);
+        for _ in 0..tx_count {
+            // For each transaction, first read the size (u64).
+            if bytes.remaining() < 8 {
+                return None;
+            }
+            let tx_size = bytes.get_u64() as usize;
+            // Ensure there are enough bytes left.
+            if bytes.remaining() < tx_size {
+                return None;
+            }
+            // Extract tx_size bytes.
+            let tx_bytes = bytes.copy_to_bytes(tx_size);
+            txs.push(RawTransaction::new(tx_bytes));
+        }
+        // Compute the digest from the transactions.
+        let digest = Self::compute_digest(&txs);
+        // Since serialize did not include accepted and timestamp, we set accepted to false
+        // and set timestamp to the current time.
+        Some(Self {
+            txs,
+            digest,
+            accepted: false,
+            timestamp: SystemTime::now(),
+        })
+    }
+
+    pub fn contain_tx(&self, digest: &Digest) -> bool {
+        self.txs.iter().any(|tx| &tx.digest == digest) 
+    }
+
+    pub fn tx(&self, digest: &Digest) -> Option<RawTransaction> {
+        self.txs.iter().find(|tx| &tx.digest == digest).cloned()
+    }
+}
+
+#[derive(Clone)]
+pub struct RawTransaction {
+    pub raw: Bytes,
+
+    pub digest: Digest
+}
+
+impl RawTransaction {
+    fn compute_digest(raw: &Bytes) -> Digest {
+        let mut hasher = Sha256::new();
+        hasher.update(&raw);
+        hasher.finalize()
+    }
+
+    pub fn new(raw: Bytes) -> Self {
+        let digest = Self::compute_digest(&raw);
+        Self {
+            raw,
+            digest
+        }
+    }
+
+    pub fn validate(&self) -> bool {
+        // TODO: implement validate here
+        true
+    }
+
+    pub fn size(&self) -> u64 {
+        self.raw.len() as u64
+    }
+}
+
+pub enum Message {
+    // mark batch as accepted by the netowrk through the broadcast protocol
+    BatchAccepted {
+        digest: Digest,
+    },
+    // from rpc or websocket
+    SubmitTx {
+        payload: RawTransaction,
+        response: oneshot::Sender<bool>
+    },
+    GetTx {
+        digest: Digest,
+        response: oneshot::Sender<Option<RawTransaction>>
+    },
+}
+
+#[derive(Clone)]
+pub struct Mailbox {
+    sender: mpsc::Sender<Message>
+}
+
+impl Mailbox {
+    pub fn new(sender: mpsc::Sender<Message>) -> Self {
+        Self {
+            sender
+        }
+    }
+
+    pub async fn issue_tx(&mut self, tx: RawTransaction) -> bool {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::SubmitTx { payload: tx, response: response })
+            .await
+            .expect("failed to issue tx");
+
+        receiver.await.expect("failed to receive tx issue status")
+    }
+
+    pub async fn get_tx(&mut self, digest: Digest) -> Option<RawTransaction> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::GetTx { digest: digest, response: response })
+            .await
+            .expect("failed to get tx");
+
+        receiver.await.expect("failed to receive tx")
+    }
+}
+
+pub struct Config {
+    pub batch_propose_interval: Duration,
+    pub batch_size_limit: u64,
+}
+
+pub struct Mempool<R: Clock + Spawner> {
+    cfg: Config,
+    context: R,
+
+    batches: HashMap<Digest, Batch>,
+    txs: Vec<RawTransaction>,
+
+    mailbox: mpsc::Receiver<Message>
+}
+
+impl<R: Clock + Spawner> Mempool<R> {
+    pub fn new(context: R, cfg: Config) -> (Self, Mailbox) {
+        let (sender, receiver) = mpsc::channel(1024);
+        (Self {
+            cfg, 
+            context,
+            mailbox: receiver,
+            txs: vec![],
+            batches: HashMap::new(),
+        }, Mailbox::new(sender))
+    }
+
+    pub fn start(
+        mut self,
+        batch_network: (
+            impl Sender<PublicKey = PublicKey>,
+            impl Receiver<PublicKey = PublicKey>,
+        )
+    ) -> Handle<()> {
+        self.context.spawn_ref()(self.run(batch_network))
+    }
+
+    async fn run(
+        mut self,
+        mut batch_network: (
+            impl Sender<PublicKey = PublicKey>,
+            impl Receiver<PublicKey = PublicKey>,
+        ), 
+    ) {
+
+        let mut batch_propose_interval = time::interval(self.cfg.batch_propose_interval);
+        loop {
+            select! {
+                mailbox_message = self.mailbox.next() => {
+                    let message = mailbox_message.expect("Mailbox closed");
+                    match message {
+                        Message::SubmitTx { payload, response } => {
+                            if !payload.validate() {
+                                let _ = response.send(false);
+                                return;    
+                            }
+
+                            self.txs.push(payload);
+                            let _ = response.send(true);
+                        },
+                        Message::BatchAccepted { digest } => {
+                            if let Some(batch) = self.batches.get_mut(&digest) {
+                                batch.accepted = true;
+                            }
+                        },
+                        Message::GetTx { digest, response } => {
+                            // TODO: optimize this naive way of seaching
+                            let pair = self.batches.iter().find(|(_, batch)| batch.contain_tx(&digest));
+                            if let Some((_, batch)) = pair {
+                                let _ = response.send(batch.tx(&digest));
+                            } else {
+                                let _ = response.send(None);
+                            }
+                        },
+                    }
+                },
+                _ = batch_propose_interval.tick() => {
+                    let mut size = 0;
+                    let mut txs_cnt = 0;
+                    for tx in self.txs.iter() {
+                        size += tx.size();
+                        txs_cnt += 1;
+
+                        if size > self.cfg.batch_size_limit {
+                            break;
+                        }
+                    }
+
+                    let batch = Batch::new(self.txs.drain(..txs_cnt).collect(), self.context.current());
+                    self.batches.insert(batch.digest, batch.clone());
+                    batch_network.0.send(Recipients::All, batch.serialize().into(), true).await.expect("failed to broadcast batch");
+                },
+                batch_message = batch_network.1.recv() => {
+                    let (sender, message) = batch_message.expect("Batch broadcast closed");
+                    let Some(batch) = Batch::deserialize(&message) else {
+                        warn!(?sender, "failed to deserialize batch");
+                        continue;
+                    };
+
+                    debug!(?sender, digest=?batch.digest, "received batch");
+                    let _ = self.batches.entry(batch.digest).or_insert(batch);
+                },
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 0fb32c7c..162c121c 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -2,13 +2,16 @@ pub mod actor;
 pub mod ingress;
 pub mod coordinator;
 pub mod collector;
+pub mod mempool;
 
 #[cfg(test)]
 mod tests {
+    use core::{num, panic};
     use std::{collections::{BTreeMap, HashMap}, hash::Hash, sync::{Arc, Mutex}, time::Duration};
     use bytes::Bytes;
     use commonware_broadcast::linked::{Config, Engine};
-    use tracing::debug;
+    use commonware_storage::mmr::mem;
+    use tracing::{debug, warn};
 
     use commonware_cryptography::{bls12381::{self, dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
     use commonware_macros::test_traced;
@@ -16,9 +19,9 @@ mod tests {
     use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
     use futures::{channel::oneshot, future::join_all};
 
-    use super::collector;
+    use super::{collector, mempool::{self, Mempool, RawTransaction}};
 
-    type Registrations<P> = HashMap<P, ((Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>))>;
+    type Registrations<P> = HashMap<P, ((Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>))>;
 
     #[allow(dead_code)]
     enum Action {
@@ -33,14 +36,17 @@ mod tests {
     ) -> HashMap<PublicKey, (
         (Sender<PublicKey>, Receiver<PublicKey>),
         (Sender<PublicKey>, Receiver<PublicKey>),
+        (Sender<PublicKey>, Receiver<PublicKey>),
     )> { 
         let mut registrations = HashMap::new();        
         for validator in validators.iter() {
-            let (chunk_sender, chunk_receiver) = oracle.register(validator.clone(), 4).await.unwrap();
+            let (digest_sender, digest_receiver) = oracle.register(validator.clone(), 4).await.unwrap();
             let (ack_sender, ack_receiver) = oracle.register(validator.clone(), 5).await.unwrap();
+            let (chunk_sender, chunk_receiver) = oracle.register(validator.clone(), 6).await.unwrap();
             registrations.insert(validator.clone(), (
-                (chunk_sender, chunk_receiver),
+                (digest_sender, digest_receiver),
                 (ack_sender, ack_receiver),
+                (chunk_sender, chunk_receiver),
             ));
         }
         registrations
@@ -242,11 +248,70 @@ mod tests {
             );
 
             context.with_label("app").spawn(|_| app.run(mailbox));
-            let ((a1, a2), (b1, b2)) = registrations.remove(validator).unwrap();
+            let ((a1, a2), (b1, b2), (_, _)) = registrations.remove(validator).unwrap();
             engine.start((a1, a2), (b1, b2));
         }
     }
 
+    fn spawn_mempools(
+        context: Context,
+        validators: &[(PublicKey, Ed25519, Share)],
+        registrations: &mut Registrations<PublicKey>,
+        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox>,
+    ) {
+        for (validator, _, _) in validators.iter() {
+            let (mempool, mailbox) = Mempool::new(
+                context.with_label("mempool"), 
+                mempool::Config {
+                    batch_propose_interval: Duration::from_millis(500),
+                    batch_size_limit: 1024*1024, // 1MB
+                }
+            );
+            mailboxes.insert(validator.clone(), mailbox);
+            let ((_, _), (_, _), (c1, c2)) = registrations.remove(validator).unwrap();
+            mempool.start((c1, c2));
+        }
+    }
+
+    fn spawn_tx_issuer_and_wait(
+        context: Context,
+        mailboxes: Arc<Mutex<BTreeMap<PublicKey, mempool::Mailbox>>>,
+        num_txs: u32,
+    ) {
+        context
+            .clone()
+            .with_label("tx issuer")
+            .spawn(move |context| async move {
+                let mut mailbox_vec: Vec<mempool::Mailbox> = {
+                    let guard = mailboxes.lock().unwrap();
+                    guard.values().cloned().collect()
+                };
+
+                let Some(mut mailbox)= mailbox_vec.pop() else {
+                    panic!("no single mailbox provided");
+                };
+
+                
+                // issue tx to the first validator
+                let mut digests = Vec::new();
+                for i in 1..num_txs {
+                    let tx = RawTransaction::new(Bytes::from(format!("tx-{}", i)));
+                    let submission_res = mailbox.issue_tx(tx.clone()).await;
+                    if !submission_res {
+                        warn!(?tx.digest, "failed to submit tx");
+                    }
+                    digests.push(tx.digest);
+                }
+
+                // check if the tx appear in other validators
+                for mut mailbox in mailbox_vec {
+                    for digest in digests.iter() {
+
+                    }
+                }
+            });
+    }
+
     #[test_traced]
     fn test_all_online() {
         let num_validators: u32 = 4;
@@ -280,4 +345,20 @@ mod tests {
             await_collectors(context.with_label("collector"), &collectors, 100).await;
         });
     }
+
+    #[test_traced]
+    fn test_mempool_p2p() {
+        let num_validators: u32 = 4;
+        let quorum: u32 = 3;
+        let (runner, mut context, _) = Executor::timed(Duration::from_secs(30));
+        let (_, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
+        shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
+
+        runner.start(async move {
+            let (_oracle, validators, pks, mut registrations ) = initialize_simulation(
+                context.with_label("simulation"), 
+                num_validators, 
+                &mut shares_vec).await;
+        });
+    }
 }
\ No newline at end of file

From ed55fdf5e43c6e47c74c2320868c8a0c6d679c3c Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 20 Mar 2025 10:53:52 -0400
Subject: [PATCH 07/20] mempool p2p test

---
 chain/src/actors/mempool/actor.rs   |  6 ++---
 chain/src/actors/mempool/ingress.rs |  4 ++--
 chain/src/actors/mempool/mempool.rs |  9 ++++----
 chain/src/actors/mempool/mod.rs     | 36 +++++++++++++++++++++++------
 chain/src/bin/validator.rs          |  6 ++---
 5 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 54cc9fcb..6268ee85 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -1,10 +1,10 @@
 use super:: ingress::{Mailbox, Message};
-use commonware_broadcast::{linked::Context, Application as A, Broadcaster};
+use commonware_broadcast::Broadcaster;
 use commonware_cryptography::Digest;
 use commonware_utils::Array;
 use futures::{
-    channel::{mpsc, oneshot},
-    SinkExt, StreamExt,
+    channel::mpsc,
+    StreamExt,
 };
 use tracing::error;
 
diff --git a/chain/src/actors/mempool/ingress.rs b/chain/src/actors/mempool/ingress.rs
index ee4584b7..f2e7050e 100644
--- a/chain/src/actors/mempool/ingress.rs
+++ b/chain/src/actors/mempool/ingress.rs
@@ -1,6 +1,6 @@
-use commonware_utils::{Array, SizedSerialize};
+use commonware_utils::Array;
 use commonware_cryptography::Digest;
-use commonware_broadcast::{linked::Context, Application as A, Broadcaster};
+use commonware_broadcast::{linked::Context, Application as A};
 use futures::{ channel::{mpsc, oneshot}, SinkExt};
 
 pub struct Payload {
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 6f8cd179..790f8d9f 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,13 +1,12 @@
-use core::num;
 use std::{collections::HashMap, time::{Duration, SystemTime}};
 
-use bytes::{Buf, BufMut, Bytes};
+use bytes::{BufMut, Bytes};
 use commonware_cryptography::{ed25519::PublicKey, sha256::Digest, Hasher, Sha256};
 use commonware_p2p::{Receiver, Recipients, Sender};
 use commonware_runtime::{Clock, Handle, Spawner};
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
-use tokio::{io::AsyncReadExt, time};
+use tokio::time;
 use tracing::{debug, warn};
 
 #[derive(Clone)]
@@ -157,7 +156,7 @@ impl Mailbox {
     pub async fn issue_tx(&mut self, tx: RawTransaction) -> bool {
         let (response, receiver) = oneshot::channel();
         self.sender
-            .send(Message::SubmitTx { payload: tx, response: response })
+            .send(Message::SubmitTx { payload: tx, response })
             .await
             .expect("failed to issue tx");
 
@@ -167,7 +166,7 @@ impl Mailbox {
     pub async fn get_tx(&mut self, digest: Digest) -> Option<RawTransaction> {
         let (response, receiver) = oneshot::channel();
         self.sender
-            .send(Message::GetTx { digest: digest, response: response })
+            .send(Message::GetTx { digest, response })
             .await
             .expect("failed to get tx");
 
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 162c121c..c259b337 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -6,14 +6,14 @@ pub mod mempool;
 
 #[cfg(test)]
 mod tests {
-    use core::{num, panic};
-    use std::{collections::{BTreeMap, HashMap}, hash::Hash, sync::{Arc, Mutex}, time::Duration};
+    use core::panic;
+    use std::{collections::{BTreeMap, HashMap}, sync::{Arc, Mutex}, time::Duration};
     use bytes::Bytes;
     use commonware_broadcast::linked::{Config, Engine};
-    use commonware_storage::mmr::mem;
-    use tracing::{debug, warn};
+    
+    use tracing::{debug, info, warn};
 
-    use commonware_cryptography::{bls12381::{self, dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
+    use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
     use commonware_macros::test_traced;
     use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network};
     use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
@@ -287,6 +287,10 @@ mod tests {
                     guard.values().cloned().collect()
                 };
 
+                if mailbox_vec.len() <= 1 {
+                    panic!("insuffient mempool nodes spawned, have {}", mailbox_vec.len());
+                }
+
                 let Some(mut mailbox)= mailbox_vec.pop() else {
                     panic!("no single mailbox provided");
                 };
@@ -294,19 +298,31 @@ mod tests {
                 
                 // issue tx to the first validator
                 let mut digests = Vec::new();
-                for i in 1..num_txs {
+                for i in 0..num_txs {
                     let tx = RawTransaction::new(Bytes::from(format!("tx-{}", i)));
                     let submission_res = mailbox.issue_tx(tx.clone()).await;
                     if !submission_res {
                         warn!(?tx.digest, "failed to submit tx");
+                        continue;
                     }
+                    debug!("tx submitted: {}", tx.digest);
                     digests.push(tx.digest);
                 }
 
+                if digests.len() == 0 {
+                    panic!("zero txs issued");
+                }
+
+                context.sleep(Duration::from_secs(5)).await;
+
                 // check if the tx appear in other validators
                 for mut mailbox in mailbox_vec {
                     for digest in digests.iter() {
+                        let Some(tx) = mailbox.get_tx(digest.clone()).await else {
+                            panic!("digest: {} not found at mailbox", digest);
+                        };
 
+                        info!("tx found at mempool: {}", tx.digest);
                     }
                 }
             });
@@ -355,10 +371,16 @@ mod tests {
         shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
 
         runner.start(async move {
-            let (_oracle, validators, pks, mut registrations ) = initialize_simulation(
+            let (_oracle, validators, _, mut registrations ) = initialize_simulation(
                 context.with_label("simulation"), 
                 num_validators, 
                 &mut shares_vec).await;
+            let mailboxes = Arc::new(Mutex::new(BTreeMap::<
+                PublicKey,
+                mempool::Mailbox,
+            >::new()));
+            spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut mailboxes.lock().unwrap());
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1);
         });
     }
 }
\ No newline at end of file
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index 83a629b3..6b6546be 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -8,7 +8,7 @@ use commonware_cryptography::{
     bls12381::primitives::{
         group::{self, Element},
         poly,
-    }, ed25519::{PrivateKey, PublicKey}, sha256, Bls12381, Ed25519, Scheme
+    }, ed25519::{PrivateKey, PublicKey}, sha256, Ed25519, Scheme
 };
 use commonware_deployer::ec2::Peers;
 use commonware_p2p::authenticated;
@@ -16,7 +16,7 @@ use commonware_runtime::{tokio, Clock, Metrics, Network, Runner, Spawner};
 use commonware_utils::{from_hex_formatted, hex, quorum};
 use futures::future::try_join_all;
 use governor::Quota;
-use prometheus_client::metrics::{self, gauge::Gauge};
+use prometheus_client::metrics::{gauge::Gauge};
 use std::{
     collections::HashMap,
     net::{IpAddr, Ipv4Addr, SocketAddr},
@@ -30,7 +30,7 @@ use sysinfo::{Disks, System};
 use tracing::{error, info, Level};
 
 const SYSTEM_METRICS_REFRESH: Duration = Duration::from_secs(5);
-const METRICS_PORT: u16 = 9090;
+// const METRICS_PORT: u16 = 9090;
 
 const VOTER_CHANNEL: u32 = 0;
 const RESOLVER_CHANNEL: u32 = 1;

From 511c3b769e533a235f743837fb773b81c19a0a6c Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 20 Mar 2025 13:23:10 -0400
Subject: [PATCH 08/20] before broadcast

---
 Cargo.lock                          |  1 -
 Cargo.toml                          |  1 -
 chain/Cargo.toml                    |  1 -
 chain/src/actors/mempool/mempool.rs | 25 +++++++++++++++++++------
 chain/src/actors/mempool/mod.rs     | 14 ++++++++++----
 5 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b8bfe071..fb8ce354 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -64,7 +64,6 @@ dependencies = [
  "serde_yaml",
  "sysinfo",
  "thiserror 2.0.12",
- "tokio",
  "tracing",
  "tracing-subscriber",
  "uuid",
diff --git a/Cargo.toml b/Cargo.toml
index a5fe08db..a700ed5a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,7 +34,6 @@ tracing-subscriber = "0.3.19"
 governor = "0.6.3"
 prometheus-client = "0.22.3"
 clap = "4.5.18"
-tokio = "1.43.0"
 
 [profile.bench]
 # Because we enable overflow checks in "release," we should benchmark with them.
diff --git a/chain/Cargo.toml b/chain/Cargo.toml
index 1acf082a..127d3fde 100644
--- a/chain/Cargo.toml
+++ b/chain/Cargo.toml
@@ -32,7 +32,6 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["fmt", "json"] }
 governor = { workspace = true }
 prometheus-client = { workspace = true }
-tokio = { workspace = true , features = ["time"] }
 clap = { workspace = true }
 sysinfo = "0.33.1"
 axum = "0.8.1"
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 790f8d9f..b2ea4852 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,12 +1,12 @@
-use std::{collections::HashMap, time::{Duration, SystemTime}};
+use std::{alloc::System, collections::HashMap, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
+use clap::error::ContextKind;
 use commonware_cryptography::{ed25519::PublicKey, sha256::Digest, Hasher, Sha256};
 use commonware_p2p::{Receiver, Recipients, Sender};
 use commonware_runtime::{Clock, Handle, Spawner};
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
-use tokio::time;
 use tracing::{debug, warn};
 
 #[derive(Clone)]
@@ -186,7 +186,7 @@ pub struct Mempool<R: Clock + Spawner> {
     batches: HashMap<Digest, Batch>,
     txs: Vec<RawTransaction>,
 
-    mailbox: mpsc::Receiver<Message>
+    mailbox: mpsc::Receiver<Message>,
 }
 
 impl<R: Clock + Spawner> Mempool<R> {
@@ -219,11 +219,16 @@ impl<R: Clock + Spawner> Mempool<R> {
         ), 
     ) {
 
-        let mut batch_propose_interval = time::interval(self.cfg.batch_propose_interval);
+        let mut propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
         loop {
             select! {
                 mailbox_message = self.mailbox.next() => {
-                    let message = mailbox_message.expect("Mailbox closed");
+                    // let message = mailbox_message.expect("Mailbox closed");
+                    let Some(message) = mailbox_message else {
+                        // TODO: revisit this branch, it was .expect("Mailbox closed") and will panic after unit test is finished
+                        debug!("Mailbox closed, terminating...");
+                        return;
+                    };
                     match message {
                         Message::SubmitTx { payload, response } => {
                             if !payload.validate() {
@@ -250,7 +255,7 @@ impl<R: Clock + Spawner> Mempool<R> {
                         },
                     }
                 },
-                _ = batch_propose_interval.tick() => {
+                _ = self.context.sleep_until(propose_timeout) => {
                     let mut size = 0;
                     let mut txs_cnt = 0;
                     for tx in self.txs.iter() {
@@ -262,9 +267,17 @@ impl<R: Clock + Spawner> Mempool<R> {
                         }
                     }
 
+                    if txs_cnt == 0 {
+                        propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
+                        continue;
+                    }
+
                     let batch = Batch::new(self.txs.drain(..txs_cnt).collect(), self.context.current());
                     self.batches.insert(batch.digest, batch.clone());
                     batch_network.0.send(Recipients::All, batch.serialize().into(), true).await.expect("failed to broadcast batch");
+
+                    // reset the timeout
+                    propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
                 },
                 batch_message = batch_network.1.recv() => {
                     let (sender, message) = batch_message.expect("Batch broadcast closed");
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index c259b337..8a998ab2 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -11,6 +11,7 @@ mod tests {
     use bytes::Bytes;
     use commonware_broadcast::linked::{Config, Engine};
     
+    use prometheus_client::metrics::info;
     use tracing::{debug, info, warn};
 
     use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
@@ -273,7 +274,7 @@ mod tests {
         }
     }
 
-    fn spawn_tx_issuer_and_wait(
+    async fn spawn_tx_issuer_and_wait(
         context: Context,
         mailboxes: Arc<Mutex<BTreeMap<PublicKey, mempool::Mailbox>>>,
         num_txs: u32,
@@ -325,7 +326,7 @@ mod tests {
                         info!("tx found at mempool: {}", tx.digest);
                     }
                 }
-            });
+            }).await.unwrap();
     }
 
     #[test_traced]
@@ -366,10 +367,12 @@ mod tests {
     fn test_mempool_p2p() {
         let num_validators: u32 = 4;
         let quorum: u32 = 3;
-        let (runner, mut context, _) = Executor::timed(Duration::from_secs(30));
+        let (runner, mut context, auditor) = Executor::timed(Duration::from_secs(30));
         let (_, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
         shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
 
+        info!("mempool p2p test started");
+
         runner.start(async move {
             let (_oracle, validators, _, mut registrations ) = initialize_simulation(
                 context.with_label("simulation"), 
@@ -380,7 +383,10 @@ mod tests {
                 mempool::Mailbox,
             >::new()));
             spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut mailboxes.lock().unwrap());
-            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1);
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1).await;
+
+            debug!("checking mempool states");
+            auditor.state()
         });
     }
 }
\ No newline at end of file

From 89594617bb41c2505666ea739b935e3362f943b4 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 20 Mar 2025 15:27:52 -0400
Subject: [PATCH 09/20] mempool: get_batch

---
 chain/src/actors/mempool/actor.rs   |  8 ++++++--
 chain/src/actors/mempool/mempool.rs | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 6268ee85..2dd91f81 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -1,4 +1,4 @@
-use super:: ingress::{Mailbox, Message};
+use super::{ ingress::{Mailbox, Message}, mempool};
 use commonware_broadcast::Broadcaster;
 use commonware_cryptography::Digest;
 use commonware_utils::Array;
@@ -20,7 +20,10 @@ impl<D: Digest, P: Array> Actor<D, P> {
         (Actor { mailbox: receiver }, Mailbox::new(sender))
     }
 
-    pub async fn run(mut self, mut engine: impl Broadcaster<Digest = D>) {
+    pub async fn run(mut self, 
+        mut engine: impl Broadcaster<Digest = D>,
+        mut mempool: mempool::Mailbox,
+    ) {
         // it passes msgs in the mailbox of the actor to the engine mailbox
         while let Some(msg) = self.mailbox.next().await {
             match msg {
@@ -39,6 +42,7 @@ impl<D: Digest, P: Array> Actor<D, P> {
                 }
                 Message::Verify(_context, _payload, sender) => {
                     // TODO: add handler here to process batch received
+                    mempool.get_batch(_payload);
                     let result = sender.send(true);
                     if result.is_err() {
                         error!("verify dropped");
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index b2ea4852..fc6aed5b 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -139,6 +139,10 @@ pub enum Message {
         digest: Digest,
         response: oneshot::Sender<Option<RawTransaction>>
     },
+    GetBatch {
+        digest: Digest,
+        response: oneshot::Sender<Option<Batch>>
+    },
 }
 
 #[derive(Clone)]
@@ -172,6 +176,16 @@ impl Mailbox {
 
         receiver.await.expect("failed to receive tx")
     }
+
+    pub async fn get_batch(&mut self, digest: Digest) -> Option<Batch> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::GetBatch { digest, response })
+            .await
+            .expect("failed to get batch");
+
+        receiver.await.expect("failed to receive batch")
+    }
 }
 
 pub struct Config {
@@ -244,6 +258,10 @@ impl<R: Clock + Spawner> Mempool<R> {
                                 batch.accepted = true;
                             }
                         },
+                        Message::GetBatch { digest, response } => { 
+                            let batch = self.batches.get(&digest).cloned();
+                            let _ = response.send(batch);
+                        },
                         Message::GetTx { digest, response } => {
                             // TODO: optimize this naive way of seaching
                             let pair = self.batches.iter().find(|(_, batch)| batch.contain_tx(&digest));

From e7da75489ee38767eda4beac7ada8b3942a0e8d6 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 20 Mar 2025 16:12:49 -0400
Subject: [PATCH 10/20] generic type fix

---
 chain/src/actors/mempool/actor.rs   | 12 +++--
 chain/src/actors/mempool/mempool.rs | 76 ++++++++++++++++-------------
 2 files changed, 49 insertions(+), 39 deletions(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 2dd91f81..da3157ea 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -1,4 +1,4 @@
-use super::{ ingress::{Mailbox, Message}, mempool};
+use super::{ ingress::{Mailbox, Message}, mempool::{self, Batch}};
 use commonware_broadcast::Broadcaster;
 use commonware_cryptography::Digest;
 use commonware_utils::Array;
@@ -6,7 +6,7 @@ use futures::{
     channel::mpsc,
     StreamExt,
 };
-use tracing::error;
+use tracing::{error, warn};
 
 
 pub struct Actor<D: Digest, P: Array> {
@@ -22,7 +22,7 @@ impl<D: Digest, P: Array> Actor<D, P> {
 
     pub async fn run(mut self, 
         mut engine: impl Broadcaster<Digest = D>,
-        mut mempool: mempool::Mailbox,
+        mut mempool: mempool::Mailbox<D>
     ) {
         // it passes msgs in the mailbox of the actor to the engine mailbox
         while let Some(msg) = self.mailbox.next().await {
@@ -42,7 +42,11 @@ impl<D: Digest, P: Array> Actor<D, P> {
                 }
                 Message::Verify(_context, _payload, sender) => {
                     // TODO: add handler here to process batch received
-                    mempool.get_batch(_payload);
+                    let Some(_) = mempool.get_batch(_payload).await else {
+                        warn!(?_payload, "batch not exists");
+                        let _ = sender.send(false);
+                        continue;
+                    };
                     let result = sender.send(true);
                     if result.is_err() {
                         error!("verify dropped");
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index fc6aed5b..9f2e79d3 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,8 +1,7 @@
-use std::{alloc::System, collections::HashMap, time::{Duration, SystemTime}};
+use std::{collections::HashMap, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
-use clap::error::ContextKind;
-use commonware_cryptography::{ed25519::PublicKey, sha256::Digest, Hasher, Sha256};
+use commonware_cryptography::{ed25519::PublicKey, sha256::{self}, Digest, Hasher, Sha256};
 use commonware_p2p::{Receiver, Recipients, Sender};
 use commonware_runtime::{Clock, Handle, Spawner};
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
@@ -10,18 +9,21 @@ use commonware_macros::select;
 use tracing::{debug, warn};
 
 #[derive(Clone)]
-pub struct Batch  {
-    pub txs: Vec<RawTransaction>,
+pub struct Batch<D: Digest>  {
+    pub txs: Vec<RawTransaction<D>>,
 
-    pub digest: Digest,
+    pub digest: D,
     // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
     pub accepted: bool, 
     pub timestamp: SystemTime,
 }
 
-impl Batch {
-    fn compute_digest(txs: &Vec<RawTransaction>) -> Digest {
+impl<D: Digest> Batch<D> 
+    where Sha256: Hasher<Digest = D>
+{
+    fn compute_digest(txs: &Vec<RawTransaction<D>>) -> D {
         let mut hasher = Sha256::new();
+
         for tx in txs.into_iter() {
             hasher.update(tx.raw.as_ref());
         }
@@ -29,7 +31,7 @@ impl Batch {
         hasher.finalize()
     }
 
-    pub fn new(txs: Vec<RawTransaction>, timestamp: SystemTime) -> Self {
+    pub fn new(txs: Vec<RawTransaction<D>>, timestamp: SystemTime) -> Self {
         let digest = Self::compute_digest(&txs);
 
         Self {
@@ -84,24 +86,26 @@ impl Batch {
         })
     }
 
-    pub fn contain_tx(&self, digest: &Digest) -> bool {
+    pub fn contain_tx(&self, digest: &D) -> bool {
         self.txs.iter().any(|tx| &tx.digest == digest) 
     }
 
-    pub fn tx(&self, digest: &Digest) -> Option<RawTransaction> {
+    pub fn tx(&self, digest: &D) -> Option<RawTransaction<D>> {
         self.txs.iter().find(|tx| &tx.digest == digest).cloned()
     }
 }
 
 #[derive(Clone)]
-pub struct RawTransaction {
+pub struct RawTransaction<D: Digest> {
     pub raw: Bytes,
 
-    pub digest: Digest
+    pub digest: D
 }
 
-impl RawTransaction {
-    fn compute_digest(raw: &Bytes) -> Digest {
+impl<D: Digest> RawTransaction<D> 
+    where Sha256: Hasher<Digest = D>
+{
+    fn compute_digest(raw: &Bytes) -> D {
         let mut hasher = Sha256::new();
         hasher.update(&raw);
         hasher.finalize()
@@ -125,39 +129,39 @@ impl RawTransaction {
     }
 }
 
-pub enum Message {
+pub enum Message<D: Digest> {
     // mark batch as accepted by the netowrk through the broadcast protocol
     BatchAccepted {
-        digest: Digest,
+        digest: D,
     },
     // from rpc or websocket
     SubmitTx {
-        payload: RawTransaction,
+        payload: RawTransaction<D>,
         response: oneshot::Sender<bool>
     },
     GetTx {
-        digest: Digest,
-        response: oneshot::Sender<Option<RawTransaction>>
+        digest: D,
+        response: oneshot::Sender<Option<RawTransaction<D>>>
     },
     GetBatch {
-        digest: Digest,
-        response: oneshot::Sender<Option<Batch>>
+        digest: D,
+        response: oneshot::Sender<Option<Batch<D>>>
     },
 }
 
 #[derive(Clone)]
-pub struct Mailbox {
-    sender: mpsc::Sender<Message>
+pub struct Mailbox<D: Digest> {
+    sender: mpsc::Sender<Message<D>>
 }
 
-impl Mailbox {
-    pub fn new(sender: mpsc::Sender<Message>) -> Self {
+impl<D: Digest> Mailbox<D> {
+    pub fn new(sender: mpsc::Sender<Message<D>>) -> Self {
         Self {
             sender
         }
     }
 
-    pub async fn issue_tx(&mut self, tx: RawTransaction) -> bool {
+    pub async fn issue_tx(&mut self, tx: RawTransaction<D>) -> bool {
         let (response, receiver) = oneshot::channel();
         self.sender
             .send(Message::SubmitTx { payload: tx, response })
@@ -167,7 +171,7 @@ impl Mailbox {
         receiver.await.expect("failed to receive tx issue status")
     }
 
-    pub async fn get_tx(&mut self, digest: Digest) -> Option<RawTransaction> {
+    pub async fn get_tx(&mut self, digest: D) -> Option<RawTransaction<D>> {
         let (response, receiver) = oneshot::channel();
         self.sender
             .send(Message::GetTx { digest, response })
@@ -177,7 +181,7 @@ impl Mailbox {
         receiver.await.expect("failed to receive tx")
     }
 
-    pub async fn get_batch(&mut self, digest: Digest) -> Option<Batch> {
+    pub async fn get_batch(&mut self, digest: D) -> Option<Batch<D>> {
         let (response, receiver) = oneshot::channel();
         self.sender
             .send(Message::GetBatch { digest, response })
@@ -193,18 +197,20 @@ pub struct Config {
     pub batch_size_limit: u64,
 }
 
-pub struct Mempool<R: Clock + Spawner> {
+pub struct Mempool<R: Clock + Spawner, D: Digest> {
     cfg: Config,
     context: R,
 
-    batches: HashMap<Digest, Batch>,
-    txs: Vec<RawTransaction>,
+    batches: HashMap<D, Batch<D>>,
+    txs: Vec<RawTransaction<D>>,
 
-    mailbox: mpsc::Receiver<Message>,
+    mailbox: mpsc::Receiver<Message<D>>,
 }
 
-impl<R: Clock + Spawner> Mempool<R> {
-    pub fn new(context: R, cfg: Config) -> (Self, Mailbox) {
+impl<R: Clock + Spawner, D: Digest> Mempool<R, D> 
+    where Sha256: Hasher<Digest = D>,
+{
+    pub fn new(context: R, cfg: Config) -> (Self, Mailbox<D>) {
         let (sender, receiver) = mpsc::channel(1024);
         (Self {
             cfg, 

From c35afa197cc7cc50fcf03512ec13158100e65d9f Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 20 Mar 2025 16:51:40 -0400
Subject: [PATCH 11/20] broadcast e2e

---
 chain/src/actors/mempool/actor.rs     |  6 ++--
 chain/src/actors/mempool/collector.rs | 10 +++++-
 chain/src/actors/mempool/mempool.rs   | 42 ++++++++++++++++++++--
 chain/src/actors/mempool/mod.rs       | 51 +++++++++++++++++----------
 4 files changed, 86 insertions(+), 23 deletions(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index da3157ea..2d0402bf 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -6,7 +6,7 @@ use futures::{
     channel::mpsc,
     StreamExt,
 };
-use tracing::{error, warn};
+use tracing::{error, warn, debug};
 
 
 pub struct Actor<D: Digest, P: Array> {
@@ -28,6 +28,7 @@ impl<D: Digest, P: Array> Actor<D, P> {
         while let Some(msg) = self.mailbox.next().await {
             match msg {
                 Message::Broadcast(payload) => {
+                    debug!("broadcasting batch {}", payload);
                     let receiver = engine.broadcast(payload).await;
                     let result = receiver.await;
                     match result {
@@ -41,12 +42,13 @@ impl<D: Digest, P: Array> Actor<D, P> {
                     }
                 }
                 Message::Verify(_context, _payload, sender) => {
-                    // TODO: add handler here to process batch received
+                    // TODO: backfill inplace?
                     let Some(_) = mempool.get_batch(_payload).await else {
                         warn!(?_payload, "batch not exists");
                         let _ = sender.send(false);
                         continue;
                     };
+                    debug!("issue verfication to batch {}", _payload);
                     let result = sender.send(true);
                     if result.is_err() {
                         error!("verify dropped");
diff --git a/chain/src/actors/mempool/collector.rs b/chain/src/actors/mempool/collector.rs
index 3c947958..28785554 100644
--- a/chain/src/actors/mempool/collector.rs
+++ b/chain/src/actors/mempool/collector.rs
@@ -10,6 +10,8 @@ use std::{
 };
 use tracing::error;
 
+use super::mempool;
+
 enum Message<C: Scheme, D: Digest> {
     Acknowledged(Proof, D),
     GetTip(C::PublicKey, oneshot::Sender<Option<u64>>),
@@ -52,7 +54,7 @@ impl<C: Scheme, D: Digest> Collector<C, D> {
         )
     }
 
-    pub async fn run(mut self) {
+    pub async fn run(mut self, mut mempool: mempool::Mailbox<D>) {
         while let Some(msg) = self.mailbox.next().await {
             match msg {
                 Message::Acknowledged(proof, payload) => {
@@ -67,6 +69,12 @@ impl<C: Scheme, D: Digest> Collector<C, D> {
                         }
                     };
 
+                    // Acknowledge batch in mempool, mark the batch as ready for pickup
+                    let acknowledge = mempool.acknowledge_batch(payload).await;
+                    if !acknowledge {
+                        error!("unable to acknowledge batch {}", payload)
+                    }
+
                     // Update the collector
                     let digests = self.digests.entry(context.sequencer.clone()).or_default();
                     digests.insert(context.height, payload);
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 9f2e79d3..7998d7c2 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -131,8 +131,9 @@ impl<D: Digest> RawTransaction<D>
 
 pub enum Message<D: Digest> {
     // mark batch as accepted by the netowrk through the broadcast protocol
-    BatchAccepted {
+    BatchAcknowledged {
         digest: D,
+        response: oneshot::Sender<bool>
     },
     // from rpc or websocket
     SubmitTx {
@@ -147,6 +148,10 @@ pub enum Message<D: Digest> {
         digest: D,
         response: oneshot::Sender<Option<Batch<D>>>
     },
+    GetBatchContainTx {
+        digest: D,
+        response: oneshot::Sender<Option<Batch<D>>>
+    }
 }
 
 #[derive(Clone)]
@@ -161,6 +166,16 @@ impl<D: Digest> Mailbox<D> {
         }
     }
 
+    pub async fn acknowledge_batch(&mut self, digest: D) -> bool {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::BatchAcknowledged { digest, response})
+            .await
+            .expect("failed to acknowledge batch");
+
+        receiver.await.expect("failed to receive batch acknowledge")
+    }
+
     pub async fn issue_tx(&mut self, tx: RawTransaction<D>) -> bool {
         let (response, receiver) = oneshot::channel();
         self.sender
@@ -190,6 +205,16 @@ impl<D: Digest> Mailbox<D> {
 
         receiver.await.expect("failed to receive batch")
     }
+
+    pub async fn get_batch_contain_tx(&mut self, digest: D) -> Option<Batch<D>> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::GetBatchContainTx { digest, response })
+            .await
+            .expect("failed to get batch");
+
+        receiver.await.expect("failed to receive batch")
+    }
 }
 
 pub struct Config {
@@ -259,15 +284,28 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                             self.txs.push(payload);
                             let _ = response.send(true);
                         },
-                        Message::BatchAccepted { digest } => {
+                        Message::BatchAcknowledged { digest, response } => {
                             if let Some(batch) = self.batches.get_mut(&digest) {
                                 batch.accepted = true;
+                                let _ = response.send(true);
+                                debug!("batch accepted by the network: {}", digest);
+                            } else {
+                                let _ = response.send(false);
                             }
                         },
                         Message::GetBatch { digest, response } => { 
                             let batch = self.batches.get(&digest).cloned();
                             let _ = response.send(batch);
                         },
+                        Message::GetBatchContainTx { digest, response } => {
+                            // TODO: optimize this naive way of seaching
+                            let pair = self.batches.iter().find(|(_, batch)| batch.contain_tx(&digest));
+                            if let Some((_, batch)) = pair {
+                                let _ = response.send(Some(batch.clone()));
+                            } else {
+                                let _ = response.send(None);
+                            }
+                        },
                         Message::GetTx { digest, response } => {
                             // TODO: optimize this naive way of seaching
                             let pair = self.batches.iter().find(|(_, batch)| batch.contain_tx(&digest));
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 8a998ab2..0eadf7ef 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -14,7 +14,7 @@ mod tests {
     use prometheus_client::metrics::info;
     use tracing::{debug, info, warn};
 
-    use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Hasher, Scheme};
+    use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Digest, Ed25519, Hasher, Scheme};
     use commonware_macros::test_traced;
     use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network};
     use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
@@ -201,13 +201,20 @@ mod tests {
         pks: &[PublicKey],
         validators: &[(PublicKey, Ed25519, Share)],
         registrations: &mut Registrations<PublicKey>,
-        mailboxes: &mut BTreeMap<PublicKey, super::ingress::Mailbox<sha256::Digest, PublicKey>>,
+        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
         collectors: &mut BTreeMap<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>,
         refresh_epoch_timeout: Duration,
         rebroadcast_timeout: Duration,
     ) {
         let namespace = b"my testing namespace";
         for (validator, scheme, share) in validators.iter() {
+            let (mempool, mempool_mailbox) = Mempool::new(context.with_label("mempool"), mempool::Config { 
+                batch_propose_interval: Duration::from_millis(500), 
+                batch_size_limit: 1024*1024, 
+            });
+            mailboxes.insert(validator.clone(), mempool_mailbox.clone());
+
+
             let context = context.with_label(&validator.to_string());
             let mut coordinator = super::coordinator::Coordinator::<PublicKey>::new(
                 identity.clone(),
@@ -218,14 +225,14 @@ mod tests {
 
             let (app, app_mailbox) =
                 super::actor::Actor::<sha256::Digest, PublicKey>::new();
-            mailboxes.insert(validator.clone(), app_mailbox.clone());
 
+            let collector_mempool_mailbox = mempool_mailbox.clone();
             let (collector, collector_mailbox) =
                 super::collector::Collector::<Ed25519, sha256::Digest>::new(
                     namespace,
                     *poly::public(&identity),
                 );
-            context.with_label("collector").spawn(|_| collector.run());
+            context.with_label("collector").spawn(move |_| collector.run(collector_mempool_mailbox));
             collectors.insert(validator.clone(), collector_mailbox);
 
             let (engine, mailbox) = Engine::new(
@@ -248,9 +255,10 @@ mod tests {
                 },
             );
 
-            context.with_label("app").spawn(|_| app.run(mailbox));
-            let ((a1, a2), (b1, b2), (_, _)) = registrations.remove(validator).unwrap();
+            context.with_label("app").spawn(move |_| app.run(mailbox, mempool_mailbox));
+            let ((a1, a2), (b1, b2), (c1, c2)) = registrations.remove(validator).unwrap();
             engine.start((a1, a2), (b1, b2));
+            mempool.start((c1, c2));
         }
     }
 
@@ -258,7 +266,7 @@ mod tests {
         context: Context,
         validators: &[(PublicKey, Ed25519, Share)],
         registrations: &mut Registrations<PublicKey>,
-        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox>,
+        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
     ) {
         for (validator, _, _) in validators.iter() {
             let (mempool, mailbox) = Mempool::new(
@@ -276,14 +284,15 @@ mod tests {
 
     async fn spawn_tx_issuer_and_wait(
         context: Context,
-        mailboxes: Arc<Mutex<BTreeMap<PublicKey, mempool::Mailbox>>>,
+        mailboxes: Arc<Mutex<BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>>>,
         num_txs: u32,
+        wait_batch_acknowlegement: bool,
     ) {
         context
             .clone()
             .with_label("tx issuer")
             .spawn(move |context| async move {
-                let mut mailbox_vec: Vec<mempool::Mailbox> = {
+                let mut mailbox_vec: Vec<mempool::Mailbox<sha256::Digest>> = {
                     let guard = mailboxes.lock().unwrap();
                     guard.values().cloned().collect()
                 };
@@ -324,6 +333,16 @@ mod tests {
                         };
 
                         info!("tx found at mempool: {}", tx.digest);
+
+                        if wait_batch_acknowlegement {
+                            let Some(batch) = mailbox.get_batch_contain_tx(digest.clone()).await else {
+                                panic!("batch not found");
+                            };
+                            if !batch.accepted {
+                                panic!("batch {} not acknowledged", batch.digest);
+                            }
+                            info!("batch contain tx {} acknowledged", batch.digest);
+                        }
                     }
                 }
             }).await.unwrap();
@@ -344,7 +363,7 @@ mod tests {
                 &mut shares_vec).await;
             let mailboxes = Arc::new(Mutex::new(BTreeMap::<
                 PublicKey,
-                super::ingress::Mailbox<sha256::Digest, PublicKey>,
+                mempool::Mailbox<sha256::Digest>,
             >::new()));
             let mut collectors = BTreeMap::<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>::new();
             spawn_validator_engines(
@@ -358,8 +377,7 @@ mod tests {
                 Duration::from_millis(100), 
                 Duration::from_secs(5)
             );
-            spawn_proposer(context.with_label("proposer"), mailboxes.clone(), |_| false);
-            await_collectors(context.with_label("collector"), &collectors, 100).await;
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1, true).await;
         });
     }
 
@@ -367,7 +385,7 @@ mod tests {
     fn test_mempool_p2p() {
         let num_validators: u32 = 4;
         let quorum: u32 = 3;
-        let (runner, mut context, auditor) = Executor::timed(Duration::from_secs(30));
+        let (runner, mut context, _) = Executor::timed(Duration::from_secs(30));
         let (_, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
         shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
 
@@ -380,13 +398,10 @@ mod tests {
                 &mut shares_vec).await;
             let mailboxes = Arc::new(Mutex::new(BTreeMap::<
                 PublicKey,
-                mempool::Mailbox,
+                mempool::Mailbox<sha256::Digest>,
             >::new()));
             spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut mailboxes.lock().unwrap());
-            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1).await;
-
-            debug!("checking mempool states");
-            auditor.state()
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1, false).await;
         });
     }
 }
\ No newline at end of file

From d185edc2ed40a590f895e39f5abab2ff28f579ff Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Mon, 24 Mar 2025 09:49:08 -0400
Subject: [PATCH 12/20] mempool batches

---
 chain/src/actors/mempool/mempool.rs | 14 ++++++++++----
 chain/src/actors/mempool/mod.rs     | 15 ++++++++++-----
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 7998d7c2..fb49d1f4 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,13 +1,15 @@
 use std::{collections::HashMap, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
-use commonware_cryptography::{ed25519::PublicKey, sha256::{self}, Digest, Hasher, Sha256};
+use commonware_cryptography::{bls12381::primitives::group::Public, ed25519::PublicKey, sha256::{self}, Digest, Hasher, Sha256};
 use commonware_p2p::{Receiver, Recipients, Sender};
 use commonware_runtime::{Clock, Handle, Spawner};
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
 use tracing::{debug, warn};
 
+use super::{actor, ingress};
+
 #[derive(Clone)]
 pub struct Batch<D: Digest>  {
     pub txs: Vec<RawTransaction<D>>,
@@ -251,9 +253,10 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
         batch_network: (
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
-        )
+        ),
+        mut app_mailbox: ingress::Mailbox<D, PublicKey>
     ) -> Handle<()> {
-        self.context.spawn_ref()(self.run(batch_network))
+        self.context.spawn_ref()(self.run(batch_network, app_mailbox))
     }
 
     async fn run(
@@ -262,8 +265,8 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
         ), 
+        mut app_mailbox: ingress::Mailbox<D, PublicKey>
     ) {
-
         let mut propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
         loop {
             select! {
@@ -334,9 +337,12 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                         continue;
                     }
 
+                    // 1. send raw batch over p2p
                     let batch = Batch::new(self.txs.drain(..txs_cnt).collect(), self.context.current());
                     self.batches.insert(batch.digest, batch.clone());
                     batch_network.0.send(Recipients::All, batch.serialize().into(), true).await.expect("failed to broadcast batch");
+                    // 2. send batch digest over broadcast layer
+                    app_mailbox.broadcast(batch.digest).await;
 
                     // reset the timeout
                     propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 0eadf7ef..755a25ff 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -3,6 +3,8 @@ pub mod ingress;
 pub mod coordinator;
 pub mod collector;
 pub mod mempool;
+pub mod handler;
+pub mod key;
 
 #[cfg(test)]
 mod tests {
@@ -18,9 +20,9 @@ mod tests {
     use commonware_macros::test_traced;
     use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network};
     use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
-    use futures::{channel::oneshot, future::join_all};
+    use futures::{channel::{mpsc, oneshot}, future::join_all};
 
-    use super::{collector, mempool::{self, Mempool, RawTransaction}};
+    use super::{collector, ingress, mempool::{self, Mempool, RawTransaction}};
 
     type Registrations<P> = HashMap<P, ((Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>))>;
 
@@ -258,7 +260,7 @@ mod tests {
             context.with_label("app").spawn(move |_| app.run(mailbox, mempool_mailbox));
             let ((a1, a2), (b1, b2), (c1, c2)) = registrations.remove(validator).unwrap();
             engine.start((a1, a2), (b1, b2));
-            mempool.start((c1, c2));
+            mempool.start((c1, c2), app_mailbox.clone());
         }
     }
 
@@ -266,6 +268,7 @@ mod tests {
         context: Context,
         validators: &[(PublicKey, Ed25519, Share)],
         registrations: &mut Registrations<PublicKey>,
+        app_mailbox: &mut ingress::Mailbox<sha256::Digest, PublicKey>,
         mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
     ) {
         for (validator, _, _) in validators.iter() {
@@ -278,7 +281,7 @@ mod tests {
             );
             mailboxes.insert(validator.clone(), mailbox);
             let ((_, _), (_, _), (c1, c2)) = registrations.remove(validator).unwrap();
-            mempool.start((c1, c2));
+            mempool.start((c1, c2), app_mailbox.clone());
         }
     }
 
@@ -390,6 +393,8 @@ mod tests {
         shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
 
         info!("mempool p2p test started");
+        let (app_mailbox_sender, _) = mpsc::channel(1024);
+        let mut app_mailbox = ingress::Mailbox::new(app_mailbox_sender);
 
         runner.start(async move {
             let (_oracle, validators, _, mut registrations ) = initialize_simulation(
@@ -400,7 +405,7 @@ mod tests {
                 PublicKey,
                 mempool::Mailbox<sha256::Digest>,
             >::new()));
-            spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut mailboxes.lock().unwrap());
+            spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut app_mailbox, &mut mailboxes.lock().unwrap());
             spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1, false).await;
         });
     }

From 8f21249051a45ff3f1372d48c4027f6ee9f69d9e Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Mon, 24 Mar 2025 16:03:11 -0400
Subject: [PATCH 13/20] backfill & archiving batches that formed accepted
 blocks

---
 chain/src/actors/mempool/archive.rs     |  65 ++++++
 chain/src/actors/mempool/coordinator.rs |  13 ++
 chain/src/actors/mempool/handler.rs     |  67 ++++++
 chain/src/actors/mempool/key.rs         | 132 ++++++++++++
 chain/src/actors/mempool/mempool.rs     | 269 +++++++++++++++++++++---
 chain/src/actors/mempool/mod.rs         |   1 +
 chain/src/actors/syncer/actor.rs        |   2 +-
 7 files changed, 518 insertions(+), 31 deletions(-)
 create mode 100644 chain/src/actors/mempool/archive.rs
 create mode 100644 chain/src/actors/mempool/handler.rs
 create mode 100644 chain/src/actors/mempool/key.rs

diff --git a/chain/src/actors/mempool/archive.rs b/chain/src/actors/mempool/archive.rs
new file mode 100644
index 00000000..7ceb04eb
--- /dev/null
+++ b/chain/src/actors/mempool/archive.rs
@@ -0,0 +1,65 @@
+use bytes::Bytes;
+use commonware_runtime::{Blob, Metrics, Storage};
+use commonware_storage::archive::{self, Archive, Identifier, Translator};
+use commonware_utils::Array;
+use futures::lock::Mutex;
+use std::sync::Arc;
+
+const BATCH_INDEX: u64 = 0;
+
+/// Archive wrapper that handles all locking.
+#[derive(Clone)]
+pub struct Wrapped<T, K, B, R>
+where
+    T: Translator,
+    K: Array,
+    B: Blob,
+    R: Storage<B> + Metrics,
+{
+    inner: Arc<Mutex<Archive<T, K, B, R>>>,
+}
+
+impl<T, K, B, R> Wrapped<T, K, B, R>
+where
+    T: Translator,
+    K: Array,
+    B: Blob,
+    R: Storage<B> + Metrics,
+{
+    /// Creates a new `Wrapped` from an existing `Archive`.
+    pub fn new(archive: Archive<T, K, B, R>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(archive)),
+        }
+    }
+
+    /// Retrieves a value from the archive by identifier.
+    pub async fn get(
+        &self,
+        identifier: Identifier<'_, K>,
+    ) -> Result<Option<Bytes>, archive::Error> {
+        let archive = self.inner.lock().await;
+        archive.get(identifier).await
+    }
+
+    /// Inserts a value into the archive with the given index and key.
+    pub async fn put(&self, key: K, data: Bytes) -> Result<(), archive::Error> {
+        let mut archive = self.inner.lock().await;
+        archive.put(BATCH_INDEX, key, data).await?;
+        Ok(())
+    }
+
+    // TODO: revisit: do we need to prune as a DA?
+    // /// Prunes entries from the archive up to the specified minimum index.
+    // pub async fn prune(&self, min_index: u64) -> Result<(), archive::Error> {
+    //     let mut archive = self.inner.lock().await;
+    //     archive.prune(min_index).await?;
+    //     Ok(())
+    // }
+
+    // /// Retrieves the next gap in the archive.
+    // pub async fn next_gap(&self, start: u64) -> (Option<u64>, Option<u64>) {
+    //     let archive = self.inner.lock().await;
+    //     archive.next_gap(start)
+    // }
+}
diff --git a/chain/src/actors/mempool/coordinator.rs b/chain/src/actors/mempool/coordinator.rs
index 3db14817..587a7cb6 100644
--- a/chain/src/actors/mempool/coordinator.rs
+++ b/chain/src/actors/mempool/coordinator.rs
@@ -3,6 +3,7 @@ use commonware_cryptography::bls12381::primitives::{
     group::{Public, Share},
     poly::Poly,
 };
+use commonware_resolver::{p2p};
 use commonware_utils::Array;
 use std::collections::HashMap;
 
@@ -78,3 +79,15 @@ impl<P: Array> T for Coordinator<P> {
         Some(&self.share)
     }
 }
+
+impl <P: Array> p2p::Coordinator for Coordinator<P>  {
+    type PublicKey = P; 
+
+    fn peers(&self) -> &Vec<Self::PublicKey> {
+        &self.signers        
+    }
+
+    fn peer_set_id(&self) -> u64 {
+        0    
+    }
+}
\ No newline at end of file
diff --git a/chain/src/actors/mempool/handler.rs b/chain/src/actors/mempool/handler.rs
new file mode 100644
index 00000000..9e5d2883
--- /dev/null
+++ b/chain/src/actors/mempool/handler.rs
@@ -0,0 +1,67 @@
+use super::key::MultiIndex;
+use bytes::Bytes;
+use commonware_resolver::{p2p::Producer, Consumer};
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt,
+};
+
+pub enum Message {
+    Deliver {
+        key: MultiIndex,
+        value: Bytes,
+        response: oneshot::Sender<bool>,
+    },
+    Produce {
+        key: MultiIndex,
+        response: oneshot::Sender<Bytes>,
+    },
+}
+
+/// Mailbox for resolver
+#[derive(Clone)]
+pub struct Handler {
+    sender: mpsc::Sender<Message>,
+}
+
+impl Handler {
+    pub(super) fn new(sender: mpsc::Sender<Message>) -> Self {
+        Self { sender }
+    }
+}
+
+impl Consumer for Handler {
+    type Key = MultiIndex;
+    type Value = Bytes;
+    type Failure = ();
+
+    async fn deliver(&mut self, key: Self::Key, value: Self::Value) -> bool {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::Deliver {
+                key,
+                value,
+                response,
+            })
+            .await
+            .expect("Failed to send deliver");
+        receiver.await.expect("Failed to receive deliver")
+    }
+
+    async fn failed(&mut self, _: Self::Key, _: Self::Failure) {
+        // Ignore any failure
+    }
+}
+
+impl Producer for Handler {
+    type Key = MultiIndex;
+
+    async fn produce(&mut self, key: Self::Key) -> oneshot::Receiver<Bytes> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::Produce { key, response })
+            .await
+            .expect("Failed to send produce");
+        receiver
+    }
+}
diff --git a/chain/src/actors/mempool/key.rs b/chain/src/actors/mempool/key.rs
new file mode 100644
index 00000000..d6ed6b98
--- /dev/null
+++ b/chain/src/actors/mempool/key.rs
@@ -0,0 +1,132 @@
+use commonware_cryptography::sha256::Digest;
+use commonware_utils::{Array, SizedSerialize};
+use std::{
+    cmp::{Ord, PartialOrd},
+    fmt::{Debug, Display},
+    hash::Hash,
+    ops::Deref,
+};
+use thiserror::Error;
+
+const SERIALIZED_LEN: usize = 1 + Digest::SERIALIZED_LEN;
+
+#[derive(Error, Debug, PartialEq)]
+pub enum Error {
+    #[error("invalid length")]
+    InvalidLength,
+}
+
+pub enum Value {
+    Digest(Digest),
+}
+
+#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[repr(transparent)]
+pub struct MultiIndex([u8; SERIALIZED_LEN]);
+
+impl MultiIndex {
+    pub fn new(value: Value) -> Self {
+        let mut bytes = [0; SERIALIZED_LEN];
+        match value {
+            Value::Digest(digest) => {
+                bytes[0] = 0;
+                bytes[1..].copy_from_slice(&digest);
+            }
+        }
+        Self(bytes)
+    }
+
+    pub fn to_value(&self) -> Value {
+        match self.0[0] {
+            0 => {
+                let bytes: [u8; Digest::SERIALIZED_LEN] = self.0[1..].try_into().unwrap();
+                let digest = Digest::from(bytes);
+                Value::Digest(digest)
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl Array for MultiIndex {
+    type Error = Error;
+}
+
+impl SizedSerialize for MultiIndex {
+    const SERIALIZED_LEN: usize = SERIALIZED_LEN;
+}
+
+impl From<[u8; MultiIndex::SERIALIZED_LEN]> for MultiIndex {
+    fn from(value: [u8; MultiIndex::SERIALIZED_LEN]) -> Self {
+        Self(value)
+    }
+}
+
+impl TryFrom<&[u8]> for MultiIndex {
+    type Error = Error;
+
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        if value.len() != MultiIndex::SERIALIZED_LEN {
+            return Err(Error::InvalidLength);
+        }
+        let array: [u8; MultiIndex::SERIALIZED_LEN] =
+            value.try_into().map_err(|_| Error::InvalidLength)?;
+        Ok(Self(array))
+    }
+}
+
+impl TryFrom<&Vec<u8>> for MultiIndex {
+    type Error = Error;
+
+    fn try_from(value: &Vec<u8>) -> Result<Self, Self::Error> {
+        Self::try_from(value.as_slice())
+    }
+}
+
+impl TryFrom<Vec<u8>> for MultiIndex {
+    type Error = Error;
+
+    fn try_from(value: Vec<u8>) -> Result<Self, Self::Error> {
+        if value.len() != MultiIndex::SERIALIZED_LEN {
+            return Err(Error::InvalidLength);
+        }
+
+        // If the length is correct, we can safely convert the vector into a boxed slice without any
+        // copies.
+        let boxed_slice = value.into_boxed_slice();
+        let boxed_array: Box<[u8; MultiIndex::SERIALIZED_LEN]> =
+            boxed_slice.try_into().map_err(|_| Error::InvalidLength)?;
+        Ok(Self(*boxed_array))
+    }
+}
+
+impl AsRef<[u8]> for MultiIndex {
+    fn as_ref(&self) -> &[u8] {
+        &self.0
+    }
+}
+
+impl Deref for MultiIndex {
+    type Target = [u8];
+    fn deref(&self) -> &[u8] {
+        &self.0
+    }
+}
+
+impl Debug for MultiIndex {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0[0] {
+            0 => {
+                let bytes: [u8; Digest::SERIALIZED_LEN] = self.0[1..].try_into().unwrap();
+                write!(f, "digest({})", Digest::from(bytes))
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl Display for MultiIndex {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Debug::fmt(self, f)
+    }
+}
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index fb49d1f4..2cf968e5 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,23 +1,34 @@
-use std::{collections::HashMap, time::{Duration, SystemTime}};
+use std::{alloc::System, collections::HashMap, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
-use commonware_cryptography::{bls12381::primitives::group::Public, ed25519::PublicKey, sha256::{self}, Digest, Hasher, Sha256};
-use commonware_p2p::{Receiver, Recipients, Sender};
-use commonware_runtime::{Clock, Handle, Spawner};
+use commonware_cryptography::{bls12381::primitives::group::Public, ed25519::PublicKey, sha256, Digest, Hasher, Sha256};
+use commonware_p2p::{utils::requester, Receiver, Recipients, Sender};
+use commonware_runtime::{Blob, Clock, Handle, Metrics, Spawner, Storage};
+use commonware_resolver::{p2p, Resolver};
+use commonware_storage::{
+    archive::{self, translator::TwoCap, Archive, Identifier}, 
+    journal::{self, variable::Journal},
+};
+use commonware_utils::{Array, SystemTimeExt};
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
+use governor::Quota;
+use rand::Rng;
 use tracing::{debug, warn};
-
-use super::{actor, ingress};
+use governor::clock::Clock as GClock;
+use tracing_subscriber::fmt::time;
+use super::{actor, handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped};
 
 #[derive(Clone)]
 pub struct Batch<D: Digest>  {
-    pub txs: Vec<RawTransaction<D>>,
-
-    pub digest: D,
+    pub timestamp: SystemTime,
     // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
     pub accepted: bool, 
-    pub timestamp: SystemTime,
+    // mark if the batch is used to produce a block and the block is accepted by the network
+    pub consumed: bool,
+
+    pub txs: Vec<RawTransaction<D>>,
+    pub digest: D,
 }
 
 impl<D: Digest> Batch<D> 
@@ -40,12 +51,16 @@ impl<D: Digest> Batch<D>
             txs,
             digest,
             accepted: false,
+            consumed: false,
             timestamp 
         }
     }
 
     pub fn serialize(&self) -> Vec<u8> {
         let mut bytes = Vec::new();
+        bytes.put_u8(self.accepted as u8);
+        bytes.put_u8(self.consumed as u8);
+        bytes.put_u64(self.timestamp.epoch_millis());
         bytes.put_u64(self.txs.len() as u64);
         for tx in self.txs.iter() {
             bytes.put_u64(tx.size());
@@ -56,10 +71,15 @@ impl<D: Digest> Batch<D>
 
     pub fn deserialize(mut bytes: &[u8]) -> Option<Self> {
         use bytes::Buf;
-        // We expect at least 8 bytes for the number of transactions.
-        if bytes.remaining() < 8 {
+        // We expect at least 18 bytes for the header
+        if bytes.remaining() < 18 {
             return None;
         }
+        let accepted: bool = bytes.get_u8() != 0;
+        let consumed = bytes.get_u8() != 0;
+        let timestamp = bytes.get_u64();
+        let timestamp = SystemTime::UNIX_EPOCH + Duration::from_millis(timestamp);
+
         let tx_count = bytes.get_u64();
         let mut txs = Vec::with_capacity(tx_count as usize);
         for _ in 0..tx_count {
@@ -81,10 +101,11 @@ impl<D: Digest> Batch<D>
         // Since serialize did not include accepted and timestamp, we set accepted to false
         // and set timestamp to the current time.
         Some(Self {
+            timestamp,
+            accepted,
+            consumed,
             txs,
             digest,
-            accepted: false,
-            timestamp: SystemTime::now(),
         })
     }
 
@@ -137,11 +158,18 @@ pub enum Message<D: Digest> {
         digest: D,
         response: oneshot::Sender<bool>
     },
+    BatchConsumed {
+        digests: Vec<D>,
+    },
     // from rpc or websocket
     SubmitTx {
         payload: RawTransaction<D>,
         response: oneshot::Sender<bool>
     },
+    // proposer consume batches to produce a block
+    ConsumeBatches {
+        response: oneshot::Sender<Vec<Batch<D>>>
+    },
     GetTx {
         digest: D,
         response: oneshot::Sender<Option<RawTransaction<D>>>
@@ -188,6 +216,26 @@ impl<D: Digest> Mailbox<D> {
         receiver.await.expect("failed to receive tx issue status")
     }
 
+    pub async fn consume_batches(&mut self) -> Vec<Batch<D>> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::ConsumeBatches { response: response })
+            .await
+            .expect("failed to consume batches");
+
+        receiver.await.expect("failed to receive batches")
+    }
+
+    pub async fn consumed_batches(&mut self, digests: Vec<D>) -> Vec<Batch<D>> {
+        let (response, receiver) = oneshot::channel();
+        self.sender
+            .send(Message::BatchConsumed { digests: digests })
+            .await
+            .expect("failed to mark batches as consumed");
+
+        receiver.await.expect("failed to mark batches as consumed")
+    }
+
     pub async fn get_tx(&mut self, digest: D) -> Option<RawTransaction<D>> {
         let (response, receiver) = oneshot::channel();
         self.sender
@@ -222,29 +270,77 @@ impl<D: Digest> Mailbox<D> {
 pub struct Config {
     pub batch_propose_interval: Duration,
     pub batch_size_limit: u64,
+    pub backfill_quota: Quota,
+    pub mailbox_size: usize,
+    pub public_key: PublicKey,
+    pub partition_prefix: String,
 }
 
-pub struct Mempool<R: Clock + Spawner, D: Digest> {
-    cfg: Config,
+pub struct Mempool<
+    B: Blob,
+    R: Rng + Clock + GClock + Spawner + Metrics + Storage<B>, 
+    D: Digest + Into<sha256::Digest> + From<sha256::Digest>
+> {
     context: R,
 
+    public_key: PublicKey,
+
     batches: HashMap<D, Batch<D>>,
+    accepted: Archive<TwoCap, D, B, R>,
+
     txs: Vec<RawTransaction<D>>,
 
     mailbox: mpsc::Receiver<Message<D>>,
+    mailbox_size: usize,
+
+    batch_propose_interval: Duration,
+    batch_size_limit: u64,
+    backfill_quota: Quota,
 }
 
-impl<R: Clock + Spawner, D: Digest> Mempool<R, D> 
-    where Sha256: Hasher<Digest = D>,
+impl<
+    B: Blob,
+    R: Rng + Clock + GClock + Spawner + Metrics + Storage<B>, 
+    D: Digest + Into<sha256::Digest> + From<sha256::Digest>
+> Mempool<B, R, D> 
+    where 
+        Sha256: Hasher<Digest = D>,
 {
-    pub fn new(context: R, cfg: Config) -> (Self, Mailbox<D>) {
+    pub async fn init(context: R, cfg: Config) -> (Self, Mailbox<D>) {
+        let accepted_journal = Journal::init(
+            context.with_label("accepted_journal"), 
+            journal::variable::Config {
+                partition: format!("{}-acceptances", cfg.partition_prefix),
+            })
+            .await
+            .expect("Failed to initialize accepted journal");
+        let accepted_archive = Archive::init(
+        context.with_label("accepted_archive"),
+            accepted_journal, 
+            archive::Config {
+                translator: TwoCap,
+                section_mask: 0xffff_ffff_fff0_0000u64,
+                pending_writes: 0,
+                replay_concurrency: 4,
+                compression: Some(3),
+            })
+            .await
+            .expect("Failed to initialize accepted archive");
         let (sender, receiver) = mpsc::channel(1024);
         (Self {
-            cfg, 
             context,
-            mailbox: receiver,
-            txs: vec![],
+            public_key: cfg.public_key,
             batches: HashMap::new(),
+            accepted: accepted_archive,
+
+            txs: vec![],
+
+            mailbox: receiver,
+            mailbox_size: cfg.mailbox_size,
+
+            batch_propose_interval: cfg.batch_propose_interval,
+            batch_size_limit: cfg.batch_size_limit,
+            backfill_quota: cfg.backfill_quota,
         }, Mailbox::new(sender))
     }
 
@@ -254,9 +350,14 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
         ),
-        mut app_mailbox: ingress::Mailbox<D, PublicKey>
+        mut backfill_network: (
+            impl Sender<PublicKey = PublicKey>,
+            impl Receiver<PublicKey = PublicKey>,
+        ), 
+        coordinator: Coordinator<PublicKey>,
+        app_mailbox: ingress::Mailbox<D, PublicKey>
     ) -> Handle<()> {
-        self.context.spawn_ref()(self.run(batch_network, app_mailbox))
+        self.context.spawn_ref()(self.run(batch_network,backfill_network, coordinator, app_mailbox))
     }
 
     async fn run(
@@ -265,10 +366,45 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
         ), 
+        backfill_network: (
+            impl Sender<PublicKey = PublicKey>,
+            impl Receiver<PublicKey = PublicKey>,
+        ), 
+        coordinator: Coordinator<PublicKey>,
         mut app_mailbox: ingress::Mailbox<D, PublicKey>
     ) {
-        let mut propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
+        let (handler_sender, mut handler_receiver) = mpsc::channel(self.mailbox_size);
+        let handler = Handler::new(handler_sender);
+        let (resolver_engine, mut resolver) = p2p::Engine::new(
+            self.context.with_label("resolver"),
+            p2p::Config {
+                coordinator: coordinator,
+                consumer: handler.clone(),
+                producer: handler,
+                mailbox_size: self.mailbox_size,
+                requester_config: requester::Config {
+                    public_key: self.public_key,
+                    rate_limit: self.backfill_quota,
+                    initial: Duration::from_secs(1),
+                    timeout: Duration::from_secs(2),
+                },
+                fetch_retry_timeout: Duration::from_millis(100), // prevent busy loop
+                priority_requests: false,
+                priority_responses: false,
+            },
+        );
+        resolver_engine.start(backfill_network);
+
+        let mut waiters: HashMap<D, Vec<oneshot::Sender<Option<Batch<D>>>>> = HashMap::new();
+        let mut propose_timeout = self.context.current() + self.batch_propose_interval;
+        let accepted = Wrapped::new(self.accepted);
         loop {
+            // Clear dead waiters
+            waiters.retain(|_, waiters| {
+                waiters.retain(|waiter| !waiter.is_canceled());
+                !waiters.is_empty()
+            });
+
             select! {
                 mailbox_message = self.mailbox.next() => {
                     // let message = mailbox_message.expect("Mailbox closed");
@@ -296,9 +432,42 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                                 let _ = response.send(false);
                             }
                         },
+                        Message::ConsumeBatches { response } => { 
+                            let batches = self.batches.iter().filter(|(_, batch)| batch.accepted).map(|(_, batch)| batch.clone()).collect();
+                            let _ = response.send(batches);
+                        },
+                        Message:: BatchConsumed { digests } => {
+                            let consumed_keys: Vec<D> = self.batches.iter()
+                                .filter_map(|(digest, batch)| {
+                                    if digests.contains(&batch.digest) {
+                                        Some(digest.clone())
+                                    } else {
+                                        None
+                                    }
+                                })
+                                .collect();
+
+                            // Then remove those entries, updating their `consumed` flag.
+                            let consumed_batches: Vec<Batch<D>> = consumed_keys.into_iter()
+                                .filter_map(|key| {
+                                    self.batches.remove(&key).map(|mut batch| {
+                                        batch.consumed = true;
+                                        batch
+                                    })
+                                })
+                                .collect();
+
+                            for batch in consumed_batches.iter() {
+                                accepted.put(batch.digest, batch.serialize().into()).await.expect("Failed to insert accepted batch");
+                            }
+                        },
                         Message::GetBatch { digest, response } => { 
-                            let batch = self.batches.get(&digest).cloned();
-                            let _ = response.send(batch);
+                            if let Some(batch) = self.batches.get(&digest).cloned() {
+                                let _ = response.send(Some(batch));
+                                continue;
+                            };
+                            resolver.fetch(MultiIndex::new(Value::Digest(digest.into()))).await;
+                            waiters.entry(digest).or_default().push(response);
                         },
                         Message::GetBatchContainTx { digest, response } => {
                             // TODO: optimize this naive way of seaching
@@ -327,13 +496,13 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                         size += tx.size();
                         txs_cnt += 1;
 
-                        if size > self.cfg.batch_size_limit {
+                        if size > self.batch_size_limit {
                             break;
                         }
                     }
 
                     if txs_cnt == 0 {
-                        propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
+                        propose_timeout = self.context.current() + self.batch_propose_interval;
                         continue;
                     }
 
@@ -345,7 +514,7 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                     app_mailbox.broadcast(batch.digest).await;
 
                     // reset the timeout
-                    propose_timeout = self.context.current() + self.cfg.batch_propose_interval;
+                    propose_timeout = self.context.current() + self.batch_propose_interval;
                 },
                 batch_message = batch_network.1.recv() => {
                     let (sender, message) = batch_message.expect("Batch broadcast closed");
@@ -357,6 +526,46 @@ impl<R: Clock + Spawner, D: Digest> Mempool<R, D>
                     debug!(?sender, digest=?batch.digest, "received batch");
                     let _ = self.batches.entry(batch.digest).or_insert(batch);
                 },
+
+                // handle batch request
+                handler_message = handler_receiver.next() => {
+                    let message = handler_message.expect("Handler closed");
+                    match message {
+                        handler::Message::Produce { key, response } => {
+                            match key.to_value() {
+                                // TODO: add a buffer
+                                key::Value::Digest(digest) => {
+                                    let accepted = accepted.get(Identifier::Key(&D::from(digest)))
+                                        .await 
+                                        .expect("Failed to get accepted batch");
+                                    if let Some(accepted) = accepted {
+                                        let _ = response.send(accepted);
+                                    }
+                                }
+                            }         
+                        },
+                        handler::Message::Deliver { key, value, response } => {
+                            match key.to_value() {
+                                key::Value::Digest(digest) => {
+                                    let batch = Batch::deserialize(&value).expect("Failed to deserialize batch");
+                                    if batch.digest.into() != digest {
+                                        let _ = response.send(false);
+                                        continue;
+                                    }
+
+                                    if let Some(waiters) = waiters.remove(&batch.digest) {
+                                        debug!(?batch.digest, "waiters resolved via batch");
+                                        for waiter in waiters {
+                                            let _ = waiter.send(Some(batch.clone()));
+                                        }
+                                    }
+
+                                    accepted.put(batch.digest, value).await.expect("Failed to insert accepted batch");
+                                }
+                            }         
+                        },
+                    }
+                }
             }
         }
     }
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 755a25ff..18096d8b 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -5,6 +5,7 @@ pub mod collector;
 pub mod mempool;
 pub mod handler;
 pub mod key;
+pub mod archive;
 
 #[cfg(test)]
 mod tests {
diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs
index 1d64d4fb..6ae3feab 100644
--- a/chain/src/actors/syncer/actor.rs
+++ b/chain/src/actors/syncer/actor.rs
@@ -287,7 +287,7 @@ impl<B: Blob, R: Rng + Spawner + Metrics + Clock + GClock + Storage<B>, I: Index
             let mut resolver = resolver.clone();
             let last_view_processed = last_view_processed.clone();
             let verified = verified.clone();
-            let notarized = notarized.clone();
+            let notarized: Wrapped<TwoCap, Digest, B, R> = notarized.clone();
             let finalized = finalized.clone();
             let blocks = blocks.clone();
             move |_| async move {

From 686b728368cf619e810da9dc13e3197f0a3eee6b Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Tue, 25 Mar 2025 11:07:40 -0400
Subject: [PATCH 14/20] ckp before replay accepted chunks

---
 chain/src/actors/mempool/archive.rs |  29 ++++---
 chain/src/actors/mempool/mempool.rs | 113 ++++++++++++++++++++--------
 2 files changed, 93 insertions(+), 49 deletions(-)

diff --git a/chain/src/actors/mempool/archive.rs b/chain/src/actors/mempool/archive.rs
index 7ceb04eb..9898cc46 100644
--- a/chain/src/actors/mempool/archive.rs
+++ b/chain/src/actors/mempool/archive.rs
@@ -5,8 +5,6 @@ use commonware_utils::Array;
 use futures::lock::Mutex;
 use std::sync::Arc;
 
-const BATCH_INDEX: u64 = 0;
-
 /// Archive wrapper that handles all locking.
 #[derive(Clone)]
 pub struct Wrapped<T, K, B, R>
@@ -43,23 +41,22 @@ where
     }
 
     /// Inserts a value into the archive with the given index and key.
-    pub async fn put(&self, key: K, data: Bytes) -> Result<(), archive::Error> {
+    pub async fn put(&self, index: u64, key: K, data: Bytes) -> Result<(), archive::Error> {
         let mut archive = self.inner.lock().await;
-        archive.put(BATCH_INDEX, key, data).await?;
+        archive.put(index, key, data).await?;
         Ok(())
     }
 
-    // TODO: revisit: do we need to prune as a DA?
-    // /// Prunes entries from the archive up to the specified minimum index.
-    // pub async fn prune(&self, min_index: u64) -> Result<(), archive::Error> {
-    //     let mut archive = self.inner.lock().await;
-    //     archive.prune(min_index).await?;
-    //     Ok(())
-    // }
+    /// Prunes entries from the archive up to the specified minimum index.
+    pub async fn prune(&self, min_index: u64) -> Result<(), archive::Error> {
+        let mut archive = self.inner.lock().await;
+        archive.prune(min_index).await?;
+        Ok(())
+    }
 
-    // /// Retrieves the next gap in the archive.
-    // pub async fn next_gap(&self, start: u64) -> (Option<u64>, Option<u64>) {
-    //     let archive = self.inner.lock().await;
-    //     archive.next_gap(start)
-    // }
+    /// Retrieves the next gap in the archive.
+    pub async fn next_gap(&self, start: u64) -> (Option<u64>, Option<u64>) {
+        let archive = self.inner.lock().await;
+        archive.next_gap(start)
+    }
 }
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 2cf968e5..1d53391b 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -24,8 +24,6 @@ pub struct Batch<D: Digest>  {
     pub timestamp: SystemTime,
     // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
     pub accepted: bool, 
-    // mark if the batch is used to produce a block and the block is accepted by the network
-    pub consumed: bool,
 
     pub txs: Vec<RawTransaction<D>>,
     pub digest: D,
@@ -51,15 +49,12 @@ impl<D: Digest> Batch<D>
             txs,
             digest,
             accepted: false,
-            consumed: false,
             timestamp 
         }
     }
 
     pub fn serialize(&self) -> Vec<u8> {
         let mut bytes = Vec::new();
-        bytes.put_u8(self.accepted as u8);
-        bytes.put_u8(self.consumed as u8);
         bytes.put_u64(self.timestamp.epoch_millis());
         bytes.put_u64(self.txs.len() as u64);
         for tx in self.txs.iter() {
@@ -75,8 +70,6 @@ impl<D: Digest> Batch<D>
         if bytes.remaining() < 18 {
             return None;
         }
-        let accepted: bool = bytes.get_u8() != 0;
-        let consumed = bytes.get_u8() != 0;
         let timestamp = bytes.get_u64();
         let timestamp = SystemTime::UNIX_EPOCH + Duration::from_millis(timestamp);
 
@@ -102,8 +95,7 @@ impl<D: Digest> Batch<D>
         // and set timestamp to the current time.
         Some(Self {
             timestamp,
-            accepted,
-            consumed,
+            accepted: false,
             txs,
             digest,
         })
@@ -158,14 +150,15 @@ pub enum Message<D: Digest> {
         digest: D,
         response: oneshot::Sender<bool>
     },
-    BatchConsumed {
-        digests: Vec<D>,
-    },
     // from rpc or websocket
     SubmitTx {
         payload: RawTransaction<D>,
         response: oneshot::Sender<bool>
     },
+    BatchConsumed {
+        digests: Vec<D>,
+        block_number: u64,
+    },
     // proposer consume batches to produce a block
     ConsumeBatches {
         response: oneshot::Sender<Vec<Batch<D>>>
@@ -219,17 +212,17 @@ impl<D: Digest> Mailbox<D> {
     pub async fn consume_batches(&mut self) -> Vec<Batch<D>> {
         let (response, receiver) = oneshot::channel();
         self.sender
-            .send(Message::ConsumeBatches { response: response })
+            .send(Message::ConsumeBatches { response })
             .await
             .expect("failed to consume batches");
 
         receiver.await.expect("failed to receive batches")
     }
 
-    pub async fn consumed_batches(&mut self, digests: Vec<D>) -> Vec<Batch<D>> {
-        let (response, receiver) = oneshot::channel();
+    pub async fn consumed_batches(&mut self, digests: Vec<D>, block_number: u64) -> Vec<Batch<D>> {
+        let (_, receiver) = oneshot::channel();
         self.sender
-            .send(Message::BatchConsumed { digests: digests })
+            .send(Message::BatchConsumed { digests, block_number})
             .await
             .expect("failed to mark batches as consumed");
 
@@ -274,6 +267,7 @@ pub struct Config {
     pub mailbox_size: usize,
     pub public_key: PublicKey,
     pub partition_prefix: String,
+    pub block_height: u64,
 }
 
 pub struct Mempool<
@@ -287,12 +281,15 @@ pub struct Mempool<
 
     batches: HashMap<D, Batch<D>>,
     accepted: Archive<TwoCap, D, B, R>,
+    consumed: Archive<TwoCap, D, B, R>,
 
     txs: Vec<RawTransaction<D>>,
 
     mailbox: mpsc::Receiver<Message<D>>,
     mailbox_size: usize,
 
+    block_height_seen: u64,
+
     batch_propose_interval: Duration,
     batch_size_limit: u64,
     backfill_quota: Quota,
@@ -326,15 +323,39 @@ impl<
             })
             .await
             .expect("Failed to initialize accepted archive");
+
+        let consumed_journal = Journal::init(
+            context.with_label("consumed_journal"), 
+            journal::variable::Config {
+                partition: format!("{}-consumptions", cfg.partition_prefix),
+            })
+            .await
+            .expect("Failed to initialize consumed journal");
+        let consumed_archive = Archive::init(
+        context.with_label("consumed_archive"),
+            consumed_journal, 
+            archive::Config {
+                translator: TwoCap,
+                section_mask: 0xffff_ffff_fff0_0000u64,
+                pending_writes: 0,
+                replay_concurrency: 4,
+                compression: Some(3),
+            })
+            .await
+            .expect("Failed to initialize consumed archive");
+
         let (sender, receiver) = mpsc::channel(1024);
         (Self {
             context,
             public_key: cfg.public_key,
             batches: HashMap::new(),
             accepted: accepted_archive,
+            consumed: consumed_archive,
 
             txs: vec![],
 
+            block_height_seen: cfg.block_height,
+
             mailbox: receiver,
             mailbox_size: cfg.mailbox_size,
 
@@ -350,14 +371,14 @@ impl<
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
         ),
-        mut backfill_network: (
+        backfill_network: (
             impl Sender<PublicKey = PublicKey>,
             impl Receiver<PublicKey = PublicKey>,
         ), 
         coordinator: Coordinator<PublicKey>,
         app_mailbox: ingress::Mailbox<D, PublicKey>
     ) -> Handle<()> {
-        self.context.spawn_ref()(self.run(batch_network,backfill_network, coordinator, app_mailbox))
+        self.context.spawn_ref()(self.run(batch_network, backfill_network, coordinator, app_mailbox))
     }
 
     async fn run(
@@ -398,6 +419,7 @@ impl<
         let mut waiters: HashMap<D, Vec<oneshot::Sender<Option<Batch<D>>>>> = HashMap::new();
         let mut propose_timeout = self.context.current() + self.batch_propose_interval;
         let accepted = Wrapped::new(self.accepted);
+        let consumed = Wrapped::new(self.consumed);
         loop {
             // Clear dead waiters
             waiters.retain(|_, waiters| {
@@ -423,8 +445,10 @@ impl<
                             self.txs.push(payload);
                             let _ = response.send(true);
                         },
+                        // batch ackowledged by the network, put in the accepted archive and mark batch as accepted
                         Message::BatchAcknowledged { digest, response } => {
                             if let Some(batch) = self.batches.get_mut(&digest) {
+                                accepted.put(self.block_height_seen, digest, batch.serialize().into()).await.expect("unable to store accepted batch");
                                 batch.accepted = true;
                                 let _ = response.send(true);
                                 debug!("batch accepted by the network: {}", digest);
@@ -436,7 +460,12 @@ impl<
                             let batches = self.batches.iter().filter(|(_, batch)| batch.accepted).map(|(_, batch)| batch.clone()).collect();
                             let _ = response.send(batches);
                         },
-                        Message:: BatchConsumed { digests } => {
+                        // received when a block is finalized, i.e. finalization message is received, 
+                        // remove consumed batches from buffer & put them in consumed archive
+                        Message:: BatchConsumed { digests, block_number } => {
+                            // update the seen height
+                            self.block_height_seen = block_number;
+
                             let consumed_keys: Vec<D> = self.batches.iter()
                                 .filter_map(|(digest, batch)| {
                                     if digests.contains(&batch.digest) {
@@ -447,25 +476,35 @@ impl<
                                 })
                                 .collect();
 
-                            // Then remove those entries, updating their `consumed` flag.
                             let consumed_batches: Vec<Batch<D>> = consumed_keys.into_iter()
-                                .filter_map(|key| {
-                                    self.batches.remove(&key).map(|mut batch| {
-                                        batch.consumed = true;
-                                        batch
-                                    })
-                                })
+                                .filter_map(|key| self.batches.remove(&key))
                                 .collect();
 
                             for batch in consumed_batches.iter() {
-                                accepted.put(batch.digest, batch.serialize().into()).await.expect("Failed to insert accepted batch");
+                                consumed.put(block_number, batch.digest, batch.serialize().into()).await.expect("Failed to insert accepted batch");
                             }
                         },
+                        // for validators, this should be only called when they receive 
+                        // 1. a digest from broadcast primitive
+                        // 2. a block with a list of references of batches
+                        // both of the above should only happen at their verification stage
                         Message::GetBatch { digest, response } => { 
+                            // fetch in buffer, i.e. accepted
                             if let Some(batch) = self.batches.get(&digest).cloned() {
                                 let _ = response.send(Some(batch));
                                 continue;
                             };
+                            // fetch in consumed
+                            let consumed = consumed.get(Identifier::Key(&digest))
+                                .await 
+                                .expect("Failed to get consumed batch");
+                            if let Some(consumed) = consumed {
+                                let consumed = Batch::deserialize(&consumed).expect("unable to deserialize batch");
+                                let _ = response.send(Some(consumed));
+                                continue;
+                            }
+                            
+                            // not found in the above, request from other peers
                             resolver.fetch(MultiIndex::new(Value::Digest(digest.into()))).await;
                             waiters.entry(digest).or_default().push(response);
                         },
@@ -489,6 +528,7 @@ impl<
                         },
                     }
                 },
+                // propose a batch in a given interval
                 _ = self.context.sleep_until(propose_timeout) => {
                     let mut size = 0;
                     let mut txs_cnt = 0;
@@ -527,19 +567,24 @@ impl<
                     let _ = self.batches.entry(batch.digest).or_insert(batch);
                 },
 
-                // handle batch request
+                // handle batch request, a validator will issue batches it has
                 handler_message = handler_receiver.next() => {
                     let message = handler_message.expect("Handler closed");
                     match message {
                         handler::Message::Produce { key, response } => {
                             match key.to_value() {
-                                // TODO: add a buffer
                                 key::Value::Digest(digest) => {
-                                    let accepted = accepted.get(Identifier::Key(&D::from(digest)))
+                                    if let Some(batch) = self.batches.get(&D::from(digest)).cloned() {
+                                        let _ = response.send(batch.serialize().into());
+                                        continue;
+                                    };
+
+                                    let consumed = consumed.get(Identifier::Key(&D::from(digest)))
                                         .await 
                                         .expect("Failed to get accepted batch");
-                                    if let Some(accepted) = accepted {
-                                        let _ = response.send(accepted);
+                                    if let Some(consumed) = consumed {
+                                        let _ = response.send(consumed);
+                                        continue;
                                     }
                                 }
                             }         
@@ -560,7 +605,9 @@ impl<
                                         }
                                     }
 
-                                    accepted.put(batch.digest, value).await.expect("Failed to insert accepted batch");
+                                    debug!(?batch.digest, "receive a batch from resolver");
+                                    // add received batch to buffer
+                                    self.batches.insert(batch.digest, batch);
                                 }
                             }         
                         },

From e7f9b4eb74bfbbbd38e071a1bf6af3bbbe1778fb Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Tue, 25 Mar 2025 15:01:33 -0400
Subject: [PATCH 15/20] backfill e2e & consume batches e2e

---
 chain/src/actors/mempool/actor.rs   |   4 +-
 chain/src/actors/mempool/mempool.rs |  51 +++++++----
 chain/src/actors/mempool/mod.rs     | 132 ++++++++++++++++++++--------
 chain/src/lib.rs                    |   1 +
 chain/src/macros/mod.rs             |  45 ++++++++++
 5 files changed, 175 insertions(+), 58 deletions(-)
 create mode 100644 chain/src/macros/mod.rs

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 2d0402bf..c7844cfe 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -42,13 +42,13 @@ impl<D: Digest, P: Array> Actor<D, P> {
                     }
                 }
                 Message::Verify(_context, _payload, sender) => {
-                    // TODO: backfill inplace?
+                    debug!(digest=?_payload, "incoming verification request");
                     let Some(_) = mempool.get_batch(_payload).await else {
                         warn!(?_payload, "batch not exists");
                         let _ = sender.send(false);
                         continue;
                     };
-                    debug!("issue verfication to batch {}", _payload);
+                    debug!("issue verfication to batch={}", _payload);
                     let result = sender.send(true);
                     if result.is_err() {
                         error!("verify dropped");
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 1d53391b..98fa087e 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -14,12 +14,12 @@ use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
 use governor::Quota;
 use rand::Rng;
-use tracing::{debug, warn};
+use tracing::{debug, warn, info};
 use governor::clock::Clock as GClock;
-use tracing_subscriber::fmt::time;
 use super::{actor, handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped};
+use crate::maybe_delay_between;
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Batch<D: Digest>  {
     pub timestamp: SystemTime,
     // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
@@ -110,7 +110,7 @@ impl<D: Digest> Batch<D>
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct RawTransaction<D: Digest> {
     pub raw: Bytes,
 
@@ -158,6 +158,7 @@ pub enum Message<D: Digest> {
     BatchConsumed {
         digests: Vec<D>,
         block_number: u64,
+        response: oneshot::Sender<bool>,
     },
     // proposer consume batches to produce a block
     ConsumeBatches {
@@ -219,10 +220,10 @@ impl<D: Digest> Mailbox<D> {
         receiver.await.expect("failed to receive batches")
     }
 
-    pub async fn consumed_batches(&mut self, digests: Vec<D>, block_number: u64) -> Vec<Batch<D>> {
-        let (_, receiver) = oneshot::channel();
+    pub async fn consumed_batches(&mut self, digests: Vec<D>, block_number: u64) -> bool {
+        let (response, receiver) = oneshot::channel();
         self.sender
-            .send(Message::BatchConsumed { digests, block_number})
+            .send(Message::BatchConsumed { digests, block_number, response })
             .await
             .expect("failed to mark batches as consumed");
 
@@ -323,7 +324,7 @@ impl<
             })
             .await
             .expect("Failed to initialize accepted archive");
-
+            
         let consumed_journal = Journal::init(
             context.with_label("consumed_journal"), 
             journal::variable::Config {
@@ -404,7 +405,7 @@ impl<
                 producer: handler,
                 mailbox_size: self.mailbox_size,
                 requester_config: requester::Config {
-                    public_key: self.public_key,
+                    public_key: self.public_key.clone(),
                     rate_limit: self.backfill_quota,
                     initial: Duration::from_secs(1),
                     timeout: Duration::from_secs(2),
@@ -462,7 +463,7 @@ impl<
                         },
                         // received when a block is finalized, i.e. finalization message is received, 
                         // remove consumed batches from buffer & put them in consumed archive
-                        Message:: BatchConsumed { digests, block_number } => {
+                        Message:: BatchConsumed { digests, block_number, response } => {
                             // update the seen height
                             self.block_height_seen = block_number;
 
@@ -475,6 +476,12 @@ impl<
                                     }
                                 })
                                 .collect();
+                            let consumed_keys_len = consumed_keys.len();
+
+                            // not all provided keys are consumed, there must be state corruption, panic immediately
+                            if consumed_keys_len != digests.len() {
+                                panic!("not all provided batch digests consumed, provided={:?}, consumed={:?}", digests, consumed_keys);
+                            }
 
                             let consumed_batches: Vec<Batch<D>> = consumed_keys.into_iter()
                                 .filter_map(|key| self.batches.remove(&key))
@@ -483,6 +490,8 @@ impl<
                             for batch in consumed_batches.iter() {
                                 consumed.put(block_number, batch.digest, batch.serialize().into()).await.expect("Failed to insert accepted batch");
                             }
+
+                            let _ = response.send(true);
                         },
                         // for validators, this should be only called when they receive 
                         // 1. a digest from broadcast primitive
@@ -546,12 +555,15 @@ impl<
                         continue;
                     }
 
-                    // 1. send raw batch over p2p
                     let batch = Batch::new(self.txs.drain(..txs_cnt).collect(), self.context.current());
                     self.batches.insert(batch.digest, batch.clone());
-                    batch_network.0.send(Recipients::All, batch.serialize().into(), true).await.expect("failed to broadcast batch");
-                    // 2. send batch digest over broadcast layer
-                    app_mailbox.broadcast(batch.digest).await;
+
+                    debug!("broadcasting batch & digest={}, peer={}", batch.digest, self.public_key.clone());
+                    maybe_delay_between! {
+                        self.context,
+                        app_mailbox.broadcast(batch.digest).await;
+                        batch_network.0.send(Recipients::All, batch.serialize().into(), true).await.expect("failed to broadcast batch");
+                    }
 
                     // reset the timeout
                     propose_timeout = self.context.current() + self.batch_propose_interval;
@@ -563,7 +575,7 @@ impl<
                         continue;
                     };
 
-                    debug!(?sender, digest=?batch.digest, "received batch");
+                    debug!(?sender, digest=?batch.digest, receiver=?self.public_key, "received batch");
                     let _ = self.batches.entry(batch.digest).or_insert(batch);
                 },
 
@@ -586,6 +598,7 @@ impl<
                                         let _ = response.send(consumed);
                                         continue;
                                     }
+                                    debug!(?digest, "missing batch");
                                 }
                             }         
                         },
@@ -599,15 +612,17 @@ impl<
                                     }
 
                                     if let Some(waiters) = waiters.remove(&batch.digest) {
-                                        debug!(?batch.digest, "waiters resolved via batch");
+                                        debug!(?batch.digest, ?self.public_key, "waiters resolved via batch");
                                         for waiter in waiters {
                                             let _ = waiter.send(Some(batch.clone()));
                                         }
                                     }
 
                                     debug!(?batch.digest, "receive a batch from resolver");
-                                    // add received batch to buffer
-                                    self.batches.insert(batch.digest, batch);
+                                    // add received batch to buffer if not exists
+                                    self.batches.entry(batch.digest).or_insert(batch);
+
+                                    let _ = response.send(true);
                                 }
                             }         
                         },
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index 18096d8b..c77c3896 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -10,10 +10,11 @@ pub mod archive;
 #[cfg(test)]
 mod tests {
     use core::panic;
-    use std::{collections::{BTreeMap, HashMap}, sync::{Arc, Mutex}, time::Duration};
+    use std::{collections::{BTreeMap, HashMap}, num::NonZeroU32, sync::{Arc, Mutex}, time::Duration};
     use bytes::Bytes;
     use commonware_broadcast::linked::{Config, Engine};
     
+    use governor::Quota;
     use prometheus_client::metrics::info;
     use tracing::{debug, info, warn};
 
@@ -25,7 +26,12 @@ mod tests {
 
     use super::{collector, ingress, mempool::{self, Mempool, RawTransaction}};
 
-    type Registrations<P> = HashMap<P, ((Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>), (Sender<P>, Receiver<P>))>;
+    type Registrations<P> = HashMap<P, (
+        (Sender<P>, Receiver<P>), 
+        (Sender<P>, Receiver<P>), 
+        (Sender<P>, Receiver<P>),
+        (Sender<P>, Receiver<P>)
+    )>;
 
     #[allow(dead_code)]
     enum Action {
@@ -41,16 +47,19 @@ mod tests {
         (Sender<PublicKey>, Receiver<PublicKey>),
         (Sender<PublicKey>, Receiver<PublicKey>),
         (Sender<PublicKey>, Receiver<PublicKey>),
+        (Sender<PublicKey>, Receiver<PublicKey>),
     )> { 
         let mut registrations = HashMap::new();        
         for validator in validators.iter() {
             let (digest_sender, digest_receiver) = oracle.register(validator.clone(), 4).await.unwrap();
             let (ack_sender, ack_receiver) = oracle.register(validator.clone(), 5).await.unwrap();
-            let (chunk_sender, chunk_receiver) = oracle.register(validator.clone(), 6).await.unwrap();
+            let (batch_sender, batch_receiver) = oracle.register(validator.clone(), 6).await.unwrap();
+            let (batch_backfill_sender, batch_backfill_receiver) = oracle.register(validator.clone(), 7).await.unwrap();
             registrations.insert(validator.clone(), (
                 (digest_sender, digest_receiver),
                 (ack_sender, ack_receiver),
-                (chunk_sender, chunk_receiver),
+                (batch_sender, batch_receiver),
+                (batch_backfill_sender, batch_backfill_receiver),
             ));
         }
         registrations
@@ -198,23 +207,28 @@ mod tests {
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn spawn_validator_engines(
+    async fn spawn_validator_engines(
         context: Context,
         identity: poly::Public,
         pks: &[PublicKey],
         validators: &[(PublicKey, Ed25519, Share)],
         registrations: &mut Registrations<PublicKey>,
-        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
         collectors: &mut BTreeMap<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>,
         refresh_epoch_timeout: Duration,
         rebroadcast_timeout: Duration,
-    ) {
+    ) -> BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>> {
+        let mut mailboxes = BTreeMap::new();
         let namespace = b"my testing namespace";
         for (validator, scheme, share) in validators.iter() {
-            let (mempool, mempool_mailbox) = Mempool::new(context.with_label("mempool"), mempool::Config { 
+            let (mempool, mempool_mailbox) = Mempool::init(context.with_label("mempool"), mempool::Config { 
                 batch_propose_interval: Duration::from_millis(500), 
                 batch_size_limit: 1024*1024, 
-            });
+                backfill_quota: Quota::per_second(NonZeroU32::new(10).unwrap()),
+                mailbox_size: 1024,
+                public_key: scheme.public_key(),
+                block_height: 0,
+                partition_prefix: format!("mempool"),
+            }).await;
             mailboxes.insert(validator.clone(), mempool_mailbox.clone());
 
 
@@ -244,7 +258,7 @@ mod tests {
                     crypto: scheme.clone(),
                     application: app_mailbox.clone(),
                     collector: collectors.get(validator).unwrap().clone(),
-                    coordinator,
+                    coordinator: coordinator.clone(),
                     mailbox_size: 1024,
                     verify_concurrent: 1024,
                     namespace: namespace.to_vec(),
@@ -259,31 +273,50 @@ mod tests {
             );
 
             context.with_label("app").spawn(move |_| app.run(mailbox, mempool_mailbox));
-            let ((a1, a2), (b1, b2), (c1, c2)) = registrations.remove(validator).unwrap();
+            let ((a1, a2), (b1, b2), (c1, c2), (d1, d2)) = registrations.remove(validator).unwrap();
             engine.start((a1, a2), (b1, b2));
-            mempool.start((c1, c2), app_mailbox.clone());
+            mempool.start((c1, c2), (d1, d2), coordinator, app_mailbox.clone());
         }
+        mailboxes
     }
 
-    fn spawn_mempools(
+    async fn spawn_mempools(
         context: Context,
+        identity: poly::Public,
+        pks: &[PublicKey],
         validators: &[(PublicKey, Ed25519, Share)],
         registrations: &mut Registrations<PublicKey>,
         app_mailbox: &mut ingress::Mailbox<sha256::Digest, PublicKey>,
-        mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
-    ) {
-        for (validator, _, _) in validators.iter() {
-            let (mempool, mailbox) = Mempool::new(
+        // mailboxes: &mut BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>,
+    ) -> BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>> {
+        let mut mailboxes= BTreeMap::new();
+        for (validator, _, share) in validators.iter() {
+            let context = context.with_label(&validator.to_string());
+            let mut coordinator = super::coordinator::Coordinator::<PublicKey>::new(
+                identity.clone(),
+                pks.to_vec(),
+                *share,
+            );
+            coordinator.set_view(111);
+
+            let (mempool, mailbox) = Mempool::init(
                 context.with_label("mempool"), 
                 mempool::Config {
-                    batch_propose_interval: Duration::from_millis(500),
-                    batch_size_limit: 1024*1024, // 1MB
+                    batch_propose_interval: Duration::from_millis(500), 
+                    batch_size_limit: 1024*1024, 
+                    backfill_quota: Quota::per_second(NonZeroU32::new(10).unwrap()),
+                    mailbox_size: 1024,
+                    public_key: validator.clone(),
+                    block_height: 0,
+                    partition_prefix: format!("mempool"),
                 }
-            );
+            ).await;
             mailboxes.insert(validator.clone(), mailbox);
-            let ((_, _), (_, _), (c1, c2)) = registrations.remove(validator).unwrap();
-            mempool.start((c1, c2), app_mailbox.clone());
+            let ((_, _), (_, _), (c1, c2), (d1, d2)) = registrations.remove(validator).unwrap();
+            mempool.start((c1, c2), (d1, d2),coordinator, app_mailbox.clone());
         }
+
+        mailboxes
     }
 
     async fn spawn_tx_issuer_and_wait(
@@ -291,6 +324,7 @@ mod tests {
         mailboxes: Arc<Mutex<BTreeMap<PublicKey, mempool::Mailbox<sha256::Digest>>>>,
         num_txs: u32,
         wait_batch_acknowlegement: bool,
+        consume_batch: bool,
     ) {
         context
             .clone()
@@ -347,6 +381,20 @@ mod tests {
                             }
                             info!("batch contain tx {} acknowledged", batch.digest);
                         }
+                        if consume_batch {
+                            // 1. consume the batches
+                            let batches = mailbox.consume_batches().await;
+                            assert!(batches.len() != 0, "expected some batches to consume");
+                            info!("consumed batches: {:?}", batches);
+                            // 2. mark the batches as consumed
+                            info!("marking batches as consumed");
+                            let digests = batches.iter().map(|batch| batch.digest).collect();
+                            let _ = mailbox.consumed_batches(digests, 0).await;
+                            // 3. try consume batches again
+                            let batches = mailbox.consume_batches().await;
+                            assert!(batches.len() == 0, "expected zero batches to consume");
+                            info!("zero batches left after consumption");
+                        }
                     }
                 }
             }).await.unwrap();
@@ -365,23 +413,23 @@ mod tests {
                 context.with_label("simulation"), 
                 num_validators, 
                 &mut shares_vec).await;
-            let mailboxes = Arc::new(Mutex::new(BTreeMap::<
-                PublicKey,
-                mempool::Mailbox<sha256::Digest>,
-            >::new()));
+            // let mailboxes = Arc::new(Mutex::new(BTreeMap::<
+            //     PublicKey,
+            //     mempool::Mailbox<sha256::Digest>,
+            // >::new()));
             let mut collectors = BTreeMap::<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>::new();
-            spawn_validator_engines(
+            let mailboxes = spawn_validator_engines(
                 context.with_label("validator"), 
                 identity.clone(), 
                 &pks, 
                 &validators, 
                 &mut registrations, 
-                &mut mailboxes.lock().unwrap(), 
                 &mut collectors, 
                 Duration::from_millis(100), 
                 Duration::from_secs(5)
-            );
-            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1, true).await;
+            ).await;
+            let guarded_mailboxes = Arc::new(Mutex::new(mailboxes));
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), guarded_mailboxes, 1, true, true).await;
         });
     }
 
@@ -390,7 +438,7 @@ mod tests {
         let num_validators: u32 = 4;
         let quorum: u32 = 3;
         let (runner, mut context, _) = Executor::timed(Duration::from_secs(30));
-        let (_, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
+        let (identity, mut shares_vec) = dkg::ops::generate_shares(&mut context, None, num_validators, quorum);        
         shares_vec.sort_by(|a, b| a.index.cmp(&b.index));
 
         info!("mempool p2p test started");
@@ -398,16 +446,24 @@ mod tests {
         let mut app_mailbox = ingress::Mailbox::new(app_mailbox_sender);
 
         runner.start(async move {
-            let (_oracle, validators, _, mut registrations ) = initialize_simulation(
+            let (_oracle, validators, pks, mut registrations ) = initialize_simulation(
                 context.with_label("simulation"), 
                 num_validators, 
                 &mut shares_vec).await;
-            let mailboxes = Arc::new(Mutex::new(BTreeMap::<
-                PublicKey,
-                mempool::Mailbox<sha256::Digest>,
-            >::new()));
-            spawn_mempools(context.with_label("mempool"), &validators, &mut registrations, &mut app_mailbox, &mut mailboxes.lock().unwrap());
-            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), mailboxes, 1, false).await;
+            // let mailboxes = Arc::new(Mutex::new(BTreeMap::<
+            //     PublicKey,
+            //     mempool::Mailbox<sha256::Digest>,
+            // >::new()));
+            let mailboxes = spawn_mempools(
+                context.with_label("mempool"), 
+                identity,
+                &pks, 
+                &validators, 
+                &mut registrations, 
+                &mut app_mailbox, 
+            ).await;
+            let guarded_mailboxes = Arc::new(Mutex::new(mailboxes));
+            spawn_tx_issuer_and_wait(context.with_label("tx_issuer"), guarded_mailboxes, 1, false, false).await;
         });
     }
 }
\ No newline at end of file
diff --git a/chain/src/lib.rs b/chain/src/lib.rs
index 9b10089c..02d3a20b 100644
--- a/chain/src/lib.rs
+++ b/chain/src/lib.rs
@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
 
 pub mod actors;
 pub mod engine;
+pub mod macros;
 
 /// Trait for interacting with an indexer.
 pub trait Indexer: Clone + Send + Sync + 'static {
diff --git a/chain/src/macros/mod.rs b/chain/src/macros/mod.rs
new file mode 100644
index 00000000..a7e8e3e6
--- /dev/null
+++ b/chain/src/macros/mod.rs
@@ -0,0 +1,45 @@
+/// A macro that conditionally inserts a random delay between two asynchronous expressions during tests.
+///
+/// When compiled in test mode (`#[cfg(test)]`), a random delay is inserted using the provided context's
+/// `sleep` method. In non-test builds, the expressions are executed sequentially without delay.
+/// The macro discards the return values of the expressions so that it always returns `()`.
+///
+/// # Example
+///
+/// ```rust
+/// // Assume that `self.context` is defined and provides an async sleep method,
+/// // and that `batch_network` and `app_mailbox` are in scope.
+/// maybe_delay_between!(self.context,
+///     app_mailbox.broadcast(batch.digest).await;
+///     batch_network.0.send(Recipients::All, batch.serialize().into(), true)
+///         .await.expect("failed to broadcast batch")
+/// );
+/// ```
+///
+/// You can also include an optional trailing semicolon after the second expression:
+///
+/// ```rust
+/// maybe_delay_between!(self.context,
+///     app_mailbox.broadcast(batch.digest).await;
+///     batch_network.0.send(Recipients::All, batch.serialize().into(), true)
+///         .await.expect("failed to broadcast batch");
+/// );
+/// ```
+#[macro_export]
+macro_rules! maybe_delay_between {
+    ($context:expr, $first:expr; $second:expr $(;)?) => {{
+        // Execute the first asynchronous expression and discard its value.
+        let _ = $first;
+        // If we're in a test build, add a random delay using the provided context.
+        #[cfg(test)]
+        {
+            use rand::Rng;
+            let delay_ms = rand::thread_rng().gen_range(50..150); // random delay between 50 and 150 ms
+            $context.sleep(std::time::Duration::from_millis(delay_ms)).await;
+        }
+        // Execute the second asynchronous expression and discard its value.
+        let _ = $second;
+        // Return unit.
+        ()
+    }};
+}

From 92c6ef0753bd35eb6a5f8e9d675b42b63d482621 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Wed, 26 Mar 2025 14:49:14 -0400
Subject: [PATCH 16/20] resolve launching

---
 chain/src/actors/mempool/collector.rs | 87 ++-------------------------
 chain/src/actors/mempool/mempool.rs   |  2 +-
 chain/src/actors/mempool/mod.rs       | 72 +---------------------
 chain/src/bin/validator.rs            | 53 +++++++++++++---
 4 files changed, 50 insertions(+), 164 deletions(-)

diff --git a/chain/src/actors/mempool/collector.rs b/chain/src/actors/mempool/collector.rs
index 28785554..4bf72efd 100644
--- a/chain/src/actors/mempool/collector.rs
+++ b/chain/src/actors/mempool/collector.rs
@@ -14,9 +14,7 @@ use super::mempool;
 
 enum Message<C: Scheme, D: Digest> {
     Acknowledged(Proof, D),
-    GetTip(C::PublicKey, oneshot::Sender<Option<u64>>),
-    GetContiguousTip(C::PublicKey, oneshot::Sender<Option<u64>>),
-    Get(C::PublicKey, u64, oneshot::Sender<Option<D>>),
+    _Phantom(C::PublicKey),    
 }
 
 pub struct Collector<C: Scheme, D: Digest> {
@@ -27,15 +25,6 @@ pub struct Collector<C: Scheme, D: Digest> {
 
     // Public key of the group
     public: group::Public,
-
-    // All known digests
-    digests: HashMap<C::PublicKey, BTreeMap<u64, D>>,
-
-    // Highest contiguous known height for each sequencer
-    contiguous: HashMap<C::PublicKey, u64>,
-
-    // Highest known height for each sequencer
-    highest: HashMap<C::PublicKey, u64>,
 }
 
 impl<C: Scheme, D: Digest> Collector<C, D> {
@@ -46,9 +35,6 @@ impl<C: Scheme, D: Digest> Collector<C, D> {
                 mailbox: receiver,
                 namespace: namespace.to_vec(),
                 public,
-                digests: HashMap::new(),
-                contiguous: HashMap::new(),
-                highest: HashMap::new(),
             },
             Mailbox { sender },
         )
@@ -61,7 +47,7 @@ impl<C: Scheme, D: Digest> Collector<C, D> {
                     // Check proof.
                     // The prover checks the validity of the threshold signature when deserializing
                     let prover = Prover::<C, D>::new(&self.namespace, self.public);
-                    let context = match prover.deserialize_threshold(proof) {
+                    let _ = match prover.deserialize_threshold(proof) {
                         Some((context, _payload, _epoch, _threshold)) => context,
                         None => {
                             error!("invalid proof");
@@ -74,44 +60,8 @@ impl<C: Scheme, D: Digest> Collector<C, D> {
                     if !acknowledge {
                         error!("unable to acknowledge batch {}", payload)
                     }
-
-                    // Update the collector
-                    let digests = self.digests.entry(context.sequencer.clone()).or_default();
-                    digests.insert(context.height, payload);
-
-                    // Update the highest height
-                    let highest = self.highest.get(&context.sequencer).copied().unwrap_or(0);
-                    self.highest
-                        .insert(context.sequencer.clone(), max(highest, context.height));
-
-                    // Update the highest contiguous height
-                    let highest = self.contiguous.get(&context.sequencer);
-                    if (highest.is_none() && context.height == 0)
-                        || (highest.is_some() && context.height == highest.unwrap() + 1)
-                    {
-                        let mut contiguous = context.height;
-                        while digests.contains_key(&(contiguous + 1)) {
-                            contiguous += 1;
-                        }
-                        self.contiguous.insert(context.sequencer, contiguous);
-                    }
-                }
-                Message::GetTip(sequencer, sender) => {
-                    let hi = self.highest.get(&sequencer).copied();
-                    sender.send(hi).unwrap();
-                }
-                Message::GetContiguousTip(sequencer, sender) => {
-                    let contiguous = self.contiguous.get(&sequencer).copied();
-                    sender.send(contiguous).unwrap();
-                }
-                Message::Get(sequencer, height, sender) => {
-                    let digest = self
-                        .digests
-                        .get(&sequencer)
-                        .and_then(|map| map.get(&height))
-                        .cloned();
-                    sender.send(digest).unwrap();
                 }
+                _ => unreachable!()
             }
         }
     }
@@ -130,33 +80,4 @@ impl<C: Scheme, D: Digest> Z for Mailbox<C, D> {
             .await
             .expect("Failed to send acknowledged");
     }
-}
-
-impl<C: Scheme, D: Digest> Mailbox<C, D> {
-    pub async fn get_tip(&mut self, sequencer: C::PublicKey) -> Option<u64> {
-        let (sender, receiver) = oneshot::channel();
-        self.sender
-            .send(Message::GetTip(sequencer, sender))
-            .await
-            .unwrap();
-        receiver.await.unwrap()
-    }
-
-    pub async fn get_contiguous_tip(&mut self, sequencer: C::PublicKey) -> Option<u64> {
-        let (sender, receiver) = oneshot::channel();
-        self.sender
-            .send(Message::GetContiguousTip(sequencer, sender))
-            .await
-            .unwrap();
-        receiver.await.unwrap()
-    }
-
-    pub async fn get(&mut self, sequencer: C::PublicKey, height: u64) -> Option<D> {
-        let (sender, receiver) = oneshot::channel();
-        self.sender
-            .send(Message::Get(sequencer, height, sender))
-            .await
-            .unwrap();
-        receiver.await.unwrap()
-    }
-}
+}
\ No newline at end of file
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 98fa087e..41d21442 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -382,7 +382,7 @@ impl<
         self.context.spawn_ref()(self.run(batch_network, backfill_network, coordinator, app_mailbox))
     }
 
-    async fn run(
+    pub async fn run(
         mut self,
         mut batch_network: (
             impl Sender<PublicKey = PublicKey>,
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index c77c3896..e793dfd6 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -94,41 +94,7 @@ mod tests {
         }
     }
 
-    async fn await_collectors(
-        context: Context,
-        collectors: &BTreeMap<PublicKey, collector::Mailbox<Ed25519, sha256::Digest>>,
-        threshold: u64,
-    ) {
-        let mut receivers = Vec::new();
-        for (sequencer, mailbox) in collectors.iter() {
-            // Create a oneshot channel to signal when the collector has reached the threshold.
-            let (tx, rx) = oneshot::channel();
-            receivers.push(rx);
-
-            // Spawn a watcher for the collector.
-            context.with_label("collector_watcher").spawn({
-                let sequencer = sequencer.clone();
-                let mut mailbox = mailbox.clone();
-                move |context| async move {
-                    loop {
-                        let tip = mailbox.get_tip(sequencer.clone()).await.unwrap_or(0);
-                        debug!(tip, ?sequencer, "collector");
-                        if tip >= threshold {
-                            let _ = tx.send(sequencer.clone());
-                            break;
-                        }
-                        context.sleep(Duration::from_millis(100)).await;
-                    }
-                }
-            });
-        }
-
-        // Wait for all oneshot receivers to complete.
-        let results = join_all(receivers).await;
-        assert_eq!(results.len(), collectors.len());
-    }
-
-        async fn initialize_simulation(
+    async fn initialize_simulation(
         context: Context,
         num_validators: u32,
         shares_vec: &mut [Share],
@@ -170,42 +136,6 @@ mod tests {
         (oracle, validators, pks, registrations)
     }
 
-    fn spawn_proposer(
-        context: Context,
-        mailboxes: Arc<
-            Mutex<BTreeMap<PublicKey, super::ingress::Mailbox<sha256::Digest, PublicKey>>>,
-        >,
-        invalid_when: fn(u64) -> bool,
-    ) {
-        context
-            .clone()
-            .with_label("invalid signature proposer")
-            .spawn(move |context| async move {
-                let mut iter = 0;
-                loop {
-                    iter += 1;
-                    let mailbox_vec: Vec<super::ingress::Mailbox<sha256::Digest, PublicKey>> = {
-                        let guard = mailboxes.lock().unwrap();
-                        guard.values().cloned().collect()
-                    };
-                    for mut mailbox in mailbox_vec {
-                        let payload = Bytes::from(format!("hello world, iter {}", iter));
-                        let mut hasher = sha256::Sha256::default();
-                        hasher.update(&payload);
-
-                        // Inject an invalid digest by updating with the payload again.
-                        if invalid_when(iter) {
-                            hasher.update(&payload);
-                        }
-
-                        let digest = hasher.finalize();
-                        mailbox.broadcast(digest).await;
-                    }
-                    context.sleep(Duration::from_millis(250)).await;
-                }
-            });
-    }
-
     #[allow(clippy::too_many_arguments)]
     async fn spawn_validator_engines(
         context: Context,
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index 6b6546be..be6a1c9f 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -1,4 +1,4 @@
-use alto_chain::{actors::mempool, engine, Config};
+use alto_chain::{actors::mempool::{self, mempool::Mempool}, engine, Config};
 use alto_client::Client;
 use alto_types::P2P_NAMESPACE;
 use axum::{routing::get, serve, Extension, Router};
@@ -7,7 +7,7 @@ use commonware_broadcast::linked;
 use commonware_cryptography::{
     bls12381::primitives::{
         group::{self, Element},
-        poly,
+        poly::{self, public},
     }, ed25519::{PrivateKey, PublicKey}, sha256, Ed25519, Scheme
 };
 use commonware_deployer::ec2::Peers;
@@ -36,8 +36,10 @@ const VOTER_CHANNEL: u32 = 0;
 const RESOLVER_CHANNEL: u32 = 1;
 const BROADCASTER_CHANNEL: u32 = 2;
 const BACKFILLER_CHANNEL: u32 = 3;
-const MEMPOOL_CHANNEL: u32 = 4;
+const MEMPOOL_DIGEST_CHANNEL: u32 = 4;
 const MEMPOOL_ACK_CHANNEL: u32 = 5;
+const MEMPOOL_BATCH_CHANNEL: u32 = 6;
+const MMEPOOL_BACKFILL_CHANNEL: u32 = 7;
 
 
 const LEADER_TIMEOUT: Duration = Duration::from_secs(1);
@@ -204,7 +206,7 @@ fn main() {
         // Register mempool broadcast channel
         let mempool_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
         let mempool_broadcaster = network.register(
-            MEMPOOL_CHANNEL,
+            MEMPOOL_DIGEST_CHANNEL,
             mempool_limit,
             config.message_backlog,
             Some(3),
@@ -217,6 +219,23 @@ fn main() {
             config.message_backlog,
             Some(3),
         );
+
+        let mempool_batch_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
+        let mempool_batch_broadcaster = network.register(
+            MEMPOOL_BATCH_CHANNEL,
+            mempool_ack_limit,
+            config.message_backlog,
+            Some(3),
+        );
+        
+
+        let mempool_backfill_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
+        let mempool_backfill_broadcaster = network.register(
+            MMEPOOL_BACKFILL_CHANNEL,
+            mempool_ack_limit,
+            config.message_backlog,
+            Some(3),
+        );
         
 
         // Create network
@@ -230,13 +249,13 @@ fn main() {
 
         // Create mempool/broadcast/Proof of Availability engine
         let mempool_namespace = b"mempool";
-        let (mempool_application, mempool_mailbox) = mempool::actor::Actor::<sha256::Digest, PublicKey>::new();
+        let (mempool_application, mempool_app_mailbox) = mempool::actor::Actor::<sha256::Digest, PublicKey>::new();
         let broadcast_coordinator = mempool::coordinator::Coordinator::new(identity.clone(), peer_keys.clone(), share);
         let (_, collector_mailbox) = mempool::collector::Collector::<Ed25519, sha256::Digest>::new(mempool_namespace, identity_public);
         let (broadcast_engine, broadcast_mailbox) = linked::Engine::new(context.with_label("broadcast_engine"), linked::Config { 
             crypto:  signer.clone(), 
-            coordinator: broadcast_coordinator, 
-            application: mempool_mailbox, 
+            coordinator: broadcast_coordinator.clone(), 
+            application: mempool_app_mailbox.clone(), 
             collector: collector_mailbox, 
             mailbox_size: 1024, 
             verify_concurrent: 1024,
@@ -249,9 +268,25 @@ fn main() {
             journal_heights_per_section: 10, 
             journal_replay_concurrency: 1 
         });
-        context.with_label("mempool").spawn(|_| mempool_application.run(broadcast_mailbox));
+
+        let (mempool, mempool_mailbox) = Mempool::init(
+            context.with_label("mempoool"), 
+            mempool::mempool::Config { 
+                batch_propose_interval: Duration::from_millis(500), 
+                batch_size_limit: 1024*1024, 
+                backfill_quota: Quota::per_second(NonZeroU32::new(10).unwrap()),
+                mailbox_size: 1024,
+                public_key: public_key,
+                block_height: 0,
+                partition_prefix: format!("mempool"),
+            }
+        ).await;
+
         let broadcast_engine = broadcast_engine.start(mempool_broadcaster, mempool_ack_broadcaster);
 
+        let mempool_handler = mempool.start(mempool_batch_broadcaster, mempool_backfill_broadcaster, broadcast_coordinator, mempool_app_mailbox);
+        let mempool_broadcast_app_handler = context.with_label("mempool_app").spawn(|_| mempool_application.run(broadcast_mailbox, mempool_mailbox));
+
         // Create engine
         let config = engine::Config {
             partition_prefix: "engine".to_string(),
@@ -350,7 +385,7 @@ fn main() {
         });
 
         // Wait for any task to error
-        if let Err(e) = try_join_all(vec![p2p, engine, broadcast_engine, system, metrics]).await {
+        if let Err(e) = try_join_all(vec![p2p, engine, broadcast_engine, system, metrics, mempool_handler, mempool_broadcast_app_handler]).await {
             error!(?e, "task failed");
         }
     });

From 4e30439303d72073ecaa9081e065d625d180deaa Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Wed, 26 Mar 2025 14:52:59 -0400
Subject: [PATCH 17/20] lint

---
 chain/src/actors/mempool/actor.rs       |  2 +-
 chain/src/actors/mempool/collector.rs   |  6 +-----
 chain/src/actors/mempool/coordinator.rs |  2 +-
 chain/src/actors/mempool/ingress.rs     |  1 +
 chain/src/actors/mempool/mempool.rs     | 10 +++++-----
 chain/src/actors/mempool/mod.rs         |  7 +++----
 chain/src/bin/validator.rs              |  8 ++++----
 7 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index c7844cfe..40e9da8b 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -1,4 +1,4 @@
-use super::{ ingress::{Mailbox, Message}, mempool::{self, Batch}};
+use super::{ ingress::{Mailbox, Message}, mempool};
 use commonware_broadcast::Broadcaster;
 use commonware_cryptography::Digest;
 use commonware_utils::Array;
diff --git a/chain/src/actors/mempool/collector.rs b/chain/src/actors/mempool/collector.rs
index 4bf72efd..9b746887 100644
--- a/chain/src/actors/mempool/collector.rs
+++ b/chain/src/actors/mempool/collector.rs
@@ -1,13 +1,9 @@
 use commonware_broadcast::{linked::Prover, Collector as Z, Proof, };
 use commonware_cryptography::{bls12381::primitives::group, Digest, Scheme};
 use futures::{
-    channel::{mpsc, oneshot},
+    channel::mpsc,
     SinkExt, StreamExt,
 };
-use std::{
-    cmp::max,
-    collections::{BTreeMap, HashMap},
-};
 use tracing::error;
 
 use super::mempool;
diff --git a/chain/src/actors/mempool/coordinator.rs b/chain/src/actors/mempool/coordinator.rs
index 587a7cb6..da796a12 100644
--- a/chain/src/actors/mempool/coordinator.rs
+++ b/chain/src/actors/mempool/coordinator.rs
@@ -3,7 +3,7 @@ use commonware_cryptography::bls12381::primitives::{
     group::{Public, Share},
     poly::Poly,
 };
-use commonware_resolver::{p2p};
+use commonware_resolver::p2p;
 use commonware_utils::Array;
 use std::collections::HashMap;
 
diff --git a/chain/src/actors/mempool/ingress.rs b/chain/src/actors/mempool/ingress.rs
index f2e7050e..e8dd1def 100644
--- a/chain/src/actors/mempool/ingress.rs
+++ b/chain/src/actors/mempool/ingress.rs
@@ -4,6 +4,7 @@ use commonware_broadcast::{linked::Context, Application as A};
 use futures::{ channel::{mpsc, oneshot}, SinkExt};
 
 pub struct Payload {
+    #[allow(dead_code)]
     data: Vec<u8>
 }
 
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 41d21442..ddab49cf 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,7 +1,7 @@
-use std::{alloc::System, collections::HashMap, time::{Duration, SystemTime}};
+use std::{collections::HashMap, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
-use commonware_cryptography::{bls12381::primitives::group::Public, ed25519::PublicKey, sha256, Digest, Hasher, Sha256};
+use commonware_cryptography::{ed25519::PublicKey, sha256, Digest, Hasher, Sha256};
 use commonware_p2p::{utils::requester, Receiver, Recipients, Sender};
 use commonware_runtime::{Blob, Clock, Handle, Metrics, Spawner, Storage};
 use commonware_resolver::{p2p, Resolver};
@@ -9,14 +9,14 @@ use commonware_storage::{
     archive::{self, translator::TwoCap, Archive, Identifier}, 
     journal::{self, variable::Journal},
 };
-use commonware_utils::{Array, SystemTimeExt};
+use commonware_utils::SystemTimeExt;
 use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
 use governor::Quota;
 use rand::Rng;
-use tracing::{debug, warn, info};
+use tracing::{debug, warn};
 use governor::clock::Clock as GClock;
-use super::{actor, handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped};
+use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped};
 use crate::maybe_delay_between;
 
 #[derive(Clone, Debug)]
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index e793dfd6..db6f38ca 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -15,16 +15,15 @@ mod tests {
     use commonware_broadcast::linked::{Config, Engine};
     
     use governor::Quota;
-    use prometheus_client::metrics::info;
     use tracing::{debug, info, warn};
 
-    use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Digest, Ed25519, Hasher, Scheme};
+    use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Scheme};
     use commonware_macros::test_traced;
     use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network};
     use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner};
-    use futures::{channel::{mpsc, oneshot}, future::join_all};
+    use futures::channel::mpsc;
 
-    use super::{collector, ingress, mempool::{self, Mempool, RawTransaction}};
+    use super::{ingress, mempool::{self, Mempool, RawTransaction}};
 
     type Registrations<P> = HashMap<P, (
         (Sender<P>, Receiver<P>), 
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index be6a1c9f..8903177e 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -7,7 +7,7 @@ use commonware_broadcast::linked;
 use commonware_cryptography::{
     bls12381::primitives::{
         group::{self, Element},
-        poly::{self, public},
+        poly,
     }, ed25519::{PrivateKey, PublicKey}, sha256, Ed25519, Scheme
 };
 use commonware_deployer::ec2::Peers;
@@ -16,7 +16,7 @@ use commonware_runtime::{tokio, Clock, Metrics, Network, Runner, Spawner};
 use commonware_utils::{from_hex_formatted, hex, quorum};
 use futures::future::try_join_all;
 use governor::Quota;
-use prometheus_client::metrics::{gauge::Gauge};
+use prometheus_client::metrics::gauge::Gauge;
 use std::{
     collections::HashMap,
     net::{IpAddr, Ipv4Addr, SocketAddr},
@@ -223,7 +223,7 @@ fn main() {
         let mempool_batch_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
         let mempool_batch_broadcaster = network.register(
             MEMPOOL_BATCH_CHANNEL,
-            mempool_ack_limit,
+            mempool_batch_limit,
             config.message_backlog,
             Some(3),
         );
@@ -232,7 +232,7 @@ fn main() {
         let mempool_backfill_limit = Quota::per_second(NonZeroU32::new(8).unwrap());
         let mempool_backfill_broadcaster = network.register(
             MMEPOOL_BACKFILL_CHANNEL,
-            mempool_ack_limit,
+            mempool_backfill_limit,
             config.message_backlog,
             Some(3),
         );

From fc828651b094bf3e5f183b999699f85f28a3221e Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Thu, 27 Mar 2025 10:22:44 -0400
Subject: [PATCH 18/20] retain batch order

---
 chain/src/actors/mempool/mempool.rs | 35 +++++++++++++++++++----------
 chain/src/actors/mempool/mod.rs     | 11 ---------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index ddab49cf..08b5655f 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, time::{Duration, SystemTime}};
+use std::{collections::{HashMap}, time::{Duration, SystemTime}};
 
 use bytes::{BufMut, Bytes};
 use commonware_cryptography::{ed25519::PublicKey, sha256, Digest, Hasher, Sha256};
@@ -22,9 +22,6 @@ use crate::maybe_delay_between;
 #[derive(Clone, Debug)]
 pub struct Batch<D: Digest>  {
     pub timestamp: SystemTime,
-    // mark if the batch is accepted by the network, i.e. the network has received & verified the batch 
-    pub accepted: bool, 
-
     pub txs: Vec<RawTransaction<D>>,
     pub digest: D,
 }
@@ -48,7 +45,6 @@ impl<D: Digest> Batch<D>
         Self {
             txs,
             digest,
-            accepted: false,
             timestamp 
         }
     }
@@ -95,7 +91,6 @@ impl<D: Digest> Batch<D>
         // and set timestamp to the current time.
         Some(Self {
             timestamp,
-            accepted: false,
             txs,
             digest,
         })
@@ -280,7 +275,10 @@ pub struct Mempool<
 
     public_key: PublicKey,
 
+
     batches: HashMap<D, Batch<D>>,
+    
+    acknowledged: Vec<D>,
     accepted: Archive<TwoCap, D, B, R>,
     consumed: Archive<TwoCap, D, B, R>,
 
@@ -350,6 +348,7 @@ impl<
             context,
             public_key: cfg.public_key,
             batches: HashMap::new(),
+            acknowledged: Vec::new(),
             accepted: accepted_archive,
             consumed: consumed_archive,
 
@@ -446,19 +445,27 @@ impl<
                             self.txs.push(payload);
                             let _ = response.send(true);
                         },
-                        // batch ackowledged by the network, put in the accepted archive and mark batch as accepted
+                        // batch ackowledged by the network 
                         Message::BatchAcknowledged { digest, response } => {
+                            debug!("batch accepted by the network: {}", digest);
+
+                            self.acknowledged.push(digest);
                             if let Some(batch) = self.batches.get_mut(&digest) {
                                 accepted.put(self.block_height_seen, digest, batch.serialize().into()).await.expect("unable to store accepted batch");
-                                batch.accepted = true;
-                                let _ = response.send(true);
-                                debug!("batch accepted by the network: {}", digest);
                             } else {
-                                let _ = response.send(false);
+                                panic!("batch not found: {}", digest);
                             }
+                            let _ = response.send(true);
                         },
                         Message::ConsumeBatches { response } => { 
-                            let batches = self.batches.iter().filter(|(_, batch)| batch.accepted).map(|(_, batch)| batch.clone()).collect();
+                            // we do not remove anything from the mempool state as the digests/batches may not be consumed
+                            let batches = self.acknowledged.iter().filter_map(|digest| {
+                                let Some(batch) = self.batches.get(&digest) else {
+                                    // shouldn't happen
+                                    panic!("batch not found: {}", digest);
+                                };
+                                Some(batch.clone())
+                            }).collect();
                             let _ = response.send(batches);
                         },
                         // received when a block is finalized, i.e. finalization message is received, 
@@ -483,14 +490,18 @@ impl<
                                 panic!("not all provided batch digests consumed, provided={:?}, consumed={:?}", digests, consumed_keys);
                             }
 
+                            // remove digests and batches
                             let consumed_batches: Vec<Batch<D>> = consumed_keys.into_iter()
                                 .filter_map(|key| self.batches.remove(&key))
                                 .collect();
 
+                            self.acknowledged.retain(|digest| !digests.contains(digest));
+
                             for batch in consumed_batches.iter() {
                                 consumed.put(block_number, batch.digest, batch.serialize().into()).await.expect("Failed to insert accepted batch");
                             }
 
+
                             let _ = response.send(true);
                         },
                         // for validators, this should be only called when they receive 
diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs
index db6f38ca..0b73aece 100644
--- a/chain/src/actors/mempool/mod.rs
+++ b/chain/src/actors/mempool/mod.rs
@@ -305,9 +305,6 @@ mod tests {
                             let Some(batch) = mailbox.get_batch_contain_tx(digest.clone()).await else {
                                 panic!("batch not found");
                             };
-                            if !batch.accepted {
-                                panic!("batch {} not acknowledged", batch.digest);
-                            }
                             info!("batch contain tx {} acknowledged", batch.digest);
                         }
                         if consume_batch {
@@ -342,10 +339,6 @@ mod tests {
                 context.with_label("simulation"), 
                 num_validators, 
                 &mut shares_vec).await;
-            // let mailboxes = Arc::new(Mutex::new(BTreeMap::<
-            //     PublicKey,
-            //     mempool::Mailbox<sha256::Digest>,
-            // >::new()));
             let mut collectors = BTreeMap::<PublicKey, super::collector::Mailbox<Ed25519, sha256::Digest>>::new();
             let mailboxes = spawn_validator_engines(
                 context.with_label("validator"), 
@@ -379,10 +372,6 @@ mod tests {
                 context.with_label("simulation"), 
                 num_validators, 
                 &mut shares_vec).await;
-            // let mailboxes = Arc::new(Mutex::new(BTreeMap::<
-            //     PublicKey,
-            //     mempool::Mailbox<sha256::Digest>,
-            // >::new()));
             let mailboxes = spawn_mempools(
                 context.with_label("mempool"), 
                 identity,

From a6cda1416d9b3d60bdc2f013084b1d9965cd03c9 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Fri, 28 Mar 2025 10:57:30 -0400
Subject: [PATCH 19/20] comments[no ci]

---
 chain/src/actors/mempool/actor.rs   | 1 -
 chain/src/actors/mempool/mempool.rs | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index 40e9da8b..cafbb146 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -11,7 +11,6 @@ use tracing::{error, warn, debug};
 
 pub struct Actor<D: Digest, P: Array> {
     mailbox: mpsc::Receiver<Message<D, P>>,
-    // TODO: add a mempool structure here
 }
 
 impl<D: Digest, P: Array> Actor<D, P> {
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 08b5655f..36153d60 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -279,6 +279,8 @@ pub struct Mempool<
     batches: HashMap<D, Batch<D>>,
     
     acknowledged: Vec<D>,
+
+    //TODO: replace the following two
     accepted: Archive<TwoCap, D, B, R>,
     consumed: Archive<TwoCap, D, B, R>,
 

From 6884d0007cef0acb7a7fb15a38836d20a61e3785 Mon Sep 17 00:00:00 2001
From: bianyuanop <chen.me.nan@gmail.com>
Date: Fri, 28 Mar 2025 11:21:57 -0400
Subject: [PATCH 20/20] recover deploy & nits

---
 Cargo.lock                          |  2 ++
 Cargo.toml                          |  3 +--
 chain/src/actors/mempool/actor.rs   |  2 --
 chain/src/actors/mempool/handler.rs |  5 +++--
 chain/src/actors/mempool/mempool.rs |  6 ++----
 chain/src/bin/setup.rs              |  1 -
 chain/src/bin/validator.rs          | 20 ++++++--------------
 7 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index fb8ce354..28d57788 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -876,6 +876,8 @@ dependencies = [
 [[package]]
 name = "commonware-deployer"
 version = "0.0.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f986a14b443a53c96ad1831c9826e35dbb0aeccaad34a289fe4000d61d0a9c6"
 dependencies = [
  "aws-config",
  "aws-sdk-ec2",
diff --git a/Cargo.toml b/Cargo.toml
index a700ed5a..334ede8e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,8 +13,7 @@ alto-types = { version = "0.0.6", path = "types" }
 commonware-broadcast = { version = "0.0.43" }
 commonware-consensus = { version = "0.0.43" }
 commonware-cryptography = { version = "0.0.43" }
-# commonware-deployer = { version = "0.0.43" }
-commonware-deployer = { path = "..//cm-monorop/deployer" }
+commonware-deployer = { version = "0.0.43" }
 commonware-macros = { version = "0.0.43" }
 commonware-p2p = { version = "0.0.43" }
 commonware-resolver = { version = "0.0.43" }
diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs
index cafbb146..ae47cd76 100644
--- a/chain/src/actors/mempool/actor.rs
+++ b/chain/src/actors/mempool/actor.rs
@@ -56,6 +56,4 @@ impl<D: Digest, P: Array> Actor<D, P> {
             }
         }
     }
-
-    // TODO: implement handler for data received
 }
diff --git a/chain/src/actors/mempool/handler.rs b/chain/src/actors/mempool/handler.rs
index 9e5d2883..e0f014e2 100644
--- a/chain/src/actors/mempool/handler.rs
+++ b/chain/src/actors/mempool/handler.rs
@@ -5,6 +5,7 @@ use futures::{
     channel::{mpsc, oneshot},
     SinkExt,
 };
+use tracing::warn;
 
 pub enum Message {
     Deliver {
@@ -48,8 +49,8 @@ impl Consumer for Handler {
         receiver.await.expect("Failed to receive deliver")
     }
 
-    async fn failed(&mut self, _: Self::Key, _: Self::Failure) {
-        // Ignore any failure
+    async fn failed(&mut self, key: Self::Key, failture: Self::Failure) {
+        warn!(?key, ?failture, "failed at consumer");
     }
 }
 
diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs
index 36153d60..a743977e 100644
--- a/chain/src/actors/mempool/mempool.rs
+++ b/chain/src/actors/mempool/mempool.rs
@@ -14,7 +14,7 @@ use futures::{channel::{mpsc, oneshot}, SinkExt, StreamExt};
 use commonware_macros::select;
 use governor::Quota;
 use rand::Rng;
-use tracing::{debug, warn};
+use tracing::{debug, warn, info};
 use governor::clock::Clock as GClock;
 use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped};
 use crate::maybe_delay_between;
@@ -431,10 +431,8 @@ impl<
 
             select! {
                 mailbox_message = self.mailbox.next() => {
-                    // let message = mailbox_message.expect("Mailbox closed");
                     let Some(message) = mailbox_message else {
-                        // TODO: revisit this branch, it was .expect("Mailbox closed") and will panic after unit test is finished
-                        debug!("Mailbox closed, terminating...");
+                        info!("Mailbox closed, terminating...");
                         return;
                     };
                     match message {
diff --git a/chain/src/bin/setup.rs b/chain/src/bin/setup.rs
index df234629..32c72cd0 100644
--- a/chain/src/bin/setup.rs
+++ b/chain/src/bin/setup.rs
@@ -488,7 +488,6 @@ fn generate_local(sub_matches: &ArgMatches) {
             name: scheme.public_key().to_string(),
             region: "local".to_string(),
             ip: "127.0.0.1".parse().expect("invalid IP address"),
-            port: PORT + index as u16,
         };
         peers.push(peer);
     }
diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs
index 8903177e..d1d730c4 100644
--- a/chain/src/bin/validator.rs
+++ b/chain/src/bin/validator.rs
@@ -66,11 +66,6 @@ fn parse_log_level(level: &str) -> Option<Level> {
 }
 
 fn main() {
-    struct PeerAddr {
-        ip: IpAddr,
-        port: u16
-    }
-
     // Parse arguments
     let matches = Command::new("validator")
         .about("Validator for an alto chain.")
@@ -82,16 +77,13 @@ fn main() {
     let peer_file = matches.get_one::<String>("peers").unwrap();
     let peers_file = std::fs::read_to_string(peer_file).expect("Could not read peers file");
     let peers: Peers = serde_yaml::from_str(&peers_file).expect("Could not parse peers file");
-    let peers: HashMap<PublicKey, PeerAddr> = peers
+    let peers: HashMap<PublicKey, IpAddr> = peers
         .peers
         .into_iter()
         .map(|peer| {
             let key = from_hex_formatted(&peer.name).expect("Could not parse peer key");
             let key = PublicKey::try_from(key).expect("Peer key is invalid");
-            (key, PeerAddr {
-                ip: peer.ip,
-                port: peer.port
-            })
+            (key, peer.ip)
         })
         .collect();
     info!(peers = peers.len(), "loaded peers");
@@ -112,7 +104,7 @@ fn main() {
     let identity_public = *poly::public(&identity);
     let public_key = signer.public_key();
     let metrics_port = config.metrics_port;
-    let ip = peers.get(&public_key).expect("Could not find self in IPs").ip;
+    let ip = peers.get(&public_key).expect("Could not find self in IPs");
     info!(
         ?public_key,
         identity = hex(&identity_public.serialize()),
@@ -136,8 +128,8 @@ fn main() {
     for bootstrapper in &config.bootstrappers {
         let key = from_hex_formatted(bootstrapper).expect("Could not parse bootstrapper key");
         let key = PublicKey::try_from(key).expect("Bootstrapper key is invalid");
-        let peer_addr = peers.get(&key).expect("Could not find bootstrapper in IPs");
-        let bootstrapper_socket = format!("{}:{}", peer_addr.ip, peer_addr.port);
+        let ip = peers.get(&key).expect("Could not find bootstrapper in IPs");
+        let bootstrapper_socket = format!("{}:{}", ip, config.port);
         let bootstrapper_socket = SocketAddr::from_str(&bootstrapper_socket)
             .expect("Could not parse bootstrapper socket");
         bootstrappers.push((key, bootstrapper_socket));
@@ -157,7 +149,7 @@ fn main() {
         signer.clone(),
         P2P_NAMESPACE,
         SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), config.port),
-        SocketAddr::new(ip, config.port),
+        SocketAddr::new(*ip, config.port),
         bootstrappers,
         MAX_MESSAGE_SIZE,
     );