From 0950e5fe5d55831b968527f8d2a4410054f04644 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 01:43:26 +0000 Subject: [PATCH 01/15] Initial working stream catchup --- chain-signatures/node/src/indexer_eth/mod.rs | 772 +++++++++--------- chain-signatures/node/src/indexer_sol.rs | 18 +- chain-signatures/node/src/stream/mod.rs | 532 ++++++++++-- .../tests/cases/ethereum_stream.rs | 375 ++++++++- 4 files changed, 1241 insertions(+), 456 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index febb804c..01e86208 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -8,7 +8,9 @@ use crate::metrics::requests::{record_request_latency, SignRequestStep}; use crate::protocol::{Chain, IndexedSignRequest}; use crate::respond_bidirectional::CompletedTx; use crate::sign_bidirectional::SignStatus; -use crate::stream::{ChainEvent, ChainStream, ExecutionOutcome}; +use crate::stream::{ + ChainBufferedStream, ChainEvent, ChainStream, ExecutionOutcome, +}; use alloy::eips::BlockNumberOrTag; use alloy::primitives::hex::{self, ToHexExt}; @@ -20,7 +22,6 @@ use k256::{AffinePoint as K256AffinePoint, EncodedPoint, FieldBytes, Scalar}; use mpc_crypto::{kdf::derive_epsilon_eth, ScalarExt as _}; use mpc_primitives::{SignArgs, SignId, Signature as MpcSignature, LATEST_MPC_KEY_VERSION}; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use std::fmt; use std::str::FromStr; use std::sync::Arc; @@ -42,49 +43,23 @@ pub(crate) static MAX_SECP256K1_SCALAR: LazyLock = LazyLock::new(|| { // This is the maximum number of blocks that Helios can look back to const MAX_CATCHUP_BLOCKS: u64 = 8191; -const MAX_BLOCKS_TO_PROCESS: usize = 10000; +const MAX_LIVE_BLOCK_BUFFER: usize = 10000; -fn blocks_to_process_channel() -> (mpsc::Sender, mpsc::Receiver) { - mpsc::channel(MAX_BLOCKS_TO_PROCESS) -} - -const MAX_INDEXED_REQUESTS: usize = 1024; - -fn indexed_channel() -> ( - mpsc::Sender, - mpsc::Receiver, -) { - mpsc::channel(MAX_INDEXED_REQUESTS) -} - -const MAX_FAILED_BLOCKS: usize = 1024; - -fn failed_blocks_channel() -> ( +fn live_blocks_channel() -> ( mpsc::Sender, mpsc::Receiver, ) { - mpsc::channel(MAX_FAILED_BLOCKS) -} - -const MAX_FINALIZED_BLOCKS: usize = 1024; - -fn finalized_block_channel() -> (mpsc::Sender, mpsc::Receiver) { - mpsc::channel(MAX_FINALIZED_BLOCKS) + mpsc::channel(MAX_LIVE_BLOCK_BUFFER) } type BlockNumber = u64; -pub enum BlockToProcess { - Catchup(BlockNumber), - NewBlock(Box), -} - -#[derive(Clone)] pub struct BlockAndRequests { block_number: u64, block_hash: alloy::primitives::B256, indexed_requests: Vec, respond_logs: Vec, + execution_events: Vec, } impl BlockAndRequests { @@ -93,16 +68,34 @@ impl BlockAndRequests { block_hash: alloy::primitives::B256, indexed_requests: Vec, respond_logs: Vec, + execution_events: Vec, ) -> Self { Self { block_number, block_hash, indexed_requests, respond_logs, + execution_events, } } } +pub struct EthereumBufferedStream { + live_blocks_rx: mpsc::Receiver, +} + +impl ChainBufferedStream for EthereumBufferedStream { + type Item = alloy::rpc::types::Block; + + async fn initial(&mut self) -> Option { + self.live_blocks_rx.recv().await + } + + async fn next(&mut self) -> Option { + self.live_blocks_rx.recv().await + } +} + #[derive(Clone)] pub struct EthConfig { /// The ethereum account secret key used to sign eth respond txn. @@ -737,198 +730,138 @@ impl EthereumIndexer { } pub async fn run(self, events_tx: mpsc::Sender) { - let backlog = self.backlog; - let eth = self.eth; - let client = Arc::new(self.client); - tracing::info!("running ethereum indexer"); - let Ok(contract_address) = Address::from_str(&format!("0x{}", eth.contract_address)) else { - tracing::error!("Failed to parse contract address: {}", eth.contract_address); - return; - }; - let (blocks_failed_send, blocks_failed_recv) = failed_blocks_channel(); - - let (requests_indexed_send, requests_indexed_recv) = indexed_channel(); - - let (finalized_block_send, finalized_block_recv) = finalized_block_channel(); - - let (blocks_to_process_send, mut blocks_to_process_recv) = blocks_to_process_channel(); - - let client_clone = Arc::clone(&client); - let finalized_block_send_clone = finalized_block_send.clone(); - let refresh_interval = eth.refresh_finalized_interval; - tokio::spawn(async move { - tracing::info!("Spawned task to refresh the latest finalized block"); - Self::refresh_finalized_block( - client_clone, - finalized_block_send_clone, - refresh_interval, - ) - .await; - }); - - let client_clone = Arc::clone(&client); - let optimistic_requests = eth.optimistic_requests; - tokio::spawn(Self::send_requests_when_final( - client_clone, - requests_indexed_recv, - finalized_block_recv, - events_tx.clone(), - optimistic_requests, - )); - - tokio::spawn(Self::retry_failed_blocks( - Arc::clone(&client), - blocks_failed_recv, - blocks_failed_send.clone(), - contract_address, - requests_indexed_send.clone(), - backlog.clone(), - events_tx.clone(), - )); - - let last_processed_block = backlog.processed_block(Chain::Ethereum).await; - let mut expected_catchup_blocks = 0usize; - let mut processed_catchup_blocks = HashSet::new(); - let mut catchup_completed_emitted = false; - - let blocks_to_process_send_clone = blocks_to_process_send.clone(); - if let Some(last_processed_block) = last_processed_block { - match Self::catchup_end_block_number(Arc::clone(&client)).await { - Some(end_block_number) => { - expected_catchup_blocks = end_block_number - .saturating_sub(last_processed_block) - .saturating_add(1) as usize; - Self::add_catchup_blocks_to_process( - blocks_to_process_send_clone, - last_processed_block, - end_block_number, - ) - .await - } - None => { - tracing::error!("Failed to get catchup end block number"); - } - } - } - - if expected_catchup_blocks == 0 { - if let Err(err) = events_tx.send(ChainEvent::CatchupCompleted).await { - tracing::warn!(?err, "failed to emit ethereum catchup completion event"); - } else { - catchup_completed_emitted = true; - } - } + let _ = events_tx; + } - tokio::spawn(Self::add_new_block_to_process( - Arc::clone(&client), - blocks_to_process_send.clone(), - )); + async fn buffer_live_blocks( + client: Arc, + live_blocks: mpsc::Sender, + ) { + tracing::info!("buffering ethereum live blocks"); + let mut next_block_number: Option = None; - let mut interval = tokio::time::interval(Duration::from_millis(200)); - let requests_indexed_send_clone = requests_indexed_send.clone(); loop { - let Some(block_to_process) = blocks_to_process_recv.recv().await else { - interval.tick().await; + let Some(latest_block_number) = client.get_latest_block_number().await else { + tokio::time::sleep(Duration::from_millis(500)).await; continue; }; - let (block, is_catchup) = match block_to_process { - BlockToProcess::Catchup(block_number) => { - let block = client - .get_block(alloy::rpc::types::BlockId::Number( - BlockNumberOrTag::Number(block_number), - )) - .await; - if let Some(block) = block { - (block, true) - } else { - tracing::warn!("Block {block_number} not found from Ethereum client"); - continue; - } - } - BlockToProcess::NewBlock(block) => ((*block).clone(), false), - }; - let block_number = block.header.number; - if let Err(err) = Self::process_block( - client.clone(), - block.clone(), - contract_address, - requests_indexed_send_clone.clone(), - backlog.clone(), - events_tx.clone(), - ) - .await - { - tracing::warn!( - "Eth indexer failed to process block number {block_number}: {err:?}" - ); - Self::add_failed_block(blocks_failed_send.clone(), block).await; + + let mut block_number = next_block_number.unwrap_or(latest_block_number); + if block_number > latest_block_number { + tokio::time::sleep(Duration::from_millis(500)).await; continue; } - if block_number % 10 == 0 { - if is_catchup { - tracing::info!("Processed catchup block number {block_number}"); - } else { - tracing::info!("Processed new block number {block_number}"); - } - } - if is_catchup && !catchup_completed_emitted { - processed_catchup_blocks.insert(block_number); - if processed_catchup_blocks.len() >= expected_catchup_blocks { - if let Err(err) = events_tx.send(ChainEvent::CatchupCompleted).await { - tracing::warn!(?err, "failed to emit ethereum catchup completion event"); - } else { - catchup_completed_emitted = true; - } + while block_number <= latest_block_number { + let Some(block) = client + .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Number( + block_number, + ))) + .await + else { + tracing::warn!(block_number, "ethereum live block not yet available"); + break; + }; + + if let Err(err) = live_blocks.send(block).await { + tracing::warn!(?err, block_number, "failed to buffer ethereum live block"); + return; } + + next_block_number = Some(block_number.saturating_add(1)); + block_number = block_number.saturating_add(1); } - crate::metrics::indexers::LATEST_BLOCK_NUMBER - .with_label_values(&[Chain::Ethereum.as_str(), "indexed"]) - .set(block_number as i64); + tokio::time::sleep(Duration::from_millis(500)).await; } } - async fn add_new_block_to_process( - client: Arc, - blocks_to_process: mpsc::Sender, - ) { - tracing::info!("Adding new blocks to process..."); - let mut current_block = 0; - loop { - let Some(latest_block) = client - .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Latest)) - .await - else { - continue; - }; - let block_number = latest_block.header.number; - if block_number <= current_block { - tokio::time::sleep(Duration::from_millis(500)).await; - continue; - } - if let Err(err) = blocks_to_process - .send(BlockToProcess::NewBlock(Box::new(latest_block))) - .await - { - tracing::warn!("Failed to send new block to process: {err:?}"); - } - current_block = block_number; + fn catchup_start_block_number( + last_processed_block: Option, + anchor_height: BlockNumber, + ) -> BlockNumber { + let requested_start = last_processed_block + .map(|height| height.saturating_add(1)) + .unwrap_or(anchor_height); + + let catchup_end = anchor_height.saturating_sub(1); + let oldest_supported = catchup_end.saturating_sub(MAX_CATCHUP_BLOCKS); + + if requested_start < oldest_supported { + tracing::warn!( + requested_start, + anchor_height, + oldest_supported, + "ethereum catchup start is older than supported range; clamping" + ); + oldest_supported + } else { + requested_start } } - async fn catchup_end_block_number(client: Arc) -> Option { - client.get_latest_block_number().await + + async fn process_height( + &self, + block_number: u64, + events_tx: mpsc::Sender, + ) -> anyhow::Result<()> { + let Some(block) = self + .client + .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Number( + block_number, + ))) + .await + else { + anyhow::bail!("ethereum block {block_number} not found"); + }; + + self.process_live_block(block, events_tx).await + } + + async fn process_live_block( + &self, + block: alloy::rpc::types::Block, + events_tx: mpsc::Sender, + ) -> anyhow::Result<()> { + let block_number = block.header.number; + let contract_address = Address::from_str(&format!("0x{}", self.eth.contract_address)) + .map_err(|_| { + anyhow::anyhow!( + "failed to parse ethereum contract address: {}", + self.eth.contract_address + ) + })?; + + let processed = Self::process_block( + Arc::new(self.client.clone()), + block, + contract_address, + self.backlog.clone(), + ) + .await?; + + Self::emit_processed_block( + Arc::new(self.client.clone()), + events_tx, + &self.eth, + processed, + ) + .await?; + + crate::metrics::indexers::LATEST_BLOCK_NUMBER + .with_label_values(&[Chain::Ethereum.as_str(), "indexed"]) + .set(block_number as i64); + + Ok(()) } async fn process_block( client: Arc, block: alloy::rpc::types::Block, contract_address: Address, - requests_indexed: mpsc::Sender, backlog: Backlog, - events_tx: mpsc::Sender, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { let block_number = block.header.number; let block_hash = block.header.hash; let block_timestamp = block.header.timestamp; @@ -998,11 +931,6 @@ impl EthereumIndexer { block_receipts.clone(), ) .await?; - for ev in exec_events { - if let Err(err) = events_tx.send(ev).await { - tracing::error!(?err, "failed to emit ExecutionConfirmed event"); - } - } for _request in &sign_requests { record_request_latency( @@ -1015,17 +943,13 @@ impl EthereumIndexer { // Always forward the processed block to the "finalization" stage so it can emit // `ChainEvent::Block` even when there are no relevant contract logs. - requests_indexed - .send(BlockAndRequests::new( - block_number, - block_hash, - sign_requests, - respond_logs, - )) - .await - .map_err(|err| anyhow::anyhow!("Failed to send indexed requests: {:?}", err))?; - - Ok(()) + Ok(BlockAndRequests::new( + block_number, + block_hash, + sign_requests, + respond_logs, + exec_events, + )) } async fn collect_execution_confirmations( @@ -1153,206 +1077,104 @@ impl EthereumIndexer { Ok(events) } - /// Sends a request to the sign queue when the block where the request is in is finalized. - async fn send_requests_when_final( + /// Emits the processed block in-order once the configured buffer policy allows it. + async fn emit_processed_block( client: Arc, - mut requests_indexed: mpsc::Receiver, - mut finalized_block_rx: mpsc::Receiver, events_tx: mpsc::Sender, - optimistic_requests: bool, - ) { - let mut finalized_block_number: Option = None; - - loop { - let Some(BlockAndRequests { + eth: &EthConfig, + BlockAndRequests { + block_number, + block_hash, + indexed_requests, + respond_logs, + execution_events, + }: BlockAndRequests, + ) -> anyhow::Result<()> { + if !eth.optimistic_requests { + Self::wait_for_finalized_block( + Arc::clone(&client), + eth.refresh_finalized_interval, block_number, - block_hash, - indexed_requests, - respond_logs, - }) = requests_indexed.recv().await - else { - tracing::error!("Failed to receive indexed requests"); - return; - }; - - if !optimistic_requests { - // Wait for finalized block if needed - while finalized_block_number.is_none_or(|n| block_number > n) { - let Some(new_finalized_block) = finalized_block_rx.recv().await else { - tracing::error!("Failed to receive finalized blocks"); - return; - }; - finalized_block_number.replace(new_finalized_block); - } - } - - // Verify block hash and send requests - let block = client - .as_ref() - .get_block(alloy::rpc::types::BlockId::Number( - BlockNumberOrTag::Number(block_number), - )) - .await; + ) + .await?; + } - let Some(block) = block else { - tracing::warn!("Block {block_number} not found from Ethereum client, skipping this block and its requests"); - continue; - }; + let Some(block) = client + .as_ref() + .get_block(alloy::rpc::types::BlockId::Number( + BlockNumberOrTag::Number(block_number), + )) + .await + else { + anyhow::bail!("ethereum block {block_number} not found during emission"); + }; - if block.header.hash == block_hash { - tracing::info!("Block {block_number} is finalized!"); + if block.header.hash != block_hash { + anyhow::bail!( + "block {block_number} hash mismatch: expected {block_hash:?}, got {:?}", + block.header.hash + ); + } - for req in indexed_requests.clone() { - if let Err(err) = events_tx.send(ChainEvent::SignRequest(req)).await { - tracing::error!(?err, "failed to emit SignRequest event"); - } - } + for event in execution_events { + events_tx + .send(event) + .await + .map_err(|err| anyhow::anyhow!("failed to emit ExecutionConfirmed event: {err:?}"))?; + } - if !respond_logs.is_empty() { - emit_respond_events(&respond_logs, events_tx.clone()).await; - } + for req in indexed_requests { + events_tx + .send(ChainEvent::SignRequest(req)) + .await + .map_err(|err| anyhow::anyhow!("failed to emit SignRequest event: {err:?}"))?; + } - if let Err(err) = events_tx.send(ChainEvent::Block(block_number)).await { - tracing::error!(?err, "failed to emit block event"); - } - } else { - // no special handling for chain reorg, just log the error - // This is because when such chain reorg happens, the new canonical chain will have already been emitted by helios's block header stream, and we can safely skip this block here. - tracing::error!( - "Block {block_number} hash mismatch: expected {block_hash:?}, got {:?}. Chain re-orged.", - block.header.hash - ); - } + if !respond_logs.is_empty() { + emit_respond_events(&respond_logs, events_tx.clone()).await; } - } - #[allow(clippy::too_many_arguments)] - async fn retry_failed_blocks( - client: Arc, - mut blocks_failed_rx: mpsc::Receiver, - blocks_failed_tx: mpsc::Sender, - contract_address: Address, - requests_indexed: mpsc::Sender, - backlog: Backlog, - events_tx: mpsc::Sender, - ) { - loop { - let Some(block) = blocks_failed_rx.recv().await else { - tracing::warn!("Failed to receive block and requests from requests_indexed"); - break; - }; - let block_number = block.header.number; - if let Err(err) = Self::process_block( - client.clone(), - block.clone(), - contract_address, - requests_indexed.clone(), - backlog.clone(), - events_tx.clone(), - ) + events_tx + .send(ChainEvent::Block(block_number)) .await - { - tracing::warn!("Retry failed for block {block_number}: {err:?}"); - Self::add_failed_block(blocks_failed_tx.clone(), block).await; - } else { - tracing::info!("Successfully retried block: {block_number}"); - } - } - } + .map_err(|err| anyhow::anyhow!("failed to emit block event: {err:?}"))?; - async fn add_failed_block( - blocks_failed: mpsc::Sender, - block: alloy::rpc::types::Block, - ) { - blocks_failed.send(block).await.unwrap_or_else(|err| { - tracing::warn!("Failed to send failed block: {:?}", err); - }); + Ok(()) } - /// Polls for the latest finalized block and update finalized block channel. - async fn refresh_finalized_block( + async fn wait_for_finalized_block( client: Arc, - finalized_block_send: mpsc::Sender, refresh_finalized_interval: u64, - ) { - let mut interval = tokio::time::interval(Duration::from_millis(refresh_finalized_interval)); - let mut final_block_number: Option = None; + block_number: BlockNumber, + ) -> anyhow::Result<()> { + let mut last_finalized_block_number: Option = None; loop { - interval.tick().await; - tracing::info!("Refreshing finalized epoch"); - - let new_finalized_block = match client + let Some(finalized_block) = client .as_ref() .get_block(alloy::rpc::types::BlockId::Number( BlockNumberOrTag::Finalized, )) .await - { - Some(block) => block, - None => { - tracing::warn!("Finalized block not found from Ethereum client"); - continue; - } + else { + tracing::warn!(block_number, "finalized ethereum block not found; retrying"); + tokio::time::sleep(Duration::from_millis(refresh_finalized_interval)).await; + continue; }; - let new_final_block_number = new_finalized_block.header.number; - tracing::info!( - "New finalized block number: {new_final_block_number}, last finalized block number: {final_block_number:?}" - ); - - if final_block_number.is_none_or(|n| new_final_block_number > n) { - tracing::info!("Found new finalized block!"); - if let Err(err) = finalized_block_send.send(new_final_block_number).await { - tracing::warn!("Failed to send finalized block: {err:?}"); - continue; - } - final_block_number.replace(new_final_block_number); + let finalized_block_number = finalized_block.header.number; + if last_finalized_block_number.is_none_or(|n| finalized_block_number > n) { + last_finalized_block_number = Some(finalized_block_number); crate::metrics::indexers::LATEST_BLOCK_NUMBER .with_label_values(&[Chain::Ethereum.as_str(), "finalized"]) - .set(new_final_block_number as i64); - continue; + .set(finalized_block_number as i64); } - let Some(last_final_block_number) = final_block_number else { - continue; + if finalized_block_number >= block_number { + return Ok(()); }; - if new_final_block_number < last_final_block_number { - tracing::warn!( - "New finalized block number overflowed range of u64 and has wrapped around!" - ); - } - - if last_final_block_number == new_final_block_number { - tracing::info!("No new finalized block"); - } - } - } - - async fn add_catchup_blocks_to_process( - blocks_to_process: mpsc::Sender, - start_block_number: u64, - end_block_number: u64, - ) { - // helios can only go back maximum MAX_CATCHUP_BLOCKS blocks, so we need to adjust the start block number if it's too far behind - let helios_oldest_block_number = end_block_number.saturating_sub(MAX_CATCHUP_BLOCKS); - let start_block_number = if start_block_number < helios_oldest_block_number { - tracing::warn!( - "Start block number {start_block_number} is too far behind the latest block {end_block_number}, adjusting to {helios_oldest_block_number}" - ); - helios_oldest_block_number - } else { - start_block_number - }; - - for block_number in start_block_number..=end_block_number { - if let Err(err) = blocks_to_process - .send(BlockToProcess::Catchup(block_number)) - .await - { - tracing::warn!("Failed to send block to process: {err:?}"); - } + tokio::time::sleep(Duration::from_millis(refresh_finalized_interval)).await; } } } @@ -1361,11 +1183,17 @@ impl EthereumIndexer { /// Construction is side-effect free; the shared `run_stream()` loop calls /// `start()` after recovery has completed. pub struct EthereumStream { - events_rx: mpsc::Receiver, - indexer: Option<(EthereumIndexer, mpsc::Sender)>, + events_tx: mpsc::Sender, + events_rx: Option>, + indexer: EthereumIndexer, tasks: Vec>, } +struct EthereumBackgroundStream { + events_tx: mpsc::Sender, + indexer: EthereumIndexer, +} + impl EthereumStream { pub async fn new(eth: Option, backlog: Backlog) -> anyhow::Result { let Some(eth) = eth else { @@ -1387,22 +1215,22 @@ impl EthereumStream { let (events_tx, events_rx) = crate::stream::channel(); Ok(Self { - events_rx, - indexer: Some((indexer, events_tx)), + events_tx, + events_rx: Some(events_rx), + indexer, tasks: Vec::new(), }) } pub fn start(&mut self) { - let Some((indexer, events_tx)) = self.indexer.take() else { - return; + let mut stream = EthereumBackgroundStream { + events_tx: self.events_tx.clone(), + indexer: self.indexer.clone(), }; - let t_indexer: JoinHandle<()> = tokio::spawn(async move { - indexer.run(events_tx).await; - }); - - self.tasks.push(t_indexer); + self.tasks.push(tokio::spawn(async move { + crate::stream::start_chain_stream(&mut stream).await; + })); } } @@ -1416,12 +1244,162 @@ impl Drop for EthereumStream { impl ChainStream for EthereumStream { const CHAIN: Chain = Chain::Ethereum; + type BufferedStream = EthereumBufferedStream; + + fn take_event_receiver(&mut self) -> Option> { + self.events_rx.take() + } async fn start(&mut self) { self.start(); } + async fn livestream(&mut self) -> anyhow::Result> { + let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); + tokio::spawn(EthereumIndexer::buffer_live_blocks( + Arc::new(self.indexer.client.clone()), + live_blocks_tx, + )); + + Ok(Some(EthereumBufferedStream { live_blocks_rx })) + } + + fn buffered_item_height(item: &::Item) -> u64 { + item.header.number + } + + async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + let catchup_start = EthereumIndexer::catchup_start_block_number( + self.indexer.backlog.processed_block(Chain::Ethereum).await, + anchor_height, + ); + + Ok(catchup_start..anchor_height) + } + + async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + if height % 10 == 0 { + tracing::info!(height, "processed ethereum catchup height attempt"); + } + + self.indexer + .process_height(height, self.events_tx.clone()) + .await + } + + async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { + self.events_tx + .send(ChainEvent::CatchupCompleted) + .await + .map_err(|err| anyhow::anyhow!("failed to emit ethereum catchup completion event: {err:?}")) + } + + async fn process_buffered_item( + &mut self, + item: ::Item, + ) -> anyhow::Result<()> { + self.indexer + .process_live_block(item, self.events_tx.clone()) + .await + } + + fn retry_delay(&self) -> Duration { + Duration::from_millis(500) + } + async fn next_event(&mut self) -> Option { - self.events_rx.recv().await + match self.events_rx.as_mut() { + Some(rx) => rx.recv().await, + None => None, + } + } +} + +impl ChainStream for EthereumBackgroundStream { + const CHAIN: Chain = Chain::Ethereum; + type BufferedStream = EthereumBufferedStream; + + async fn livestream(&mut self) -> anyhow::Result> { + let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); + tokio::spawn(EthereumIndexer::buffer_live_blocks( + Arc::new(self.indexer.client.clone()), + live_blocks_tx, + )); + + Ok(Some(EthereumBufferedStream { live_blocks_rx })) + } + + fn buffered_item_height(item: &::Item) -> u64 { + item.header.number + } + + async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + let catchup_start = EthereumIndexer::catchup_start_block_number( + self.indexer.backlog.processed_block(Chain::Ethereum).await, + anchor_height, + ); + + Ok(catchup_start..anchor_height) + } + + async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + if height % 10 == 0 { + tracing::info!(height, "processed ethereum catchup height attempt"); + } + + self.indexer + .process_height(height, self.events_tx.clone()) + .await + } + + async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { + self.events_tx + .send(ChainEvent::CatchupCompleted) + .await + .map_err(|err| anyhow::anyhow!("failed to emit ethereum catchup completion event: {err:?}")) + } + + async fn process_buffered_item( + &mut self, + item: ::Item, + ) -> anyhow::Result<()> { + self.indexer + .process_live_block(item, self.events_tx.clone()) + .await + } + + fn retry_delay(&self) -> Duration { + Duration::from_millis(500) + } + + async fn next_event(&mut self) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::EthereumIndexer; + + #[test] + fn catchup_starts_after_processed_height() { + assert_eq!(EthereumIndexer::catchup_start_block_number(Some(41), 50), 42); + } + + #[test] + fn catchup_without_checkpoint_starts_from_anchor() { + assert_eq!(EthereumIndexer::catchup_start_block_number(None, 50), 50); + } + + #[test] + fn catchup_start_is_clamped_to_supported_window() { + let anchor_height = 10_000; + let catchup_end = anchor_height - 1; + let expected_oldest = catchup_end - super::MAX_CATCHUP_BLOCKS; + + assert_eq!( + EthereumIndexer::catchup_start_block_number(Some(1), anchor_height), + expected_oldest, + ); } } diff --git a/chain-signatures/node/src/indexer_sol.rs b/chain-signatures/node/src/indexer_sol.rs index 3614fd24..f571907d 100644 --- a/chain-signatures/node/src/indexer_sol.rs +++ b/chain-signatures/node/src/indexer_sol.rs @@ -293,7 +293,7 @@ type Result = anyhow::Result; /// Solana stream that implements the new ChainStream abstraction pub struct SolanaStream { - rx: mpsc::Receiver, + rx: Option>, start_state: Option, tasks: Vec>, } @@ -328,7 +328,7 @@ impl SolanaStream { let (tx, rx) = crate::stream::channel(); Some(SolanaStream { - rx, + rx: Some(rx), start_state: Some(SolanaStreamStartState { program_id, rpc_http_url: sol.rpc_http_url.clone(), @@ -342,6 +342,15 @@ impl SolanaStream { impl ChainStream for SolanaStream { const CHAIN: Chain = Chain::Solana; + type BufferedStream = crate::stream::DisabledBufferedStream; + + fn take_event_receiver(&mut self) -> Option> { + self.rx.take() + } + + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } async fn start(&mut self) { let Some(start_state) = self.start_state.take() else { @@ -363,7 +372,10 @@ impl ChainStream for SolanaStream { } async fn next_event(&mut self) -> Option { - self.rx.recv().await + match self.rx.as_mut() { + Some(rx) => rx.recv().await, + None => None, + } } } diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index b549964c..9f29f781 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -11,6 +11,8 @@ use crate::stream::ops::{ RespondBidirectionalEvent, SignatureRespondedEvent, }; +use std::time::Duration; +use std::{ops::Range, vec::Vec}; use tokio::sync::mpsc; use tokio::sync::watch; @@ -91,13 +93,138 @@ pub enum ExecutionOutcome { Failed, } +pub struct DisabledBufferedStream; + +impl ChainBufferedStream for DisabledBufferedStream { + type Item = (); + + async fn initial(&mut self) -> Option { + None + } + + async fn next(&mut self) -> Option { + None + } +} + +#[allow(async_fn_in_trait)] +pub trait ChainBufferedStream: Send + 'static { + type Item: Clone + Send + 'static; + + async fn initial(&mut self) -> Option; + async fn next(&mut self) -> Option; +} + #[allow(async_fn_in_trait)] pub trait ChainStream: Send + 'static { const CHAIN: Chain; + type BufferedStream: ChainBufferedStream; + + fn take_event_receiver(&mut self) -> Option> { + None + } + async fn start(&mut self) {} + + async fn livestream(&mut self) -> anyhow::Result> { + Ok(None) + } + + fn buffered_item_height(_item: &::Item) -> u64; + + async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn catchup_range(&mut self, _anchor_height: u64) -> anyhow::Result> { + Ok(0..0) + } + + async fn process_catchup_height(&mut self, _height: u64) -> anyhow::Result<()> { + Ok(()) + } + + async fn process_buffered_item( + &mut self, + _item: ::Item, + ) -> anyhow::Result<()> { + Ok(()) + } + + fn retry_delay(&self) -> Duration { + Duration::from_millis(500) + } + async fn next_event(&mut self) -> Option; } +pub(crate) async fn start_chain_stream(stream: &mut S) { + tracing::info!(chain = %S::CHAIN, "starting chain stream orchestration"); + + match stream.livestream().await { + Ok(Some(mut buffered)) => { + let Some(anchor_item) = buffered.initial().await else { + tracing::warn!(chain = %S::CHAIN, "buffered livestream ended before anchor item"); + return; + }; + + let anchor_height = S::buffered_item_height(&anchor_item); + let catchup_range = loop { + match stream.catchup_range(anchor_height).await { + Ok(range) => break range, + Err(err) => { + tracing::warn!(?err, chain = %S::CHAIN, anchor_height, "failed to determine catchup range; retrying"); + tokio::time::sleep(stream.retry_delay()).await; + } + } + }; + + for height in catchup_range { + loop { + match stream.process_catchup_height(height).await { + Ok(()) => break, + Err(err) => { + tracing::warn!(?err, chain = %S::CHAIN, height, "catchup height processing failed; retrying"); + tokio::time::sleep(stream.retry_delay()).await; + } + } + } + } + + if let Err(err) = stream.emit_catchup_completed().await { + tracing::warn!(?err, chain = %S::CHAIN, "failed to emit catchup completed event"); + return; + } + + let mut next_item = Some(anchor_item); + loop { + let item = match next_item.take() { + Some(item) => item, + None => match buffered.next().await { + Some(item) => item, + None => break, + }, + }; + + match stream.process_buffered_item(item.clone()).await { + Ok(()) => {} + Err(err) => { + tracing::warn!(?err, chain = %S::CHAIN, "buffered item processing failed; retrying"); + tokio::time::sleep(stream.retry_delay()).await; + next_item = Some(item); + } + } + } + } + Ok(None) => { + stream.start().await; + } + Err(err) => { + tracing::error!(?err, chain = %S::CHAIN, "failed to initialize livestream"); + } + } +} + /// Shared indexer loop: recovers backlog then processes events from the stream pub async fn run_stream( mut stream: S, @@ -124,73 +251,162 @@ pub async fn run_stream( // NOTE: we need to start after we recover entries from backlog and starting the run_stream task // such that we can guarantee getting the CatchupCompleted event from this task to modify the // recovered entries. - stream.start().await; - - while let Some(event) = stream.next_event().await { - match event { - ChainEvent::SignRequest(req) => { - // process sign request (insert into backlog + send sign request) - if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await - { - tracing::error!(?err, chain = %chain, "failed to process sign request"); + let mut event_rx = stream.take_event_receiver(); + + if let Some(mut rx) = event_rx.take() { + let orchestration = start_chain_stream(&mut stream); + tokio::pin!(orchestration); + let mut orchestration_done = false; + + loop { + let event = tokio::select! { + _ = &mut orchestration, if !orchestration_done => { + orchestration_done = true; + continue; } - } - ChainEvent::Respond(ev) => { - if let Err(err) = - process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) - .await - { - tracing::error!(?err, chain = %chain, "failed to process respond event"); + event = rx.recv() => event, + }; + + let Some(event) = event else { + if orchestration_done { + break; } - } - ChainEvent::RespondBidirectional(ev) => { - if let Err(err) = - process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await - { - tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + continue; + }; + + match event { + ChainEvent::SignRequest(req) => { + if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await + { + tracing::error!(?err, chain = %chain, "failed to process sign request"); + } } - } - ChainEvent::CatchupCompleted => { - if recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { - requeue_recovered_sign_requests( + ChainEvent::Respond(ev) => { + if let Err(err) = + process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) + .await + { + tracing::error!(?err, chain = %chain, "failed to process respond event"); + } + } + ChainEvent::RespondBidirectional(ev) => { + if let Err(err) = + process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await + { + tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + } + } + ChainEvent::CatchupCompleted => { + if recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + requeue_recovered_sign_requests( + &backlog, + chain, + sign_tx.clone(), + &recovered.pending, + ) + .await; + recovered.pending.clear(); + } + } + ChainEvent::Block(block) => { + if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { + tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + } + crate::metrics::indexers::LATEST_BLOCK_NUMBER + .with_label_values(&[S::CHAIN.as_str(), "indexed"]) + .set(block as i64); + } + ChainEvent::ExecutionConfirmed { + tx_id, + sign_id, + source_chain, + block_height, + result, + } => { + if let Err(err) = process_execution_confirmed( + tx_id, + sign_id, + source_chain, + block_height, + result, &backlog, - chain, sign_tx.clone(), - &recovered.pending, + S::CHAIN, ) - .await; - recovered.pending.clear(); + .await + { + tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } } } - ChainEvent::Block(block) => { - // central checkpointing for all chains - if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { - tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + } + } else { + start_chain_stream(&mut stream).await; + + while let Some(event) = stream.next_event().await { + match event { + ChainEvent::SignRequest(req) => { + if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await + { + tracing::error!(?err, chain = %chain, "failed to process sign request"); + } } - crate::metrics::indexers::LATEST_BLOCK_NUMBER - .with_label_values(&[S::CHAIN.as_str(), "indexed"]) - .set(block as i64); - } - ChainEvent::ExecutionConfirmed { - tx_id, - sign_id, - source_chain, - block_height, - result, - } => { - if let Err(err) = process_execution_confirmed( + ChainEvent::Respond(ev) => { + if let Err(err) = + process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) + .await + { + tracing::error!(?err, chain = %chain, "failed to process respond event"); + } + } + ChainEvent::RespondBidirectional(ev) => { + if let Err(err) = + process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await + { + tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + } + } + ChainEvent::CatchupCompleted => { + if recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + requeue_recovered_sign_requests( + &backlog, + chain, + sign_tx.clone(), + &recovered.pending, + ) + .await; + recovered.pending.clear(); + } + } + ChainEvent::Block(block) => { + if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { + tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + } + crate::metrics::indexers::LATEST_BLOCK_NUMBER + .with_label_values(&[S::CHAIN.as_str(), "indexed"]) + .set(block as i64); + } + ChainEvent::ExecutionConfirmed { tx_id, sign_id, source_chain, block_height, result, - &backlog, - sign_tx.clone(), - S::CHAIN, - ) - .await - { - tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } => { + if let Err(err) = process_execution_confirmed( + tx_id, + sign_id, + source_chain, + block_height, + result, + &backlog, + sign_tx.clone(), + S::CHAIN, + ) + .await + { + tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } } } } @@ -219,6 +435,8 @@ mod tests { use mpc_primitives::SignId; use mpc_primitives::Signature; use near_primitives::types::AccountId; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::timeout; @@ -229,6 +447,152 @@ mod tests { impl ChainStream for TestEventStream { const CHAIN: Chain = Chain::Solana; + type BufferedStream = DisabledBufferedStream; + + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } + + async fn next_event(&mut self) -> Option { + if self.events.is_empty() { + return None; + } + self.events.remove(0) + } + } + + struct TestBufferedStream { + items: Vec, + } + + impl ChainBufferedStream for TestBufferedStream { + type Item = u64; + + async fn initial(&mut self) -> Option { + if self.items.is_empty() { + return None; + } + Some(self.items.remove(0)) + } + + async fn next(&mut self) -> Option { + if self.items.is_empty() { + return None; + } + Some(self.items.remove(0)) + } + } + + #[derive(Clone)] + struct TestLinearControl { + persisted_height: Option, + live_items: Vec, + catchup_failures: Arc>>, + live_failures: Arc>>, + } + + impl TestLinearControl { + fn new(persisted_height: Option, live_items: Vec) -> Self { + Self { + persisted_height, + live_items, + catchup_failures: Arc::new(Mutex::new(HashMap::new())), + live_failures: Arc::new(Mutex::new(HashMap::new())), + } + } + + fn fail_catchup_once(self, height: u64) -> Self { + self.catchup_failures.lock().unwrap().insert(height, 1); + self + } + + fn fail_live_once(self, height: u64) -> Self { + self.live_failures.lock().unwrap().insert(height, 1); + self + } + + fn consume_failure(map: &Mutex>, height: u64) -> bool { + let mut failures = map.lock().unwrap(); + let Some(remaining) = failures.get_mut(&height) else { + return false; + }; + if *remaining == 0 { + return false; + } + *remaining -= 1; + true + } + + fn retry_delay(&self) -> Duration { + Duration::from_millis(1) + } + } + + struct TestLinearStream { + control: TestLinearControl, + events: Vec>, + } + + impl TestLinearStream { + fn new(control: TestLinearControl) -> Self { + Self { + control, + events: Vec::new(), + } + } + } + + impl ChainStream for TestLinearStream { + const CHAIN: Chain = Chain::Ethereum; + type BufferedStream = TestBufferedStream; + + async fn livestream(&mut self) -> anyhow::Result> { + Ok(Some(TestBufferedStream { + items: self.control.live_items.clone(), + })) + } + + fn buffered_item_height(item: &::Item) -> u64 { + *item + } + + async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { + self.events.push(Some(ChainEvent::CatchupCompleted)); + Ok(()) + } + + async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + let start = self + .control + .persisted_height + .map(|height| height + 1) + .unwrap_or(anchor_height); + Ok(start..anchor_height) + } + + async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + if TestLinearControl::consume_failure(&self.control.catchup_failures, height) { + anyhow::bail!("synthetic catchup failure at height {height}"); + } + self.events.push(Some(ChainEvent::Block(height))); + Ok(()) + } + + async fn process_buffered_item( + &mut self, + item: ::Item, + ) -> anyhow::Result<()> { + if TestLinearControl::consume_failure(&self.control.live_failures, item) { + anyhow::bail!("synthetic live failure at height {item}"); + } + self.events.push(Some(ChainEvent::Block(item))); + Ok(()) + } + + fn retry_delay(&self) -> Duration { + self.control.retry_delay() + } + async fn next_event(&mut self) -> Option { if self.events.is_empty() { return None; @@ -237,6 +601,52 @@ mod tests { } } + #[tokio::test] + async fn test_run_linearized_source_orders_catchup_before_live() { + let mut stream = TestLinearStream::new(TestLinearControl::new(Some(1), vec![4, 5])); + start_chain_stream(&mut stream).await; + + let mut observed = Vec::new(); + while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) + .await + .ok() + .flatten() + { + observed.push(event); + } + + assert!(matches!(observed[0], ChainEvent::Block(2))); + assert!(matches!(observed[1], ChainEvent::Block(3))); + assert!(matches!(observed[2], ChainEvent::CatchupCompleted)); + assert!(matches!(observed[3], ChainEvent::Block(4))); + assert!(matches!(observed[4], ChainEvent::Block(5))); + } + + #[tokio::test] + async fn test_run_linearized_source_retries_without_reordering() { + let mut stream = TestLinearStream::new( + TestLinearControl::new(Some(1), vec![4, 5]) + .fail_catchup_once(3) + .fail_live_once(4), + ); + start_chain_stream(&mut stream).await; + + let mut observed = Vec::new(); + while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) + .await + .ok() + .flatten() + { + observed.push(event); + } + + assert!(matches!(observed[0], ChainEvent::Block(2))); + assert!(matches!(observed[1], ChainEvent::Block(3))); + assert!(matches!(observed[2], ChainEvent::CatchupCompleted)); + assert!(matches!(observed[3], ChainEvent::Block(4))); + assert!(matches!(observed[4], ChainEvent::Block(5))); + } + #[tokio::test] async fn test_stream_handles_sign_and_respond() { let backlog = Backlog::new(); @@ -331,6 +741,11 @@ mod tests { impl ChainStream for StartAwareStream { const CHAIN: Chain = Chain::Solana; + type BufferedStream = DisabledBufferedStream; + + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } async fn start(&mut self) { self.started = true; @@ -416,6 +831,12 @@ mod tests { impl ChainStream for LocalStream { const CHAIN: Chain = Chain::Solana; + type BufferedStream = DisabledBufferedStream; + + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } + async fn next_event(&mut self) -> Option { self.rx.recv().await } @@ -657,6 +1078,11 @@ mod tests { impl ChainStream for EthereumLocalStream { const CHAIN: Chain = Chain::Ethereum; + type BufferedStream = DisabledBufferedStream; + + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } async fn next_event(&mut self) -> Option { if self.events.is_empty() { diff --git a/integration-tests/tests/cases/ethereum_stream.rs b/integration-tests/tests/cases/ethereum_stream.rs index ed9abf03..594c073b 100644 --- a/integration-tests/tests/cases/ethereum_stream.rs +++ b/integration-tests/tests/cases/ethereum_stream.rs @@ -1,5 +1,6 @@ use alloy::primitives::{Address as AlloyAddress, B256}; use anyhow::{Context, Result}; +use cait_sith::protocol::Participant; use ethers::middleware::{Middleware, SignerMiddleware}; use ethers::providers::{Http, Provider}; use ethers::signers::{LocalWallet, Signer}; @@ -13,13 +14,21 @@ use integration_tests::eth::{ use k256::elliptic_curve::sec1::ToEncodedPoint as _; use mpc_node::backlog::Backlog; use mpc_node::indexer_eth::{EthConfig, EthereumStream}; -use mpc_node::protocol::{Chain, SignKind}; +use mpc_node::mesh::{connection::NodeStatus, MeshState}; +use mpc_node::node_client::NodeClient; +use mpc_node::protocol::{Chain, ParticipantInfo, Sign, SignKind}; +use mpc_node::rpc::ContractStateWatcher; +use mpc_node::storage::checkpoint_storage::CheckpointStorage; +use mpc_node::stream::ops::SignBidirectionalEvent as NodeSignBidirectionalEvent; use mpc_node::stream::ops::SignatureRespondedEvent; -use mpc_node::stream::{ChainEvent, ChainStream}; -use mpc_primitives::{SignId, LATEST_MPC_KEY_VERSION}; +use mpc_node::stream::{run_stream, ChainEvent, ChainStream}; +use mpc_node::util::current_unix_timestamp; +use mpc_primitives::{SignArgs, SignId, LATEST_MPC_KEY_VERSION}; +use near_primitives::types::AccountId; use rand::thread_rng; use std::sync::Arc; use std::time::Duration; +use tokio::sync::{mpsc, watch}; use tokio::time::timeout; fn signature_deposit() -> U256 { @@ -163,6 +172,133 @@ async fn submit_sign_request( Ok(receipt.transaction_hash) } +async fn submit_sign_request_with_block( + ctx: &EthereumTestEnvironment, + payload: [u8; 32], + path: &str, +) -> Result<(H256, u64)> { + let contract = ctx.contract(); + let sign_request = SignRequest { + payload, + path: path.to_string(), + key_version: LATEST_MPC_KEY_VERSION, + algo: "secp256k1".to_string(), + dest: "".to_string(), + params: "".to_string(), + }; + + let call = contract.sign(sign_request).value(signature_deposit()); + let pending_tx = call.send().await?; + let receipt = pending_tx + .await + .context("failed to mine sign transaction")? + .context("sign transaction dropped from mempool")?; + + Ok(( + receipt.transaction_hash, + receipt + .block_number + .context("sign transaction missing block number")? + .as_u64(), + )) +} + +async fn submit_eth_transfer(ctx: &EthereumTestEnvironment) -> Result { + let pending_tx = ctx + .signer + .send_transaction( + TransactionRequest::new().to(ctx.wallet).value(U256::zero()), + None, + ) + .await?; + let receipt = pending_tx + .await + .context("failed to mine eth transfer transaction")? + .context("eth transfer transaction dropped from mempool")?; + Ok(receipt.transaction_hash) +} + +async fn submit_respond_for_request_id( + contract: ChainSignaturesContract>, + request_id: [u8; 32], +) -> Result +where + M: Middleware + 'static, +{ + let enc = k256::ProjectivePoint::GENERATOR.to_encoded_point(false); + let x = enc.x().expect("generator must have x coordinate"); + let y = enc.y().expect("generator must have y coordinate"); + let s = U256::from_big_endian(k256::Scalar::from(11u64).to_bytes().as_slice()); + + let response = chain_signatures_contract::Response { + request_id, + signature: chain_signatures_contract::Signature { + big_r: chain_signatures_contract::AffinePoint { + x: U256::from_big_endian(x), + y: U256::from_big_endian(y), + }, + s, + recovery_id: 1, + }, + }; + + let respond_call = contract.respond(vec![response]); + let pending_tx = respond_call.send().await?; + let receipt = pending_tx + .await + .context("respond transaction execution failed")? + .context("respond transaction dropped from mempool")?; + Ok(receipt.transaction_hash) +} + +async fn next_sign_message_within( + rx: &mut mpsc::Receiver, + duration: Duration, +) -> Result { + timeout(duration, rx.recv()) + .await + .context("timed out waiting for sign message")? + .context("sign channel closed unexpectedly") +} + +fn test_sign_args(seed: u8) -> SignArgs { + SignArgs { + entropy: [seed; 32], + epsilon: k256::Scalar::from(1u64), + payload: k256::Scalar::from((seed as u64) + 1), + path: format!("test-path-{seed}"), + key_version: LATEST_MPC_KEY_VERSION, + } +} + +fn test_bidirectional_event() -> NodeSignBidirectionalEvent { + let mut rlp_s = rlp::RlpStream::new_list(9); + rlp_s.append(&0u64); + rlp_s.append(&0u64); + rlp_s.append(&0u64); + rlp_s.append(&Vec::::new()); + rlp_s.append(&0u64); + rlp_s.append(&Vec::::new()); + rlp_s.append(&1u64); + rlp_s.append(&0u64); + rlp_s.append(&0u64); + + NodeSignBidirectionalEvent::Solana(signet_program::SignBidirectionalEvent { + sender: solana_sdk::pubkey::Pubkey::new_unique(), + serialized_transaction: rlp_s.out().to_vec(), + dest: Chain::Ethereum.to_string(), + caip2_id: "eip155:31337".to_string(), + key_version: LATEST_MPC_KEY_VERSION, + deposit: 1, + path: "bidirectional-test-path".to_string(), + algo: "secp256k1".to_string(), + params: "{}".to_string(), + program_id: solana_sdk::pubkey::Pubkey::new_unique(), + output_deserialization_schema: vec![], + respond_serialization_schema: vec![], + }) +} + async fn next_event_within(client: &mut EthereumStream, duration: Duration) -> Result { timeout(duration, async { loop { @@ -184,6 +320,239 @@ async fn stream_ethereum( Ok(stream) } +#[test_log::test(tokio::test)] +async fn test_ethereum_stream_resume_starts_after_checkpoint_height() -> Result<()> { + let ctx = EthereumTestEnvironment::new().await?; + let storage = CheckpointStorage::in_memory(); + let seeded_backlog = Backlog::persisted(storage.clone()); + + let replayed_payload = [0x31; 32]; + let (_, processed_height) = + submit_sign_request_with_block(&ctx, replayed_payload, "resume-processed-path").await?; + + seeded_backlog + .set_processed_block(Chain::Ethereum, processed_height) + .await; + seeded_backlog.checkpoint(Chain::Ethereum).await; + + let mut expected_payload = [0x32; 32]; + loop { + let (_, block_height) = + submit_sign_request_with_block(&ctx, expected_payload, "resume-new-path").await?; + if block_height > processed_height { + break; + } + + expected_payload[0] = expected_payload[0].saturating_add(1); + } + + let backlog = Backlog::persisted(storage); + let stream = EthereumStream::new(Some(ctx.config(true)), backlog.clone()).await?; + let (sign_tx, mut sign_rx) = mpsc::channel(16); + let (contract_watcher, _contract_tx) = ContractStateWatcher::with_running( + &"test.near".parse::().unwrap(), + k256::ProjectivePoint::GENERATOR.to_affine(), + 1, + Default::default(), + ); + + let mut mesh_state = MeshState::default(); + let mut info = ParticipantInfo::new(0); + info.url = "http://127.0.0.1:1".to_string(); + mesh_state.update(Participant::from(0u32), NodeStatus::Active, info); + let (_mesh_tx, mesh_rx) = watch::channel(mesh_state); + + let run_handle = tokio::spawn(run_stream( + stream, + sign_tx, + backlog, + contract_watcher, + mesh_rx, + NodeClient::new(&Default::default()), + )); + + let mut saw_replayed_payload = false; + let mut saw_expected_payload = false; + for _ in 0..12 { + match next_sign_message_within(&mut sign_rx, Duration::from_secs(10)).await? { + Sign::Request(req) => { + let payload: [u8; 32] = req.args.payload.to_bytes().into(); + if payload == replayed_payload { + saw_replayed_payload = true; + } + if payload == expected_payload { + saw_expected_payload = true; + break; + } + } + _ => continue, + } + } + + run_handle.abort(); + + assert!(!saw_replayed_payload, "stream replayed the stored processed block"); + assert!(saw_expected_payload, "stream did not catch up the next block"); + Ok(()) +} + +#[test_log::test(tokio::test)] +async fn test_ethereum_stream_linear_catchup_from_checkpoint() -> Result<()> { + let ctx = EthereumTestEnvironment::new().await?; + + let responder_wallet = LocalWallet::new(&mut thread_rng()).with_chain_id(ctx.sandbox.chain_id); + let responder_address = responder_wallet.address(); + let responder_provider = + Provider::::try_from(ctx.sandbox.external_http_endpoint.as_str())?; + let responder_signer: Arc, LocalWallet>> = + Arc::new(SignerMiddleware::new(responder_provider, responder_wallet)); + let fund_tx = TransactionRequest::new() + .to(responder_address) + .value(U256::from(1_000_000_000_000_000u64)); + let pending_fund = ctx.signer.send_transaction(fund_tx, None).await?; + let _ = pending_fund + .await + .context("failed to mine responder funding transaction")? + .context("responder funding transaction dropped from mempool")?; + + let checkpoint_height = ctx.signer.get_block_number().await?.as_u64(); + let checkpoint_nonce = ctx + .signer + .get_transaction_count(ctx.wallet, None) + .await? + .as_u64(); + + let storage = CheckpointStorage::in_memory(); + let seeded_backlog = Backlog::persisted(storage.clone()); + + let resolved_sign_id = SignId::new([0x11; 32]); + let requeued_sign_id = SignId::new([0x22; 32]); + seeded_backlog + .insert(mpc_node::protocol::IndexedSignRequest::sign( + resolved_sign_id, + test_sign_args(0x11), + Chain::Ethereum, + current_unix_timestamp(), + )) + .await; + seeded_backlog + .insert(mpc_node::protocol::IndexedSignRequest::sign( + requeued_sign_id, + test_sign_args(0x22), + Chain::Ethereum, + current_unix_timestamp(), + )) + .await; + seeded_backlog + .set_processed_block(Chain::Ethereum, checkpoint_height) + .await; + seeded_backlog.checkpoint(Chain::Ethereum).await; + + let backlog = Backlog::persisted(storage.clone()); + + let execution_sign_id = SignId::new([0x33; 32]); + backlog + .insert(mpc_node::protocol::IndexedSignRequest::sign_bidirectional( + execution_sign_id, + test_sign_args(0x33), + Chain::Solana, + current_unix_timestamp(), + test_bidirectional_event(), + )) + .await; + + let execution_tx = mpc_node::sign_bidirectional::BidirectionalTx { + id: mpc_node::sign_bidirectional::BidirectionalTxId(B256::from([0x44; 32])), + sender: [0u8; 32], + serialized_transaction: vec![], + source_chain: Chain::Solana, + target_chain: Chain::Ethereum, + caip2_id: "eip155:31337".to_string(), + key_version: LATEST_MPC_KEY_VERSION, + deposit: 1, + path: "bidirectional-test-path".to_string(), + algo: "secp256k1".to_string(), + dest: Chain::Ethereum.to_string(), + params: "{}".to_string(), + output_deserialization_schema: vec![], + respond_serialization_schema: vec![], + request_id: execution_sign_id.request_id, + from_address: AlloyAddress::from_slice(ctx.wallet.as_bytes()), + nonce: checkpoint_nonce, + status: mpc_node::sign_bidirectional::SignStatus::PendingExecution, + }; + backlog + .advance(Chain::Solana, execution_sign_id, execution_tx) + .await + .context("failed to seed execution watcher")?; + + let responder_contract = + ChainSignaturesContract::new(ctx.contract_address, responder_signer.clone().into()); + + submit_respond_for_request_id(responder_contract, resolved_sign_id.request_id).await?; + submit_eth_transfer(&ctx).await?; + let catchup_payload = [0x55; 32]; + submit_sign_request(&ctx, catchup_payload, "catchup-linear-path").await?; + + let stream = EthereumStream::new(Some(ctx.config(true)), backlog.clone()).await?; + let (sign_tx, mut sign_rx) = mpsc::channel(16); + let (contract_watcher, _contract_tx) = ContractStateWatcher::with_running( + &"test.near".parse::().unwrap(), + k256::ProjectivePoint::GENERATOR.to_affine(), + 1, + Default::default(), + ); + + let mut mesh_state = MeshState::default(); + let mut info = ParticipantInfo::new(0); + info.url = "http://127.0.0.1:1".to_string(); + mesh_state.update(Participant::from(0u32), NodeStatus::Active, info); + let (_mesh_tx, mesh_rx) = watch::channel(mesh_state); + + let run_handle = tokio::spawn(run_stream( + stream, + sign_tx, + backlog.clone(), + contract_watcher, + mesh_rx, + NodeClient::new(&Default::default()), + )); + + let first = next_sign_message_within(&mut sign_rx, Duration::from_secs(20)).await?; + match first { + Sign::Completion(sign_id) => assert_eq!(sign_id, resolved_sign_id), + other => panic!("expected recovered respond completion first, got {other:?}"), + } + + let second = next_sign_message_within(&mut sign_rx, Duration::from_secs(20)).await?; + match second { + Sign::Request(req) => { + assert_eq!(req.id, execution_sign_id); + assert!(matches!(req.kind, SignKind::RespondBidirectional(_))); + assert_eq!(req.chain, Chain::Solana); + } + other => panic!("expected execution follow-up request second, got {other:?}"), + } + + let third = next_sign_message_within(&mut sign_rx, Duration::from_secs(20)).await?; + match third { + Sign::Request(req) => { + assert_eq!(req.chain, Chain::Ethereum); + assert_eq!(req.args.payload.to_bytes(), catchup_payload.into()); + } + other => panic!("expected catchup sign request third, got {other:?}"), + } + + let fourth = next_sign_message_within(&mut sign_rx, Duration::from_secs(20)).await?; + match fourth { + Sign::Request(req) => assert_eq!(req.id, requeued_sign_id), + other => panic!("expected deferred recovered requeue fourth, got {other:?}"), + } + + run_handle.abort(); + Ok(()) +} + #[test_log::test(tokio::test)] async fn test_ethereum_stream_parse_sign_event() -> Result<()> { let _ = tracing_subscriber::fmt::try_init(); From 3c444e380467fca84a7dda691d25ff724981460a Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 02:45:49 +0000 Subject: [PATCH 02/15] Cleaner setup for eth --- chain-signatures/node/src/indexer_eth/mod.rs | 139 +++++------- chain-signatures/node/src/stream/mod.rs | 210 +++++++++---------- 2 files changed, 160 insertions(+), 189 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 01e86208..a210324c 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -1189,11 +1189,6 @@ pub struct EthereumStream { tasks: Vec>, } -struct EthereumBackgroundStream { - events_tx: mpsc::Sender, - indexer: EthereumIndexer, -} - impl EthereumStream { pub async fn new(eth: Option, backlog: Backlog) -> anyhow::Result { let Some(eth) = eth else { @@ -1223,15 +1218,67 @@ impl EthereumStream { } pub fn start(&mut self) { - let mut stream = EthereumBackgroundStream { - events_tx: self.events_tx.clone(), - indexer: self.indexer.clone(), - }; - + let events_tx = self.events_tx.clone(); + let indexer = self.indexer.clone(); self.tasks.push(tokio::spawn(async move { - crate::stream::start_chain_stream(&mut stream).await; + Self::run_background(indexer, events_tx).await; })); } + + async fn run_background(indexer: EthereumIndexer, events_tx: mpsc::Sender) { + let (live_blocks_tx, mut live_blocks_rx) = live_blocks_channel(); + tokio::spawn(EthereumIndexer::buffer_live_blocks( + Arc::new(indexer.client.clone()), + live_blocks_tx, + )); + + let Some(anchor_block) = live_blocks_rx.recv().await else { + tracing::warn!("ethereum livestream ended before anchor block"); + return; + }; + + let anchor_height = anchor_block.header.number; + let catchup_start = EthereumIndexer::catchup_start_block_number( + indexer.backlog.processed_block(Chain::Ethereum).await, + anchor_height, + ); + + for height in catchup_start..anchor_height { + loop { + if height % 10 == 0 { + tracing::info!(height, "processed ethereum catchup height attempt"); + } + + match indexer.process_height(height, events_tx.clone()).await { + Ok(()) => break, + Err(err) => { + tracing::warn!(?err, height, "catchup height processing failed; retrying"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + } + } + } + + let mut next_block = Some(anchor_block); + loop { + let block = match next_block.take() { + Some(block) => block, + None => match live_blocks_rx.recv().await { + Some(block) => block, + None => break, + }, + }; + + match indexer.process_live_block(block.clone(), events_tx.clone()).await { + Ok(()) => {} + Err(err) => { + tracing::warn!(?err, "buffered item processing failed; retrying"); + tokio::time::sleep(Duration::from_millis(500)).await; + next_block = Some(block); + } + } + } + } } impl Drop for EthereumStream { @@ -1287,13 +1334,6 @@ impl ChainStream for EthereumStream { .await } - async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { - self.events_tx - .send(ChainEvent::CatchupCompleted) - .await - .map_err(|err| anyhow::anyhow!("failed to emit ethereum catchup completion event: {err:?}")) - } - async fn process_buffered_item( &mut self, item: ::Item, @@ -1314,69 +1354,6 @@ impl ChainStream for EthereumStream { } } } - -impl ChainStream for EthereumBackgroundStream { - const CHAIN: Chain = Chain::Ethereum; - type BufferedStream = EthereumBufferedStream; - - async fn livestream(&mut self) -> anyhow::Result> { - let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); - tokio::spawn(EthereumIndexer::buffer_live_blocks( - Arc::new(self.indexer.client.clone()), - live_blocks_tx, - )); - - Ok(Some(EthereumBufferedStream { live_blocks_rx })) - } - - fn buffered_item_height(item: &::Item) -> u64 { - item.header.number - } - - async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { - let catchup_start = EthereumIndexer::catchup_start_block_number( - self.indexer.backlog.processed_block(Chain::Ethereum).await, - anchor_height, - ); - - Ok(catchup_start..anchor_height) - } - - async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { - if height % 10 == 0 { - tracing::info!(height, "processed ethereum catchup height attempt"); - } - - self.indexer - .process_height(height, self.events_tx.clone()) - .await - } - - async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { - self.events_tx - .send(ChainEvent::CatchupCompleted) - .await - .map_err(|err| anyhow::anyhow!("failed to emit ethereum catchup completion event: {err:?}")) - } - - async fn process_buffered_item( - &mut self, - item: ::Item, - ) -> anyhow::Result<()> { - self.indexer - .process_live_block(item, self.events_tx.clone()) - .await - } - - fn retry_delay(&self) -> Duration { - Duration::from_millis(500) - } - - async fn next_event(&mut self) -> Option { - None - } -} - #[cfg(test)] mod tests { use super::EthereumIndexer; diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 9f29f781..dbf64eb5 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -14,6 +14,7 @@ use crate::stream::ops::{ use std::time::Duration; use std::{ops::Range, vec::Vec}; use tokio::sync::mpsc; +use tokio::sync::oneshot; use tokio::sync::watch; pub mod ops; @@ -31,9 +32,6 @@ pub enum ChainEvent { Respond(SignatureRespondedEvent), RespondBidirectional(RespondBidirectionalEvent), - /// The stream has finished replaying catch-up data for this chain. - CatchupCompleted, - /// Block height indicating the client has observed/processed up to `u64` (slot/block) Block(u64), @@ -67,7 +65,6 @@ impl std::fmt::Debug for ChainEvent { .field(&ev.request_id()) .field(&ev.source_chain().as_str()) .finish(), - ChainEvent::CatchupCompleted => f.debug_tuple("CatchupCompleted").finish(), ChainEvent::Block(b) => write!(f, "Block({b})"), ChainEvent::ExecutionConfirmed { tx_id, @@ -132,10 +129,6 @@ pub trait ChainStream: Send + 'static { fn buffered_item_height(_item: &::Item) -> u64; - async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { - Ok(()) - } - async fn catchup_range(&mut self, _anchor_height: u64) -> anyhow::Result> { Ok(0..0) } @@ -158,7 +151,10 @@ pub trait ChainStream: Send + 'static { async fn next_event(&mut self) -> Option; } -pub(crate) async fn start_chain_stream(stream: &mut S) { +pub(crate) async fn start_chain_stream( + stream: &mut S, + mut catchup_completed_tx: Option>, +) { tracing::info!(chain = %S::CHAIN, "starting chain stream orchestration"); match stream.livestream().await { @@ -191,9 +187,8 @@ pub(crate) async fn start_chain_stream(stream: &mut S) { } } - if let Err(err) = stream.emit_catchup_completed().await { - tracing::warn!(?err, chain = %S::CHAIN, "failed to emit catchup completed event"); - return; + if let Some(tx) = catchup_completed_tx.take() { + let _ = tx.send(()); } let mut next_item = Some(anchor_item); @@ -218,6 +213,10 @@ pub(crate) async fn start_chain_stream(stream: &mut S) { } Ok(None) => { stream.start().await; + + if let Some(tx) = catchup_completed_tx.take() { + let _ = tx.send(()); + } } Err(err) => { tracing::error!(?err, chain = %S::CHAIN, "failed to initialize livestream"); @@ -248,56 +247,27 @@ pub async fn run_stream( ) .await; - // NOTE: we need to start after we recover entries from backlog and starting the run_stream task - // such that we can guarantee getting the CatchupCompleted event from this task to modify the - // recovered entries. let mut event_rx = stream.take_event_receiver(); if let Some(mut rx) = event_rx.take() { - let orchestration = start_chain_stream(&mut stream); + let (catchup_completed_tx, mut catchup_completed_rx) = oneshot::channel(); + let orchestration = start_chain_stream(&mut stream, Some(catchup_completed_tx)); tokio::pin!(orchestration); let mut orchestration_done = false; + let mut catchup_completed = false; loop { - let event = tokio::select! { + tokio::select! { _ = &mut orchestration, if !orchestration_done => { orchestration_done = true; continue; } - event = rx.recv() => event, - }; + result = &mut catchup_completed_rx, if !catchup_completed => { + catchup_completed = true; - let Some(event) = event else { - if orchestration_done { - break; - } - continue; - }; - - match event { - ChainEvent::SignRequest(req) => { - if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await - { - tracing::error!(?err, chain = %chain, "failed to process sign request"); - } - } - ChainEvent::Respond(ev) => { - if let Err(err) = - process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) - .await - { - tracing::error!(?err, chain = %chain, "failed to process respond event"); - } - } - ChainEvent::RespondBidirectional(ev) => { - if let Err(err) = - process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await + if result.is_ok() + && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { - tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); - } - } - ChainEvent::CatchupCompleted => { - if recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { requeue_recovered_sign_requests( &backlog, chain, @@ -307,41 +277,77 @@ pub async fn run_stream( .await; recovered.pending.clear(); } + + continue; } - ChainEvent::Block(block) => { - if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { - tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); - } - crate::metrics::indexers::LATEST_BLOCK_NUMBER - .with_label_values(&[S::CHAIN.as_str(), "indexed"]) - .set(block as i64); - } - ChainEvent::ExecutionConfirmed { - tx_id, - sign_id, - source_chain, - block_height, - result, - } => { - if let Err(err) = process_execution_confirmed( - tx_id, - sign_id, - source_chain, - block_height, - result, - &backlog, - sign_tx.clone(), - S::CHAIN, - ) - .await - { - tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + event = rx.recv() => { + let Some(event) = event else { + if orchestration_done { + break; + } + continue; + }; + + match event { + ChainEvent::SignRequest(req) => { + if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await + { + tracing::error!(?err, chain = %chain, "failed to process sign request"); + } + } + ChainEvent::Respond(ev) => { + if let Err(err) = + process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) + .await + { + tracing::error!(?err, chain = %chain, "failed to process respond event"); + } + } + ChainEvent::RespondBidirectional(ev) => { + if let Err(err) = + process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await + { + tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + } + } + ChainEvent::Block(block) => { + if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { + tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + } + crate::metrics::indexers::LATEST_BLOCK_NUMBER + .with_label_values(&[S::CHAIN.as_str(), "indexed"]) + .set(block as i64); + } + ChainEvent::ExecutionConfirmed { + tx_id, + sign_id, + source_chain, + block_height, + result, + } => { + if let Err(err) = process_execution_confirmed( + tx_id, + sign_id, + source_chain, + block_height, + result, + &backlog, + sign_tx.clone(), + S::CHAIN, + ) + .await + { + tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } + } } } } } } else { - start_chain_stream(&mut stream).await; + let (catchup_completed_tx, catchup_completed_rx) = oneshot::channel(); + start_chain_stream(&mut stream, Some(catchup_completed_tx)).await; + let catchup_completed = catchup_completed_rx.await.is_ok(); while let Some(event) = stream.next_event().await { match event { @@ -366,18 +372,6 @@ pub async fn run_stream( tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); } } - ChainEvent::CatchupCompleted => { - if recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { - requeue_recovered_sign_requests( - &backlog, - chain, - sign_tx.clone(), - &recovered.pending, - ) - .await; - recovered.pending.clear(); - } - } ChainEvent::Block(block) => { if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); @@ -410,6 +404,17 @@ pub async fn run_stream( } } } + + if catchup_completed && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + requeue_recovered_sign_requests( + &backlog, + chain, + sign_tx.clone(), + &recovered.pending, + ) + .await; + recovered.pending.clear(); + } } tracing::warn!(%chain, "indexer shut down"); @@ -556,11 +561,6 @@ mod tests { *item } - async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { - self.events.push(Some(ChainEvent::CatchupCompleted)); - Ok(()) - } - async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { let start = self .control @@ -604,7 +604,7 @@ mod tests { #[tokio::test] async fn test_run_linearized_source_orders_catchup_before_live() { let mut stream = TestLinearStream::new(TestLinearControl::new(Some(1), vec![4, 5])); - start_chain_stream(&mut stream).await; + start_chain_stream(&mut stream, None).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -617,9 +617,8 @@ mod tests { assert!(matches!(observed[0], ChainEvent::Block(2))); assert!(matches!(observed[1], ChainEvent::Block(3))); - assert!(matches!(observed[2], ChainEvent::CatchupCompleted)); - assert!(matches!(observed[3], ChainEvent::Block(4))); - assert!(matches!(observed[4], ChainEvent::Block(5))); + assert!(matches!(observed[2], ChainEvent::Block(4))); + assert!(matches!(observed[3], ChainEvent::Block(5))); } #[tokio::test] @@ -629,7 +628,7 @@ mod tests { .fail_catchup_once(3) .fail_live_once(4), ); - start_chain_stream(&mut stream).await; + start_chain_stream(&mut stream, None).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -642,9 +641,8 @@ mod tests { assert!(matches!(observed[0], ChainEvent::Block(2))); assert!(matches!(observed[1], ChainEvent::Block(3))); - assert!(matches!(observed[2], ChainEvent::CatchupCompleted)); - assert!(matches!(observed[3], ChainEvent::Block(4))); - assert!(matches!(observed[4], ChainEvent::Block(5))); + assert!(matches!(observed[2], ChainEvent::Block(4))); + assert!(matches!(observed[3], ChainEvent::Block(5))); } #[tokio::test] @@ -1099,11 +1097,7 @@ mod tests { }); let client = EthereumLocalStream { - events: vec![ - Some(ChainEvent::Respond(respond)), - Some(ChainEvent::CatchupCompleted), - None, - ], + events: vec![Some(ChainEvent::Respond(respond)), None], }; let backlog = Backlog::persisted(storage); From 5d6bc2af8a95bdc3f494df94c04f4b609bf5d5a6 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 02:46:04 +0000 Subject: [PATCH 03/15] Cleaner generalized setup --- chain-signatures/node/src/indexer_eth/mod.rs | 83 ++----- chain-signatures/node/src/indexer_sol.rs | 22 +- chain-signatures/node/src/stream/mod.rs | 233 +++++++------------ 3 files changed, 114 insertions(+), 224 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index a210324c..79159663 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -729,11 +729,6 @@ impl EthereumIndexer { }) } - pub async fn run(self, events_tx: mpsc::Sender) { - tracing::info!("running ethereum indexer"); - let _ = events_tx; - } - async fn buffer_live_blocks( client: Arc, live_blocks: mpsc::Sender, @@ -1184,7 +1179,8 @@ impl EthereumIndexer { /// `start()` after recovery has completed. pub struct EthereumStream { events_tx: mpsc::Sender, - events_rx: Option>, + events_rx: mpsc::Receiver, + catchup_completed_rx: Option>, indexer: EthereumIndexer, tasks: Vec>, } @@ -1211,21 +1207,28 @@ impl EthereumStream { Ok(Self { events_tx, - events_rx: Some(events_rx), + events_rx, + catchup_completed_rx: None, indexer, tasks: Vec::new(), }) } pub fn start(&mut self) { + let (catchup_completed_tx, catchup_completed_rx) = tokio::sync::oneshot::channel(); let events_tx = self.events_tx.clone(); let indexer = self.indexer.clone(); self.tasks.push(tokio::spawn(async move { - Self::run_background(indexer, events_tx).await; + Self::run_background(indexer, events_tx, catchup_completed_tx).await; })); + self.catchup_completed_rx = Some(catchup_completed_rx); } - async fn run_background(indexer: EthereumIndexer, events_tx: mpsc::Sender) { + async fn run_background( + indexer: EthereumIndexer, + events_tx: mpsc::Sender, + catchup_completed_tx: tokio::sync::oneshot::Sender<()>, + ) { let (live_blocks_tx, mut live_blocks_rx) = live_blocks_channel(); tokio::spawn(EthereumIndexer::buffer_live_blocks( Arc::new(indexer.client.clone()), @@ -1259,6 +1262,8 @@ impl EthereumStream { } } + let _ = catchup_completed_tx.send(()); + let mut next_block = Some(anchor_block); loop { let block = match next_block.take() { @@ -1293,65 +1298,15 @@ impl ChainStream for EthereumStream { const CHAIN: Chain = Chain::Ethereum; type BufferedStream = EthereumBufferedStream; - fn take_event_receiver(&mut self) -> Option> { - self.events_rx.take() - } - - async fn start(&mut self) { + async fn start(&mut self) -> tokio::sync::oneshot::Receiver<()> { self.start(); - } - - async fn livestream(&mut self) -> anyhow::Result> { - let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); - tokio::spawn(EthereumIndexer::buffer_live_blocks( - Arc::new(self.indexer.client.clone()), - live_blocks_tx, - )); - - Ok(Some(EthereumBufferedStream { live_blocks_rx })) - } - - fn buffered_item_height(item: &::Item) -> u64 { - item.header.number - } - - async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { - let catchup_start = EthereumIndexer::catchup_start_block_number( - self.indexer.backlog.processed_block(Chain::Ethereum).await, - anchor_height, - ); - - Ok(catchup_start..anchor_height) - } - - async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { - if height % 10 == 0 { - tracing::info!(height, "processed ethereum catchup height attempt"); - } - - self.indexer - .process_height(height, self.events_tx.clone()) - .await - } - - async fn process_buffered_item( - &mut self, - item: ::Item, - ) -> anyhow::Result<()> { - self.indexer - .process_live_block(item, self.events_tx.clone()) - .await - } - - fn retry_delay(&self) -> Duration { - Duration::from_millis(500) + self.catchup_completed_rx + .take() + .expect("ethereum stream start() called without catchup receiver") } async fn next_event(&mut self) -> Option { - match self.events_rx.as_mut() { - Some(rx) => rx.recv().await, - None => None, - } + self.events_rx.recv().await } } #[cfg(test)] diff --git a/chain-signatures/node/src/indexer_sol.rs b/chain-signatures/node/src/indexer_sol.rs index f571907d..b204953b 100644 --- a/chain-signatures/node/src/indexer_sol.rs +++ b/chain-signatures/node/src/indexer_sol.rs @@ -293,7 +293,7 @@ type Result = anyhow::Result; /// Solana stream that implements the new ChainStream abstraction pub struct SolanaStream { - rx: Option>, + rx: mpsc::Receiver, start_state: Option, tasks: Vec>, } @@ -328,7 +328,7 @@ impl SolanaStream { let (tx, rx) = crate::stream::channel(); Some(SolanaStream { - rx: Some(rx), + rx, start_state: Some(SolanaStreamStartState { program_id, rpc_http_url: sol.rpc_http_url.clone(), @@ -344,17 +344,15 @@ impl ChainStream for SolanaStream { const CHAIN: Chain = Chain::Solana; type BufferedStream = crate::stream::DisabledBufferedStream; - fn take_event_receiver(&mut self) -> Option> { - self.rx.take() - } - fn buffered_item_height(_item: &::Item) -> u64 { 0 } - async fn start(&mut self) { + async fn start(&mut self) -> tokio::sync::oneshot::Receiver<()> { + let (tx, rx) = tokio::sync::oneshot::channel(); let Some(start_state) = self.start_state.take() else { - return; + let _ = tx.send(()); + return rx; }; self.tasks.push(spawn_cpi_sign_events( @@ -369,13 +367,13 @@ impl ChainStream for SolanaStream { start_state.rpc_ws_url, start_state.tx, )); + + let _ = tx.send(()); + rx } async fn next_event(&mut self) -> Option { - match self.rx.as_mut() { - Some(rx) => rx.recv().await, - None => None, - } + self.rx.recv().await } } diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index dbf64eb5..1a868583 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -117,17 +117,19 @@ pub trait ChainStream: Send + 'static { const CHAIN: Chain; type BufferedStream: ChainBufferedStream; - fn take_event_receiver(&mut self) -> Option> { - None + async fn start(&mut self) -> oneshot::Receiver<()> { + let (tx, rx) = oneshot::channel(); + let _ = tx.send(()); + rx } - async fn start(&mut self) {} - async fn livestream(&mut self) -> anyhow::Result> { Ok(None) } - fn buffered_item_height(_item: &::Item) -> u64; + fn buffered_item_height(_item: &::Item) -> u64 { + 0 + } async fn catchup_range(&mut self, _anchor_height: u64) -> anyhow::Result> { Ok(0..0) @@ -151,10 +153,12 @@ pub trait ChainStream: Send + 'static { async fn next_event(&mut self) -> Option; } +#[cfg(test)] pub(crate) async fn start_chain_stream( stream: &mut S, - mut catchup_completed_tx: Option>, + catchup_completed_tx: oneshot::Sender<()>, ) { + let mut catchup_completed_tx = Some(catchup_completed_tx); tracing::info!(chain = %S::CHAIN, "starting chain stream orchestration"); match stream.livestream().await { @@ -247,165 +251,93 @@ pub async fn run_stream( ) .await; - let mut event_rx = stream.take_event_receiver(); + let mut catchup_completed_rx = stream.start().await; + let mut catchup_completed = false; - if let Some(mut rx) = event_rx.take() { - let (catchup_completed_tx, mut catchup_completed_rx) = oneshot::channel(); - let orchestration = start_chain_stream(&mut stream, Some(catchup_completed_tx)); - tokio::pin!(orchestration); - let mut orchestration_done = false; - let mut catchup_completed = false; + loop { + tokio::select! { + result = &mut catchup_completed_rx, if !catchup_completed => { + catchup_completed = true; - loop { - tokio::select! { - _ = &mut orchestration, if !orchestration_done => { - orchestration_done = true; - continue; + if result.is_ok() + && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup + { + requeue_recovered_sign_requests( + &backlog, + chain, + sign_tx.clone(), + &recovered.pending, + ) + .await; + recovered.pending.clear(); } - result = &mut catchup_completed_rx, if !catchup_completed => { - catchup_completed = true; - if result.is_ok() - && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup - { - requeue_recovered_sign_requests( - &backlog, - chain, - sign_tx.clone(), - &recovered.pending, - ) - .await; - recovered.pending.clear(); - } + continue; + } + event = stream.next_event() => { + let Some(event) = event else { + break; + }; - continue; - } - event = rx.recv() => { - let Some(event) = event else { - if orchestration_done { - break; - } - continue; - }; - - match event { - ChainEvent::SignRequest(req) => { - if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await - { - tracing::error!(?err, chain = %chain, "failed to process sign request"); - } - } - ChainEvent::Respond(ev) => { - if let Err(err) = - process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) - .await - { - tracing::error!(?err, chain = %chain, "failed to process respond event"); - } - } - ChainEvent::RespondBidirectional(ev) => { - if let Err(err) = - process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await - { - tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); - } - } - ChainEvent::Block(block) => { - if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { - tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); - } - crate::metrics::indexers::LATEST_BLOCK_NUMBER - .with_label_values(&[S::CHAIN.as_str(), "indexed"]) - .set(block as i64); - } - ChainEvent::ExecutionConfirmed { - tx_id, - sign_id, - source_chain, - block_height, - result, - } => { - if let Err(err) = process_execution_confirmed( - tx_id, - sign_id, - source_chain, - block_height, - result, - &backlog, - sign_tx.clone(), - S::CHAIN, - ) - .await - { - tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); - } + match event { + ChainEvent::SignRequest(req) => { + if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await + { + tracing::error!(?err, chain = %chain, "failed to process sign request"); } } - } - } - } - } else { - let (catchup_completed_tx, catchup_completed_rx) = oneshot::channel(); - start_chain_stream(&mut stream, Some(catchup_completed_tx)).await; - let catchup_completed = catchup_completed_rx.await.is_ok(); - - while let Some(event) = stream.next_event().await { - match event { - ChainEvent::SignRequest(req) => { - if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await - { - tracing::error!(?err, chain = %chain, "failed to process sign request"); - } - } - ChainEvent::Respond(ev) => { - if let Err(err) = - process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) - .await - { - tracing::error!(?err, chain = %chain, "failed to process respond event"); + ChainEvent::Respond(ev) => { + if let Err(err) = + process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) + .await + { + tracing::error!(?err, chain = %chain, "failed to process respond event"); + } } - } - ChainEvent::RespondBidirectional(ev) => { - if let Err(err) = - process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await - { - tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + ChainEvent::RespondBidirectional(ev) => { + if let Err(err) = + process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await + { + tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + } } - } - ChainEvent::Block(block) => { - if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { - tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + ChainEvent::Block(block) => { + if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { + tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + } + crate::metrics::indexers::LATEST_BLOCK_NUMBER + .with_label_values(&[S::CHAIN.as_str(), "indexed"]) + .set(block as i64); } - crate::metrics::indexers::LATEST_BLOCK_NUMBER - .with_label_values(&[S::CHAIN.as_str(), "indexed"]) - .set(block as i64); - } - ChainEvent::ExecutionConfirmed { - tx_id, - sign_id, - source_chain, - block_height, - result, - } => { - if let Err(err) = process_execution_confirmed( + ChainEvent::ExecutionConfirmed { tx_id, sign_id, source_chain, block_height, result, - &backlog, - sign_tx.clone(), - S::CHAIN, - ) - .await - { - tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } => { + if let Err(err) = process_execution_confirmed( + tx_id, + sign_id, + source_chain, + block_height, + result, + &backlog, + sign_tx.clone(), + S::CHAIN, + ) + .await + { + tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + } } } } } + } - if catchup_completed && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + if catchup_completed && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + if !recovered.pending.is_empty() { requeue_recovered_sign_requests( &backlog, chain, @@ -604,7 +536,8 @@ mod tests { #[tokio::test] async fn test_run_linearized_source_orders_catchup_before_live() { let mut stream = TestLinearStream::new(TestLinearControl::new(Some(1), vec![4, 5])); - start_chain_stream(&mut stream, None).await; + let (tx, _rx) = oneshot::channel(); + start_chain_stream(&mut stream, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -628,7 +561,8 @@ mod tests { .fail_catchup_once(3) .fail_live_once(4), ); - start_chain_stream(&mut stream, None).await; + let (tx, _rx) = oneshot::channel(); + start_chain_stream(&mut stream, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -745,8 +679,11 @@ mod tests { 0 } - async fn start(&mut self) { + async fn start(&mut self) -> oneshot::Receiver<()> { self.started = true; + let (tx, rx) = oneshot::channel(); + let _ = tx.send(()); + rx } async fn next_event(&mut self) -> Option { From 22673490aead94f0fac88c081c7dbd52d7fc9f13 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 03:04:37 +0000 Subject: [PATCH 04/15] Closer to the generalized view --- chain-signatures/node/src/indexer_eth/mod.rs | 133 +++++++++---------- chain-signatures/node/src/stream/mod.rs | 1 - 2 files changed, 66 insertions(+), 68 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 79159663..b6ef60c2 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -1179,12 +1179,24 @@ impl EthereumIndexer { /// `start()` after recovery has completed. pub struct EthereumStream { events_tx: mpsc::Sender, - events_rx: mpsc::Receiver, + events_rx: Option>, catchup_completed_rx: Option>, indexer: EthereumIndexer, tasks: Vec>, } +impl Clone for EthereumStream { + fn clone(&self) -> Self { + Self { + events_tx: self.events_tx.clone(), + events_rx: None, + catchup_completed_rx: None, + indexer: self.indexer.clone(), + tasks: Vec::new(), + } + } +} + impl EthereumStream { pub async fn new(eth: Option, backlog: Backlog) -> anyhow::Result { let Some(eth) = eth else { @@ -1207,7 +1219,7 @@ impl EthereumStream { Ok(Self { events_tx, - events_rx, + events_rx: Some(events_rx), catchup_completed_rx: None, indexer, tasks: Vec::new(), @@ -1216,74 +1228,12 @@ impl EthereumStream { pub fn start(&mut self) { let (catchup_completed_tx, catchup_completed_rx) = tokio::sync::oneshot::channel(); - let events_tx = self.events_tx.clone(); - let indexer = self.indexer.clone(); + let mut producer = self.clone(); self.tasks.push(tokio::spawn(async move { - Self::run_background(indexer, events_tx, catchup_completed_tx).await; + crate::stream::start_chain_stream(&mut producer, catchup_completed_tx).await; })); self.catchup_completed_rx = Some(catchup_completed_rx); } - - async fn run_background( - indexer: EthereumIndexer, - events_tx: mpsc::Sender, - catchup_completed_tx: tokio::sync::oneshot::Sender<()>, - ) { - let (live_blocks_tx, mut live_blocks_rx) = live_blocks_channel(); - tokio::spawn(EthereumIndexer::buffer_live_blocks( - Arc::new(indexer.client.clone()), - live_blocks_tx, - )); - - let Some(anchor_block) = live_blocks_rx.recv().await else { - tracing::warn!("ethereum livestream ended before anchor block"); - return; - }; - - let anchor_height = anchor_block.header.number; - let catchup_start = EthereumIndexer::catchup_start_block_number( - indexer.backlog.processed_block(Chain::Ethereum).await, - anchor_height, - ); - - for height in catchup_start..anchor_height { - loop { - if height % 10 == 0 { - tracing::info!(height, "processed ethereum catchup height attempt"); - } - - match indexer.process_height(height, events_tx.clone()).await { - Ok(()) => break, - Err(err) => { - tracing::warn!(?err, height, "catchup height processing failed; retrying"); - tokio::time::sleep(Duration::from_millis(500)).await; - } - } - } - } - - let _ = catchup_completed_tx.send(()); - - let mut next_block = Some(anchor_block); - loop { - let block = match next_block.take() { - Some(block) => block, - None => match live_blocks_rx.recv().await { - Some(block) => block, - None => break, - }, - }; - - match indexer.process_live_block(block.clone(), events_tx.clone()).await { - Ok(()) => {} - Err(err) => { - tracing::warn!(?err, "buffered item processing failed; retrying"); - tokio::time::sleep(Duration::from_millis(500)).await; - next_block = Some(block); - } - } - } - } } impl Drop for EthereumStream { @@ -1305,8 +1255,57 @@ impl ChainStream for EthereumStream { .expect("ethereum stream start() called without catchup receiver") } + async fn livestream(&mut self) -> anyhow::Result> { + let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); + tokio::spawn(EthereumIndexer::buffer_live_blocks( + Arc::new(self.indexer.client.clone()), + live_blocks_tx, + )); + + Ok(Some(EthereumBufferedStream { live_blocks_rx })) + } + + fn buffered_item_height(item: &::Item) -> u64 { + item.header.number + } + + async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + let catchup_start = EthereumIndexer::catchup_start_block_number( + self.indexer.backlog.processed_block(Chain::Ethereum).await, + anchor_height, + ); + + Ok(catchup_start..anchor_height) + } + + async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + if height % 10 == 0 { + tracing::info!(height, "processed ethereum catchup height attempt"); + } + + self.indexer + .process_height(height, self.events_tx.clone()) + .await + } + + async fn process_buffered_item( + &mut self, + item: ::Item, + ) -> anyhow::Result<()> { + self.indexer + .process_live_block(item, self.events_tx.clone()) + .await + } + + fn retry_delay(&self) -> Duration { + Duration::from_millis(500) + } + async fn next_event(&mut self) -> Option { - self.events_rx.recv().await + match self.events_rx.as_mut() { + Some(rx) => rx.recv().await, + None => None, + } } } #[cfg(test)] diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 1a868583..9a036868 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -153,7 +153,6 @@ pub trait ChainStream: Send + 'static { async fn next_event(&mut self) -> Option; } -#[cfg(test)] pub(crate) async fn start_chain_stream( stream: &mut S, catchup_completed_tx: oneshot::Sender<()>, From 7868ecfc6eabbb1ba0a6c970360be5df124f664b Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 03:54:02 +0000 Subject: [PATCH 05/15] Cleaner now --- chain-signatures/node/src/indexer_eth/mod.rs | 69 +++---- chain-signatures/node/src/indexer_sol.rs | 28 ++- chain-signatures/node/src/stream/mod.rs | 182 ++++++++++++------ .../tests/cases/ethereum_stream.rs | 34 +++- .../tests/cases/solana_stream.rs | 2 +- 5 files changed, 191 insertions(+), 124 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index b6ef60c2..fea20b7f 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -9,8 +9,9 @@ use crate::protocol::{Chain, IndexedSignRequest}; use crate::respond_bidirectional::CompletedTx; use crate::sign_bidirectional::SignStatus; use crate::stream::{ - ChainBufferedStream, ChainEvent, ChainStream, ExecutionOutcome, + ChainBufferedStream, ChainEvent, ChainIndexer, ChainStream, ExecutionOutcome, }; +use async_trait::async_trait; use alloy::eips::BlockNumberOrTag; use alloy::primitives::hex::{self, ToHexExt}; @@ -27,7 +28,6 @@ use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; use tokio::sync::mpsc; -use tokio::task::JoinHandle; use tokio::time::Duration; pub(crate) static MAX_SECP256K1_SCALAR: LazyLock = LazyLock::new(|| { @@ -84,6 +84,7 @@ pub struct EthereumBufferedStream { live_blocks_rx: mpsc::Receiver, } +#[async_trait] impl ChainBufferedStream for EthereumBufferedStream { type Item = alloy::rpc::types::Block; @@ -1178,23 +1179,13 @@ impl EthereumIndexer { /// Construction is side-effect free; the shared `run_stream()` loop calls /// `start()` after recovery has completed. pub struct EthereumStream { - events_tx: mpsc::Sender, events_rx: Option>, - catchup_completed_rx: Option>, - indexer: EthereumIndexer, - tasks: Vec>, + start_state: Option, } -impl Clone for EthereumStream { - fn clone(&self) -> Self { - Self { - events_tx: self.events_tx.clone(), - events_rx: None, - catchup_completed_rx: None, - indexer: self.indexer.clone(), - tasks: Vec::new(), - } - } +pub struct EthereumStreamIndexer { + events_tx: mpsc::Sender, + indexer: EthereumIndexer, } impl EthereumStream { @@ -1218,43 +1209,16 @@ impl EthereumStream { let (events_tx, events_rx) = crate::stream::channel(); Ok(Self { - events_tx, events_rx: Some(events_rx), - catchup_completed_rx: None, - indexer, - tasks: Vec::new(), + start_state: Some(EthereumStreamIndexer { events_tx, indexer }), }) } - - pub fn start(&mut self) { - let (catchup_completed_tx, catchup_completed_rx) = tokio::sync::oneshot::channel(); - let mut producer = self.clone(); - self.tasks.push(tokio::spawn(async move { - crate::stream::start_chain_stream(&mut producer, catchup_completed_tx).await; - })); - self.catchup_completed_rx = Some(catchup_completed_rx); - } -} - -impl Drop for EthereumStream { - fn drop(&mut self) { - for t in &self.tasks { - t.abort(); - } - } } -impl ChainStream for EthereumStream { - const CHAIN: Chain = Chain::Ethereum; +#[async_trait] +impl ChainIndexer for EthereumStreamIndexer { type BufferedStream = EthereumBufferedStream; - async fn start(&mut self) -> tokio::sync::oneshot::Receiver<()> { - self.start(); - self.catchup_completed_rx - .take() - .expect("ethereum stream start() called without catchup receiver") - } - async fn livestream(&mut self) -> anyhow::Result> { let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); tokio::spawn(EthereumIndexer::buffer_live_blocks( @@ -1301,6 +1265,19 @@ impl ChainStream for EthereumStream { Duration::from_millis(500) } +} + +#[async_trait] +impl ChainStream for EthereumStream { + const CHAIN: Chain = Chain::Ethereum; + type Indexer = EthereumStreamIndexer; + + async fn start(&mut self) -> anyhow::Result { + self.start_state + .take() + .ok_or_else(|| anyhow::anyhow!("ethereum stream already started")) + } + async fn next_event(&mut self) -> Option { match self.events_rx.as_mut() { Some(rx) => rx.recv().await, diff --git a/chain-signatures/node/src/indexer_sol.rs b/chain-signatures/node/src/indexer_sol.rs index b204953b..7ba4c5bc 100644 --- a/chain-signatures/node/src/indexer_sol.rs +++ b/chain-signatures/node/src/indexer_sol.rs @@ -1,9 +1,10 @@ use crate::protocol::{Chain, IndexedSignRequest}; use crate::sign_bidirectional::hash_rlp_data; use crate::stream::ops::{SignatureEvent, SignatureEventBox}; -use crate::stream::{ChainEvent, ChainStream}; +use crate::stream::{ChainEvent, ChainStream, DisabledChainIndexer}; use crate::util::retry::{retry_async, RetryConfig, RetryError, RetryReason}; +use async_trait::async_trait; use std::collections::HashMap; use std::fmt; use std::str::FromStr; @@ -293,7 +294,7 @@ type Result = anyhow::Result; /// Solana stream that implements the new ChainStream abstraction pub struct SolanaStream { - rx: mpsc::Receiver, + rx: Option>, start_state: Option, tasks: Vec>, } @@ -328,7 +329,7 @@ impl SolanaStream { let (tx, rx) = crate::stream::channel(); Some(SolanaStream { - rx, + rx: Some(rx), start_state: Some(SolanaStreamStartState { program_id, rpc_http_url: sol.rpc_http_url.clone(), @@ -340,19 +341,14 @@ impl SolanaStream { } } +#[async_trait] impl ChainStream for SolanaStream { const CHAIN: Chain = Chain::Solana; - type BufferedStream = crate::stream::DisabledBufferedStream; + type Indexer = DisabledChainIndexer; - fn buffered_item_height(_item: &::Item) -> u64 { - 0 - } - - async fn start(&mut self) -> tokio::sync::oneshot::Receiver<()> { - let (tx, rx) = tokio::sync::oneshot::channel(); + async fn start(&mut self) -> anyhow::Result { let Some(start_state) = self.start_state.take() else { - let _ = tx.send(()); - return rx; + anyhow::bail!("solana stream already started"); }; self.tasks.push(spawn_cpi_sign_events( @@ -368,12 +364,14 @@ impl ChainStream for SolanaStream { start_state.tx, )); - let _ = tx.send(()); - rx + Ok(DisabledChainIndexer) } async fn next_event(&mut self) -> Option { - self.rx.recv().await + match self.rx.as_mut() { + Some(rx) => rx.recv().await, + None => None, + } } } diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 9a036868..925c9b29 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -11,6 +11,7 @@ use crate::stream::ops::{ RespondBidirectionalEvent, SignatureRespondedEvent, }; +use async_trait::async_trait; use std::time::Duration; use std::{ops::Range, vec::Vec}; use tokio::sync::mpsc; @@ -92,6 +93,7 @@ pub enum ExecutionOutcome { pub struct DisabledBufferedStream; +#[async_trait] impl ChainBufferedStream for DisabledBufferedStream { type Item = (); @@ -104,7 +106,7 @@ impl ChainBufferedStream for DisabledBufferedStream { } } -#[allow(async_fn_in_trait)] +#[async_trait] pub trait ChainBufferedStream: Send + 'static { type Item: Clone + Send + 'static; @@ -112,15 +114,12 @@ pub trait ChainBufferedStream: Send + 'static { async fn next(&mut self) -> Option; } -#[allow(async_fn_in_trait)] -pub trait ChainStream: Send + 'static { - const CHAIN: Chain; +#[async_trait] +pub trait ChainIndexer: Send + 'static { type BufferedStream: ChainBufferedStream; - async fn start(&mut self) -> oneshot::Receiver<()> { - let (tx, rx) = oneshot::channel(); - let _ = tx.send(()); - rx + async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { + Ok(()) } async fn livestream(&mut self) -> anyhow::Result> { @@ -150,46 +149,68 @@ pub trait ChainStream: Send + 'static { Duration::from_millis(500) } +} + +pub struct DisabledChainIndexer; + +#[async_trait] +impl ChainIndexer for DisabledChainIndexer { + type BufferedStream = DisabledBufferedStream; +} + +#[async_trait] +pub trait ChainStream: Send + 'static { + const CHAIN: Chain; + type Indexer: ChainIndexer; + + async fn start(&mut self) -> anyhow::Result; + async fn next_event(&mut self) -> Option; } -pub(crate) async fn start_chain_stream( - stream: &mut S, +pub(crate) async fn start_chain_stream( + chain: Chain, + indexer: &mut I, catchup_completed_tx: oneshot::Sender<()>, ) { let mut catchup_completed_tx = Some(catchup_completed_tx); - tracing::info!(chain = %S::CHAIN, "starting chain stream orchestration"); + tracing::info!(chain = %chain, "starting chain stream orchestration"); - match stream.livestream().await { + match indexer.livestream().await { Ok(Some(mut buffered)) => { let Some(anchor_item) = buffered.initial().await else { - tracing::warn!(chain = %S::CHAIN, "buffered livestream ended before anchor item"); + tracing::warn!(chain = %chain, "buffered livestream ended before anchor item"); return; }; - let anchor_height = S::buffered_item_height(&anchor_item); + let anchor_height = I::buffered_item_height(&anchor_item); let catchup_range = loop { - match stream.catchup_range(anchor_height).await { + match indexer.catchup_range(anchor_height).await { Ok(range) => break range, Err(err) => { - tracing::warn!(?err, chain = %S::CHAIN, anchor_height, "failed to determine catchup range; retrying"); - tokio::time::sleep(stream.retry_delay()).await; + tracing::warn!(?err, chain = %chain, anchor_height, "failed to determine catchup range; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; } } }; for height in catchup_range { loop { - match stream.process_catchup_height(height).await { + match indexer.process_catchup_height(height).await { Ok(()) => break, Err(err) => { - tracing::warn!(?err, chain = %S::CHAIN, height, "catchup height processing failed; retrying"); - tokio::time::sleep(stream.retry_delay()).await; + tracing::warn!(?err, chain = %chain, height, "catchup height processing failed; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; } } } } + if let Err(err) = indexer.emit_catchup_completed().await { + tracing::warn!(?err, chain = %chain, "failed to emit catchup completion event"); + return; + } + if let Some(tx) = catchup_completed_tx.take() { let _ = tx.send(()); } @@ -204,29 +225,39 @@ pub(crate) async fn start_chain_stream( }, }; - match stream.process_buffered_item(item.clone()).await { + match indexer.process_buffered_item(item.clone()).await { Ok(()) => {} Err(err) => { - tracing::warn!(?err, chain = %S::CHAIN, "buffered item processing failed; retrying"); - tokio::time::sleep(stream.retry_delay()).await; + tracing::warn!(?err, chain = %chain, "buffered item processing failed; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; next_item = Some(item); } } } } Ok(None) => { - stream.start().await; - if let Some(tx) = catchup_completed_tx.take() { let _ = tx.send(()); } } Err(err) => { - tracing::error!(?err, chain = %S::CHAIN, "failed to initialize livestream"); + tracing::error!(?err, chain = %chain, "failed to initialize livestream"); } } } +pub async fn spawn_stream_indexer( + stream: &mut S, +) -> anyhow::Result> { + let mut indexer = stream.start().await?; + let chain = S::CHAIN; + + Ok(tokio::spawn(async move { + let (catchup_completed_tx, _catchup_completed_rx) = oneshot::channel(); + start_chain_stream(chain, &mut indexer, catchup_completed_tx).await; + })) +} + /// Shared indexer loop: recovers backlog then processes events from the stream pub async fn run_stream( mut stream: S, @@ -250,9 +281,20 @@ pub async fn run_stream( ) .await; - let mut catchup_completed_rx = stream.start().await; - let mut catchup_completed = false; + let mut indexer = match stream.start().await { + Ok(indexer) => indexer, + Err(err) => { + tracing::error!(?err, chain = %chain, "failed to start stream"); + return; + } + }; + let (catchup_completed_tx, mut catchup_completed_rx) = oneshot::channel(); + tokio::spawn(async move { + start_chain_stream(chain, &mut indexer, catchup_completed_tx).await; + }); + + let mut catchup_completed = false; loop { tokio::select! { result = &mut catchup_completed_rx, if !catchup_completed => { @@ -335,6 +377,10 @@ pub async fn run_stream( } } + if !catchup_completed { + catchup_completed = catchup_completed_rx.await.is_ok(); + } + if catchup_completed && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { if !recovered.pending.is_empty() { requeue_recovered_sign_requests( @@ -381,12 +427,14 @@ mod tests { events: Vec>, } + #[async_trait] impl ChainStream for TestEventStream { const CHAIN: Chain = Chain::Solana; - type BufferedStream = DisabledBufferedStream; - fn buffered_item_height(_item: &::Item) -> u64 { - 0 + type Indexer = DisabledChainIndexer; + + async fn start(&mut self) -> anyhow::Result { + Ok(DisabledChainIndexer) } async fn next_event(&mut self) -> Option { @@ -401,6 +449,7 @@ mod tests { items: Vec, } + #[async_trait] impl ChainBufferedStream for TestBufferedStream { type Item = u64; @@ -466,20 +515,28 @@ mod tests { struct TestLinearStream { control: TestLinearControl, - events: Vec>, + rx: mpsc::Receiver, + tx: mpsc::Sender, } impl TestLinearStream { fn new(control: TestLinearControl) -> Self { + let (tx, rx) = mpsc::channel(16); Self { control, - events: Vec::new(), + rx, + tx, } } } - impl ChainStream for TestLinearStream { - const CHAIN: Chain = Chain::Ethereum; + struct TestLinearIndexer { + control: TestLinearControl, + tx: mpsc::Sender, + } + + #[async_trait] + impl ChainIndexer for TestLinearIndexer { type BufferedStream = TestBufferedStream; async fn livestream(&mut self) -> anyhow::Result> { @@ -505,7 +562,7 @@ mod tests { if TestLinearControl::consume_failure(&self.control.catchup_failures, height) { anyhow::bail!("synthetic catchup failure at height {height}"); } - self.events.push(Some(ChainEvent::Block(height))); + self.tx.send(ChainEvent::Block(height)).await?; Ok(()) } @@ -516,27 +573,38 @@ mod tests { if TestLinearControl::consume_failure(&self.control.live_failures, item) { anyhow::bail!("synthetic live failure at height {item}"); } - self.events.push(Some(ChainEvent::Block(item))); + self.tx.send(ChainEvent::Block(item)).await?; Ok(()) } fn retry_delay(&self) -> Duration { self.control.retry_delay() } + } + + #[async_trait] + impl ChainStream for TestLinearStream { + const CHAIN: Chain = Chain::Ethereum; + type Indexer = TestLinearIndexer; + + async fn start(&mut self) -> anyhow::Result { + Ok(TestLinearIndexer { + control: self.control.clone(), + tx: self.tx.clone(), + }) + } async fn next_event(&mut self) -> Option { - if self.events.is_empty() { - return None; - } - self.events.remove(0) + self.rx.recv().await } } #[tokio::test] async fn test_run_linearized_source_orders_catchup_before_live() { let mut stream = TestLinearStream::new(TestLinearControl::new(Some(1), vec![4, 5])); + let mut indexer = stream.start().await.unwrap(); let (tx, _rx) = oneshot::channel(); - start_chain_stream(&mut stream, tx).await; + start_chain_stream(Chain::Ethereum, &mut indexer, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -560,8 +628,9 @@ mod tests { .fail_catchup_once(3) .fail_live_once(4), ); + let mut indexer = stream.start().await.unwrap(); let (tx, _rx) = oneshot::channel(); - start_chain_stream(&mut stream, tx).await; + start_chain_stream(Chain::Ethereum, &mut indexer, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -670,19 +739,14 @@ mod tests { event: Option, } + #[async_trait] impl ChainStream for StartAwareStream { const CHAIN: Chain = Chain::Solana; - type BufferedStream = DisabledBufferedStream; + type Indexer = DisabledChainIndexer; - fn buffered_item_height(_item: &::Item) -> u64 { - 0 - } - - async fn start(&mut self) -> oneshot::Receiver<()> { + async fn start(&mut self) -> anyhow::Result { self.started = true; - let (tx, rx) = oneshot::channel(); - let _ = tx.send(()); - rx + Ok(DisabledChainIndexer) } async fn next_event(&mut self) -> Option { @@ -763,12 +827,13 @@ mod tests { rx: mpsc::Receiver, } + #[async_trait] impl ChainStream for LocalStream { const CHAIN: Chain = Chain::Solana; - type BufferedStream = DisabledBufferedStream; + type Indexer = DisabledChainIndexer; - fn buffered_item_height(_item: &::Item) -> u64 { - 0 + async fn start(&mut self) -> anyhow::Result { + Ok(DisabledChainIndexer) } async fn next_event(&mut self) -> Option { @@ -1010,12 +1075,13 @@ mod tests { events: Vec>, } + #[async_trait] impl ChainStream for EthereumLocalStream { const CHAIN: Chain = Chain::Ethereum; - type BufferedStream = DisabledBufferedStream; + type Indexer = DisabledChainIndexer; - fn buffered_item_height(_item: &::Item) -> u64 { - 0 + async fn start(&mut self) -> anyhow::Result { + Ok(DisabledChainIndexer) } async fn next_event(&mut self) -> Option { diff --git a/integration-tests/tests/cases/ethereum_stream.rs b/integration-tests/tests/cases/ethereum_stream.rs index 594c073b..6f496219 100644 --- a/integration-tests/tests/cases/ethereum_stream.rs +++ b/integration-tests/tests/cases/ethereum_stream.rs @@ -21,6 +21,7 @@ use mpc_node::rpc::ContractStateWatcher; use mpc_node::storage::checkpoint_storage::CheckpointStorage; use mpc_node::stream::ops::SignBidirectionalEvent as NodeSignBidirectionalEvent; use mpc_node::stream::ops::SignatureRespondedEvent; +use mpc_node::stream::spawn_stream_indexer; use mpc_node::stream::{run_stream, ChainEvent, ChainStream}; use mpc_node::util::current_unix_timestamp; use mpc_primitives::{SignArgs, SignId, LATEST_MPC_KEY_VERSION}; @@ -299,7 +300,29 @@ fn test_bidirectional_event() -> NodeSignBidirectionalEvent { }) } -async fn next_event_within(client: &mut EthereumStream, duration: Duration) -> Result { +struct StartedEthereumStream { + stream: EthereumStream, + _indexer_task: tokio::task::JoinHandle<()>, +} + +impl std::ops::Deref for StartedEthereumStream { + type Target = EthereumStream; + + fn deref(&self) -> &Self::Target { + &self.stream + } +} + +impl std::ops::DerefMut for StartedEthereumStream { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.stream + } +} + +async fn next_event_within( + client: &mut StartedEthereumStream, + duration: Duration, +) -> Result { timeout(duration, async { loop { if let Some(event) = client.next_event().await { @@ -314,10 +337,13 @@ async fn next_event_within(client: &mut EthereumStream, duration: Duration) -> R async fn stream_ethereum( ctx: &EthereumTestEnvironment, backlog: Backlog, -) -> Result { +) -> Result { let mut stream = EthereumStream::new(Some(ctx.config(true)), backlog).await?; - ChainStream::start(&mut stream).await; - Ok(stream) + let indexer_task = spawn_stream_indexer(&mut stream).await?; + Ok(StartedEthereumStream { + stream, + _indexer_task: indexer_task, + }) } #[test_log::test(tokio::test)] diff --git a/integration-tests/tests/cases/solana_stream.rs b/integration-tests/tests/cases/solana_stream.rs index 2b63dd70..5a0b9fcf 100644 --- a/integration-tests/tests/cases/solana_stream.rs +++ b/integration-tests/tests/cases/solana_stream.rs @@ -30,7 +30,7 @@ fn test_dependencies() -> (Backlog, watch::Receiver, NodeClient) { async fn stream_solana(config: SolConfig) -> Result { let mut stream = SolanaStream::new(Some(config)).context("failed to create SolanaStream")?; - ChainStream::start(&mut stream).await; + let _ = ChainStream::start(&mut stream).await?; Ok(stream) } From b592def4897da5cd107c03a93bab511150030e18 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 08:30:02 +0000 Subject: [PATCH 06/15] Merge previous abstractions --- chain-signatures/node/src/indexer_eth/mod.rs | 52 ++++++++------------ chain-signatures/node/src/stream/mod.rs | 16 +++--- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index fea20b7f..0938def5 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -717,16 +717,22 @@ pub struct EthereumIndexer { eth: EthConfig, backlog: Backlog, client: EthereumClient, + events_tx: mpsc::Sender, } impl EthereumIndexer { - pub async fn new(eth: EthConfig, backlog: Backlog) -> anyhow::Result { + pub async fn new( + eth: EthConfig, + backlog: Backlog, + events_tx: mpsc::Sender, + ) -> anyhow::Result { let client = EthereumClient::new(eth.clone()).await?; Ok(Self { eth, backlog, client, + events_tx, }) } @@ -797,11 +803,7 @@ impl EthereumIndexer { } } - async fn process_height( - &self, - block_number: u64, - events_tx: mpsc::Sender, - ) -> anyhow::Result<()> { + async fn process_height(&self, block_number: u64) -> anyhow::Result<()> { let Some(block) = self .client .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Number( @@ -812,14 +814,10 @@ impl EthereumIndexer { anyhow::bail!("ethereum block {block_number} not found"); }; - self.process_live_block(block, events_tx).await + self.process_live_block(block).await } - async fn process_live_block( - &self, - block: alloy::rpc::types::Block, - events_tx: mpsc::Sender, - ) -> anyhow::Result<()> { + async fn process_live_block(&self, block: alloy::rpc::types::Block) -> anyhow::Result<()> { let block_number = block.header.number; let contract_address = Address::from_str(&format!("0x{}", self.eth.contract_address)) .map_err(|_| { @@ -839,7 +837,7 @@ impl EthereumIndexer { Self::emit_processed_block( Arc::new(self.client.clone()), - events_tx, + self.events_tx.clone(), &self.eth, processed, ) @@ -1180,12 +1178,7 @@ impl EthereumIndexer { /// `start()` after recovery has completed. pub struct EthereumStream { events_rx: Option>, - start_state: Option, -} - -pub struct EthereumStreamIndexer { - events_tx: mpsc::Sender, - indexer: EthereumIndexer, + start_state: Option, } impl EthereumStream { @@ -1204,25 +1197,24 @@ impl EthereumStream { "creating ethereum indexer stream" ); - let indexer = EthereumIndexer::new(eth, backlog).await?; - let (events_tx, events_rx) = crate::stream::channel(); + let indexer = EthereumIndexer::new(eth, backlog, events_tx).await?; Ok(Self { events_rx: Some(events_rx), - start_state: Some(EthereumStreamIndexer { events_tx, indexer }), + start_state: Some(indexer), }) } } #[async_trait] -impl ChainIndexer for EthereumStreamIndexer { +impl ChainIndexer for EthereumIndexer { type BufferedStream = EthereumBufferedStream; async fn livestream(&mut self) -> anyhow::Result> { let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); tokio::spawn(EthereumIndexer::buffer_live_blocks( - Arc::new(self.indexer.client.clone()), + Arc::new(self.client.clone()), live_blocks_tx, )); @@ -1235,7 +1227,7 @@ impl ChainIndexer for EthereumStreamIndexer { async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { let catchup_start = EthereumIndexer::catchup_start_block_number( - self.indexer.backlog.processed_block(Chain::Ethereum).await, + self.backlog.processed_block(Chain::Ethereum).await, anchor_height, ); @@ -1247,18 +1239,14 @@ impl ChainIndexer for EthereumStreamIndexer { tracing::info!(height, "processed ethereum catchup height attempt"); } - self.indexer - .process_height(height, self.events_tx.clone()) - .await + self.process_height(height).await } async fn process_buffered_item( &mut self, item: ::Item, ) -> anyhow::Result<()> { - self.indexer - .process_live_block(item, self.events_tx.clone()) - .await + self.process_live_block(item).await } fn retry_delay(&self) -> Duration { @@ -1270,7 +1258,7 @@ impl ChainIndexer for EthereumStreamIndexer { #[async_trait] impl ChainStream for EthereumStream { const CHAIN: Chain = Chain::Ethereum; - type Indexer = EthereumStreamIndexer; + type Indexer = EthereumIndexer; async fn start(&mut self) -> anyhow::Result { self.start_state diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 925c9b29..4eaa5fc7 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -268,8 +268,7 @@ pub async fn run_stream( node_client: NodeClient, ) { let chain = S::CHAIN; - - tracing::info!(%chain, "starting indexer loop"); + tracing::info!(%chain, "starting stream"); let mut recovered = recover_backlog( &backlog, @@ -284,7 +283,7 @@ pub async fn run_stream( let mut indexer = match stream.start().await { Ok(indexer) => indexer, Err(err) => { - tracing::error!(?err, chain = %chain, "failed to start stream"); + tracing::error!(?err, %chain, "failed to start stream"); return; } }; @@ -317,6 +316,7 @@ pub async fn run_stream( } event = stream.next_event() => { let Some(event) = event else { + tracing::info!(%chain, "stream dropped event channel"); break; }; @@ -324,7 +324,7 @@ pub async fn run_stream( ChainEvent::SignRequest(req) => { if let Err(err) = process_sign_request(req, sign_tx.clone(), backlog.clone()).await { - tracing::error!(?err, chain = %chain, "failed to process sign request"); + tracing::error!(?err, %chain, "failed to process sign request"); } } ChainEvent::Respond(ev) => { @@ -332,19 +332,19 @@ pub async fn run_stream( process_respond_event(ev, sign_tx.clone(), &mut contract_watcher, &backlog) .await { - tracing::error!(?err, chain = %chain, "failed to process respond event"); + tracing::error!(?err, %chain, "failed to process respond event"); } } ChainEvent::RespondBidirectional(ev) => { if let Err(err) = process_respond_bidirectional_event(ev, sign_tx.clone(), &backlog).await { - tracing::error!(?err, chain = %chain, "failed to process respond bidirectional event"); + tracing::error!(?err, %chain, "failed to process respond bidirectional event"); } } ChainEvent::Block(block) => { if let Some(checkpoint) = backlog.set_processed_block(S::CHAIN, block).await { - tracing::info!(block, ?checkpoint, chain = %chain, "created checkpoint"); + tracing::info!(block, ?checkpoint, %chain, "created checkpoint"); } crate::metrics::indexers::LATEST_BLOCK_NUMBER .with_label_values(&[S::CHAIN.as_str(), "indexed"]) @@ -369,7 +369,7 @@ pub async fn run_stream( ) .await { - tracing::error!(?err, chain = %chain, "failed to process execution confirmation"); + tracing::error!(?err, %chain, "failed to process execution confirmation"); } } } From ab4a5f25d4d30a7709c68317957da2c31da46576 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 08:49:32 +0000 Subject: [PATCH 07/15] Cleanup --- chain-signatures/node/src/indexer_eth/mod.rs | 70 ++++++++++---------- chain-signatures/node/src/stream/mod.rs | 10 +-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 0938def5..ebd2f552 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -43,7 +43,7 @@ pub(crate) static MAX_SECP256K1_SCALAR: LazyLock = LazyLock::new(|| { // This is the maximum number of blocks that Helios can look back to const MAX_CATCHUP_BLOCKS: u64 = 8191; -const MAX_LIVE_BLOCK_BUFFER: usize = 10000; +const MAX_LIVE_BLOCK_BUFFER: usize = 16384; fn live_blocks_channel() -> ( mpsc::Sender, @@ -1173,40 +1173,6 @@ impl EthereumIndexer { } } -/// Ethereum indexer stream implementing the `ChainStream` trait. -/// Construction is side-effect free; the shared `run_stream()` loop calls -/// `start()` after recovery has completed. -pub struct EthereumStream { - events_rx: Option>, - start_state: Option, -} - -impl EthereumStream { - pub async fn new(eth: Option, backlog: Backlog) -> anyhow::Result { - let Some(eth) = eth else { - tracing::warn!( - "ethereum indexer is disabled: no EthConfig provided \ - (check that all --eth-* CLI flags were supplied)" - ); - return Err(anyhow::anyhow!( - "ethereum indexer is disabled: no EthConfig provided" - )); - }; - tracing::info!( - eth_config = ?eth, - "creating ethereum indexer stream" - ); - - let (events_tx, events_rx) = crate::stream::channel(); - let indexer = EthereumIndexer::new(eth, backlog, events_tx).await?; - - Ok(Self { - events_rx: Some(events_rx), - start_state: Some(indexer), - }) - } -} - #[async_trait] impl ChainIndexer for EthereumIndexer { type BufferedStream = EthereumBufferedStream; @@ -1255,6 +1221,40 @@ impl ChainIndexer for EthereumIndexer { } +/// Ethereum indexer stream implementing the `ChainStream` trait. +/// Construction is side-effect free; the shared `run_stream()` loop calls +/// `start()` after recovery has completed. +pub struct EthereumStream { + events_rx: Option>, + start_state: Option, +} + +impl EthereumStream { + pub async fn new(eth: Option, backlog: Backlog) -> anyhow::Result { + let Some(eth) = eth else { + tracing::warn!( + "ethereum indexer is disabled: no EthConfig provided \ + (check that all --eth-* CLI flags were supplied)" + ); + return Err(anyhow::anyhow!( + "ethereum indexer is disabled: no EthConfig provided" + )); + }; + tracing::info!( + eth_config = ?eth, + "creating ethereum indexer stream" + ); + + let (events_tx, events_rx) = crate::stream::channel(); + let indexer = EthereumIndexer::new(eth, backlog, events_tx).await?; + + Ok(Self { + events_rx: Some(events_rx), + start_state: Some(indexer), + }) + } +} + #[async_trait] impl ChainStream for EthereumStream { const CHAIN: Chain = Chain::Ethereum; diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 4eaa5fc7..b440e4a0 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -168,7 +168,7 @@ pub trait ChainStream: Send + 'static { async fn next_event(&mut self) -> Option; } -pub(crate) async fn start_chain_stream( +pub(crate) async fn catchup_then_livestream( chain: Chain, indexer: &mut I, catchup_completed_tx: oneshot::Sender<()>, @@ -254,7 +254,7 @@ pub async fn spawn_stream_indexer( Ok(tokio::spawn(async move { let (catchup_completed_tx, _catchup_completed_rx) = oneshot::channel(); - start_chain_stream(chain, &mut indexer, catchup_completed_tx).await; + catchup_then_livestream(chain, &mut indexer, catchup_completed_tx).await; })) } @@ -290,7 +290,7 @@ pub async fn run_stream( let (catchup_completed_tx, mut catchup_completed_rx) = oneshot::channel(); tokio::spawn(async move { - start_chain_stream(chain, &mut indexer, catchup_completed_tx).await; + catchup_then_livestream(chain, &mut indexer, catchup_completed_tx).await; }); let mut catchup_completed = false; @@ -604,7 +604,7 @@ mod tests { let mut stream = TestLinearStream::new(TestLinearControl::new(Some(1), vec![4, 5])); let mut indexer = stream.start().await.unwrap(); let (tx, _rx) = oneshot::channel(); - start_chain_stream(Chain::Ethereum, &mut indexer, tx).await; + catchup_then_livestream(Chain::Ethereum, &mut indexer, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) @@ -630,7 +630,7 @@ mod tests { ); let mut indexer = stream.start().await.unwrap(); let (tx, _rx) = oneshot::channel(); - start_chain_stream(Chain::Ethereum, &mut indexer, tx).await; + catchup_then_livestream(Chain::Ethereum, &mut indexer, tx).await; let mut observed = Vec::new(); while let Some(event) = timeout(Duration::from_millis(20), stream.next_event()) From 60ee8be5416b0d9bc0537ea92d0ac748afe4b1db Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 10:26:04 +0000 Subject: [PATCH 08/15] More cleanup --- chain-signatures/node/src/stream/mod.rs | 117 ++++++++++++------------ 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index b440e4a0..33876c11 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -174,74 +174,73 @@ pub(crate) async fn catchup_then_livestream( catchup_completed_tx: oneshot::Sender<()>, ) { let mut catchup_completed_tx = Some(catchup_completed_tx); - tracing::info!(chain = %chain, "starting chain stream orchestration"); + tracing::info!(%chain, "starting chain stream orchestration"); - match indexer.livestream().await { - Ok(Some(mut buffered)) => { - let Some(anchor_item) = buffered.initial().await else { - tracing::warn!(chain = %chain, "buffered livestream ended before anchor item"); - return; - }; + let buffered = match indexer.livestream().await { + Ok(buffered) => buffered, + Err(err) => { + tracing::error!(?err, %chain, "failed to initialize livestream"); + return; + } + }; + let Some(mut buffered) = buffered else { + if let Some(tx) = catchup_completed_tx.take() { + let _ = tx.send(()); + } + return; + }; - let anchor_height = I::buffered_item_height(&anchor_item); - let catchup_range = loop { - match indexer.catchup_range(anchor_height).await { - Ok(range) => break range, - Err(err) => { - tracing::warn!(?err, chain = %chain, anchor_height, "failed to determine catchup range; retrying"); - tokio::time::sleep(indexer.retry_delay()).await; - } - } - }; + let Some(anchor_item) = buffered.initial().await else { + tracing::warn!(%chain, "buffered livestream ended before anchor item"); + return; + }; - for height in catchup_range { - loop { - match indexer.process_catchup_height(height).await { - Ok(()) => break, - Err(err) => { - tracing::warn!(?err, chain = %chain, height, "catchup height processing failed; retrying"); - tokio::time::sleep(indexer.retry_delay()).await; - } - } - } + let anchor_height = I::buffered_item_height(&anchor_item); + let catchup_range = loop { + match indexer.catchup_range(anchor_height).await { + Ok(range) => break range, + Err(err) => { + tracing::warn!(?err, %chain, anchor_height, "failed to determine catchup range; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; } + } + }; - if let Err(err) = indexer.emit_catchup_completed().await { - tracing::warn!(?err, chain = %chain, "failed to emit catchup completion event"); - return; + for height in catchup_range { + loop { + match indexer.process_catchup_height(height).await { + Ok(()) => break, + Err(err) => { + tracing::warn!(?err, %chain, height, "catchup height processing failed; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; + } } + } + } - if let Some(tx) = catchup_completed_tx.take() { - let _ = tx.send(()); - } + if let Err(err) = indexer.emit_catchup_completed().await { + tracing::warn!(?err, %chain, "failed to emit catchup completion event"); + return; + } - let mut next_item = Some(anchor_item); - loop { - let item = match next_item.take() { - Some(item) => item, - None => match buffered.next().await { - Some(item) => item, - None => break, - }, - }; + if let Some(tx) = catchup_completed_tx.take() { + let _ = tx.send(()); + } - match indexer.process_buffered_item(item.clone()).await { - Ok(()) => {} - Err(err) => { - tracing::warn!(?err, chain = %chain, "buffered item processing failed; retrying"); - tokio::time::sleep(indexer.retry_delay()).await; - next_item = Some(item); - } - } - } - } - Ok(None) => { - if let Some(tx) = catchup_completed_tx.take() { - let _ = tx.send(()); - } - } - Err(err) => { - tracing::error!(?err, chain = %chain, "failed to initialize livestream"); + let mut next_item = Some(anchor_item); + loop { + let item = match next_item.take() { + Some(item) => item, + None => match buffered.next().await { + Some(item) => item, + None => break, + }, + }; + + if let Err(err) = indexer.process_buffered_item(item.clone()).await { + tracing::warn!(?err, %chain, "buffered item processing failed; retrying"); + tokio::time::sleep(indexer.retry_delay()).await; + next_item = Some(item); } } } From 6ad26c6a0f79b374726ba18ec36d01ca19684da9 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 10:27:31 +0000 Subject: [PATCH 09/15] fmt --- chain-signatures/node/src/indexer_eth/mod.rs | 29 +++++++++---------- chain-signatures/node/src/stream/mod.rs | 24 +++++---------- .../tests/cases/ethereum_stream.rs | 10 +++++-- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index ebd2f552..6b4c6c8f 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -8,9 +8,7 @@ use crate::metrics::requests::{record_request_latency, SignRequestStep}; use crate::protocol::{Chain, IndexedSignRequest}; use crate::respond_bidirectional::CompletedTx; use crate::sign_bidirectional::SignStatus; -use crate::stream::{ - ChainBufferedStream, ChainEvent, ChainIndexer, ChainStream, ExecutionOutcome, -}; +use crate::stream::{ChainBufferedStream, ChainEvent, ChainIndexer, ChainStream, ExecutionOutcome}; use async_trait::async_trait; use alloy::eips::BlockNumberOrTag; @@ -757,9 +755,9 @@ impl EthereumIndexer { while block_number <= latest_block_number { let Some(block) = client - .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Number( - block_number, - ))) + .get_block(alloy::rpc::types::BlockId::Number( + BlockNumberOrTag::Number(block_number), + )) .await else { tracing::warn!(block_number, "ethereum live block not yet available"); @@ -806,9 +804,9 @@ impl EthereumIndexer { async fn process_height(&self, block_number: u64) -> anyhow::Result<()> { let Some(block) = self .client - .get_block(alloy::rpc::types::BlockId::Number(BlockNumberOrTag::Number( - block_number, - ))) + .get_block(alloy::rpc::types::BlockId::Number( + BlockNumberOrTag::Number(block_number), + )) .await else { anyhow::bail!("ethereum block {block_number} not found"); @@ -1111,10 +1109,9 @@ impl EthereumIndexer { } for event in execution_events { - events_tx - .send(event) - .await - .map_err(|err| anyhow::anyhow!("failed to emit ExecutionConfirmed event: {err:?}"))?; + events_tx.send(event).await.map_err(|err| { + anyhow::anyhow!("failed to emit ExecutionConfirmed event: {err:?}") + })?; } for req in indexed_requests { @@ -1218,7 +1215,6 @@ impl ChainIndexer for EthereumIndexer { fn retry_delay(&self) -> Duration { Duration::from_millis(500) } - } /// Ethereum indexer stream implementing the `ChainStream` trait. @@ -1279,7 +1275,10 @@ mod tests { #[test] fn catchup_starts_after_processed_height() { - assert_eq!(EthereumIndexer::catchup_start_block_number(Some(41), 50), 42); + assert_eq!( + EthereumIndexer::catchup_start_block_number(Some(41), 50), + 42 + ); } #[test] diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 33876c11..a59c92f4 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -148,7 +148,6 @@ pub trait ChainIndexer: Send + 'static { fn retry_delay(&self) -> Duration { Duration::from_millis(500) } - } pub struct DisabledChainIndexer; @@ -380,15 +379,12 @@ pub async fn run_stream( catchup_completed = catchup_completed_rx.await.is_ok(); } - if catchup_completed && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup { + if catchup_completed + && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup + { if !recovered.pending.is_empty() { - requeue_recovered_sign_requests( - &backlog, - chain, - sign_tx.clone(), - &recovered.pending, - ) - .await; + requeue_recovered_sign_requests(&backlog, chain, sign_tx.clone(), &recovered.pending) + .await; recovered.pending.clear(); } } @@ -521,11 +517,7 @@ mod tests { impl TestLinearStream { fn new(control: TestLinearControl) -> Self { let (tx, rx) = mpsc::channel(16); - Self { - control, - rx, - tx, - } + Self { control, rx, tx } } } @@ -624,8 +616,8 @@ mod tests { async fn test_run_linearized_source_retries_without_reordering() { let mut stream = TestLinearStream::new( TestLinearControl::new(Some(1), vec![4, 5]) - .fail_catchup_once(3) - .fail_live_once(4), + .fail_catchup_once(3) + .fail_live_once(4), ); let mut indexer = stream.start().await.unwrap(); let (tx, _rx) = oneshot::channel(); diff --git a/integration-tests/tests/cases/ethereum_stream.rs b/integration-tests/tests/cases/ethereum_stream.rs index 6f496219..16fd5659 100644 --- a/integration-tests/tests/cases/ethereum_stream.rs +++ b/integration-tests/tests/cases/ethereum_stream.rs @@ -417,8 +417,14 @@ async fn test_ethereum_stream_resume_starts_after_checkpoint_height() -> Result< run_handle.abort(); - assert!(!saw_replayed_payload, "stream replayed the stored processed block"); - assert!(saw_expected_payload, "stream did not catch up the next block"); + assert!( + !saw_replayed_payload, + "stream replayed the stored processed block" + ); + assert!( + saw_expected_payload, + "stream did not catch up the next block" + ); Ok(()) } From dcba29deb0fb88c0f96c401413d82abc32e8b3f4 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 10:56:09 +0000 Subject: [PATCH 10/15] Even more cleanup --- chain-signatures/node/src/indexer_eth/mod.rs | 2 +- chain-signatures/node/src/stream/mod.rs | 49 ++++++-------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 6b4c6c8f..dd40c7d8 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -1205,7 +1205,7 @@ impl ChainIndexer for EthereumIndexer { self.process_height(height).await } - async fn process_buffered_item( + async fn process_buffered_block( &mut self, item: ::Item, ) -> anyhow::Result<()> { diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index a59c92f4..3dfcfb6d 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -118,10 +118,6 @@ pub trait ChainBufferedStream: Send + 'static { pub trait ChainIndexer: Send + 'static { type BufferedStream: ChainBufferedStream; - async fn emit_catchup_completed(&mut self) -> anyhow::Result<()> { - Ok(()) - } - async fn livestream(&mut self) -> anyhow::Result> { Ok(None) } @@ -138,7 +134,7 @@ pub trait ChainIndexer: Send + 'static { Ok(()) } - async fn process_buffered_item( + async fn process_buffered_block( &mut self, _item: ::Item, ) -> anyhow::Result<()> { @@ -172,8 +168,7 @@ pub(crate) async fn catchup_then_livestream( indexer: &mut I, catchup_completed_tx: oneshot::Sender<()>, ) { - let mut catchup_completed_tx = Some(catchup_completed_tx); - tracing::info!(%chain, "starting chain stream orchestration"); + tracing::info!(%chain, "starting ChainStream catchup then livestream"); let buffered = match indexer.livestream().await { Ok(buffered) => buffered, @@ -183,18 +178,16 @@ pub(crate) async fn catchup_then_livestream( } }; let Some(mut buffered) = buffered else { - if let Some(tx) = catchup_completed_tx.take() { - let _ = tx.send(()); - } + let _ = catchup_completed_tx.send(()); return; }; - let Some(anchor_item) = buffered.initial().await else { - tracing::warn!(%chain, "buffered livestream ended before anchor item"); + let Some(anchor_block) = buffered.initial().await else { + tracing::warn!(%chain, "buffered livestream ended before anchor block"); return; }; - let anchor_height = I::buffered_item_height(&anchor_item); + let anchor_height = I::buffered_item_height(&anchor_block); let catchup_range = loop { match indexer.catchup_range(anchor_height).await { Ok(range) => break range, @@ -217,29 +210,17 @@ pub(crate) async fn catchup_then_livestream( } } - if let Err(err) = indexer.emit_catchup_completed().await { - tracing::warn!(?err, %chain, "failed to emit catchup completion event"); - return; - } + let _ = catchup_completed_tx.send(()); - if let Some(tx) = catchup_completed_tx.take() { - let _ = tx.send(()); - } - - let mut next_item = Some(anchor_item); + let mut next_block = anchor_block; loop { - let item = match next_item.take() { - Some(item) => item, - None => match buffered.next().await { - Some(item) => item, - None => break, - }, - }; + if let Err(err) = indexer.process_buffered_block(next_block).await { + tracing::warn!(?err, %chain, "buffered block processing failed"); + } - if let Err(err) = indexer.process_buffered_item(item.clone()).await { - tracing::warn!(?err, %chain, "buffered item processing failed; retrying"); - tokio::time::sleep(indexer.retry_delay()).await; - next_item = Some(item); + match buffered.next().await { + Some(block) => next_block = block, + None => break, } } } @@ -557,7 +538,7 @@ mod tests { Ok(()) } - async fn process_buffered_item( + async fn process_buffered_block( &mut self, item: ::Item, ) -> anyhow::Result<()> { From d7723cc006a1ef2fd3231b19bf17538983735c51 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 11:37:13 +0000 Subject: [PATCH 11/15] Remove some results --- chain-signatures/node/src/indexer_eth/mod.rs | 4 ++-- chain-signatures/node/src/stream/mod.rs | 18 +++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index dd40c7d8..2b503b23 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -1188,13 +1188,13 @@ impl ChainIndexer for EthereumIndexer { item.header.number } - async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + async fn catchup_range(&mut self, anchor_height: u64) -> std::ops::Range { let catchup_start = EthereumIndexer::catchup_start_block_number( self.backlog.processed_block(Chain::Ethereum).await, anchor_height, ); - Ok(catchup_start..anchor_height) + catchup_start..anchor_height } async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 3dfcfb6d..8202143b 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -126,8 +126,8 @@ pub trait ChainIndexer: Send + 'static { 0 } - async fn catchup_range(&mut self, _anchor_height: u64) -> anyhow::Result> { - Ok(0..0) + async fn catchup_range(&mut self, _anchor_height: u64) -> Range { + 0..0 } async fn process_catchup_height(&mut self, _height: u64) -> anyhow::Result<()> { @@ -188,15 +188,7 @@ pub(crate) async fn catchup_then_livestream( }; let anchor_height = I::buffered_item_height(&anchor_block); - let catchup_range = loop { - match indexer.catchup_range(anchor_height).await { - Ok(range) => break range, - Err(err) => { - tracing::warn!(?err, %chain, anchor_height, "failed to determine catchup range; retrying"); - tokio::time::sleep(indexer.retry_delay()).await; - } - } - }; + let catchup_range = indexer.catchup_range(anchor_height).await; for height in catchup_range { loop { @@ -521,13 +513,13 @@ mod tests { *item } - async fn catchup_range(&mut self, anchor_height: u64) -> anyhow::Result> { + async fn catchup_range(&mut self, anchor_height: u64) -> Range { let start = self .control .persisted_height .map(|height| height + 1) .unwrap_or(anchor_height); - Ok(start..anchor_height) + start..anchor_height } async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { From c18cdcd0b1e89878616af845cf21f94ed3a6c47f Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 11:44:24 +0000 Subject: [PATCH 12/15] fmt --- chain-signatures/node/src/indexer_eth/mod.rs | 2 +- chain-signatures/node/src/stream/mod.rs | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 2b503b23..73820d19 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -1198,7 +1198,7 @@ impl ChainIndexer for EthereumIndexer { } async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { - if height % 10 == 0 { + if height.is_multiple_of(10) { tracing::info!(height, "processed ethereum catchup height attempt"); } diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 8202143b..236bbce8 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -348,20 +348,6 @@ pub async fn run_stream( } } - if !catchup_completed { - catchup_completed = catchup_completed_rx.await.is_ok(); - } - - if catchup_completed - && recovered.requeue_mode == crate::backlog::RecoveryRequeueMode::AfterCatchup - { - if !recovered.pending.is_empty() { - requeue_recovered_sign_requests(&backlog, chain, sign_tx.clone(), &recovered.pending) - .await; - recovered.pending.clear(); - } - } - tracing::warn!(%chain, "indexer shut down"); } From 625aa93627ea0e07c25598bbeb4f458d05fa6fb8 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 12:00:55 +0000 Subject: [PATCH 13/15] Move contract_address to ethereum_indexer --- chain-signatures/node/src/indexer_eth/mod.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 73820d19..cc57163e 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -716,6 +716,7 @@ pub struct EthereumIndexer { backlog: Backlog, client: EthereumClient, events_tx: mpsc::Sender, + contract_address: Address, } impl EthereumIndexer { @@ -725,12 +726,17 @@ impl EthereumIndexer { events_tx: mpsc::Sender, ) -> anyhow::Result { let client = EthereumClient::new(eth.clone()).await?; + let contract_address = format!("0x{}", eth.contract_address); + let contract_address = Address::from_str(&contract_address).map_err(|_| { + anyhow::anyhow!("failed to parse ethereum contract address: {contract_address}") + })?; Ok(Self { eth, backlog, client, events_tx, + contract_address, }) } @@ -817,18 +823,11 @@ impl EthereumIndexer { async fn process_live_block(&self, block: alloy::rpc::types::Block) -> anyhow::Result<()> { let block_number = block.header.number; - let contract_address = Address::from_str(&format!("0x{}", self.eth.contract_address)) - .map_err(|_| { - anyhow::anyhow!( - "failed to parse ethereum contract address: {}", - self.eth.contract_address - ) - })?; let processed = Self::process_block( Arc::new(self.client.clone()), block, - contract_address, + self.contract_address, self.backlog.clone(), ) .await?; From 778278b0dde6c37e90cade9b26fa1ec30b6e96b0 Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 12:01:37 +0000 Subject: [PATCH 14/15] Move client arc to ethereum_indexer --- chain-signatures/node/src/indexer_eth/mod.rs | 12 ++++++------ chain-signatures/node/src/stream/mod.rs | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index cc57163e..52a36bee 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -714,7 +714,7 @@ impl EthereumClient { pub struct EthereumIndexer { eth: EthConfig, backlog: Backlog, - client: EthereumClient, + client: Arc, events_tx: mpsc::Sender, contract_address: Address, } @@ -725,7 +725,7 @@ impl EthereumIndexer { backlog: Backlog, events_tx: mpsc::Sender, ) -> anyhow::Result { - let client = EthereumClient::new(eth.clone()).await?; + let client = Arc::new(EthereumClient::new(eth.clone()).await?); let contract_address = format!("0x{}", eth.contract_address); let contract_address = Address::from_str(&contract_address).map_err(|_| { anyhow::anyhow!("failed to parse ethereum contract address: {contract_address}") @@ -825,7 +825,7 @@ impl EthereumIndexer { let block_number = block.header.number; let processed = Self::process_block( - Arc::new(self.client.clone()), + self.client.clone(), block, self.contract_address, self.backlog.clone(), @@ -833,7 +833,7 @@ impl EthereumIndexer { .await?; Self::emit_processed_block( - Arc::new(self.client.clone()), + self.client.clone(), self.events_tx.clone(), &self.eth, processed, @@ -1176,7 +1176,7 @@ impl ChainIndexer for EthereumIndexer { async fn livestream(&mut self) -> anyhow::Result> { let (live_blocks_tx, live_blocks_rx) = live_blocks_channel(); tokio::spawn(EthereumIndexer::buffer_live_blocks( - Arc::new(self.client.clone()), + self.client.clone(), live_blocks_tx, )); @@ -1196,7 +1196,7 @@ impl ChainIndexer for EthereumIndexer { catchup_start..anchor_height } - async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + async fn process_catchup_on_height(&mut self, height: u64) -> anyhow::Result<()> { if height.is_multiple_of(10) { tracing::info!(height, "processed ethereum catchup height attempt"); } diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index 236bbce8..c93bbc68 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -130,7 +130,7 @@ pub trait ChainIndexer: Send + 'static { 0..0 } - async fn process_catchup_height(&mut self, _height: u64) -> anyhow::Result<()> { + async fn process_catchup_on_height(&mut self, _height: u64) -> anyhow::Result<()> { Ok(()) } @@ -192,7 +192,7 @@ pub(crate) async fn catchup_then_livestream( for height in catchup_range { loop { - match indexer.process_catchup_height(height).await { + match indexer.process_catchup_on_height(height).await { Ok(()) => break, Err(err) => { tracing::warn!(?err, %chain, height, "catchup height processing failed; retrying"); @@ -508,7 +508,7 @@ mod tests { start..anchor_height } - async fn process_catchup_height(&mut self, height: u64) -> anyhow::Result<()> { + async fn process_catchup_on_height(&mut self, height: u64) -> anyhow::Result<()> { if TestLinearControl::consume_failure(&self.control.catchup_failures, height) { anyhow::bail!("synthetic catchup failure at height {height}"); } From 03e474174d2bc008c10bc9d40686ce882ba7b7ee Mon Sep 17 00:00:00 2001 From: "Phuong N." Date: Wed, 8 Apr 2026 12:41:27 +0000 Subject: [PATCH 15/15] Renaming --- chain-signatures/node/src/indexer_eth/mod.rs | 14 +++---- chain-signatures/node/src/stream/mod.rs | 39 +++++++++++--------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/chain-signatures/node/src/indexer_eth/mod.rs b/chain-signatures/node/src/indexer_eth/mod.rs index 9709d0ba..d56aa478 100644 --- a/chain-signatures/node/src/indexer_eth/mod.rs +++ b/chain-signatures/node/src/indexer_eth/mod.rs @@ -84,13 +84,13 @@ pub struct EthereumBufferedStream { #[async_trait] impl ChainBufferedStream for EthereumBufferedStream { - type Item = alloy::rpc::types::Block; + type Block = alloy::rpc::types::Block; - async fn initial(&mut self) -> Option { + async fn initial(&mut self) -> Option { self.live_blocks_rx.recv().await } - async fn next(&mut self) -> Option { + async fn next(&mut self) -> Option { self.live_blocks_rx.recv().await } } @@ -1206,8 +1206,8 @@ impl ChainIndexer for EthereumIndexer { Ok(Some(EthereumBufferedStream { live_blocks_rx })) } - fn buffered_item_height(item: &::Item) -> u64 { - item.header.number + fn buffered_item_height(block: &::Block) -> u64 { + block.header.number } async fn catchup_range(&mut self, anchor_height: u64) -> std::ops::Range { @@ -1229,9 +1229,9 @@ impl ChainIndexer for EthereumIndexer { async fn process_buffered_block( &mut self, - item: ::Item, + block: ::Block, ) -> anyhow::Result<()> { - self.process_live_block(item).await + self.process_live_block(block).await } fn retry_delay(&self) -> Duration { diff --git a/chain-signatures/node/src/stream/mod.rs b/chain-signatures/node/src/stream/mod.rs index c93bbc68..fa7fab06 100644 --- a/chain-signatures/node/src/stream/mod.rs +++ b/chain-signatures/node/src/stream/mod.rs @@ -95,23 +95,23 @@ pub struct DisabledBufferedStream; #[async_trait] impl ChainBufferedStream for DisabledBufferedStream { - type Item = (); + type Block = (); - async fn initial(&mut self) -> Option { + async fn initial(&mut self) -> Option { None } - async fn next(&mut self) -> Option { + async fn next(&mut self) -> Option { None } } #[async_trait] pub trait ChainBufferedStream: Send + 'static { - type Item: Clone + Send + 'static; + type Block: Send + 'static; - async fn initial(&mut self) -> Option; - async fn next(&mut self) -> Option; + async fn initial(&mut self) -> Option; + async fn next(&mut self) -> Option; } #[async_trait] @@ -122,7 +122,8 @@ pub trait ChainIndexer: Send + 'static { Ok(None) } - fn buffered_item_height(_item: &::Item) -> u64 { + fn buffered_item_height(block: &::Block) -> u64 { + let _ = block; 0 } @@ -136,8 +137,9 @@ pub trait ChainIndexer: Send + 'static { async fn process_buffered_block( &mut self, - _item: ::Item, + block: ::Block, ) -> anyhow::Result<()> { + let _ = block; Ok(()) } @@ -159,7 +161,6 @@ pub trait ChainStream: Send + 'static { type Indexer: ChainIndexer; async fn start(&mut self) -> anyhow::Result; - async fn next_event(&mut self) -> Option; } @@ -405,16 +406,16 @@ mod tests { #[async_trait] impl ChainBufferedStream for TestBufferedStream { - type Item = u64; + type Block = u64; - async fn initial(&mut self) -> Option { + async fn initial(&mut self) -> Option { if self.items.is_empty() { return None; } Some(self.items.remove(0)) } - async fn next(&mut self) -> Option { + async fn next(&mut self) -> Option { if self.items.is_empty() { return None; } @@ -495,8 +496,10 @@ mod tests { })) } - fn buffered_item_height(item: &::Item) -> u64 { - *item + fn buffered_item_height( + block: &::Block, + ) -> u64 { + *block } async fn catchup_range(&mut self, anchor_height: u64) -> Range { @@ -518,12 +521,12 @@ mod tests { async fn process_buffered_block( &mut self, - item: ::Item, + block: ::Block, ) -> anyhow::Result<()> { - if TestLinearControl::consume_failure(&self.control.live_failures, item) { - anyhow::bail!("synthetic live failure at height {item}"); + if TestLinearControl::consume_failure(&self.control.live_failures, block) { + anyhow::bail!("synthetic live failure at height {block}"); } - self.tx.send(ChainEvent::Block(item)).await?; + self.tx.send(ChainEvent::Block(block)).await?; Ok(()) }