best bid is replaced with CompetitionBidContext (#826)

## 📝 Summary

This PR improves the information we store in clickhouse about the
blocks:
- seen_competition_bid: Option<U256> is replaced by
competition_bid_context: CompetitionBidContext which contains not only
the competition bid we must beat but also it's time and relay and also
time and relay of the competition bid that trigger our bid.
- block_id: our internal unique block id
- block_uses: how many times we already bidded for this block_id


## ✅ I have completed the following steps:

* [X] Run `make lint`
* [X] Run `make test`
* [ ] Added tests (if applicable)
* [X] Prayed to the god of coding that this does not contain any bugs.

Author: ZanCorDX

crates/rbuilder-operator/src/bidding_service_wrapper/fast_streams/types.rs

@@ -7,8 +7,8 @@ use iceoryx2_bb_container::byte_string::FixedSizeByteString;
 use rbuilder::{
     building::builders::BuiltBlockId,
     live_builder::block_output::bidding_service_interface::{
-        BuiltBlockDescriptorForSlotBidder, PayoutInfo, RelaySet, ScrapedRelayBlockBidWithStats,
-        SlotBidderSealBidCommand,
+        BidSourceInfo, BidWithInfo, BuiltBlockDescriptorForSlotBidder, CompetitionBidContext,
+        PayoutInfo, RelaySet, ScrapedRelayBlockBidWithStats, SlotBidderSealBidCommand,
     },
     utils::{offset_datetime_to_timestamp_us, timestamp_us_to_offset_datetime},
 };
@@ -227,13 +227,92 @@ impl PayoutInfoRPC {
     }
 }
 
+#[derive(Debug, Copy, Clone, ZeroCopySend)]
+#[type_name("BidSourceInfoRPC")]
+#[repr(C)]
+pub struct BidSourceInfoRPC {
+    pub seen_time_us: u64,
+    pub relay: FixedSizeByteString<MAX_RELAY_NAME_LENGTH>,
+}
+
+impl From<BidSourceInfo> for BidSourceInfoRPC {
+    fn from(value: BidSourceInfo) -> Self {
+        Self {
+            seen_time_us: offset_datetime_to_timestamp_us(value.seen_time),
+            relay: FixedSizeByteString::<MAX_RELAY_NAME_LENGTH>::from_str_truncated(&value.relay),
+        }
+    }
+}
+
+impl From<BidSourceInfoRPC> for BidSourceInfo {
+    fn from(value: BidSourceInfoRPC) -> Self {
+        Self {
+            seen_time: timestamp_us_to_offset_datetime(value.seen_time_us),
+            relay: value.relay.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, ZeroCopySend)]
+#[type_name("BidWithInfoRPC")]
+#[repr(C)]
+pub struct BidWithInfoRPC {
+    pub value: [u8; U256_DATA_LENGTH],
+    pub info: BidSourceInfoRPC,
+}
+
+impl From<BidWithInfo> for BidWithInfoRPC {
+    fn from(bid_with_info: BidWithInfo) -> Self {
+        Self {
+            value: bid_with_info.value.to_le_bytes(),
+            info: bid_with_info.info.into(),
+        }
+    }
+}
+
+impl From<BidWithInfoRPC> for BidWithInfo {
+    fn from(bid_with_info: BidWithInfoRPC) -> Self {
+        Self {
+            value: U256::from_le_bytes(bid_with_info.value),
+            info: bid_with_info.info.into(),
+        }
+    }
+}
+
+/// Context about the competitions bids that we used to generate our bid.
+#[derive(Debug, Copy, Clone, ZeroCopySend)]
+#[type_name("CompetitionBidContextRPC")]
+#[repr(C)]
+pub struct CompetitionBidContextRPC {
+    pub seen_competition_bid: Option<BidWithInfoRPC>,
+    pub triggering_bid_source_info: Option<BidSourceInfoRPC>,
+}
+
+impl From<CompetitionBidContext> for CompetitionBidContextRPC {
+    fn from(value: CompetitionBidContext) -> Self {
+        Self {
+            seen_competition_bid: value.seen_competition_bid.map(|k| k.into()),
+            triggering_bid_source_info: value.triggering_bid_source_info.map(|k| k.into()),
+        }
+    }
+}
+
+impl From<CompetitionBidContextRPC> for CompetitionBidContext {
+    fn from(value: CompetitionBidContextRPC) -> Self {
+        Self {
+            seen_competition_bid: value.seen_competition_bid.map(|k| k.into()),
+            triggering_bid_source_info: value.triggering_bid_source_info.map(|k| k.into()),
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, ZeroCopySend)]
 #[type_name("SlotBidderSealBidCommandRPC")]
 #[repr(C)]
 pub struct SlotBidderSealBidCommandRPC {
     pub session_id: u64,
     pub block_id: u64,
-    pub seen_competition_bid: Option<[u8; U256_DATA_LENGTH]>,
+    pub competition_bid_context: CompetitionBidContextRPC,
     /// When this bid is a reaction so some event (eg: new block, new competition bid) we put here
     /// the creation time of that event so we can measure our reaction time.
     pub trigger_creation_time_us: Option<u64>,
@@ -263,7 +342,7 @@ impl SlotBidderSealBidCommandRPC {
         Some(Self {
             session_id: value.1,
             block_id: value.0.block_id.0,
-            seen_competition_bid: value.0.seen_competition_bid.map(|k| k.to_le_bytes()),
+            competition_bid_context: value.0.competition_bid_context.into(),
             trigger_creation_time_us: value
                 .0
                 .trigger_creation_time
@@ -283,7 +362,7 @@ impl SlotBidderSealBidCommandRPC {
         }
         Some(SlotBidderSealBidCommand {
             block_id: BuiltBlockId(val.block_id),
-            seen_competition_bid: val.seen_competition_bid.map(|k| U256::from_le_bytes(k)),
+            competition_bid_context: val.competition_bid_context.into(),
             trigger_creation_time: val
                 .trigger_creation_time_us
                 .map(timestamp_us_to_offset_datetime),

crates/rbuilder-operator/src/clickhouse.rs

@@ -1,6 +1,6 @@
 //! Clickhouse integration to save all the blocks we build and submit to relays.
 
-use std::{sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use alloy_primitives::{utils::format_ether, Address, U256};
 use alloy_rpc_types_beacon::relay::SubmitBlockRequest as AlloySubmitBlockRequest;
@@ -31,6 +31,10 @@ use crate::{flashbots_config::BuiltBlocksClickhouseConfig, metrics::ClickhouseMe
 #[derive(Debug, Clone, Serialize, Deserialize, Row)]
 pub struct BlockRow {
     pub block_number: u64,
+    pub block_id: u64,
+    /// Number of times this block_id was used by the builder to bid before this.
+    /// Starts with 0 on the first bid.
+    pub block_uses: u64,
     /// name of the node that submitted the block
     pub builder_name: String,
     /// git commit of the rbuilder running
@@ -70,6 +74,15 @@ pub struct BlockRow {
     pub delayed_payment_addresses: Vec<String>,
     pub sent_to_relay_at: i64,
     pub tx_hashes: Vec<String>,
+
+    /// Info about the bid that triggered the bid to be generated.
+    /// Notice this may not be the top bid since a builder lowering it's bid could trigger a new bid
+    /// against the new winner.
+    pub triggering_bid_seen_time: Option<i64>,
+    pub triggering_bid_relay: Option<String>,
+    /// Info about the top bid among all relays we are trying to beat (best_relay_value)
+    pub bid_to_beat_seen_time: Option<i64>,
+    pub bid_to_beat_seen_relay: Option<String>,
 }
 
 impl ClickhouseRowExt for BlockRow {
@@ -231,7 +244,6 @@ impl BidObserver for BuiltBlocksWriter {
         submit_block_request: Arc<AlloySubmitBlockRequest>,
         built_block_trace: Arc<BuiltBlockTrace>,
         builder_algorithm_name: String,
-        best_bid_value: U256,
         _relays: &RelaySet,
         sent_to_relay_at: OffsetDateTime,
     ) {
@@ -241,6 +253,9 @@ impl BidObserver for BuiltBlocksWriter {
         let rbuilder_commit = self.rbuilder_commit.clone();
         let builder_name = self.builder_name.clone();
         tokio::spawn(async move {
+            // How many times we submitted this block_id to the relay.
+            let mut block_uses = HashMap::new();
+
             let submit_trace = submit_block_request.bid_trace();
             let execution_payload_v1 = match submit_block_request.as_ref() {
                 AlloySubmitBlockRequest::Capella(request) => {
@@ -277,8 +292,41 @@ impl BidObserver for BuiltBlocksWriter {
                 .iter()
                 .flat_map(|res| res.tx_infos.iter().map(|info| info.tx.hash().to_string()))
                 .collect();
+            let block_uses = block_uses
+                .entry(built_block_trace.build_block_id)
+                .or_insert(0);
+
+            let (triggering_bid_seen_time, triggering_bid_relay) = built_block_trace
+                .competition_bid_context
+                .triggering_bid_source_info
+                .as_ref()
+                .map(|info| {
+                    (
+                        Some(offset_date_to_clickhouse_timestamp(info.seen_time)),
+                        Some(info.relay.clone()),
+                    )
+                })
+                .unwrap_or_default();
+
+            let (bid_to_beat_seen_time, bid_to_beat_seen_relay, best_relay_value) =
+                built_block_trace
+                    .competition_bid_context
+                    .seen_competition_bid
+                    .as_ref()
+                    .map(|bid_with_info| {
+                        (
+                            Some(offset_date_to_clickhouse_timestamp(
+                                bid_with_info.info.seen_time,
+                            )),
+                            Some(bid_with_info.info.relay.clone()),
+                            Some(bid_with_info.value),
+                        )
+                    })
+                    .unwrap_or_default();
+
             let block_row = BlockRow {
                 block_number,
+                block_id: built_block_trace.build_block_id.0,
                 builder_name,
                 rbuilder_commit,
                 profit: format_ether(submit_trace.value),
@@ -302,7 +350,7 @@ impl BidObserver for BuiltBlocksWriter {
                 sealed_at: offset_date_to_clickhouse_timestamp(built_block_trace.orders_sealed_at),
                 algorithm: builder_algorithm_name,
                 true_value: Some(built_block_trace.true_bid_value),
-                best_relay_value: Some(best_bid_value),
+                best_relay_value,
                 block_value: Some(submit_trace.value),
                 used_bundle_hashes,
                 used_bundle_uuids,
@@ -312,7 +360,14 @@ impl BidObserver for BuiltBlocksWriter {
                 delayed_payment_addresses,
                 sent_to_relay_at: offset_date_to_clickhouse_timestamp(sent_to_relay_at),
                 tx_hashes,
+                block_uses: *block_uses,
+                triggering_bid_seen_time,
+                triggering_bid_relay,
+                bid_to_beat_seen_time,
+                bid_to_beat_seen_relay,
             };
+            *block_uses += 1;
+
             if let Err(err) = blocks_tx.try_send(block_row) {
                 error!(?err, "Failed to send block to clickhouse");
             }

crates/rbuilder-operator/src/flashbots_config.rs

@@ -2,7 +2,6 @@
 //! This code has lots of copy/paste from the example config but it's not really copy/paste since we use our own private types.
 //! @Pending make this copy/paste generic code on the library
 
-use alloy_primitives::U256;
 use alloy_rpc_types_beacon::relay::SubmitBlockRequest as AlloySubmitBlockRequest;
 use alloy_signer_local::PrivateKeySigner;
 use derivative::Derivative;
@@ -458,7 +457,6 @@ impl BidObserver for RbuilderOperatorBidObserver {
         submit_block_request: Arc<AlloySubmitBlockRequest>,
         built_block_trace: Arc<BuiltBlockTrace>,
         builder_algorithm_name: String,
-        best_bid_value: U256,
         relays: &RelaySet,
         sent_to_relay_at: OffsetDateTime,
     ) {
@@ -468,7 +466,6 @@ impl BidObserver for RbuilderOperatorBidObserver {
                 submit_block_request.clone(),
                 built_block_trace.clone(),
                 builder_algorithm_name.clone(),
-                best_bid_value,
                 relays,
                 sent_to_relay_at,
             )
@@ -479,7 +476,6 @@ impl BidObserver for RbuilderOperatorBidObserver {
                 submit_block_request,
                 built_block_trace,
                 builder_algorithm_name,
-                best_bid_value,
                 relays,
                 sent_to_relay_at,
             )

crates/rbuilder-operator/src/true_block_value_push/best_true_value_observer.rs

@@ -79,7 +79,6 @@ impl BidObserver for BestTrueValueObserver {
         _submit_block_request: Arc<AlloySubmitBlockRequest>,
         built_block_trace: Arc<BuiltBlockTrace>,
         builder_algorithm_name: String,
-        _best_bid_value: alloy_primitives::U256,
         _relays: &RelaySet,
         _sent_to_relay_at: OffsetDateTime,
     ) {

crates/rbuilder/src/building/builders/block_building_helper.rs

@@ -18,6 +18,7 @@ use crate::{
         FinalizeRevertStateCurrentIteration, NullPartialBlockExecutionTracer, PartialBlock,
         PartialBlockExecutionTracer, ThreadBlockBuildingContext,
     },
+    live_builder::block_output::bidding_service_interface::CompetitionBidContext,
     telemetry::{self, add_block_fill_time, add_order_simulation_time},
     utils::{check_block_hash_reader_health, elapsed_ms, HistoricalBlockError},
 };
@@ -74,7 +75,7 @@ pub trait BlockBuildingHelper: Send + Sync {
         local_ctx: &mut ThreadBlockBuildingContext,
         payout_tx_value: U256,
         subsidy: I256,
-        seen_competition_bid: Option<U256>,
+        competition_bid_context: CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError>;
 
     /// BuiltBlockTrace for current state.
@@ -99,7 +100,7 @@ pub trait BlockBuildingHelper: Send + Sync {
         local_ctx: &mut ThreadBlockBuildingContext,
         payout_tx_value: U256,
         subsidy: I256,
-        seen_competition_bid: Option<U256>,
+        competition_bid_context: CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError>;
 }
 
@@ -381,7 +382,7 @@ impl<
         local_ctx: &mut ThreadBlockBuildingContext,
         payout_tx_value: U256,
         subsidy: I256,
-        seen_competition_bid: Option<U256>,
+        competition_bid_context: CompetitionBidContext,
         adjust_finalized_block: bool,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError> {
         if adjust_finalized_block != self.finalize_adjustment_state.is_some() {
@@ -463,7 +464,7 @@ impl<
             self.built_block_trace.root_hash_time = finalized_block.root_hash_time;
             self.built_block_trace.finalize_time = start_time.elapsed();
         }
-        self.built_block_trace.seen_competition_bid = seen_competition_bid;
+        self.built_block_trace.competition_bid_context = competition_bid_context;
         Self::trace_finalized_block(
             &finalized_block,
             &self.builder_name,
@@ -601,13 +602,13 @@ impl<
         local_ctx: &mut ThreadBlockBuildingContext,
         payout_tx_value: U256,
         subsidy: I256,
-        seen_competition_bid: Option<U256>,
+        competition_bid_context: CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError> {
         self.finalize_block_impl(
             local_ctx,
             payout_tx_value,
             subsidy,
-            seen_competition_bid,
+            competition_bid_context,
             false,
         )
     }
@@ -642,13 +643,13 @@ impl<
         local_ctx: &mut ThreadBlockBuildingContext,
         payout_tx_value: U256,
         subsidy: I256,
-        seen_competition_bid: Option<U256>,
+        competition_bid_context: CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError> {
         self.finalize_block_impl(
             local_ctx,
             payout_tx_value,
             subsidy,
-            seen_competition_bid,
+            competition_bid_context,
             true,
         )
     }

crates/rbuilder/src/building/builders/block_building_helper_stats_logger.rs

@@ -209,7 +209,7 @@ impl BlockBuildingHelper for BlockBuildingHelperStatsLogger<'_> {
         _local_ctx: &mut crate::building::ThreadBlockBuildingContext,
         _payout_tx_value: alloy_primitives::U256,
         _subsidy: alloy_primitives::I256,
-        _seen_competition_bid: Option<alloy_primitives::U256>,
+        _competition_bid_context: crate::live_builder::block_output::bidding_service_interface::CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError> {
         panic!("finalize_block not implemented. This is only for testing.");
     }
@@ -240,7 +240,7 @@ impl BlockBuildingHelper for BlockBuildingHelperStatsLogger<'_> {
         _local_ctx: &mut ThreadBlockBuildingContext,
         _payout_tx_value: U256,
         _subsidy: I256,
-        _seen_competition_bid: Option<U256>,
+        _competition_bid_context: crate::live_builder::block_output::bidding_service_interface::CompetitionBidContext,
     ) -> Result<FinalizeBlockResult, BlockBuildingHelperError> {
         unimplemented!()
     }