Feature/custom block range (#115)

* add curl installation in dockerfile

* change params of trace_replayBlockTransaction

* add custom block range

* add logs to failed http requests

* add params validation and update dependencies

Author: erwin-wee

Dockerfile

@@ -19,7 +19,7 @@ FROM python:3.10-slim-bookworm as prod
 COPY --from=venv /app/venv /app/venv/
 ENV PATH /app/venv/bin:$PATH
 
-RUN apt-get update && apt-get install -y tini htop nano
+RUN apt-get update && apt-get install -y tini htop nano curl
 
 WORKDIR /app
 COPY . ./

README.md

@@ -128,6 +128,8 @@ This will run a load test with a general BSC profile.
 - `--size`: Specifies the test data size. Available values are XS, S, M, L, XL. Default is S.
 - `--batch`: Runs the test using batch requests. This will send multiple requests in a single batch request. The number of requests in a batch can be specified using the `--batch-size` flag. Default batch size is 10.
 - `--ref-url`: Specifies the reference blockchain node URL that will be used to fetch test data from before the benchmark starts.
+- `--start-block`: Starting block number to use when generating test data. Must be used together with `--end-block`.
+- `--end-block`: Ending block number to use when generating test data. Must be greater than `--start-block`. Note: if `--use-latest-blocks` is set, this custom range is ignored.
 
 You may also run `chainbench start --help` for the full list of parameters and flags.
 

chainbench/main.py

@@ -215,6 +215,8 @@ def validate_profile_path(ctx: Context, param: Parameter, value: str) -> str:
     default=None,
     help="Reference Node URL for retrieving test data before test starts. If not specified, target url is used.",
 )
+@click.option("--start-block", default=None, help="Start block for test data", type=int)
+@click.option("--end-block", default=None, help="End block for test data", type=int)
 @click.pass_context
 def start(
     ctx: Context,
@@ -249,7 +251,16 @@ def start(
     size: str | None,
     method: str | None = None,
     ref_url: str | None = None,
+    start_block: int | None = None,
+    end_block: int | None = None,
 ) -> None:
+    if start_block is not None or end_block is not None:
+        if start_block is None or end_block is None:
+            raise ValueError("Both start-block and end-block are required for specifying custom block range.")
+        else:
+            if end_block < start_block:
+                raise ValueError("end-block must be greater than start-block.")
+
     if notify:
         click.echo(f"Notify when test is finished using topic: {notify}")
         notifier = Notifier(topic=notify)
@@ -386,6 +397,8 @@ def start(
         enable_class_picker=enable_class_picker,
         batch_size=batch_size,
         ref_url=ref_url,
+        start_block=start_block,
+        end_block=end_block,
     )
     # Start the Locust master
     master_command = locust_options.get_master_command()

chainbench/test_data/blockchain.py

@@ -174,7 +174,13 @@ def fetch_random_block(self, block_numbers: list[BlockNumber]) -> B:
         return self.fetch_block(block_number)
 
     def _get_start_and_end_blocks(self, parsed_options: Namespace) -> BlockRange:
-        raise NotImplementedError
+        if parsed_options.start_block is not None:
+            self.start_block_number = parsed_options.start_block
+        else:
+            self.start_block_number = 1
+        if parsed_options.end_block is not None:
+            self.end_block_number = parsed_options.end_block
+        return self.data.block_range
 
     def get_block_from_data(self, data: dict[str, t.Any] | str) -> B:
         raise NotImplementedError
@@ -210,10 +216,18 @@ def get_random_block_number(self, rng: RNG | None = None) -> BlockNumber:
     def start_block_number(self) -> BlockNumber:
         return self.data.block_range.start
 
+    @start_block_number.setter
+    def start_block_number(self, value: BlockNumber) -> None:
+        self.data.block_range.start = value
+
     @property
     def end_block_number(self) -> BlockNumber:
         return self.data.block_range.end
 
+    @end_block_number.setter
+    def end_block_number(self, value: BlockNumber) -> None:
+        self.data.block_range.end = value
+
 
 class SmartContract:
     def __init__(self, address: str):

chainbench/test_data/ethereum.py

@@ -134,13 +134,12 @@ def fetch_latest_block_number(self) -> BlockNumber:
         return int(self.fetch_block_header("head")["slot"])
 
     def _get_start_and_end_blocks(self, parsed_options: Namespace) -> BlockRange:
-        end_block_number = self.fetch_latest_block_number()
+        super()._get_start_and_end_blocks(parsed_options)
         if parsed_options.use_latest_blocks:
-            start_block_number = end_block_number - self.data.size.blocks_len + 1
-        else:
-            start_block_number = 1
-        logger.info("Using blocks from %s to %s as test data", start_block_number, end_block_number)
-        return BlockRange(start_block_number, end_block_number)
+            self.end_block_number = self.fetch_latest_block_number()
+            self.start_block_number = self.end_block_number - self.data.size.blocks_len + 1
+        logger.info("Using blocks from %s to %s as test data", self.start_block_number, self.end_block_number)
+        return self.data.block_range
 
     def get_block_from_data(self, data: dict[str, t.Any] | str) -> EthBeaconBlock:
         def get_committee(committee: dict[str, t.Any]) -> EthCommittee:

chainbench/test_data/evm.py

@@ -482,13 +482,29 @@ def fetch_latest_block(self) -> EvmBlock:
         return self.fetch_block("latest")
 
     def _get_start_and_end_blocks(self, parsed_options: Namespace) -> BlockRange:
-        end_block_number = self.fetch_latest_block_number()
-        if parsed_options.use_latest_blocks:
-            start_block_number = end_block_number - self.data.size.blocks_len + 1
+        latest_block_number = self.fetch_latest_block_number()
+        if parsed_options.start_block is not None:
+            self.start_block_number = parsed_options.start_block
+        else:
+            self.start_block_number = self.network.start_block
+        if parsed_options.end_block is not None:
+            self.end_block_number = parsed_options.end_block
         else:
-            start_block_number = self.network.start_block
-            logger.info("Using blocks from %s to %s as test data", start_block_number, end_block_number)
-        return BlockRange(start_block_number, end_block_number)
+            self.end_block_number = latest_block_number
+        if parsed_options.use_latest_blocks:
+            self.end_block_number = latest_block_number
+            self.start_block_number = self.end_block_number - self.data.size.blocks_len + 1
+        if self.start_block_number < 0:
+            logger.warning("start_block is before genesis block," "setting to 0.")
+            self.start_block_number = 0
+        if self.end_block_number > latest_block_number:
+            logger.warning(
+                f"end_block {self.end_block_number} is after latest_block_number {latest_block_number},"
+                "setting to latest_block_number."
+            )
+            self.end_block_number = latest_block_number
+        logger.info("Using blocks from %s to %s as test data", self.start_block_number, self.end_block_number)
+        return self.data.block_range
 
     def get_random_block_hash(self, rng: RNG | None = None) -> BlockHash:
         if rng is None:

chainbench/test_data/solana.py

@@ -98,24 +98,24 @@ def _fetch_first_available_block(self) -> Slot:
         return slot
 
     def _get_start_and_end_blocks(self, parsed_options: Namespace) -> BlockRange:
-        end_block_number = self.fetch_latest_block_number()
+        super()._get_start_and_end_blocks(parsed_options)
         earliest_available_block_number = self._fetch_first_available_block()
-
+        latest_block_number = self.fetch_latest_block_number()
         # factor in run_time and add 10% buffer to ensure blocks used in test data are
         # not removed from the ledger
         earliest_available_block_number += int((parsed_options.run_time / self.BLOCK_TIME) * 1.1)
-        start_block_number = earliest_available_block_number
-
         if parsed_options.use_latest_blocks:
-            start_block_number = end_block_number - self.data.size.blocks_len + 1
-
-        if start_block_number < earliest_available_block_number:
-            raise ValueError(
-                f"Earliest available block (with buffer) is {earliest_available_block_number}, "
-                f"but start block is {start_block_number}"
+            self.end_block_number = latest_block_number
+            self.start_block_number = self.end_block_number - self.data.size.blocks_len + 1
+        if self.start_block_number < earliest_available_block_number:
+            logger.warning(
+                "start_block is before earliest_available_block_number," "setting to earliest_available_block_number."
             )
-
-        return BlockRange(start_block_number, end_block_number)
+            self.start_block_number = earliest_available_block_number
+        if self.end_block_number > latest_block_number:
+            logger.warning("end_block is after latest_block_number," "setting to latest_block_number.")
+            self.end_block_number = latest_block_number
+        return self.data.block_range
 
     def get_random_block_hash(self, rng: RNG | None = None) -> BlockHash:
         if rng is None:

chainbench/user/http.py

@@ -61,6 +61,7 @@ def check_http_error(self, response: ResponseContextManager) -> None:
         """Check the response for errors."""
         if response.status_code != 200:
             self.logger.error(f"Request failed with {response.status_code} code")
+            self.logger.error(f"Request body of failed request: {response.request.body if response.request else ''}")
             self.logger.debug(
                 f"Request to {response.url} failed with HTTP Error {response.status_code} code: {response.text}"
             )

chainbench/user/protocol/evm.py

@@ -85,10 +85,9 @@ def _debug_trace_block_by_hash_params_factory(self, rng: RNG) -> list[BlockHash
             {"tracer": "callTracer", "timeout": self._default_trace_timeout},
         ]
 
-    @staticmethod
-    def _trace_replay_block_transaction_params_factory() -> list[str | list[str]]:
+    def _trace_replay_block_transaction_params_factory(self, rng: RNG) -> list[str | list[str]]:
         return [
-            "latest",
+            hex(self.test_data.get_random_block_number(rng)),
             ["vmTrace", "trace", "stateDiff"],
         ]
 
@@ -403,7 +402,8 @@ def trace_filter(self) -> RpcCall:
 
     def trace_replay_block_transactions(self) -> RpcCall:
         return RpcCall(
-            method="trace_replayBlockTransactions", params=self._trace_replay_block_transaction_params_factory()
+            method="trace_replayBlockTransactions",
+            params=self._trace_replay_block_transaction_params_factory(self.rng.get_rng()),
         )
 
     def trace_replay_transaction(self) -> RpcCall:

chainbench/util/cli.py

@@ -145,6 +145,8 @@ class LocustOptions:
     enable_class_picker: bool = False
     batch_size: int | None = None
     ref_url: str | None = None
+    start_block: int | None = None
+    end_block: int | None = None
 
     def get_master_command(self) -> str:
         """Generate master command."""
@@ -198,6 +200,11 @@ def get_extra_options(self, command: str):
 
         if self.use_latest_blocks:
             command += " --use-latest-blocks True"
+        else:
+            if self.start_block is not None:
+                command += f" --start-block {self.start_block}"
+            if self.end_block is not None:
+                command += f" --end-block {self.end_block}"
 
         if self.method is not None:
             command += f" --method {self.method}"