From ab8491b28a7b5b95dce8c71c435b5505ed96592c Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 5 May 2026 16:23:38 +0100
Subject: [PATCH 01/16] ci(bench): add two-validator e2e harness

---
 .github/workflows/bench-e2e.yml | 181 ++++++--
 .github/workflows/bench.yml     |  19 +
 tempo.nu                        | 756 +++++++++++++++++++++++++++++++-
 3 files changed, 905 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index 7306745d94..eba2cea690 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -1,7 +1,7 @@
 # E2E benchmark job.
 #
-# Called by bench.yml when mode=e2e. Runs `nu tempo.nu bench` for an
-# interleaved B-F-F-B comparison using synthetic transactions.
+# Called by bench.yml when mode=e2e. Runs `nu tempo.nu bench-consensus`
+# for an interleaved B-F-F-B comparison using synthetic transactions.
 
 name: bench-e2e
 
@@ -242,10 +242,19 @@ permissions:
 
 jobs:
   bench-e2e:
-    name: bench-e2e
-    runs-on: [self-hosted, Linux, X64, bare-metal]
+    name: bench-e2e-${{ matrix.role }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - role: red
+            runner: bench-a
+          - role: blue
+            runner: bench-b
+    runs-on: [self-hosted, Linux, X64, "${{ matrix.runner }}"]
     timeout-minutes: 300
     env:
+      BENCH_ROLE: ${{ matrix.role }}
       BENCH_PR: ${{ inputs.pr }}
       BENCH_ACTOR: ${{ inputs.actor || github.actor }}
       BENCH_PRESET: ${{ inputs.preset }}
@@ -287,7 +296,7 @@ jobs:
         run: echo "::add-mask::$GRAFANA_TEMPO"
 
       - name: Mask ClickHouse credentials
-        if: env.CLICKHOUSE_URL
+        if: matrix.role == 'red' && env.CLICKHOUSE_URL
         run: |
           echo "::add-mask::$CLICKHOUSE_URL"
           echo "::add-mask::$CLICKHOUSE_PASSWORD"
@@ -331,7 +340,7 @@ jobs:
           ref: ${{ steps.checkout-ref.outputs.ref }}
 
       - name: Checkout txgen repository
-        if: env.BENCH_BACKEND == 'txgen'
+        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen'
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           repository: tempoxyz/txgen
@@ -341,14 +350,14 @@ jobs:
           fetch-depth: 0
 
       - name: Checkout txgen ref
-        if: env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != ''
+        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != ''
         working-directory: txgen
         run: |
           git fetch origin "$BENCH_TXGEN_REF" --quiet
           git checkout --detach FETCH_HEAD
 
       - name: Resolve job URL and update status
-        if: env.BENCH_COMMENT_ID
+        if: matrix.role == 'red' && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           INPUT_BASELINE_NAME: ${{ inputs.baseline-name }}
@@ -361,7 +370,7 @@ jobs:
               repo: context.repo.repo,
               run_id: context.runId,
             });
-            const job = jobs.jobs.find(j => j.name === 'bench-e2e');
+            const job = jobs.jobs.find(j => j.name === 'bench-e2e-red');
             const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             core.exportVariable('BENCH_JOB_URL', jobUrl);
 
@@ -395,8 +404,19 @@ jobs:
       - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
         continue-on-error: true
 
+      - name: Validate e2e options
+        run: |
+          if [ "$BENCH_BACKEND" = "txgen" ]; then
+            echo "::error::mode=e2e currently supports backend=tempo-bench only; backend=txgen will be wired in the txgen bench PR."
+            exit 1
+          fi
+          if [ "$BENCH_FORCE_BLOAT" = "true" ]; then
+            echo "::error::mode=e2e does not support force-bloat yet because both runners must use coordinated benchmark snapshots."
+            exit 1
+          fi
+
       - name: Build txgen backend
-        if: env.BENCH_BACKEND == 'txgen'
+        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen'
         working-directory: txgen
         run: |
           cargo build --release --bin txgen-tempo --bin bench
@@ -480,6 +500,7 @@ jobs:
             core.setOutput('feature-name', featureName);
 
       - name: Resolve PR attribution
+        if: matrix.role == 'red'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
@@ -506,7 +527,7 @@ jobs:
             core.exportVariable('BENCH_PR', pr);
 
       - name: Update status (running benchmark)
-        if: success() && env.BENCH_COMMENT_ID
+        if: matrix.role == 'red' && success() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
@@ -514,36 +535,44 @@ jobs:
             const s = require('./.github/scripts/bench-update-status.js');
             await s({github, context, status: 'Running benchmark...'});
 
-      - name: Run benchmark
+      - name: Run benchmark role
         id: bench
         env:
           BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
           FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
         run: |
-          if [ "$BENCH_BACKEND" = "txgen" ]; then
-            cmd=(nu contrib/bench/bench-txgen.nu run)
-          else
-            cmd=(nu tempo.nu bench)
-          fi
+          cmd=(nu tempo.nu bench-consensus)
           cmd+=(
+            --role "$BENCH_ROLE"
             --preset "$BENCH_PRESET"
-            --mode dev
             --bloat "$BENCH_BLOAT"
             --duration "$BENCH_DURATION"
             --tps "$BENCH_TPS"
-            --no-infra
             --baseline "$BASELINE_REF"
             --feature "$FEATURE_REF"
-            --bench-datadir "/reth-bench/tempo_${BENCH_BLOAT}mb"
+            --baseline-name "${{ steps.refs.outputs.baseline-name }}"
+            --feature-name "${{ steps.refs.outputs.feature-name }}"
+            --peer-url "$BENCH_E2E_PEER_URL"
+            --bench-datadir "$BENCH_E2E_NODE_DIR"
+            --node-dir "$BENCH_E2E_NODE_DIR"
+            --genesis "$BENCH_E2E_GENESIS"
+            --trusted-peers "$BENCH_E2E_TRUSTED_PEERS"
+            --consensus-port "$BENCH_E2E_CONSENSUS_PORT"
+            --benchmark-id "bench-e2e-${GITHUB_RUN_ID}"
+            --reference-epoch "$(date +%s)"
             --tune
             --gas-limit 1000000000000
           )
+          [ -n "${BENCH_E2E_CONSENSUS_IP:-}" ] && cmd+=(--consensus-ip "$BENCH_E2E_CONSENSUS_IP")
           [ "$BENCH_SAMPLY" = "true" ] && cmd+=(--samply)
           [ "$BENCH_TRACY" != "off" ] && cmd+=(--tracy "$BENCH_TRACY" --tracy-seconds "$BENCH_TRACY_SECONDS" --tracy-offset "$BENCH_TRACY_OFFSET")
+          [ -n "$BENCH_BASELINE_HARDFORK" ] && cmd+=(--baseline-hardfork "$BENCH_BASELINE_HARDFORK" --feature-hardfork "$BENCH_FEATURE_HARDFORK")
+          [ -n "${BENCH_E2E_BASELINE_GENESIS:-}" ] && cmd+=(--baseline-genesis "$BENCH_E2E_BASELINE_GENESIS")
+          [ -n "${BENCH_E2E_FEATURE_GENESIS:-}" ] && cmd+=(--feature-genesis "$BENCH_E2E_FEATURE_GENESIS")
+          [ -n "${BENCH_E2E_BASELINE_NODE_DIR:-}" ] && cmd+=(--baseline-node-dir "$BENCH_E2E_BASELINE_NODE_DIR")
+          [ -n "${BENCH_E2E_FEATURE_NODE_DIR:-}" ] && cmd+=(--feature-node-dir "$BENCH_E2E_FEATURE_NODE_DIR")
           [ -n "$BENCH_BASELINE_ARGS" ] && cmd+=(--baseline-args="$BENCH_BASELINE_ARGS")
           [ -n "$BENCH_FEATURE_ARGS" ] && cmd+=(--feature-args="$BENCH_FEATURE_ARGS")
-          [ -n "$BENCH_BASELINE_HARDFORK" ] && cmd+=(--baseline-hardfork "$BENCH_BASELINE_HARDFORK" --feature-hardfork "$BENCH_FEATURE_HARDFORK")
-          [ "$BENCH_FORCE_BLOAT" = "true" ] && cmd+=(--force)
           [ -n "$BENCH_BENCH_ARGS" ] && cmd+=(--bench-args="$BENCH_BENCH_ARGS")
           [ -n "$BENCH_BENCH_ENV" ] && cmd+=(--bench-env="$BENCH_BENCH_ENV")
           [ -n "$BENCH_BASELINE_ENV" ] && cmd+=(--baseline-env="$BENCH_BASELINE_ENV")
@@ -563,14 +592,21 @@ jobs:
           echo "Results directory: $RESULTS_DIR"
 
       - name: Upload results
-        if: "!cancelled()"
+        if: matrix.role == 'red' && !cancelled()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: tempo-bench-red-results
+          path: bench-results/
+
+      - name: Upload blue results
+        if: matrix.role == 'blue' && !cancelled()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: tempo-bench-results
+          name: tempo-bench-blue-results
           path: bench-results/
 
       - name: Upload results to ClickHouse
-        if: success()
+        if: matrix.role == 'red' && success()
         env:
           BENCH_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
           BENCH_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
@@ -580,7 +616,7 @@ jobs:
         run: bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
 
       - name: Post results to PR
-        if: success()
+        if: matrix.role == 'red' && success()
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
@@ -655,12 +691,12 @@ jobs:
               const links = [];
               for (const run of runs) {
                 try {
-                  const url = fs.readFileSync(`${resultsDir}/profile-${run}-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run}**: [Firefox Profiler](${url})`);
+                  const url = fs.readFileSync(`${resultsDir}/profile-${run}-red-url.txt`, 'utf8').trim();
+                  if (url) links.push(`- **${run} / red**: [Firefox Profiler](${url})`);
                 } catch (e) {}
               }
               if (links.length > 0) {
-                samplySection = `\n\n### Samply Profiles\n\n${links.join('\n')}\n`;
+                samplySection = `\n\n### Red Samply Profiles\n\n${links.join('\n')}\n`;
               }
             }
 
@@ -671,12 +707,12 @@ jobs:
               const links = [];
               for (const run of runs) {
                 try {
-                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run}**: [Tracy Viewer](${url})`);
+                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-red-url.txt`, 'utf8').trim();
+                  if (url) links.push(`- **${run} / red**: [Tracy Viewer](${url})`);
                 } catch (e) {}
               }
               if (links.length > 0) {
-                tracySection = `\n\n### Tracy Profiles\n\n${links.join('\n')}\n`;
+                tracySection = `\n\n### Red Tracy Profiles\n\n${links.join('\n')}\n`;
               }
             }
 
@@ -695,8 +731,81 @@ jobs:
               await core.summary.addRaw(body).write();
             }
 
+      - name: Add blue profile links to PR
+        if: matrix.role == 'blue' && success() && env.BENCH_COMMENT_ID && steps.results-dir.outputs.path
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
+        with:
+          github-token: ${{ secrets.DEREK_PAT }}
+          script: |
+            const fs = require('fs');
+            const resultsDir = process.env.BENCH_RESULTS_DIR;
+            const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
+
+            const sections = [];
+            if (process.env.BENCH_SAMPLY === 'true') {
+              const links = [];
+              for (const run of runs) {
+                try {
+                  const url = fs.readFileSync(`${resultsDir}/profile-${run}-blue-url.txt`, 'utf8').trim();
+                  if (url) links.push(`- **${run} / blue**: [Firefox Profiler](${url})`);
+                } catch (e) {}
+              }
+              if (links.length) sections.push(`### Blue Samply Profiles\n\n${links.join('\n')}`);
+            }
+
+            if (process.env.BENCH_TRACY && process.env.BENCH_TRACY !== 'off') {
+              const links = [];
+              for (const run of runs) {
+                try {
+                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-blue-url.txt`, 'utf8').trim();
+                  if (url) links.push(`- **${run} / blue**: [Tracy Viewer](${url})`);
+                } catch (e) {}
+              }
+              if (links.length) sections.push(`### Blue Tracy Profiles\n\n${links.join('\n')}`);
+            }
+
+            if (!sections.length) return;
+
+            const markerStart = '<!-- e2e-blue-profiles:start -->';
+            const markerEnd = '<!-- e2e-blue-profiles:end -->';
+            const block = `${markerStart}\n\n${sections.join('\n\n')}\n\n${markerEnd}`;
+            const comment_id = parseInt(process.env.BENCH_COMMENT_ID);
+            const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+            for (let attempt = 0; attempt < 60; attempt++) {
+              const { data: comment } = await github.rest.issues.getComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id,
+              });
+
+              const current = comment.body || '';
+              if (!current.includes('Benchmark complete!')) {
+                await sleep(10000);
+                continue;
+              }
+
+              const escapedStart = markerStart.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+              const escapedEnd = markerEnd.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+              const withoutOldBlock = current
+                .replace(new RegExp(`\\n*${escapedStart}[\\s\\S]*?${escapedEnd}`, 'm'), '')
+                .trimEnd();
+
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id,
+                body: `${withoutOldBlock}\n\n${block}`,
+              });
+              return;
+            }
+
+            core.warning('Timed out waiting for the red result comment; blue profile links were left in the blue artifact.');
+
       - name: Send Slack notification (success)
-        if: success() && steps.results-dir.outputs.path && env.BENCH_NO_SLACK != 'true'
+        if: matrix.role == 'red' && success() && steps.results-dir.outputs.path && env.BENCH_NO_SLACK != 'true'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           BENCH_WORK_DIR: ${{ steps.results-dir.outputs.path }}
@@ -708,7 +817,7 @@ jobs:
             await notify.success({ core, context });
 
       - name: Send Slack notification (failure)
-        if: failure() && env.BENCH_NO_SLACK != 'true'
+        if: matrix.role == 'red' && failure() && env.BENCH_NO_SLACK != 'true'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
@@ -716,7 +825,7 @@ jobs:
             await notify.failure({ core, context, failedStep: 'running benchmark' });
 
       - name: Update status (failed)
-        if: failure() && env.BENCH_COMMENT_ID
+        if: matrix.role == 'red' && failure() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
@@ -730,7 +839,7 @@ jobs:
             });
 
       - name: Update status (cancelled)
-        if: cancelled() && env.BENCH_COMMENT_ID
+        if: matrix.role == 'red' && cancelled() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5d443d543e..3353977ab4 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -237,6 +237,25 @@ jobs:
               return;
             }
 
+            // E2E currently owns the two-runner consensus harness. The txgen
+            // sender plugs into this path once #3669 lands.
+            if (mode === 'e2e') {
+              const unsupported = [];
+              if (backend !== 'tempo-bench') unsupported.push(`backend=${backend}`);
+              if (forceBloat === 'true') unsupported.push('force-bloat');
+              if (unsupported.length > 0) {
+                const msg = `❌ **Invalid bench command**\n\n\`mode=e2e\` does not support ${unsupported.map(s => `\`${s}\``).join(', ')} yet. The e2e harness currently uses \`backend=tempo-bench\`; \`backend=txgen\` will be wired in the txgen bench PR.`;
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: msg,
+                });
+                core.setFailed(msg);
+                return;
+              }
+            }
+
             // Validate tracy value
             if (!['off', 'on', 'full'].includes(tracy)) {
               const msg = `❌ **Invalid bench command**\n\n\`tracy=${tracy}\` is not valid. Must be \`off\`, \`on\`, or \`full\`.\n\n${usageStr}`;
diff --git a/tempo.nu b/tempo.nu
index fb312bd4c7..9403ebc9d8 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -1400,6 +1400,100 @@ def build-consensus-args [node_dir: string, trusted_peers: string, port: int] {
     ]
 }
 
+# Build consensus args for a single validator running on an isolated e2e runner.
+def build-e2e-consensus-args [node_dir: string, trusted_peers: string, port: int, consensus_ip: string] {
+    let addr = ($node_dir | path basename)
+    let inferred_ip = if ($addr | str contains ":") {
+        $addr | split row ":" | get 0
+    } else {
+        "0.0.0.0"
+    }
+    let ip = if $consensus_ip != "" { $consensus_ip } else { $inferred_ip }
+    let signing_key = $"($node_dir)/signing.key"
+    let signing_share = $"($node_dir)/signing.share"
+    let enode_key = $"($node_dir)/enode.key"
+
+    let execution_p2p_port = $port + 1
+    let metrics_port = $port + 2
+    let authrpc_port = $port + 3
+    let discv5_port = $port + 4
+
+    [
+        "--consensus.signing-key" $signing_key
+        "--consensus.signing-share" $signing_share
+        "--consensus.listen-address" $"($ip):($port)"
+        "--consensus.metrics-address" $"($ip):($metrics_port)"
+        "--trusted-peers" $trusted_peers
+        "--port" $"($execution_p2p_port)"
+        "--discovery.port" $"($execution_p2p_port)"
+        "--discovery.v5.port" $"($discv5_port)"
+        "--p2p-secret-key" $enode_key
+        "--authrpc.port" $"($authrpc_port)"
+        "--consensus.fee-recipient" "0x0000000000000000000000000000000000000000"
+        "--consensus.use-local-defaults"
+        "--consensus.bypass-ip-check"
+    ]
+}
+
+def stop-tempo-processes-gracefully [] {
+    let pids = (find-tempo-pids)
+    if ($pids | length) > 0 {
+        print $"Stopping tempo processes: ($pids | str join ', ')"
+    }
+    for pid in $pids {
+        kill -s 2 $pid
+    }
+    for pid in $pids {
+        mut wait = 0
+        while $wait < 30 {
+            if (ps | where pid == $pid | length) == 0 { break }
+            sleep 1sec
+            $wait = $wait + 1
+        }
+        if $wait >= 30 {
+            print $"  Warning: PID ($pid) did not exit, sending SIGKILL"
+            kill -s 9 $pid
+            sleep 1sec
+        }
+    }
+    if ("/tmp/reth.ipc" | path exists) {
+        rm --force /tmp/reth.ipc
+    }
+}
+
+def stop-tracy-capture [] {
+    print "  Stopping tracy-capture..."
+    let capture_pids = (ps | where name =~ "tracy-capture" | get pid)
+    for pid in $capture_pids {
+        kill -s 2 $pid
+    }
+    mut wait_tracy = 0
+    while $wait_tracy < 30 {
+        if (ps | where name =~ "tracy-capture" | length) == 0 { break }
+        sleep 1sec
+        $wait_tracy = $wait_tracy + 1
+    }
+    if $wait_tracy >= 30 {
+        print "  Warning: tracy-capture did not exit, sending SIGKILL"
+        for pid in (ps | where name =~ "tracy-capture" | get pid) {
+            kill -s 9 $pid
+        }
+    }
+}
+
+def wait-for-samply-profile [] {
+    print "  Waiting for samply to finish saving profile..."
+    mut wait = 0
+    while $wait < 120 {
+        if (ps | where name =~ "samply" | length) == 0 { break }
+        sleep 500ms
+        $wait = $wait + 1
+    }
+    if $wait >= 120 {
+        print "  Warning: samply did not exit in time"
+    }
+}
+
 # ============================================================================
 # System tuning for benchmarks
 # ============================================================================
@@ -1552,6 +1646,536 @@ def "main bench-init" [
     print $"Virgin snapshot initialized and promoted."
 }
 
+# Run one side of a 2-runner e2e consensus benchmark phase.
+def "main bench-consensus-phase" [
+    --role: string                                      # Runner role: red sends load, blue validates only
+    --phase: string                                     # Phase label: baseline-1, feature-1, feature-2, baseline-2
+    --ref: string                                       # Git SHA/ref to build and run
+    --peer-url: string                                  # RPC URL for the validator on the other runner
+    --preset: string = ""                               # Preset: tip20, erc20, swap, order, tempo-mix
+    --tps: int = 10000                                  # Target TPS
+    --duration: int = 300                               # Duration in seconds
+    --accounts: int = 1000                              # Number of accounts
+    --max-concurrent-requests: int = 100                # Max concurrent requests
+    --bench-datadir: string = ""                        # Datadir/snapshot path to recover before the phase
+    --bloat: int = 0                                    # State bloat size in MiB; enables bloat mnemonic for the sender
+    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
+    --features: string = $DEFAULT_FEATURES              # Cargo features
+    --samply                                            # Profile this runner's validator with samply
+    --samply-args: list<string> = []                    # Additional samply arguments
+    --tracy: string = "off"                             # Tracy profiling: off, on, full
+    --tracy-filter: string = "debug"                    # Tracy tracing filter level
+    --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
+    --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
+    --node-args: string = ""                            # Additional node args for all phases
+    --baseline-args: string = ""                        # Additional node args for baseline phases
+    --feature-args: string = ""                         # Additional node args for feature phases
+    --bench-args: string = ""                           # Additional tempo-bench args
+    --baseline-env: string = ""                         # Environment vars for baseline node phases
+    --feature-env: string = ""                          # Environment vars for feature node phases
+    --bench-env: string = ""                            # Environment vars for the sender process
+    --benchmark-id: string = ""                         # Shared benchmark identifier
+    --reference-epoch: int = 0                           # Shared timestamp for observability correlation
+    --tune                                              # Apply system tuning
+    --loud                                              # Show node debug logs
+    --node-dir: string                                  # Validator node directory for this runner
+    --genesis: string                                   # Shared genesis file path
+    --trusted-peers: string                             # Comma-separated enode peers for the network
+    --consensus-port: int = 8000                        # Consensus listen port for this validator
+    --consensus-ip: string = ""                         # Optional consensus listen IP override
+    --results-dir: string = ""                          # Output root directory
+    --gas-limit: string = ""                            # Optional builder gas limit override
+    --tracing-otlp: string = ""                         # OTLP endpoint for tracing
+    --peer-hold-extra: int = 600                         # Extra seconds to wait for phase peer shutdown
+    --wait-peer-offline                                  # Wait for peer RPC to go offline before returning from sender phase
+    --no-cache                                          # Skip binary cache
+] {
+    if $role not-in ["red" "blue"] {
+        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
+        exit 1
+    }
+    if $preset == "" and $bench_args == "" {
+        print "Error: either --preset or --bench-args must be provided"
+        print $"  Available presets: ($PRESETS | columns | str join ', ')"
+        exit 1
+    }
+    if $preset != "" and not ($preset in $PRESETS) {
+        print $"Unknown preset: ($preset). Available: ($PRESETS | columns | str join ', ')"
+        exit 1
+    }
+    if $tracy not-in ["off" "on" "full"] {
+        print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
+        exit 1
+    }
+    if $samply and $tracy != "off" {
+        print "Error: --samply and --tracy are mutually exclusive. Choose one."
+        exit 1
+    }
+    if $tracy != "off" {
+        let has_tracy_capture = (which tracy-capture | length) > 0
+        if not $has_tracy_capture {
+            print "Error: tracy-capture not found. Install tracy and ensure tracy-capture is in PATH."
+            exit 1
+        }
+    }
+    if not ($node_dir | path exists) {
+        print $"Error: node dir does not exist: ($node_dir)"
+        exit 1
+    }
+    if not ($genesis | path exists) {
+        print $"Error: genesis file does not exist: ($genesis)"
+        exit 1
+    }
+    for required_file in ["signing.key" "signing.share" "enode.key"] {
+        let path = $"($node_dir)/($required_file)"
+        if not ($path | path exists) {
+            print $"Error: missing validator file: ($path)"
+            exit 1
+        }
+    }
+
+    let run_type = if ($phase | str starts-with "baseline") { "baseline" } else { "feature" }
+    let side_args = if $run_type == "baseline" { $baseline_args } else { $feature_args }
+    let side_env = if $run_type == "baseline" { $baseline_env } else { $feature_env }
+    let effective_node_args = ([$node_args $side_args] | where { |a| $a != "" } | str join " ")
+    let extra_args = if $effective_node_args == "" { [] } else { $effective_node_args | split row " " }
+    let weights = if $preset != "" { $PRESETS | get $preset } else { [0.0, 0.0, 0.0, 0.0] }
+    let datadir = if $bench_datadir != "" { $bench_datadir } else { $node_dir }
+    let phase_results_dir = if $results_dir != "" { $results_dir } else { $"($BENCH_RESULTS_DIR)/($phase)" }
+
+    main kill
+    let tuning_state = if $tune { apply-system-tuning } else { { tuned: false } }
+    bench-recover $datadir
+
+    let worktree_dir = $"($BENCH_WORKTREES_DIR)/e2e-($role)"
+    git worktree prune
+    mkdir $BENCH_WORKTREES_DIR
+    if ($worktree_dir | path exists) {
+        print $"Removing stale e2e worktree: ($worktree_dir)"
+        try { git worktree remove --force $worktree_dir } catch { rm -rf $worktree_dir }
+    }
+    git worktree add $worktree_dir $ref
+    let tbc = (tracy-build-config $features $tracy)
+    let effective_features = $tbc.features
+    let effective_extra_rustflags = $tbc.extra_rustflags
+    let effective_no_cache = $no_cache or ($tracy != "off")
+    if $effective_no_cache {
+        build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags --bench-features $features $worktree_dir $ref $profile $effective_features $ref
+    } else {
+        build-in-worktree $worktree_dir $ref $profile $effective_features $ref
+    }
+
+    let tempo_bin = (worktree-bin $worktree_dir $profile "tempo")
+    let bench_bin = (worktree-bin $worktree_dir $profile "tempo-bench")
+    if $results_dir == "" and ($phase_results_dir | path exists) { rm -rf $phase_results_dir }
+    mkdir $phase_results_dir
+    for stale in [
+        $"($phase_results_dir)/report-($phase).json"
+        $"($phase_results_dir)/profile-($phase).json.gz"
+        $"($phase_results_dir)/profile-($phase)-($role).json.gz"
+        $"($phase_results_dir)/tracy-profile-($phase).tracy"
+        $"($phase_results_dir)/tracy-profile-($phase)-($role).tracy"
+        $"($phase_results_dir)/logs-($phase)-($role)"
+    ] {
+        if ($stale | path exists) { rm -rf $stale }
+    }
+    if ("report.json" | path exists) { rm report.json }
+
+    let log_dir = $"($LOCALNET_DIR)/logs-e2e-($role)-($phase)"
+    if ($log_dir | path exists) { rm -rf $log_dir }
+    mkdir $log_dir
+
+    let tracing_url = if $tracing_otlp == "" and ($env.GRAFANA_TEMPO? | default "" | str length) > 0 {
+        let base = ($env.GRAFANA_TEMPO | str trim --right --char '/')
+        $"($base)/v1/traces"
+    } else if $tracing_otlp == "" and ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
+        let base = ($env.TEMPO_TELEMETRY_URL | str trim --right --char '/')
+        $"($base)/opentelemetry/v1/traces"
+    } else {
+        $tracing_otlp
+    }
+
+    let node_index = (port-to-node-index $consensus_port)
+    let http_port = 8545 + $node_index
+    let reth_metrics_port = 9001 + $node_index
+    let rpc_url = $"http://localhost:($http_port)"
+
+    let base_args = (build-base-args $genesis $node_dir $log_dir "0.0.0.0" $http_port $reth_metrics_port)
+        | append (build-e2e-consensus-args $node_dir $trusted_peers $consensus_port $consensus_ip)
+        | append (log-filter-args $loud)
+        | append (if $gas_limit != "" { ["--builder.gaslimit" $gas_limit] } else { [] })
+        | append (if $tracy != "off" { ["--log.tracy" "--log.tracy.filter" $tracy_filter] } else { [] })
+        | append (if $tracing_url != "" { [$"--tracing-otlp=($tracing_url)"] } else { [] })
+    let args = (dedup-args $base_args $extra_args)
+
+    let tracy_env_prefix = if $tracy == "on" {
+        "TRACY_NO_SYS_TRACE=1 "
+    } else if $tracy == "full" {
+        "TRACY_SAMPLING_HZ=1 "
+    } else { "" }
+    let profile_label = $"($phase)-($role)"
+    let full_samply_args = if $samply {
+        $samply_args | append ["--save-only" "--presymbolicate" "--output" $"($phase_results_dir)/profile-($profile_label).json.gz"]
+    } else { [] }
+    let node_cmd = wrap-samply [$tempo_bin ...$args] $samply $full_samply_args
+    let node_cmd_str = ($node_cmd | str join " ")
+    let otel_attrs = $"OTEL_RESOURCE_ATTRIBUTES=benchmark_id=($benchmark_id),benchmark_run=($phase),runner_role=($role),run_type=($run_type),git_ref=($ref),reference_epoch=($reference_epoch) "
+    let env_prefix = if $side_env != "" { $"($side_env) " } else { "" }
+
+    print $"Starting e2e validator ($role) for ($phase) at ($rpc_url)"
+    job spawn { sh -c $"($env_prefix)($otel_attrs)($tracy_env_prefix)($node_cmd_str) 2>&1" | lines | each { |line| print $"[e2e-($phase)-($role)] ($line)" } }
+
+    sleep 2sec
+    let rpc_timeout = if $bloat > 0 { 600 } else { 300 }
+    wait-for-rpc-online $rpc_url $rpc_timeout
+    wait-for-peers $rpc_url 1 300
+    wait-for-chain-advance $rpc_url 300
+
+    let tracy_output = $"($phase_results_dir)/tracy-profile-($profile_label).tracy"
+    mut tracy_capture_started = false
+    if $tracy != "off" {
+        let seconds_flag = if $tracy_seconds > 0 { $"-s ($tracy_seconds)" } else { "" }
+        let limit_msg = if $tracy_seconds > 0 { $" \(($tracy_seconds)s limit\)" } else { "" }
+        if $tracy_offset > 0 {
+            print $"  Tracy-capture will start in ($tracy_offset)s($limit_msg)..."
+            job spawn { sleep ($"($tracy_offset)sec" | into duration); sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
+        } else {
+            print $"  Starting tracy-capture($limit_msg)..."
+            job spawn { sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
+            sleep 500ms
+        }
+        $tracy_capture_started = true
+    }
+
+    mut phase_exit = 0
+    if $role == "red" {
+        wait-for-rpc-online $peer_url 300
+        let bench_cmd = [
+            $bench_bin
+            "run-max-tps"
+            "--tps" $"($tps)"
+            "--duration" $"($duration)"
+            "--accounts" $"($accounts)"
+            "--max-concurrent-requests" $"($max_concurrent_requests)"
+            "--target-urls" $"($rpc_url),($peer_url)"
+            "--faucet"
+            "--clear-txpool"
+        ]
+        | append (if $preset != "" {
+            [
+                "--tip20-weight" $"($weights | get 0)"
+                "--erc20-weight" $"($weights | get 1)"
+                "--swap-weight" $"($weights | get 2)"
+                "--place-order-weight" $"($weights | get 3)"
+            ]
+        } else { [] })
+        | append (if $bloat > 0 {
+            ["--mnemonic" $"'($BLOAT_MNEMONIC)'"]
+        } else { [] })
+        | append (if $bench_args != "" { $bench_args | split row " " } else { [] })
+        | append ["--node-commit-sha" $ref "--build-profile" $profile "--benchmark-mode" "e2e"]
+
+        let bench_env_export = if $bench_env != "" { $"export ($bench_env) && " } else { "" }
+        print $"Running e2e sender: ($bench_cmd | str join ' ')"
+        let bench_result = (bash -c $"($bench_env_export)ulimit -Sn unlimited && ($bench_cmd | str join ' ')" | complete)
+        if $bench_result.stdout != "" { print $bench_result.stdout }
+        if $bench_result.stderr != "" { print $bench_result.stderr }
+        if $bench_result.exit_code != 0 {
+            print $"Sender failed for ($phase) with exit code ($bench_result.exit_code)"
+            $phase_exit = $bench_result.exit_code
+        }
+
+        if ("report.json" | path exists) {
+            cp report.json $"($phase_results_dir)/report-($phase).json"
+            rm report.json
+        } else {
+            print $"ERROR: sender for ($phase) produced no report.json"
+            $phase_exit = 1
+        }
+        if $wait_peer_offline {
+            if $tracy_capture_started {
+                stop-tracy-capture
+                $tracy_capture_started = false
+            }
+            stop-tempo-processes-gracefully
+            if $samply { wait-for-samply-profile }
+            wait-for-rpc-offline $peer_url ($peer_hold_extra + 300)
+        }
+    } else {
+        let hold_seconds = $duration + $peer_hold_extra
+        wait-for-rpc-online $peer_url 300
+        print $"Runner blue is holding validator online until runner red stops \(timeout: ($hold_seconds)s\)..."
+        wait-for-rpc-offline $peer_url $hold_seconds
+    }
+
+    if $tracy_capture_started {
+        stop-tracy-capture
+    }
+    stop-tempo-processes-gracefully
+    if $samply { wait-for-samply-profile }
+    if ($log_dir | path exists) {
+        cp -r $log_dir $"($phase_results_dir)/logs-($phase)-($role)"
+    }
+    try { git worktree remove --force $worktree_dir } catch { }
+    restore-system-tuning $tuning_state
+
+    if $phase_exit != 0 {
+        exit $phase_exit
+    }
+}
+
+# Run the full B-F-F-B e2e sequence on one side of the 2-runner setup.
+def "main bench-consensus" [
+    --role: string                                      # Runner role: red sends load, blue validates only
+    --baseline: string                                  # Baseline git SHA/ref
+    --feature: string                                   # Feature git SHA/ref
+    --peer-url: string                                  # RPC URL for the validator on the other runner
+    --preset: string = ""                               # Preset: tip20, erc20, swap, order, tempo-mix
+    --tps: int = 10000                                  # Target TPS
+    --duration: int = 300                               # Duration in seconds
+    --accounts: int = 1000                              # Number of accounts
+    --max-concurrent-requests: int = 100                # Max concurrent requests
+    --bench-datadir: string = ""                        # Datadir/snapshot path to recover before each phase
+    --bloat: int = 0                                    # State bloat size in MiB
+    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
+    --features: string = $DEFAULT_FEATURES              # Cargo features
+    --samply                                            # Profile validators with samply
+    --samply-args: string = ""                          # Additional samply arguments
+    --tracy: string = "off"                             # Tracy profiling: off, on, full
+    --tracy-filter: string = "debug"                    # Tracy tracing filter level
+    --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
+    --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
+    --node-args: string = ""                            # Additional node args for all phases
+    --baseline-args: string = ""                        # Additional node args for baseline phases
+    --feature-args: string = ""                         # Additional node args for feature phases
+    --bench-args: string = ""                           # Additional tempo-bench args
+    --baseline-env: string = ""                         # Environment vars for baseline node phases
+    --feature-env: string = ""                          # Environment vars for feature node phases
+    --bench-env: string = ""                            # Environment vars for the sender process
+    --benchmark-id: string = ""                         # Shared benchmark identifier
+    --reference-epoch: int = 0                           # Shared timestamp for observability correlation
+    --baseline-name: string = ""                         # Baseline display name for summary
+    --feature-name: string = ""                          # Feature display name for summary
+    --tune                                              # Apply system tuning
+    --loud                                              # Show node debug logs
+    --node-dir: string                                  # Validator node directory for this runner
+    --genesis: string                                   # Shared genesis file path
+    --baseline-genesis: string = ""                     # Baseline genesis for hardfork comparison
+    --feature-genesis: string = ""                      # Feature genesis for hardfork comparison
+    --baseline-node-dir: string = ""                    # Baseline validator dir for hardfork comparison
+    --feature-node-dir: string = ""                     # Feature validator dir for hardfork comparison
+    --baseline-hardfork: string = ""                    # Latest active hardfork for baseline phases
+    --feature-hardfork: string = ""                     # Latest active hardfork for feature phases
+    --trusted-peers: string                             # Comma-separated enode peers for the network
+    --consensus-port: int = 8000                        # Consensus listen port for this validator
+    --consensus-ip: string = ""                         # Optional consensus listen IP override
+    --gas-limit: string = ""                            # Optional builder gas limit override
+    --tracing-otlp: string = ""                         # OTLP endpoint for tracing
+    --no-cache                                          # Skip binary cache
+] {
+    if $role not-in ["red" "blue"] {
+        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
+        exit 1
+    }
+
+    let timestamp = (date now | format date "%Y%m%d-%H%M%S")
+    let results_dir = $"($BENCH_RESULTS_DIR)/($timestamp)"
+    mkdir $results_dir
+    print $"BENCH_RESULTS_DIR=($results_dir)"
+
+    let effective_benchmark_id = if $benchmark_id != "" { $benchmark_id } else { $"bench-e2e-($timestamp)" }
+    let effective_reference_epoch = if $reference_epoch != 0 {
+        $reference_epoch
+    } else {
+        ((date now | into int) / 1_000_000_000 | into int)
+    }
+    if $tracy not-in ["off" "on" "full"] {
+        print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
+        exit 1
+    }
+    if $samply and $tracy != "off" {
+        print "Error: --samply and --tracy are mutually exclusive. Choose one."
+        exit 1
+    }
+    if ($baseline_hardfork != "" or $feature_hardfork != "") and ($baseline_hardfork == "" or $feature_hardfork == "") {
+        print "Error: --baseline-hardfork and --feature-hardfork must both be provided"
+        exit 1
+    }
+    if $baseline_hardfork != "" {
+        let valid = ($TEMPO_HARDFORKS | any { |f| $f == ($baseline_hardfork | str upcase) })
+        if not $valid {
+            print $"Error: unknown baseline hardfork '($baseline_hardfork)'. Valid: ($TEMPO_HARDFORKS | str join ', ')"
+            exit 1
+        }
+    }
+    if $feature_hardfork != "" {
+        let valid = ($TEMPO_HARDFORKS | any { |f| $f == ($feature_hardfork | str upcase) })
+        if not $valid {
+            print $"Error: unknown feature hardfork '($feature_hardfork)'. Valid: ($TEMPO_HARDFORKS | str join ', ')"
+            exit 1
+        }
+    }
+    let dual_hardfork = $baseline_hardfork != "" and $feature_hardfork != ""
+    if $tracy == "full" and (^uname | str trim) == "Linux" {
+        print "Configuring system for tracy CPU sampling..."
+        try { sudo sysctl -w kernel.perf_event_paranoid=-1 } catch { }
+        try { sudo mount -t tracefs tracefs /sys/kernel/tracing -o remount,mode=755 } catch { }
+        try { sudo chmod -R a+r /sys/kernel/tracing } catch { }
+    }
+
+    let env_baseline_genesis = ($env.BENCH_E2E_BASELINE_GENESIS? | default "")
+    let env_feature_genesis = ($env.BENCH_E2E_FEATURE_GENESIS? | default "")
+    let env_baseline_node_dir = ($env.BENCH_E2E_BASELINE_NODE_DIR? | default "")
+    let env_feature_node_dir = ($env.BENCH_E2E_FEATURE_NODE_DIR? | default "")
+    let genesis_dir = ($genesis | path dirname)
+    let baseline_genesis_candidate = $"($genesis_dir)/genesis-baseline.json"
+    let feature_genesis_candidate = $"($genesis_dir)/genesis-feature.json"
+    let effective_baseline_genesis = if $baseline_genesis != "" {
+        $baseline_genesis
+    } else if $env_baseline_genesis != "" {
+        $env_baseline_genesis
+    } else if ($baseline_genesis_candidate | path exists) {
+        $baseline_genesis_candidate
+    } else {
+        $genesis
+    }
+    let effective_feature_genesis = if $feature_genesis != "" {
+        $feature_genesis
+    } else if $env_feature_genesis != "" {
+        $env_feature_genesis
+    } else if ($feature_genesis_candidate | path exists) {
+        $feature_genesis_candidate
+    } else {
+        $genesis
+    }
+    let effective_baseline_node_dir = if $baseline_node_dir != "" {
+        $baseline_node_dir
+    } else if $env_baseline_node_dir != "" {
+        $env_baseline_node_dir
+    } else {
+        $node_dir
+    }
+    let effective_feature_node_dir = if $feature_node_dir != "" {
+        $feature_node_dir
+    } else if $env_feature_node_dir != "" {
+        $env_feature_node_dir
+    } else {
+        $node_dir
+    }
+    if $dual_hardfork {
+        if $effective_baseline_genesis == $genesis or $effective_feature_genesis == $genesis {
+            print "Error: hardfork comparison requires phase-specific e2e genesis files. Set BENCH_E2E_BASELINE_GENESIS and BENCH_E2E_FEATURE_GENESIS, or place genesis-baseline.json and genesis-feature.json next to BENCH_E2E_GENESIS on both runners."
+            exit 1
+        }
+    }
+    for phase_genesis in [$effective_baseline_genesis $effective_feature_genesis] {
+        if not ($phase_genesis | path exists) {
+            print $"Error: e2e genesis file does not exist: ($phase_genesis)"
+            exit 1
+        }
+    }
+    for phase_node_dir in [$effective_baseline_node_dir $effective_feature_node_dir] {
+        if not ($phase_node_dir | path exists) {
+            print $"Error: e2e node dir does not exist: ($phase_node_dir)"
+            exit 1
+        }
+    }
+    let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " }
+    let effective_baseline_datadir = if $effective_baseline_node_dir != $node_dir {
+        $effective_baseline_node_dir
+    } else {
+        $bench_datadir
+    }
+    let effective_feature_datadir = if $effective_feature_node_dir != $node_dir {
+        $effective_feature_node_dir
+    } else {
+        $bench_datadir
+    }
+
+    let runs = [
+        { phase: "baseline-1", ref: $baseline, wait_peer: true, genesis: $effective_baseline_genesis, node_dir: $effective_baseline_node_dir, bench_datadir: $effective_baseline_datadir }
+        { phase: "feature-1", ref: $feature, wait_peer: true, genesis: $effective_feature_genesis, node_dir: $effective_feature_node_dir, bench_datadir: $effective_feature_datadir }
+        { phase: "feature-2", ref: $feature, wait_peer: true, genesis: $effective_feature_genesis, node_dir: $effective_feature_node_dir, bench_datadir: $effective_feature_datadir }
+        { phase: "baseline-2", ref: $baseline, wait_peer: false, genesis: $effective_baseline_genesis, node_dir: $effective_baseline_node_dir, bench_datadir: $effective_baseline_datadir }
+    ]
+
+    for run in $runs {
+        (main bench-consensus-phase
+            --role $role
+            --phase $run.phase
+            --ref $run.ref
+            --peer-url $peer_url
+            --preset $preset
+            --tps $tps
+            --duration $duration
+            --accounts $accounts
+            --max-concurrent-requests $max_concurrent_requests
+            --bench-datadir $run.bench_datadir
+            --bloat $bloat
+            --profile $profile
+            --features $features
+            --samply=$samply
+            --samply-args $samply_args_list
+            --tracy $tracy
+            --tracy-filter $tracy_filter
+            --tracy-seconds $tracy_seconds
+            --tracy-offset $tracy_offset
+            --node-args $node_args
+            --baseline-args $baseline_args
+            --feature-args $feature_args
+            --bench-args $bench_args
+            --baseline-env $baseline_env
+            --feature-env $feature_env
+            --bench-env $bench_env
+            --benchmark-id $effective_benchmark_id
+            --reference-epoch $effective_reference_epoch
+            --tune=$tune
+            --loud=$loud
+            --node-dir $run.node_dir
+            --genesis $run.genesis
+            --trusted-peers $trusted_peers
+            --consensus-port $consensus_port
+            --consensus-ip $consensus_ip
+            --results-dir $results_dir
+            --gas-limit $gas_limit
+            --tracing-otlp $tracing_otlp
+            --peer-hold-extra 600
+            --wait-peer-offline=$run.wait_peer
+            --no-cache=$no_cache)
+    }
+
+    if $samply {
+        print $"\nUploading ($role) samply profiles to Firefox Profiler..."
+        for run in $runs {
+            let profile_label = $"($run.phase)-($role)"
+            let profile = $"($results_dir)/profile-($profile_label).json.gz"
+            let url = (upload-samply-profile $profile)
+            if $url != null {
+                $url | save -f $"($results_dir)/profile-($profile_label)-url.txt"
+            }
+        }
+    }
+    if $tracy != "off" {
+        print $"\nUploading ($role) tracy profiles to R2..."
+        for run in $runs {
+            let profile_label = $"($run.phase)-($role)"
+            let profile = $"($results_dir)/tracy-profile-($profile_label).tracy"
+            let viewer_url = (upload-tracy-profile $profile $profile_label $run.ref)
+            if $viewer_url != null {
+                $viewer_url | save -f $"($results_dir)/tracy-($profile_label)-url.txt"
+            }
+        }
+    }
+
+    if $role == "red" {
+        let baseline_label_base = if $baseline_name != "" { $baseline_name } else { $baseline }
+        let feature_label_base = if $feature_name != "" { $feature_name } else { $feature }
+        let baseline_label = if $dual_hardfork { $"($baseline_label_base) \(($baseline_hardfork | str upcase)\)" } else { $baseline_label_base }
+        let feature_label = if $dual_hardfork { $"($feature_label_base) \(($feature_hardfork | str upcase)\)" } else { $feature_label_base }
+        generate-summary $results_dir $baseline_label $feature_label $bloat $preset $tps $duration --benchmark-id $effective_benchmark_id --reference-epoch $effective_reference_epoch
+    }
+}
+
 # ============================================================================
 # Bench command
 # ============================================================================
@@ -2203,10 +2827,43 @@ def "main bench" [
     print "Done."
 }
 
-# Wait for an RPC endpoint to be ready and chain advancing
-def wait-for-rpc [url: string, max_attempts: int = 120] {
+# Fetch the current block number from an RPC endpoint.
+def rpc-block-number [url: string] {
+    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' } | complete)
+    if $result.exit_code != 0 {
+        return null
+    }
+    let parsed = (try { $result.stdout | from json } catch { null })
+    if $parsed == null {
+        return null
+    }
+    let hex = ($parsed | get -o result | default "")
+    if $hex == "" {
+        return null
+    }
+    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
+}
+
+# Fetch the current peer count from an RPC endpoint.
+def rpc-peer-count [url: string] {
+    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' } | complete)
+    if $result.exit_code != 0 {
+        return null
+    }
+    let parsed = (try { $result.stdout | from json } catch { null })
+    if $parsed == null {
+        return null
+    }
+    let hex = ($parsed | get -o result | default "")
+    if $hex == "" {
+        return null
+    }
+    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
+}
+
+# Wait for an RPC endpoint to answer eth_blockNumber.
+def wait-for-rpc-online [url: string, max_attempts: int = 120] {
     mut attempt = 0
-    mut start_block: int = -1
 
     loop {
         $attempt = $attempt + 1
@@ -2214,30 +2871,99 @@ def wait-for-rpc [url: string, max_attempts: int = 120] {
             print $"  Timeout waiting for ($url)"
             exit 1
         }
-        let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' } | complete)
-        if $result.exit_code == 0 {
-            let hex = ($result.stdout | from json | get result)
-            let block = ($hex | str replace "0x" "" | into int --radix 16)
+        let block = (rpc-block-number $url)
+        if $block != null {
+            print $"  ($url) online \(block ($block)\)"
+            break
+        } else {
+            if ($attempt mod 10) == 0 {
+                print $"  Still waiting for ($url)... \(($attempt)s\)"
+            }
+        }
+        sleep 1sec
+    }
+}
+
+# Wait for an RPC endpoint to stop answering eth_blockNumber.
+def wait-for-rpc-offline [url: string, max_attempts: int = 120] {
+    mut attempt = 0
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url) to go offline"
+            exit 1
+        }
+        let block = (rpc-block-number $url)
+        if $block == null {
+            print $"  ($url) offline"
+            break
+        }
+        if ($attempt mod 10) == 0 {
+            print $"  Waiting for ($url) to go offline... \(($attempt)s\)"
+        }
+        sleep 1sec
+    }
+}
+
+# Wait for an RPC endpoint to see at least the requested number of peers.
+def wait-for-peers [url: string, min_peers: int = 1, max_attempts: int = 120] {
+    mut attempt = 0
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url) to reach ($min_peers) peer\(s\)"
+            exit 1
+        }
+        let peers = (rpc-peer-count $url)
+        if $peers != null and $peers >= $min_peers {
+            print $"  ($url) has ($peers) peer\(s\)"
+            break
+        }
+        if ($attempt mod 10) == 0 {
+            let current = if $peers == null { "unknown" } else { $"($peers)" }
+            print $"  ($url) peers: ($current)/($min_peers)... \(($attempt)s\)"
+        }
+        sleep 1sec
+    }
+}
+
+# Wait for an RPC endpoint's chain to advance beyond its first observed block.
+def wait-for-chain-advance [url: string, max_attempts: int = 120] {
+    mut attempt = 0
+    mut start_block: int = -1
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url) chain to advance"
+            exit 1
+        }
+        let block = (rpc-block-number $url)
+        if $block != null {
             if $start_block == -1 {
                 $start_block = $block
                 print $"  ($url) connected \(block ($block)\), waiting for chain to advance..."
             } else if $block > $start_block {
                 print $"  ($url) ready \(block ($start_block) -> ($block)\)"
                 break
-            } else {
-                if ($attempt mod 10) == 0 {
-                    print $"  ($url) still at block ($block)... \(($attempt)s\)"
-                }
-            }
-        } else {
-            if ($attempt mod 10) == 0 {
-                print $"  Still waiting for ($url)... \(($attempt)s\)"
+            } else if ($attempt mod 10) == 0 {
+                print $"  ($url) still at block ($block)... \(($attempt)s\)"
             }
+        } else if ($attempt mod 10) == 0 {
+            print $"  Still waiting for ($url)... \(($attempt)s\)"
         }
         sleep 1sec
     }
 }
 
+# Wait for an RPC endpoint to be ready and chain advancing.
+def wait-for-rpc [url: string, max_attempts: int = 120] {
+    wait-for-rpc-online $url $max_attempts
+    wait-for-chain-advance $url $max_attempts
+}
+
 # ============================================================================
 # Coverage commands
 # ============================================================================

From 16c71ae77d40db9d4208fe83d3f5880fdfb903e7 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 5 May 2026 17:35:43 +0100
Subject: [PATCH 02/16] ci(bench): split e2e datadir and validator dirs

---
 .github/workflows/bench-e2e.yml |  4 +-
 tempo.nu                        | 79 ++++++++++++++++-----------------
 2 files changed, 41 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index eba2cea690..bfe44705ff 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -553,7 +553,7 @@ jobs:
             --baseline-name "${{ steps.refs.outputs.baseline-name }}"
             --feature-name "${{ steps.refs.outputs.feature-name }}"
             --peer-url "$BENCH_E2E_PEER_URL"
-            --bench-datadir "$BENCH_E2E_NODE_DIR"
+            --bench-datadir "${BENCH_E2E_DATADIR:-$BENCH_E2E_NODE_DIR}"
             --node-dir "$BENCH_E2E_NODE_DIR"
             --genesis "$BENCH_E2E_GENESIS"
             --trusted-peers "$BENCH_E2E_TRUSTED_PEERS"
@@ -571,6 +571,8 @@ jobs:
           [ -n "${BENCH_E2E_FEATURE_GENESIS:-}" ] && cmd+=(--feature-genesis "$BENCH_E2E_FEATURE_GENESIS")
           [ -n "${BENCH_E2E_BASELINE_NODE_DIR:-}" ] && cmd+=(--baseline-node-dir "$BENCH_E2E_BASELINE_NODE_DIR")
           [ -n "${BENCH_E2E_FEATURE_NODE_DIR:-}" ] && cmd+=(--feature-node-dir "$BENCH_E2E_FEATURE_NODE_DIR")
+          [ -n "${BENCH_E2E_BASELINE_DATADIR:-}" ] && cmd+=(--baseline-bench-datadir "$BENCH_E2E_BASELINE_DATADIR")
+          [ -n "${BENCH_E2E_FEATURE_DATADIR:-}" ] && cmd+=(--feature-bench-datadir "$BENCH_E2E_FEATURE_DATADIR")
           [ -n "$BENCH_BASELINE_ARGS" ] && cmd+=(--baseline-args="$BENCH_BASELINE_ARGS")
           [ -n "$BENCH_FEATURE_ARGS" ] && cmd+=(--feature-args="$BENCH_FEATURE_ARGS")
           [ -n "$BENCH_BENCH_ARGS" ] && cmd+=(--bench-args="$BENCH_BENCH_ARGS")
diff --git a/tempo.nu b/tempo.nu
index 9403ebc9d8..1bd205b549 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -1657,7 +1657,7 @@ def "main bench-consensus-phase" [
     --duration: int = 300                               # Duration in seconds
     --accounts: int = 1000                              # Number of accounts
     --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bench-datadir: string = ""                        # Datadir/snapshot path to recover before the phase
+    --bench-datadir: string = ""                        # Datadir/snapshot path to recover and pass to the node
     --bloat: int = 0                                    # State bloat size in MiB; enables bloat mnemonic for the sender
     --profile: string = $DEFAULT_PROFILE                # Cargo build profile
     --features: string = $DEFAULT_FEATURES              # Cargo features
@@ -1678,7 +1678,7 @@ def "main bench-consensus-phase" [
     --reference-epoch: int = 0                           # Shared timestamp for observability correlation
     --tune                                              # Apply system tuning
     --loud                                              # Show node debug logs
-    --node-dir: string                                  # Validator node directory for this runner
+    --node-dir: string                                  # Validator identity directory for this runner
     --genesis: string                                   # Shared genesis file path
     --trusted-peers: string                             # Comma-separated enode peers for the network
     --consensus-port: int = 8000                        # Consensus listen port for this validator
@@ -1718,22 +1718,6 @@ def "main bench-consensus-phase" [
             exit 1
         }
     }
-    if not ($node_dir | path exists) {
-        print $"Error: node dir does not exist: ($node_dir)"
-        exit 1
-    }
-    if not ($genesis | path exists) {
-        print $"Error: genesis file does not exist: ($genesis)"
-        exit 1
-    }
-    for required_file in ["signing.key" "signing.share" "enode.key"] {
-        let path = $"($node_dir)/($required_file)"
-        if not ($path | path exists) {
-            print $"Error: missing validator file: ($path)"
-            exit 1
-        }
-    }
-
     let run_type = if ($phase | str starts-with "baseline") { "baseline" } else { "feature" }
     let side_args = if $run_type == "baseline" { $baseline_args } else { $feature_args }
     let side_env = if $run_type == "baseline" { $baseline_env } else { $feature_env }
@@ -1747,6 +1731,22 @@ def "main bench-consensus-phase" [
     let tuning_state = if $tune { apply-system-tuning } else { { tuned: false } }
     bench-recover $datadir
 
+    if not ($node_dir | path exists) {
+        print $"Error: node dir does not exist after snapshot recovery: ($node_dir)"
+        exit 1
+    }
+    if not ($genesis | path exists) {
+        print $"Error: genesis file does not exist after snapshot recovery: ($genesis)"
+        exit 1
+    }
+    for required_file in ["signing.key" "signing.share" "enode.key"] {
+        let path = $"($node_dir)/($required_file)"
+        if not ($path | path exists) {
+            print $"Error: missing validator file after snapshot recovery: ($path)"
+            exit 1
+        }
+    }
+
     let worktree_dir = $"($BENCH_WORKTREES_DIR)/e2e-($role)"
     git worktree prune
     mkdir $BENCH_WORKTREES_DIR
@@ -1800,7 +1800,7 @@ def "main bench-consensus-phase" [
     let reth_metrics_port = 9001 + $node_index
     let rpc_url = $"http://localhost:($http_port)"
 
-    let base_args = (build-base-args $genesis $node_dir $log_dir "0.0.0.0" $http_port $reth_metrics_port)
+    let base_args = (build-base-args $genesis $datadir $log_dir "0.0.0.0" $http_port $reth_metrics_port)
         | append (build-e2e-consensus-args $node_dir $trusted_peers $consensus_port $consensus_ip)
         | append (log-filter-args $loud)
         | append (if $gas_limit != "" { ["--builder.gaslimit" $gas_limit] } else { [] })
@@ -1935,7 +1935,7 @@ def "main bench-consensus" [
     --duration: int = 300                               # Duration in seconds
     --accounts: int = 1000                              # Number of accounts
     --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bench-datadir: string = ""                        # Datadir/snapshot path to recover before each phase
+    --bench-datadir: string = ""                        # Datadir/snapshot path to recover and pass to the node
     --bloat: int = 0                                    # State bloat size in MiB
     --profile: string = $DEFAULT_PROFILE                # Cargo build profile
     --features: string = $DEFAULT_FEATURES              # Cargo features
@@ -1958,12 +1958,14 @@ def "main bench-consensus" [
     --feature-name: string = ""                          # Feature display name for summary
     --tune                                              # Apply system tuning
     --loud                                              # Show node debug logs
-    --node-dir: string                                  # Validator node directory for this runner
+    --node-dir: string                                  # Validator identity directory for this runner
     --genesis: string                                   # Shared genesis file path
     --baseline-genesis: string = ""                     # Baseline genesis for hardfork comparison
     --feature-genesis: string = ""                      # Feature genesis for hardfork comparison
-    --baseline-node-dir: string = ""                    # Baseline validator dir for hardfork comparison
-    --feature-node-dir: string = ""                     # Feature validator dir for hardfork comparison
+    --baseline-node-dir: string = ""                    # Baseline validator identity dir for hardfork comparison
+    --feature-node-dir: string = ""                     # Feature validator identity dir for hardfork comparison
+    --baseline-bench-datadir: string = ""               # Baseline datadir/snapshot path for hardfork comparison
+    --feature-bench-datadir: string = ""                # Feature datadir/snapshot path for hardfork comparison
     --baseline-hardfork: string = ""                    # Latest active hardfork for baseline phases
     --feature-hardfork: string = ""                     # Latest active hardfork for feature phases
     --trusted-peers: string                             # Comma-separated enode peers for the network
@@ -2027,6 +2029,8 @@ def "main bench-consensus" [
     let env_feature_genesis = ($env.BENCH_E2E_FEATURE_GENESIS? | default "")
     let env_baseline_node_dir = ($env.BENCH_E2E_BASELINE_NODE_DIR? | default "")
     let env_feature_node_dir = ($env.BENCH_E2E_FEATURE_NODE_DIR? | default "")
+    let env_baseline_datadir = ($env.BENCH_E2E_BASELINE_DATADIR? | default "")
+    let env_feature_datadir = ($env.BENCH_E2E_FEATURE_DATADIR? | default "")
     let genesis_dir = ($genesis | path dirname)
     let baseline_genesis_candidate = $"($genesis_dir)/genesis-baseline.json"
     let feature_genesis_candidate = $"($genesis_dir)/genesis-feature.json"
@@ -2068,28 +2072,21 @@ def "main bench-consensus" [
             exit 1
         }
     }
-    for phase_genesis in [$effective_baseline_genesis $effective_feature_genesis] {
-        if not ($phase_genesis | path exists) {
-            print $"Error: e2e genesis file does not exist: ($phase_genesis)"
-            exit 1
-        }
-    }
-    for phase_node_dir in [$effective_baseline_node_dir $effective_feature_node_dir] {
-        if not ($phase_node_dir | path exists) {
-            print $"Error: e2e node dir does not exist: ($phase_node_dir)"
-            exit 1
-        }
-    }
     let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " }
-    let effective_baseline_datadir = if $effective_baseline_node_dir != $node_dir {
-        $effective_baseline_node_dir
+    let effective_bench_datadir = if $bench_datadir != "" { $bench_datadir } else { $node_dir }
+    let effective_baseline_datadir = if $baseline_bench_datadir != "" {
+        $baseline_bench_datadir
+    } else if $env_baseline_datadir != "" {
+        $env_baseline_datadir
     } else {
-        $bench_datadir
+        $effective_bench_datadir
     }
-    let effective_feature_datadir = if $effective_feature_node_dir != $node_dir {
-        $effective_feature_node_dir
+    let effective_feature_datadir = if $feature_bench_datadir != "" {
+        $feature_bench_datadir
+    } else if $env_feature_datadir != "" {
+        $env_feature_datadir
     } else {
-        $bench_datadir
+        $effective_bench_datadir
     }
 
     let runs = [

From c4865086de4f0f56dcc543f2fa2d32f2041ceb9f Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Tue, 5 May 2026 18:01:18 +0100
Subject: [PATCH 03/16] ci(bench): initialize e2e bloat snapshots

---
 .github/workflows/bench-e2e.yml |  89 +++++++++++++++++++++-
 .github/workflows/bench.yml     |   1 -
 tempo.nu                        | 130 ++++++++++++++++++++++++++++++--
 3 files changed, 210 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index bfe44705ff..ca4d7f1f2a 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -237,6 +237,7 @@ env:
   RUSTC_WRAPPER: "sccache"
 
 permissions:
+  actions: read
   contents: read
   pull-requests: write
 
@@ -411,9 +412,95 @@ jobs:
             exit 1
           fi
           if [ "$BENCH_FORCE_BLOAT" = "true" ]; then
-            echo "::error::mode=e2e does not support force-bloat yet because both runners must use coordinated benchmark snapshots."
+            if [ -n "$BENCH_BASELINE_HARDFORK" ] || [ -n "$BENCH_FEATURE_HARDFORK" ]; then
+              echo "::error::mode=e2e force-bloat does not support hardfork comparison yet; use pre-provisioned phase snapshots for hardfork runs."
+              exit 1
+            fi
+          fi
+
+      - name: Refresh e2e bloat snapshot
+        if: env.BENCH_FORCE_BLOAT == 'true'
+        run: |
+          validators="${BENCH_E2E_VALIDATORS:-${{ vars.BENCH_E2E_VALIDATORS }}}"
+          seed="${BENCH_E2E_SEED:-${{ vars.BENCH_E2E_SEED }}}"
+          seed="${seed:-42}"
+          bench_datadir="${BENCH_E2E_DATADIR:-$BENCH_E2E_NODE_DIR}"
+
+          if [ -z "$validators" ]; then
+            echo "::error::force-bloat requires BENCH_E2E_VALIDATORS ordered as red_ip:port,blue_ip:port."
+            exit 1
+          fi
+          if [ -z "$bench_datadir" ]; then
+            echo "::error::force-bloat requires BENCH_E2E_DATADIR or BENCH_E2E_NODE_DIR."
+            exit 1
+          fi
+          if [ -z "$BENCH_E2E_NODE_DIR" ]; then
+            echo "::error::force-bloat requires BENCH_E2E_NODE_DIR for this runner's validator identity files."
+            exit 1
+          fi
+          if [ -z "$BENCH_E2E_GENESIS" ]; then
+            echo "::error::force-bloat requires BENCH_E2E_GENESIS."
+            exit 1
+          fi
+
+          nu tempo.nu bench-consensus-init \
+            --role "$BENCH_ROLE" \
+            --validators "$validators" \
+            --bench-datadir "$bench_datadir" \
+            --node-dir "$BENCH_E2E_NODE_DIR" \
+            --genesis "$BENCH_E2E_GENESIS" \
+            --bloat "$BENCH_BLOAT" \
+            --seed "$seed" \
+            --gas-limit 1000000000000
+
+          trusted_peers_file="$bench_datadir/.bench-meta/trusted-peers.txt"
+          if [ ! -s "$trusted_peers_file" ]; then
+            echo "::error::force-bloat did not produce $trusted_peers_file."
             exit 1
           fi
+          echo "BENCH_E2E_DATADIR=$bench_datadir" >> "$GITHUB_ENV"
+          echo "BENCH_E2E_TRUSTED_PEERS=$(cat "$trusted_peers_file")" >> "$GITHUB_ENV"
+          mkdir -p .bench-e2e-init
+          {
+            echo "role=$BENCH_ROLE"
+            echo "datadir=$bench_datadir"
+            echo "node_dir=$BENCH_E2E_NODE_DIR"
+            echo "genesis=$BENCH_E2E_GENESIS"
+            echo "trusted_peers_file=$trusted_peers_file"
+          } > ".bench-e2e-init/$BENCH_ROLE.txt"
+
+      - name: Upload e2e init marker
+        if: env.BENCH_FORCE_BLOAT == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: tempo-bench-e2e-init-${{ matrix.role }}
+          path: .bench-e2e-init/${{ matrix.role }}.txt
+          retention-days: 1
+
+      - name: Wait for e2e init markers
+        if: env.BENCH_FORCE_BLOAT == 'true'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const wanted = new Set(['tempo-bench-e2e-init-red', 'tempo-bench-e2e-init-blue']);
+            const deadline = Date.now() + 60 * 60 * 1000;
+            while (Date.now() < deadline) {
+              const { data } = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+                per_page: 100,
+              });
+              const names = new Set(data.artifacts.map(a => a.name));
+              const missing = [...wanted].filter(name => !names.has(name));
+              if (missing.length === 0) {
+                core.info('Both e2e init markers are present.');
+                return;
+              }
+              core.info(`Waiting for e2e init markers: ${missing.join(', ')}`);
+              await new Promise(resolve => setTimeout(resolve, 30_000));
+            }
+            core.setFailed('Timed out waiting for both e2e init markers.');
 
       - name: Build txgen backend
         if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen'
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 3353977ab4..febd4eecfd 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -242,7 +242,6 @@ jobs:
             if (mode === 'e2e') {
               const unsupported = [];
               if (backend !== 'tempo-bench') unsupported.push(`backend=${backend}`);
-              if (forceBloat === 'true') unsupported.push('force-bloat');
               if (unsupported.length > 0) {
                 const msg = `❌ **Invalid bench command**\n\n\`mode=e2e\` does not support ${unsupported.map(s => `\`${s}\``).join(', ')} yet. The e2e harness currently uses \`backend=tempo-bench\`; \`backend=txgen\` will be wired in the txgen bench PR.`;
                 await github.rest.issues.createComment({
diff --git a/tempo.nu b/tempo.nu
index 1bd205b549..dacb7accd4 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -275,6 +275,23 @@ def read-bench-marker [datadir: string] {
     }
 }
 
+def validator-dirs-in-localnet [localnet_dir: string] {
+    ls $localnet_dir
+    | where type == "dir"
+    | get name
+    | where { |d| ($d | path basename) =~ '^\d+\.\d+\.\d+\.\d+:\d+$' }
+}
+
+def trusted-peers-from-localnet [localnet_dir: string] {
+    validator-dirs-in-localnet $localnet_dir | each { |d|
+        let addr = ($d | path basename)
+        let ip = ($addr | split row ":" | get 0)
+        let port = ($addr | split row ":" | get 1 | into int)
+        let identity = (open $"($d)/enode.identity" | str trim)
+        $"enode://($identity)@($ip):($port + 1)"
+    } | str join ","
+}
+
 # ============================================================================
 # Comparison mode helpers
 # ============================================================================
@@ -1265,14 +1282,8 @@ def run-consensus-nodes [nodes: int, accounts: int, genesis: string, samply: boo
     let genesis_path = if $genesis != "" { $genesis } else { $"($LOCALNET_DIR)/genesis.json" }
 
     # Build trusted peers from enode.identity files
-    let validator_dirs = (ls $LOCALNET_DIR | where type == "dir" | get name | where { |d| ($d | path basename) =~ '^\d+\.\d+\.\d+\.\d+:\d+$' })
-    let trusted_peers = ($validator_dirs | each { |d|
-        let addr = ($d | path basename)
-        let ip = ($addr | split row ":" | get 0)
-        let port = ($addr | split row ":" | get 1 | into int)
-        let identity = (open $"($d)/enode.identity" | str trim)
-        $"enode://($identity)@($ip):($port + 1)"
-    } | str join ",")
+    let validator_dirs = (validator-dirs-in-localnet $LOCALNET_DIR)
+    let trusted_peers = (trusted-peers-from-localnet $LOCALNET_DIR)
 
     print $"Found ($validator_dirs | length) validator configs"
 
@@ -1646,6 +1657,109 @@ def "main bench-init" [
     print $"Virgin snapshot initialized and promoted."
 }
 
+# Initialize the schelk virgin snapshot for the 2-runner e2e consensus bench.
+#
+# This is intended for CI `force-bloat` runs. Both runners call it with the
+# same ordered validator list so they generate identical genesis/trusted-peer
+# data, then each runner copies only its role's validator identity files.
+def "main bench-consensus-init" [
+    --role: string                                      # Runner role: red uses validator 0, blue uses validator 1
+    --validators: string                                # Ordered validator consensus addresses: red_ip:port,blue_ip:port
+    --bench-datadir: string                             # Datadir/snapshot path to initialize and promote
+    --node-dir: string                                  # Validator identity directory for this runner
+    --genesis: string                                   # Destination genesis path for benchmark runs
+    --bloat: int = 0                                    # State bloat size in MiB
+    --accounts: int = 1000                              # Number of benchmark sender accounts
+    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
+    --features: string = $DEFAULT_FEATURES              # Cargo features
+    --seed: int = 42                                    # Deterministic seed shared by both runners
+    --gas-limit: string = ""                            # Optional genesis gas limit override
+] {
+    if $role not-in ["red" "blue"] {
+        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
+        exit 1
+    }
+
+    let validator_list = (
+        $validators
+        | split row ","
+        | each { |v| $v | str trim }
+        | where { |v| $v != "" }
+    )
+    if ($validator_list | length) != 2 {
+        print "Error: --validators must contain exactly two comma-separated consensus addresses ordered as red,blue"
+        exit 1
+    }
+
+    let validator_addr = if $role == "red" {
+        $validator_list | get 0
+    } else {
+        $validator_list | get 1
+    }
+    let validators_arg = ($validator_list | str join ",")
+    let genesis_accounts = ([$accounts 3] | math max) + 1
+    let init_dir = $"($LOCALNET_DIR)/e2e-consensus-init"
+    let generated_genesis = $"($init_dir)/genesis.json"
+    let generated_node_dir = $"($init_dir)/($validator_addr)"
+    let bloat_file = $"($init_dir)/state_bloat.bin"
+    let meta_dir = $"($bench_datadir)/($BENCH_META_SUBDIR)"
+    let generated_trusted_peers = $"($init_dir)/trusted-peers.txt"
+    let gas_limit_args = if $gas_limit != "" { ["--gas-limit" $gas_limit] } else { [] }
+
+    if ($init_dir | path exists) { rm -rf $init_dir }
+    mkdir $init_dir
+
+    build-tempo ["tempo"] $profile $features
+    let tempo_bin = if $profile == "dev" { "./target/debug/tempo" } else { $"./target/($profile)/tempo" }
+
+    print $"Generating e2e localnet config for validators: ($validators_arg)"
+    cargo run -p tempo-xtask --profile $profile -- generate-localnet -o $init_dir --accounts $genesis_accounts --validators $validators_arg --seed $seed --force ...$gas_limit_args
+
+    let trusted_peers = (trusted-peers-from-localnet $init_dir)
+    if $trusted_peers == "" {
+        print "Error: generated localnet did not produce trusted peers"
+        exit 1
+    }
+
+    if $bloat > 0 {
+        print $"Generating e2e state bloat \(($bloat) MiB\)..."
+        let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten)
+        cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args
+    }
+
+    bench-mount
+    bench-clean-datadir $bench_datadir
+    mkdir $bench_datadir
+    mkdir $node_dir
+
+    bench-init-db $tempo_bin $generated_genesis $bench_datadir $bloat $bloat_file
+
+    for file in ["signing.key" "signing.share" "enode.key" "enode.identity"] {
+        cp $"($generated_node_dir)/($file)" $"($node_dir)/($file)"
+    }
+
+    mkdir ($genesis | path dirname)
+    cp $generated_genesis $genesis
+    mkdir $meta_dir
+    $trusted_peers | save -f $generated_trusted_peers
+
+    bench-save-and-promote $bench_datadir $meta_dir {
+        bloat_mib: $bloat
+        accounts: $genesis_accounts
+        bench_datadir: $bench_datadir
+        node_dir: $node_dir
+        validator_role: $role
+        validator_addr: $validator_addr
+        validators: $validators_arg
+        seed: $seed
+        gas_limit: $gas_limit
+        dkg_in_genesis: true
+    } [[$generated_genesis "genesis.json"] [$generated_trusted_peers "trusted-peers.txt"]] 0 ""
+
+    print $"E2E consensus snapshot initialized and promoted for ($role)."
+    print $"Trusted peers: ($trusted_peers)"
+}
+
 # Run one side of a 2-runner e2e consensus benchmark phase.
 def "main bench-consensus-phase" [
     --role: string                                      # Runner role: red sends load, blue validates only

From e6f6f3a2ae8959c3a8fb43d4e23da9beb38fcd12 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:03:50 +0100
Subject: [PATCH 04/16] ci(bench): add single-runner e2e harness

---
 .github/workflows/bench-e2e.yml | 267 ++---------
 .github/workflows/bench.yml     |   6 +-
 bench-e2e.nu                    | 812 ++++++++++++++++++++++++++++++++
 tempo.nu                        |  18 +-
 4 files changed, 869 insertions(+), 234 deletions(-)
 create mode 100644 bench-e2e.nu

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index ca4d7f1f2a..ea234ef910 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -1,7 +1,7 @@
 # E2E benchmark job.
 #
-# Called by bench.yml when mode=e2e. Runs `nu tempo.nu bench-consensus`
-# for an interleaved B-F-F-B comparison using synthetic transactions.
+# Called by bench.yml when mode=e2e. Runs `nu bench-e2e.nu e2e`
+# for an interleaved baseline-feature-feature-baseline comparison using two local validators.
 
 name: bench-e2e
 
@@ -243,21 +243,13 @@ permissions:
 
 jobs:
   bench-e2e:
-    name: bench-e2e-${{ matrix.role }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - role: red
-            runner: bench-a
-          - role: blue
-            runner: bench-b
-    runs-on: [self-hosted, Linux, X64, "${{ matrix.runner }}"]
+    name: bench-e2e
+    runs-on: [self-hosted, Linux, X64, bare-metal-dual-schelk]
     timeout-minutes: 300
     env:
-      BENCH_ROLE: ${{ matrix.role }}
       BENCH_PR: ${{ inputs.pr }}
       BENCH_ACTOR: ${{ inputs.actor || github.actor }}
+      BENCH_MODE: ${{ inputs.mode || 'e2e' }}
       BENCH_PRESET: ${{ inputs.preset }}
       BENCH_DURATION: ${{ inputs.duration }}
       BENCH_BLOAT: ${{ inputs.bloat }}
@@ -297,7 +289,7 @@ jobs:
         run: echo "::add-mask::$GRAFANA_TEMPO"
 
       - name: Mask ClickHouse credentials
-        if: matrix.role == 'red' && env.CLICKHOUSE_URL
+        if: env.CLICKHOUSE_URL
         run: |
           echo "::add-mask::$CLICKHOUSE_URL"
           echo "::add-mask::$CLICKHOUSE_PASSWORD"
@@ -341,7 +333,7 @@ jobs:
           ref: ${{ steps.checkout-ref.outputs.ref }}
 
       - name: Checkout txgen repository
-        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen'
+        if: env.BENCH_BACKEND == 'txgen'
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           repository: tempoxyz/txgen
@@ -351,14 +343,14 @@ jobs:
           fetch-depth: 0
 
       - name: Checkout txgen ref
-        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != ''
+        if: env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != ''
         working-directory: txgen
         run: |
           git fetch origin "$BENCH_TXGEN_REF" --quiet
           git checkout --detach FETCH_HEAD
 
       - name: Resolve job URL and update status
-        if: matrix.role == 'red' && env.BENCH_COMMENT_ID
+        if: env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           INPUT_BASELINE_NAME: ${{ inputs.baseline-name }}
@@ -371,7 +363,7 @@ jobs:
               repo: context.repo.repo,
               run_id: context.runId,
             });
-            const job = jobs.jobs.find(j => j.name === 'bench-e2e-red');
+            const job = jobs.jobs.find(j => j.name === 'bench-e2e');
             const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             core.exportVariable('BENCH_JOB_URL', jobUrl);
 
@@ -411,99 +403,13 @@ jobs:
             echo "::error::mode=e2e currently supports backend=tempo-bench only; backend=txgen will be wired in the txgen bench PR."
             exit 1
           fi
-          if [ "$BENCH_FORCE_BLOAT" = "true" ]; then
-            if [ -n "$BENCH_BASELINE_HARDFORK" ] || [ -n "$BENCH_FEATURE_HARDFORK" ]; then
-              echo "::error::mode=e2e force-bloat does not support hardfork comparison yet; use pre-provisioned phase snapshots for hardfork runs."
-              exit 1
-            fi
-          fi
-
-      - name: Refresh e2e bloat snapshot
-        if: env.BENCH_FORCE_BLOAT == 'true'
-        run: |
-          validators="${BENCH_E2E_VALIDATORS:-${{ vars.BENCH_E2E_VALIDATORS }}}"
-          seed="${BENCH_E2E_SEED:-${{ vars.BENCH_E2E_SEED }}}"
-          seed="${seed:-42}"
-          bench_datadir="${BENCH_E2E_DATADIR:-$BENCH_E2E_NODE_DIR}"
-
-          if [ -z "$validators" ]; then
-            echo "::error::force-bloat requires BENCH_E2E_VALIDATORS ordered as red_ip:port,blue_ip:port."
-            exit 1
-          fi
-          if [ -z "$bench_datadir" ]; then
-            echo "::error::force-bloat requires BENCH_E2E_DATADIR or BENCH_E2E_NODE_DIR."
-            exit 1
-          fi
-          if [ -z "$BENCH_E2E_NODE_DIR" ]; then
-            echo "::error::force-bloat requires BENCH_E2E_NODE_DIR for this runner's validator identity files."
-            exit 1
-          fi
-          if [ -z "$BENCH_E2E_GENESIS" ]; then
-            echo "::error::force-bloat requires BENCH_E2E_GENESIS."
+          if [ -n "$BENCH_BASELINE_HARDFORK" ] || [ -n "$BENCH_FEATURE_HARDFORK" ]; then
+            echo "::error::mode=e2e hardfork comparison is not wired for the single-runner local harness yet."
             exit 1
           fi
 
-          nu tempo.nu bench-consensus-init \
-            --role "$BENCH_ROLE" \
-            --validators "$validators" \
-            --bench-datadir "$bench_datadir" \
-            --node-dir "$BENCH_E2E_NODE_DIR" \
-            --genesis "$BENCH_E2E_GENESIS" \
-            --bloat "$BENCH_BLOAT" \
-            --seed "$seed" \
-            --gas-limit 1000000000000
-
-          trusted_peers_file="$bench_datadir/.bench-meta/trusted-peers.txt"
-          if [ ! -s "$trusted_peers_file" ]; then
-            echo "::error::force-bloat did not produce $trusted_peers_file."
-            exit 1
-          fi
-          echo "BENCH_E2E_DATADIR=$bench_datadir" >> "$GITHUB_ENV"
-          echo "BENCH_E2E_TRUSTED_PEERS=$(cat "$trusted_peers_file")" >> "$GITHUB_ENV"
-          mkdir -p .bench-e2e-init
-          {
-            echo "role=$BENCH_ROLE"
-            echo "datadir=$bench_datadir"
-            echo "node_dir=$BENCH_E2E_NODE_DIR"
-            echo "genesis=$BENCH_E2E_GENESIS"
-            echo "trusted_peers_file=$trusted_peers_file"
-          } > ".bench-e2e-init/$BENCH_ROLE.txt"
-
-      - name: Upload e2e init marker
-        if: env.BENCH_FORCE_BLOAT == 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: tempo-bench-e2e-init-${{ matrix.role }}
-          path: .bench-e2e-init/${{ matrix.role }}.txt
-          retention-days: 1
-
-      - name: Wait for e2e init markers
-        if: env.BENCH_FORCE_BLOAT == 'true'
-        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-        with:
-          script: |
-            const wanted = new Set(['tempo-bench-e2e-init-red', 'tempo-bench-e2e-init-blue']);
-            const deadline = Date.now() + 60 * 60 * 1000;
-            while (Date.now() < deadline) {
-              const { data } = await github.rest.actions.listWorkflowRunArtifacts({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                run_id: context.runId,
-                per_page: 100,
-              });
-              const names = new Set(data.artifacts.map(a => a.name));
-              const missing = [...wanted].filter(name => !names.has(name));
-              if (missing.length === 0) {
-                core.info('Both e2e init markers are present.');
-                return;
-              }
-              core.info(`Waiting for e2e init markers: ${missing.join(', ')}`);
-              await new Promise(resolve => setTimeout(resolve, 30_000));
-            }
-            core.setFailed('Timed out waiting for both e2e init markers.');
-
       - name: Build txgen backend
-        if: matrix.role == 'red' && env.BENCH_BACKEND == 'txgen'
+        if: env.BENCH_BACKEND == 'txgen'
         working-directory: txgen
         run: |
           cargo build --release --bin txgen-tempo --bin bench
@@ -587,7 +493,6 @@ jobs:
             core.setOutput('feature-name', featureName);
 
       - name: Resolve PR attribution
-        if: matrix.role == 'red'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
@@ -614,7 +519,7 @@ jobs:
             core.exportVariable('BENCH_PR', pr);
 
       - name: Update status (running benchmark)
-        if: matrix.role == 'red' && success() && env.BENCH_COMMENT_ID
+        if: success() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
@@ -622,15 +527,14 @@ jobs:
             const s = require('./.github/scripts/bench-update-status.js');
             await s({github, context, status: 'Running benchmark...'});
 
-      - name: Run benchmark role
+      - name: Run e2e benchmark
         id: bench
         env:
           BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
           FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
         run: |
-          cmd=(nu tempo.nu bench-consensus)
+          cmd=(nu bench-e2e.nu e2e)
           cmd+=(
-            --role "$BENCH_ROLE"
             --preset "$BENCH_PRESET"
             --bloat "$BENCH_BLOAT"
             --duration "$BENCH_DURATION"
@@ -639,27 +543,11 @@ jobs:
             --feature "$FEATURE_REF"
             --baseline-name "${{ steps.refs.outputs.baseline-name }}"
             --feature-name "${{ steps.refs.outputs.feature-name }}"
-            --peer-url "$BENCH_E2E_PEER_URL"
-            --bench-datadir "${BENCH_E2E_DATADIR:-$BENCH_E2E_NODE_DIR}"
-            --node-dir "$BENCH_E2E_NODE_DIR"
-            --genesis "$BENCH_E2E_GENESIS"
-            --trusted-peers "$BENCH_E2E_TRUSTED_PEERS"
-            --consensus-port "$BENCH_E2E_CONSENSUS_PORT"
-            --benchmark-id "bench-e2e-${GITHUB_RUN_ID}"
-            --reference-epoch "$(date +%s)"
             --tune
-            --gas-limit 1000000000000
           )
-          [ -n "${BENCH_E2E_CONSENSUS_IP:-}" ] && cmd+=(--consensus-ip "$BENCH_E2E_CONSENSUS_IP")
+          [ "$BENCH_FORCE_BLOAT" = "true" ] && cmd+=(--force-bloat)
           [ "$BENCH_SAMPLY" = "true" ] && cmd+=(--samply)
           [ "$BENCH_TRACY" != "off" ] && cmd+=(--tracy "$BENCH_TRACY" --tracy-seconds "$BENCH_TRACY_SECONDS" --tracy-offset "$BENCH_TRACY_OFFSET")
-          [ -n "$BENCH_BASELINE_HARDFORK" ] && cmd+=(--baseline-hardfork "$BENCH_BASELINE_HARDFORK" --feature-hardfork "$BENCH_FEATURE_HARDFORK")
-          [ -n "${BENCH_E2E_BASELINE_GENESIS:-}" ] && cmd+=(--baseline-genesis "$BENCH_E2E_BASELINE_GENESIS")
-          [ -n "${BENCH_E2E_FEATURE_GENESIS:-}" ] && cmd+=(--feature-genesis "$BENCH_E2E_FEATURE_GENESIS")
-          [ -n "${BENCH_E2E_BASELINE_NODE_DIR:-}" ] && cmd+=(--baseline-node-dir "$BENCH_E2E_BASELINE_NODE_DIR")
-          [ -n "${BENCH_E2E_FEATURE_NODE_DIR:-}" ] && cmd+=(--feature-node-dir "$BENCH_E2E_FEATURE_NODE_DIR")
-          [ -n "${BENCH_E2E_BASELINE_DATADIR:-}" ] && cmd+=(--baseline-bench-datadir "$BENCH_E2E_BASELINE_DATADIR")
-          [ -n "${BENCH_E2E_FEATURE_DATADIR:-}" ] && cmd+=(--feature-bench-datadir "$BENCH_E2E_FEATURE_DATADIR")
           [ -n "$BENCH_BASELINE_ARGS" ] && cmd+=(--baseline-args="$BENCH_BASELINE_ARGS")
           [ -n "$BENCH_FEATURE_ARGS" ] && cmd+=(--feature-args="$BENCH_FEATURE_ARGS")
           [ -n "$BENCH_BENCH_ARGS" ] && cmd+=(--bench-args="$BENCH_BENCH_ARGS")
@@ -681,21 +569,14 @@ jobs:
           echo "Results directory: $RESULTS_DIR"
 
       - name: Upload results
-        if: matrix.role == 'red' && !cancelled()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: tempo-bench-red-results
-          path: bench-results/
-
-      - name: Upload blue results
-        if: matrix.role == 'blue' && !cancelled()
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: tempo-bench-blue-results
+          name: tempo-bench-results
           path: bench-results/
 
       - name: Upload results to ClickHouse
-        if: matrix.role == 'red' && success()
+        if: success()
         env:
           BENCH_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
           BENCH_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
@@ -705,7 +586,7 @@ jobs:
         run: bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
 
       - name: Post results to PR
-        if: matrix.role == 'red' && success()
+        if: success()
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
@@ -773,35 +654,38 @@ jobs:
               }
             } catch (e) {}
 
+            const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
+
             // Samply profile links (URLs produced by tempo.nu upload-samply-profile)
             let samplySection = '';
             if (process.env.BENCH_SAMPLY === 'true') {
-              const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
               const links = [];
               for (const run of runs) {
-                try {
-                  const url = fs.readFileSync(`${resultsDir}/profile-${run}-red-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run} / red**: [Firefox Profiler](${url})`);
-                } catch (e) {}
+                for (const role of ['a', 'b']) {
+                  try {
+                    const url = fs.readFileSync(`${resultsDir}/profile-${run}-${role}-url.txt`, 'utf8').trim();
+                    if (url) links.push(`- **${run} / ${role}**: [Firefox Profiler](${url})`);
+                  } catch (e) {}
+                }
               }
               if (links.length > 0) {
-                samplySection = `\n\n### Red Samply Profiles\n\n${links.join('\n')}\n`;
+                samplySection = `\n\n### Samply Profiles\n\n${links.join('\n')}\n`;
               }
             }
 
-            // Tracy profile links (URLs produced by tempo.nu upload-tracy-profile)
+            // Tracy profile links (URLs produced by tempo.nu upload-tracy-profile).
+            // Single-runner e2e captures both local validators in one phase-level file.
             let tracySection = '';
             if (process.env.BENCH_TRACY && process.env.BENCH_TRACY !== 'off') {
-              const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
               const links = [];
               for (const run of runs) {
                 try {
-                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-red-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run} / red**: [Tracy Viewer](${url})`);
+                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-url.txt`, 'utf8').trim();
+                  if (url) links.push(`- **${run} / local validators**: [Tracy Viewer](${url})`);
                 } catch (e) {}
               }
               if (links.length > 0) {
-                tracySection = `\n\n### Red Tracy Profiles\n\n${links.join('\n')}\n`;
+                tracySection = `\n\n### Tracy Profiles\n\n${links.join('\n')}\n`;
               }
             }
 
@@ -820,81 +704,8 @@ jobs:
               await core.summary.addRaw(body).write();
             }
 
-      - name: Add blue profile links to PR
-        if: matrix.role == 'blue' && success() && env.BENCH_COMMENT_ID && steps.results-dir.outputs.path
-        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-        env:
-          BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
-        with:
-          github-token: ${{ secrets.DEREK_PAT }}
-          script: |
-            const fs = require('fs');
-            const resultsDir = process.env.BENCH_RESULTS_DIR;
-            const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
-
-            const sections = [];
-            if (process.env.BENCH_SAMPLY === 'true') {
-              const links = [];
-              for (const run of runs) {
-                try {
-                  const url = fs.readFileSync(`${resultsDir}/profile-${run}-blue-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run} / blue**: [Firefox Profiler](${url})`);
-                } catch (e) {}
-              }
-              if (links.length) sections.push(`### Blue Samply Profiles\n\n${links.join('\n')}`);
-            }
-
-            if (process.env.BENCH_TRACY && process.env.BENCH_TRACY !== 'off') {
-              const links = [];
-              for (const run of runs) {
-                try {
-                  const url = fs.readFileSync(`${resultsDir}/tracy-${run}-blue-url.txt`, 'utf8').trim();
-                  if (url) links.push(`- **${run} / blue**: [Tracy Viewer](${url})`);
-                } catch (e) {}
-              }
-              if (links.length) sections.push(`### Blue Tracy Profiles\n\n${links.join('\n')}`);
-            }
-
-            if (!sections.length) return;
-
-            const markerStart = '<!-- e2e-blue-profiles:start -->';
-            const markerEnd = '<!-- e2e-blue-profiles:end -->';
-            const block = `${markerStart}\n\n${sections.join('\n\n')}\n\n${markerEnd}`;
-            const comment_id = parseInt(process.env.BENCH_COMMENT_ID);
-            const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
-
-            for (let attempt = 0; attempt < 60; attempt++) {
-              const { data: comment } = await github.rest.issues.getComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id,
-              });
-
-              const current = comment.body || '';
-              if (!current.includes('Benchmark complete!')) {
-                await sleep(10000);
-                continue;
-              }
-
-              const escapedStart = markerStart.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-              const escapedEnd = markerEnd.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-              const withoutOldBlock = current
-                .replace(new RegExp(`\\n*${escapedStart}[\\s\\S]*?${escapedEnd}`, 'm'), '')
-                .trimEnd();
-
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id,
-                body: `${withoutOldBlock}\n\n${block}`,
-              });
-              return;
-            }
-
-            core.warning('Timed out waiting for the red result comment; blue profile links were left in the blue artifact.');
-
       - name: Send Slack notification (success)
-        if: matrix.role == 'red' && success() && steps.results-dir.outputs.path && env.BENCH_NO_SLACK != 'true'
+        if: success() && steps.results-dir.outputs.path && env.BENCH_NO_SLACK != 'true'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           BENCH_WORK_DIR: ${{ steps.results-dir.outputs.path }}
@@ -906,7 +717,7 @@ jobs:
             await notify.success({ core, context });
 
       - name: Send Slack notification (failure)
-        if: matrix.role == 'red' && failure() && env.BENCH_NO_SLACK != 'true'
+        if: failure() && env.BENCH_NO_SLACK != 'true'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
@@ -914,7 +725,7 @@ jobs:
             await notify.failure({ core, context, failedStep: 'running benchmark' });
 
       - name: Update status (failed)
-        if: matrix.role == 'red' && failure() && env.BENCH_COMMENT_ID
+        if: failure() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
@@ -928,7 +739,7 @@ jobs:
             });
 
       - name: Update status (cancelled)
-        if: matrix.role == 'red' && cancelled() && env.BENCH_COMMENT_ID
+        if: cancelled() && env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           github-token: ${{ secrets.DEREK_PAT }}
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index febd4eecfd..3124bef00c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -1,7 +1,7 @@
 # Runs tempo benchmarks.
 #
-# Benchmarks use `nu tempo.nu bench` which runs an interleaved B-F-F-B
-# comparison against a schelk-managed snapshot on a self-hosted runner.
+# E2E benchmarks run `nu bench-e2e.nu e2e` against the dual-schelk runner.
+# Replay benchmarks use `nu tempo.nu bench-replay`.
 #
 # Trigger via PR comment (`@decofe bench` or `derek bench`).
 
@@ -237,7 +237,7 @@ jobs:
               return;
             }
 
-            // E2E currently owns the two-runner consensus harness. The txgen
+            // E2E currently owns the single-runner local consensus harness. The txgen
             // sender plugs into this path once #3669 lands.
             if (mode === 'e2e') {
               const unsupported = [];
diff --git a/bench-e2e.nu b/bench-e2e.nu
new file mode 100644
index 0000000000..b51bf387c1
--- /dev/null
+++ b/bench-e2e.nu
@@ -0,0 +1,812 @@
+#!/usr/bin/env nu
+
+# Single-runner e2e benchmark harness.
+# Shared build/cache/report helpers are sourced from tempo.nu; the replacement
+# e2e topology stays isolated here.
+source tempo.nu
+
+const E2E_A_STATE_PATH = "/var/lib/schelk/a.json"
+const E2E_B_STATE_PATH = "/var/lib/schelk/b.json"
+const E2E_A_MOUNT = "/reth-bench-a"
+const E2E_B_MOUNT = "/reth-bench-b"
+const E2E_VALIDATORS = "127.0.0.2:8000,127.0.0.3:8100"
+const E2E_SEED = 42
+const E2E_A_CPUS = "0-7,16-23"
+const E2E_B_CPUS = "8-15,24-31"
+const E2E_A_MEMORY = ""
+const E2E_B_MEMORY = ""
+const E2E_GAS_LIMIT = "1000000000000"
+const E2E_BLOAT_TMP_DIR = "/reth-bench-a/.bench-tmp/e2e-local-init"
+const E2E_BLOAT_FREE_MARGIN_MIB = 51200
+const E2E_LOCAL_RETH_ARGS = [
+    "--ipcdisable"
+    "--disable-discovery"
+    "--trusted-only"
+    "--tempo.bootnodes-endpoint" "none"
+]
+
+def schelk [state_path: string, subcommand: string, ...args: string] {
+    sudo schelk --state-path $state_path $subcommand ...$args
+}
+
+def schelk-state [state_path: string] {
+    sudo cat $state_path | from json
+}
+
+def validate-schelk-state [a_state_path: string, b_state_path: string] {
+    if (has-schelk) {
+        for state_path in [$a_state_path $b_state_path] {
+            if not ($state_path | path exists) {
+                print $"Error: schelk state file does not exist: ($state_path)"
+                exit 1
+            }
+        }
+        let a_state = (schelk-state $a_state_path)
+        let b_state = (schelk-state $b_state_path)
+        let a_dm_era = ($a_state | get --optional dm_era_name)
+        let b_dm_era = ($b_state | get --optional dm_era_name)
+        if $a_dm_era == null or $b_dm_era == null {
+            print "Error: schelk state files must include dm_era_name for parallel a/b instances."
+            print "Reinitialize schelk a and b with unique --dm-era-name values."
+            exit 1
+        }
+        if $a_dm_era == $b_dm_era {
+            print $"Error: schelk a/b state files use the same dm_era_name: ($a_dm_era)"
+            print "Reinitialize one side with a unique --dm-era-name before running e2e."
+            exit 1
+        }
+        let a_mount = ($a_state | get --optional mount_point)
+        let b_mount = ($b_state | get --optional mount_point)
+        if $a_mount != $E2E_A_MOUNT {
+            print $"Error: schelk a state mount_point is ($a_mount), expected ($E2E_A_MOUNT)"
+            exit 1
+        }
+        if $b_mount != $E2E_B_MOUNT {
+            print $"Error: schelk b state mount_point is ($b_mount), expected ($E2E_B_MOUNT)"
+            exit 1
+        }
+        if $a_mount == $b_mount {
+            print $"Error: schelk a/b state files use the same mount_point: ($a_mount)"
+            exit 1
+        }
+    }
+}
+
+def bench-restore-at [state_path: string, mount_point: string, datadir: string] {
+    if (has-schelk) {
+        print $"Restoring schelk snapshot ($mount_point)..."
+        let state = (schelk-state $state_path)
+        let state_mounted = ($state | get --optional is_mounted) == true
+        let actual_mounted = (mountpoint -q $mount_point | complete).exit_code == 0
+        try {
+            if $state_mounted or $actual_mounted {
+                schelk $state_path recover "-y" "--kill"
+            }
+            schelk $state_path mount
+        } catch {
+            print $"Schelk restore failed for ($mount_point), falling back to full-recover..."
+            schelk $state_path full-recover "-y"
+            schelk $state_path mount
+        }
+        sudo chown -R (whoami | str trim) $mount_point
+    } else {
+        print $"Restoring snapshot from ($datadir).virgin..."
+        rm -rf $datadir
+        ^cp -a $"($datadir).virgin" $datadir
+    }
+}
+
+# Promote a specific schelk scratch volume as the new virgin baseline.
+def bench-promote-at [state_path: string, datadir: string] {
+    if (has-schelk) {
+        print $"Promoting schelk scratch to virgin ($state_path)..."
+        schelk $state_path promote "-y" "--kill"
+    } else {
+        print $"Saving snapshot to ($datadir).virgin..."
+        rm -rf $"($datadir).virgin"
+        ^cp -a $datadir $"($datadir).virgin"
+    }
+}
+
+def df-available-mib [path: string] {
+    let row = (^df -Pm $path | lines | skip 1 | first | split row --regex '\s+')
+    $row | get 3 | into int
+}
+
+def ensure-bloat-space [bloat: int] {
+    if $bloat <= 0 {
+        return
+    }
+    let required_mib = $bloat + $E2E_BLOAT_FREE_MARGIN_MIB
+    for mount in [$E2E_A_MOUNT $E2E_B_MOUNT] {
+        let available_mib = (df-available-mib $mount)
+        if $available_mib < $required_mib {
+            print $"Error: ($mount) has ($available_mib) MiB free, needs at least ($required_mib) MiB for ($bloat) MiB bloat plus margin"
+            exit 1
+        }
+    }
+}
+
+def bench-save-e2e-meta [datadir: string, meta_dir: string, marker: record, genesis_files: list] {
+    mkdir $meta_dir
+    for pair in $genesis_files {
+        cp ($pair | first) $"($meta_dir)/($pair | last)"
+    }
+    let marker_path = $"($meta_dir)/marker.json"
+    $marker | insert initialized_at (date now | format date "%Y-%m-%dT%H:%M:%SZ") | to json | save -f $marker_path
+    print $"Bench marker written to ($marker_path)"
+}
+
+def systemd-scope-command [unit: string, cpus: string, memory: string, script: string] {
+    let can_scope = (^uname | str trim) == "Linux" and ((which systemd-run | length) > 0) and ($cpus != "" or $memory != "")
+    if not $can_scope {
+        return ["bash" "-lc" $script]
+    }
+
+    let cpu_args = if $cpus != "" { ["-p" $"AllowedCPUs=($cpus)"] } else { [] }
+    let memory_args = if $memory != "" { ["-p" $"MemoryMax=($memory)"] } else { [] }
+    let uid = (id -u | str trim)
+    let gid = (id -g | str trim)
+    [
+        "sudo"
+        "systemd-run"
+        "--scope"
+        "--quiet"
+        "--collect"
+        "--same-dir"
+        "--unit" $unit
+        "--uid" $uid
+        "--gid" $gid
+        "-p" "CPUWeight=100"
+        ...$cpu_args
+        ...$memory_args
+        "bash"
+        "-lc"
+        $script
+    ]
+}
+
+def start-e2e-local-node [
+    role: string,
+    phase: string,
+    tempo_bin: string,
+    args: list<string>,
+    env_prefix: string,
+    otel_attrs: string,
+    tracy_env_prefix: string,
+    samply: bool,
+    samply_args: list<string>,
+    results_dir: string,
+    cpus: string,
+    memory: string,
+] {
+    let profile_label = $"($phase)-($role)"
+    let full_samply_args = if $samply {
+        $samply_args | append ["--save-only" "--presymbolicate" "--output" $"($results_dir)/profile-($profile_label).json.gz"]
+    } else { [] }
+    let node_cmd = wrap-samply [$tempo_bin ...$args] $samply $full_samply_args
+    let node_cmd_str = ($node_cmd | str join " ")
+    let script = $"($env_prefix)($otel_attrs)($tracy_env_prefix)($node_cmd_str) 2>&1"
+    let unit_phase = ($phase | str replace -a "_" "-" | str replace -a "." "-")
+    let runner = (systemd-scope-command $"tempo-e2e-($role)-($unit_phase)" $cpus $memory $script)
+    print $"Starting local e2e validator ($role) for ($phase): ($runner | str join ' ')"
+    job spawn {
+        run-external ($runner | first) ...($runner | skip 1)
+        | lines
+        | each { |line| print $"[e2e-($phase)-($role)] ($line)" }
+    }
+}
+
+def stop-e2e-processes-gracefully [] {
+    let pids = (find-tempo-pids)
+    if ($pids | length) > 0 {
+        print $"Stopping tempo processes: ($pids | str join ', ')"
+    }
+    for pid in $pids {
+        kill -s 2 $pid
+    }
+    for pid in $pids {
+        mut wait = 0
+        while $wait < 30 {
+            if (ps | where pid == $pid | length) == 0 { break }
+            sleep 1sec
+            $wait = $wait + 1
+        }
+        if $wait >= 30 {
+            print $"  Warning: PID ($pid) did not exit, sending SIGKILL"
+            kill -s 9 $pid
+            sleep 1sec
+        }
+    }
+    if ("/tmp/reth.ipc" | path exists) {
+        rm --force /tmp/reth.ipc
+    }
+}
+
+def stop-local-e2e-systemd-scopes [] {
+    if (^uname | str trim) != "Linux" or ((which systemctl | length) == 0) {
+        return
+    }
+
+    let units = (
+        bash -lc "systemctl list-units 'tempo-e2e-*.scope' --all --plain --no-legend 2>/dev/null | awk '{print $1}'"
+        | lines
+        | where { |unit| $unit != "" }
+    )
+    for unit in $units {
+        print $"Stopping stale local e2e scope: ($unit)"
+        sudo systemctl kill --kill-whom=all $unit | ignore
+        sudo systemctl reset-failed $unit | ignore
+    }
+}
+
+def cleanup-local-e2e-processes [] {
+    stop-local-e2e-systemd-scopes
+    stop-e2e-processes-gracefully
+    stop-tracy-capture
+}
+
+def e2e-wait-for-rpc-online [url: string, max_attempts: int] {
+    mut attempt = 0
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url)"
+            return false
+        }
+        let block = (rpc-block-number $url)
+        if $block != null {
+            print $"  ($url) online \(block ($block)\)"
+            return true
+        }
+        if ($attempt mod 10) == 0 {
+            print $"  Still waiting for ($url)... \(($attempt)s\)"
+        }
+        sleep 1sec
+    }
+}
+
+def e2e-wait-for-peers [url: string, min_peers: int, max_attempts: int] {
+    mut attempt = 0
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url) to reach ($min_peers) peer\(s\)"
+            return false
+        }
+        let peers = (rpc-peer-count $url)
+        if $peers != null and $peers >= $min_peers {
+            print $"  ($url) has ($peers) peer\(s\)"
+            return true
+        }
+        if ($attempt mod 10) == 0 {
+            let current = if $peers == null { "unknown" } else { $"($peers)" }
+            print $"  ($url) peers: ($current)/($min_peers)... \(($attempt)s\)"
+        }
+        sleep 1sec
+    }
+}
+
+def e2e-wait-for-chain-advance [url: string, max_attempts: int] {
+    mut attempt = 0
+    mut start_block: int = -1
+
+    loop {
+        $attempt = $attempt + 1
+        if $attempt > $max_attempts {
+            print $"  Timeout waiting for ($url) chain to advance"
+            return false
+        }
+        let block = (rpc-block-number $url)
+        if $block != null {
+            if $start_block == -1 {
+                $start_block = $block
+                print $"  ($url) connected \(block ($block)\), waiting for chain to advance..."
+            } else if $block > $start_block {
+                print $"  ($url) ready \(block ($start_block) -> ($block)\)"
+                return true
+            } else if ($attempt mod 10) == 0 {
+                print $"  ($url) still at block ($block)... \(($attempt)s\)"
+            }
+        } else if ($attempt mod 10) == 0 {
+            print $"  ($url) unavailable while waiting for chain advance... \(($attempt)s\)"
+        }
+        sleep 1sec
+    }
+}
+
+def init-local-e2e-side [
+    role: string,
+    state_path: string,
+    mount_point: string,
+    datadir: string,
+    node_dir: string,
+    generated_node_dir: string,
+    generated_genesis: string,
+    trusted_peers: string,
+    bloat: int,
+    bloat_file: string,
+    tempo_bin: string,
+    marker: record,
+] {
+    let meta_dir = $"($datadir)/($BENCH_META_SUBDIR)"
+    let generated_trusted_peers = $"($LOCALNET_DIR)/e2e-local-init/trusted-peers.txt"
+
+    bench-clean-datadir $datadir
+    mkdir $datadir
+    mkdir $node_dir
+
+    bench-init-db $tempo_bin $generated_genesis $datadir $bloat $bloat_file
+    for file in ["signing.key" "signing.share" "enode.key" "enode.identity"] {
+        cp $"($generated_node_dir)/($file)" $"($node_dir)/($file)"
+    }
+    $trusted_peers | save -f $generated_trusted_peers
+
+    bench-save-e2e-meta $datadir $meta_dir ($marker | insert validator_role $role) [[$generated_genesis "genesis.json"] [$generated_trusted_peers "trusted-peers.txt"]]
+}
+
+def run-local-e2e-phase [run: record, ctx: record] {
+    let phase = $run.phase
+    print $"=== Starting local e2e phase: ($phase) ==="
+    let run_type = if ($phase | str starts-with "baseline") { "baseline" } else { "feature" }
+    let side_args = if $run_type == "baseline" { $ctx.baseline_args } else { $ctx.feature_args }
+    let side_env = if $run_type == "baseline" { $ctx.baseline_env } else { $ctx.feature_env }
+    let effective_node_args = ([$ctx.node_args $side_args] | where { |a| $a != "" } | str join " ")
+    let extra_args = if $effective_node_args == "" { [] } else { $effective_node_args | split row " " }
+    let weights = if $ctx.preset != "" { $PRESETS | get $ctx.preset } else { [0.0, 0.0, 0.0, 0.0] }
+
+    cleanup-local-e2e-processes
+    bench-restore-at $ctx.a.state_path $ctx.a.mount $ctx.a.datadir
+    bench-restore-at $ctx.b.state_path $ctx.b.mount $ctx.b.datadir
+
+    for path in [$ctx.genesis $ctx.a.node_dir $ctx.b.node_dir] {
+        if not ($path | path exists) {
+            print $"Error: required e2e path does not exist after snapshot recovery: ($path)"
+            exit 1
+        }
+    }
+    for role_info in [
+        { role: "a", node_dir: $ctx.a.node_dir }
+        { role: "b", node_dir: $ctx.b.node_dir }
+    ] {
+        for required_file in ["signing.key" "signing.share" "enode.key"] {
+            let path = $"($role_info.node_dir)/($required_file)"
+            if not ($path | path exists) {
+                print $"Error: missing ($role_info.role) validator file after snapshot recovery: ($path)"
+                exit 1
+            }
+        }
+    }
+
+    let a_log_dir = $"($LOCALNET_DIR)/logs-e2e-local-($phase)-a"
+    let b_log_dir = $"($LOCALNET_DIR)/logs-e2e-local-($phase)-b"
+    for dir in [$a_log_dir $b_log_dir] {
+        if ($dir | path exists) { rm -rf $dir }
+        mkdir $dir
+    }
+
+    for stale in [
+        $"($ctx.results_dir)/report-($phase).json"
+        $"($ctx.results_dir)/profile-($phase)-a.json.gz"
+        $"($ctx.results_dir)/profile-($phase)-b.json.gz"
+        $"($ctx.results_dir)/tracy-profile-($phase).tracy"
+        $"($ctx.results_dir)/logs-($phase)-a"
+        $"($ctx.results_dir)/logs-($phase)-b"
+    ] {
+        if ($stale | path exists) { rm -rf $stale }
+    }
+    if ("report.json" | path exists) { rm report.json }
+    let tuning_state = if $ctx.tune { apply-system-tuning } else { { tuned: false } }
+
+    let a_rpc = "http://127.0.0.1:8545"
+    let b_rpc = "http://127.0.0.1:8645"
+    let a_base_args = (build-base-args $ctx.genesis $ctx.a.datadir $a_log_dir "0.0.0.0" 8545 9001)
+        | append (build-e2e-consensus-args $ctx.a.node_dir $ctx.trusted_peers $ctx.a.consensus_port $ctx.a.ip)
+        | append $E2E_LOCAL_RETH_ARGS
+        | append (log-filter-args $ctx.loud)
+        | append (if $ctx.gas_limit != "" { ["--builder.gaslimit" $ctx.gas_limit] } else { [] })
+        | append (if $ctx.tracy != "off" { ["--log.tracy" "--log.tracy.filter" $ctx.tracy_filter] } else { [] })
+    let b_base_args = (build-base-args $ctx.genesis $ctx.b.datadir $b_log_dir "0.0.0.0" 8645 9101)
+        | append (build-e2e-consensus-args $ctx.b.node_dir $ctx.trusted_peers $ctx.b.consensus_port $ctx.b.ip)
+        | append $E2E_LOCAL_RETH_ARGS
+        | append (log-filter-args $ctx.loud)
+        | append (if $ctx.gas_limit != "" { ["--builder.gaslimit" $ctx.gas_limit] } else { [] })
+        | append (if $ctx.tracy != "off" { ["--log.tracy" "--log.tracy.filter" $ctx.tracy_filter] } else { [] })
+    let a_args = (dedup-args $a_base_args $extra_args)
+    let b_args = (dedup-args $b_base_args $extra_args)
+
+    let tracy_env_prefix = if $ctx.tracy == "on" {
+        "TRACY_NO_SYS_TRACE=1 "
+    } else if $ctx.tracy == "full" {
+        "TRACY_SAMPLING_HZ=1 "
+    } else { "" }
+    let env_prefix = if $side_env != "" { $"($side_env) " } else { "" }
+    let a_otel = $"OTEL_RESOURCE_ATTRIBUTES=benchmark_id=($ctx.benchmark_id),benchmark_run=($phase),runner_role=a,run_type=($run_type),git_ref=($run.ref),reference_epoch=($ctx.reference_epoch) "
+    let b_otel = $"OTEL_RESOURCE_ATTRIBUTES=benchmark_id=($ctx.benchmark_id),benchmark_run=($phase),runner_role=b,run_type=($run_type),git_ref=($run.ref),reference_epoch=($ctx.reference_epoch) "
+
+    start-e2e-local-node a $phase $run.tempo $a_args $env_prefix $a_otel $tracy_env_prefix $ctx.samply $ctx.samply_args $ctx.results_dir $ctx.a.cpus $ctx.a.memory
+    start-e2e-local-node b $phase $run.tempo $b_args $env_prefix $b_otel $tracy_env_prefix $ctx.samply $ctx.samply_args $ctx.results_dir $ctx.b.cpus $ctx.b.memory
+
+    sleep 2sec
+    let rpc_timeout = if $ctx.bloat > 0 { 600 } else { 300 }
+    mut phase_exit = 0
+    if ((find-tempo-pids) | length) < 2 {
+        print $"Error: local e2e validators exited before readiness checks completed for ($phase)"
+        $phase_exit = 1
+    }
+    if $phase_exit == 0 and not (e2e-wait-for-rpc-online $a_rpc $rpc_timeout) { $phase_exit = 1 }
+    if $phase_exit == 0 and not (e2e-wait-for-rpc-online $b_rpc $rpc_timeout) { $phase_exit = 1 }
+    if $phase_exit == 0 and not (e2e-wait-for-peers $a_rpc 1 300) { $phase_exit = 1 }
+    if $phase_exit == 0 and not (e2e-wait-for-peers $b_rpc 1 300) { $phase_exit = 1 }
+    if $phase_exit == 0 and not (e2e-wait-for-chain-advance $a_rpc 300) { $phase_exit = 1 }
+    if $phase_exit == 0 and not (e2e-wait-for-chain-advance $b_rpc 300) { $phase_exit = 1 }
+
+    let tracy_output = $"($ctx.results_dir)/tracy-profile-($phase).tracy"
+    mut tracy_capture_started = false
+    if $phase_exit == 0 and $ctx.tracy != "off" {
+        let seconds_flag = if $ctx.tracy_seconds > 0 { $"-s ($ctx.tracy_seconds)" } else { "" }
+        let limit_msg = if $ctx.tracy_seconds > 0 { $" \(($ctx.tracy_seconds)s limit\)" } else { "" }
+        if $ctx.tracy_offset > 0 {
+            print $"  Tracy-capture will start in ($ctx.tracy_offset)s($limit_msg)..."
+            job spawn { sleep ($"($ctx.tracy_offset)sec" | into duration); sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
+        } else {
+            print $"  Starting tracy-capture($limit_msg)..."
+            job spawn { sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
+            sleep 500ms
+        }
+        $tracy_capture_started = true
+    }
+
+    let bench_cmd = [
+        $run.bench
+        "run-max-tps"
+        "--tps" $"($ctx.tps)"
+        "--duration" $"($ctx.duration)"
+        "--accounts" $"($ctx.accounts)"
+        "--max-concurrent-requests" $"($ctx.max_concurrent_requests)"
+        "--target-urls" $"($a_rpc),($b_rpc)"
+        "--faucet"
+        "--clear-txpool"
+    ]
+    | append (if $ctx.preset != "" {
+        [
+            "--tip20-weight" $"($weights | get 0)"
+            "--erc20-weight" $"($weights | get 1)"
+            "--swap-weight" $"($weights | get 2)"
+            "--place-order-weight" $"($weights | get 3)"
+        ]
+    } else { [] })
+    | append (if $ctx.bloat > 0 { ["--mnemonic" $"'($BLOAT_MNEMONIC)'"] } else { [] })
+    | append (if $ctx.bench_args != "" { $ctx.bench_args | split row " " } else { [] })
+    | append ["--node-commit-sha" $run.ref "--build-profile" $ctx.profile "--benchmark-mode" "e2e"]
+
+    if $phase_exit == 0 {
+        let bench_env_export = if $ctx.bench_env != "" { $"export ($ctx.bench_env) && " } else { "" }
+        print $"Running local e2e sender: ($bench_cmd | str join ' ')"
+        let bench_result = (bash -c $"($bench_env_export)ulimit -Sn unlimited && ($bench_cmd | str join ' ')" | complete)
+        if $bench_result.stdout != "" { print $bench_result.stdout }
+        if $bench_result.stderr != "" { print $bench_result.stderr }
+        $phase_exit = $bench_result.exit_code
+
+        if ("report.json" | path exists) {
+            cp report.json $"($ctx.results_dir)/report-($phase).json"
+            rm report.json
+        } else {
+            print $"ERROR: sender for ($phase) produced no report.json"
+            $phase_exit = 1
+        }
+    } else {
+        print $"Skipping local e2e sender for ($phase) because readiness checks failed"
+    }
+
+    if $tracy_capture_started {
+        stop-tracy-capture
+    }
+    stop-e2e-processes-gracefully
+    if $ctx.samply { wait-for-samply-profile }
+    if ($a_log_dir | path exists) { cp -r $a_log_dir $"($ctx.results_dir)/logs-($phase)-a" }
+    if ($b_log_dir | path exists) { cp -r $b_log_dir $"($ctx.results_dir)/logs-($phase)-b" }
+    restore-system-tuning $tuning_state
+
+    if $phase_exit != 0 {
+        return $phase_exit
+    }
+    print $"=== Local e2e phase complete: ($phase) ==="
+    return 0
+}
+
+# Run the baseline-feature-feature-baseline e2e sequence on one runner.
+def "main e2e" [
+    --baseline: string                                  # Baseline git SHA/ref
+    --feature: string                                   # Feature git SHA/ref
+    --preset: string = ""                               # Preset: tip20, erc20, swap, order, tempo-mix
+    --tps: int = 10000                                  # Target TPS
+    --duration: int = 300                               # Duration in seconds
+    --accounts: int = 1000                              # Number of accounts
+    --max-concurrent-requests: int = 100                # Max concurrent requests
+    --bloat: int = 0                                    # State bloat size in MiB
+    --force-bloat                                      # Regenerate and promote both local e2e snapshots
+    --init-only                                         # Refresh snapshots and exit without running benchmark phases
+    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
+    --features: string = $DEFAULT_FEATURES              # Cargo features
+    --samply                                            # Profile validators with samply
+    --samply-args: string = ""                          # Additional samply arguments
+    --tracy: string = "off"                             # Tracy profiling: off, on, full
+    --tracy-filter: string = "debug"                    # Tracy tracing filter level
+    --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
+    --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
+    --node-args: string = ""                            # Additional node args for all phases
+    --baseline-args: string = ""                        # Additional node args for baseline phases
+    --feature-args: string = ""                         # Additional node args for feature phases
+    --bench-args: string = ""                           # Additional tempo-bench args
+    --baseline-env: string = ""                         # Environment vars for baseline node phases
+    --feature-env: string = ""                          # Environment vars for feature node phases
+    --bench-env: string = ""                            # Environment vars for the sender process
+    --baseline-name: string = ""                         # Baseline display name for summary
+    --feature-name: string = ""                          # Feature display name for summary
+    --tune                                              # Apply system tuning
+    --loud                                              # Show node debug logs
+    --no-cache                                           # Skip binary cache
+] {
+    if $preset == "" and $bench_args == "" {
+        print "Error: either --preset or --bench-args must be provided"
+        print $"  Available presets: ($PRESETS | columns | str join ', ')"
+        exit 1
+    }
+    if $preset != "" and not ($preset in $PRESETS) {
+        print $"Unknown preset: ($preset). Available: ($PRESETS | columns | str join ', ')"
+        exit 1
+    }
+    if $tracy not-in ["off" "on" "full"] {
+        print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
+        exit 1
+    }
+    if $samply and $tracy != "off" {
+        print "Error: --samply and --tracy are mutually exclusive. Choose one."
+        exit 1
+    }
+    if $init_only and not $force_bloat {
+        print "Error: --init-only requires --force-bloat"
+        exit 1
+    }
+    if $tracy != "off" and ((which tracy-capture | length) == 0) {
+        print "Error: tracy-capture not found. Install tracy and ensure tracy-capture is in PATH."
+        exit 1
+    }
+
+    let validator_list = (
+        $E2E_VALIDATORS
+        | split row ","
+        | each { |v| $v | str trim }
+        | where { |v| $v != "" }
+    )
+    if ($validator_list | length) != 2 {
+        print "Error: E2E_VALIDATORS must contain exactly two comma-separated consensus addresses ordered as a,b"
+        exit 1
+    }
+    let a_validator = ($validator_list | get 0)
+    let b_validator = ($validator_list | get 1)
+    let a_ip = ($a_validator | split row ":" | get 0)
+    let a_consensus_port = ($a_validator | split row ":" | get 1 | into int)
+    let b_ip = ($b_validator | split row ":" | get 0)
+    let b_consensus_port = ($b_validator | split row ":" | get 1 | into int)
+    let a_db = $"($E2E_A_MOUNT)/tempo_e2e_($bloat)mb"
+    let b_db = $"($E2E_B_MOUNT)/tempo_e2e_($bloat)mb"
+    let a_identity = $a_db
+    let b_identity = $b_db
+    let genesis_path = $"($a_db)/($BENCH_META_SUBDIR)/genesis.json"
+    let a_trusted_peers_path = $"($a_db)/($BENCH_META_SUBDIR)/trusted-peers.txt"
+    let run_started_at = (date now)
+    let timestamp = ($run_started_at | format date "%Y%m%d-%H%M%S-%3f")
+    let benchmark_id = $"bench-e2e-local-($timestamp)"
+    let reference_epoch = (($run_started_at | into int) / 1_000_000_000 | into int)
+    let gas_limit_args = if $E2E_GAS_LIMIT != "" { ["--gas-limit" $E2E_GAS_LIMIT] } else { [] }
+
+    validate-schelk-state $E2E_A_STATE_PATH $E2E_B_STATE_PATH
+    cleanup-local-e2e-processes
+
+    if $force_bloat {
+        let init_dir = $"($LOCALNET_DIR)/e2e-local-init"
+        let generated_genesis = $"($init_dir)/genesis.json"
+        let bloat_file = $"($E2E_BLOAT_TMP_DIR)/state_bloat.bin"
+        if ($init_dir | path exists) { rm -rf $init_dir }
+        mkdir $init_dir
+        bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
+        bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
+        if ($E2E_BLOAT_TMP_DIR | path exists) { rm -rf $E2E_BLOAT_TMP_DIR }
+        mkdir $E2E_BLOAT_TMP_DIR
+
+        build-tempo ["tempo"] $profile $features
+        let tempo_bin = if $profile == "dev" { "./target/debug/tempo" } else { $"./target/($profile)/tempo" }
+        let genesis_accounts = ([$accounts 3] | math max) + 1
+        print $"Generating local e2e localnet config for validators: ($E2E_VALIDATORS)"
+        cargo run -p tempo-xtask --profile $profile -- generate-localnet -o $init_dir --accounts $genesis_accounts --validators $E2E_VALIDATORS --seed $E2E_SEED --force ...$gas_limit_args
+
+        let trusted_peers = (trusted-peers-from-localnet $init_dir)
+        if $trusted_peers == "" {
+            print "Error: generated localnet did not produce trusted peers"
+            exit 1
+        }
+        if $bloat > 0 {
+            ensure-bloat-space $bloat
+            print $"Generating local e2e state bloat \(($bloat) MiB\)..."
+            let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten)
+            cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args
+        }
+
+        let marker = {
+            bloat_mib: $bloat
+            accounts: $genesis_accounts
+            validators: $E2E_VALIDATORS
+            seed: $E2E_SEED
+            gas_limit: $E2E_GAS_LIMIT
+            dkg_in_genesis: true
+            topology: "single-runner"
+        }
+        init-local-e2e-side a $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db $a_identity $"($init_dir)/($a_validator)" $generated_genesis $trusted_peers $bloat $bloat_file $tempo_bin ($marker | insert bench_datadir $a_db | insert node_dir $a_identity | insert validator_addr $a_validator)
+        init-local-e2e-side b $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db $b_identity $"($init_dir)/($b_validator)" $generated_genesis $trusted_peers $bloat $bloat_file $tempo_bin ($marker | insert bench_datadir $b_db | insert node_dir $b_identity | insert validator_addr $b_validator)
+        if ($E2E_BLOAT_TMP_DIR | path exists) {
+            rm -rf $E2E_BLOAT_TMP_DIR
+        }
+        bench-promote-at $E2E_A_STATE_PATH $a_db
+        bench-promote-at $E2E_B_STATE_PATH $b_db
+    }
+
+    bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
+    bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
+    if $init_only {
+        cleanup-local-e2e-processes
+        return
+    }
+    let trusted_peers = if ($a_trusted_peers_path | path exists) {
+        open $a_trusted_peers_path | str trim
+    } else {
+        let b_trusted_peers_path = $"($b_db)/($BENCH_META_SUBDIR)/trusted-peers.txt"
+        if ($b_trusted_peers_path | path exists) {
+            open $b_trusted_peers_path | str trim
+        } else {
+            print $"Error: trusted peers file not found in ($a_trusted_peers_path) or ($b_trusted_peers_path)"
+            exit 1
+        }
+    }
+
+    let results_dir = $"($BENCH_RESULTS_DIR)/($timestamp)"
+    mkdir $results_dir
+    print $"BENCH_RESULTS_DIR=($results_dir)"
+
+    git worktree prune
+    mkdir $BENCH_WORKTREES_DIR
+    let baseline_wt = $"($BENCH_WORKTREES_DIR)/e2e-local-baseline"
+    let feature_wt = $"($BENCH_WORKTREES_DIR)/e2e-local-feature"
+    for wt in [$baseline_wt $feature_wt] {
+        if ($wt | path exists) {
+            print $"Removing stale local e2e worktree: ($wt)"
+            try { git worktree remove --force $wt } catch { rm -rf $wt }
+        }
+    }
+    git worktree add $baseline_wt $baseline
+    git worktree add $feature_wt $feature
+
+    let tbc = (tracy-build-config $features $tracy)
+    let effective_features = $tbc.features
+    let effective_extra_rustflags = $tbc.extra_rustflags
+    let effective_no_cache = $no_cache or ($tracy != "off")
+    if $effective_no_cache {
+        build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags --bench-features $features $baseline_wt $baseline $profile $effective_features $baseline
+        build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags --bench-features $features $feature_wt $feature $profile $effective_features $feature
+    } else {
+        build-in-worktree $baseline_wt $baseline $profile $effective_features $baseline
+        build-in-worktree $feature_wt $feature $profile $effective_features $feature
+    }
+    let baseline_tempo = (worktree-bin $baseline_wt $profile "tempo")
+    let baseline_bench = (worktree-bin $baseline_wt $profile "tempo-bench")
+    let feature_tempo = (worktree-bin $feature_wt $profile "tempo")
+    let feature_bench = (worktree-bin $feature_wt $profile "tempo-bench")
+    let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " }
+    let ctx = {
+        genesis: $genesis_path
+        trusted_peers: $trusted_peers
+        a: {
+            state_path: $E2E_A_STATE_PATH
+            mount: $E2E_A_MOUNT
+            datadir: $a_db
+            node_dir: $a_identity
+            ip: $a_ip
+            consensus_port: $a_consensus_port
+            cpus: $E2E_A_CPUS
+            memory: $E2E_A_MEMORY
+        }
+        b: {
+            state_path: $E2E_B_STATE_PATH
+            mount: $E2E_B_MOUNT
+            datadir: $b_db
+            node_dir: $b_identity
+            ip: $b_ip
+            consensus_port: $b_consensus_port
+            cpus: $E2E_B_CPUS
+            memory: $E2E_B_MEMORY
+        }
+        preset: $preset
+        tps: $tps
+        duration: $duration
+        accounts: $accounts
+        max_concurrent_requests: $max_concurrent_requests
+        bloat: $bloat
+        results_dir: $results_dir
+        profile: $profile
+        samply: $samply
+        samply_args: $samply_args_list
+        tracy: $tracy
+        tracy_filter: $tracy_filter
+        tracy_seconds: $tracy_seconds
+        tracy_offset: $tracy_offset
+        node_args: $node_args
+        baseline_args: $baseline_args
+        feature_args: $feature_args
+        bench_args: $bench_args
+        baseline_env: $baseline_env
+        feature_env: $feature_env
+        bench_env: $bench_env
+        benchmark_id: $benchmark_id
+        reference_epoch: $reference_epoch
+        tune: $tune
+        loud: $loud
+        gas_limit: $E2E_GAS_LIMIT
+    }
+
+    let runs = [
+        { phase: "baseline-1", ref: $baseline, tempo: $baseline_tempo, bench: $baseline_bench }
+        { phase: "feature-1", ref: $feature, tempo: $feature_tempo, bench: $feature_bench }
+        { phase: "feature-2", ref: $feature, tempo: $feature_tempo, bench: $feature_bench }
+        { phase: "baseline-2", ref: $baseline, tempo: $baseline_tempo, bench: $baseline_bench }
+    ]
+    mut e2e_exit = 0
+    for run in $runs {
+        let phase_exit = (run-local-e2e-phase $run $ctx)
+        if $phase_exit != 0 {
+            $e2e_exit = $phase_exit
+            break
+        }
+    }
+
+    if $e2e_exit == 0 and $samply {
+        print "\nUploading local e2e samply profiles to Firefox Profiler..."
+        for run in $runs {
+            for role in ["a" "b"] {
+                let profile_label = $"($run.phase)-($role)"
+                let profile = $"($results_dir)/profile-($profile_label).json.gz"
+                let url = (upload-samply-profile $profile)
+                if $url != null {
+                    $url | save -f $"($results_dir)/profile-($profile_label)-url.txt"
+                }
+            }
+        }
+    }
+    if $e2e_exit == 0 and $tracy != "off" {
+        print "\nUploading local e2e tracy profiles to R2..."
+        for run in $runs {
+            let profile = $"($results_dir)/tracy-profile-($run.phase).tracy"
+            let viewer_url = (upload-tracy-profile $profile $run.phase $run.ref)
+            if $viewer_url != null {
+                $viewer_url | save -f $"($results_dir)/tracy-($run.phase)-url.txt"
+            }
+        }
+    }
+
+    let baseline_label = if $baseline_name != "" { $baseline_name } else { $baseline }
+    let feature_label = if $feature_name != "" { $feature_name } else { $feature }
+    if $e2e_exit == 0 {
+        generate-summary $results_dir $baseline_label $feature_label $bloat $preset $tps $duration --benchmark-id $benchmark_id --reference-epoch $reference_epoch
+    }
+
+    try { git worktree remove --force $baseline_wt } catch { }
+    try { git worktree remove --force $feature_wt } catch { }
+    cleanup-local-e2e-processes
+    bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
+    bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
+    if $e2e_exit != 0 {
+        exit $e2e_exit
+    }
+}
diff --git a/tempo.nu b/tempo.nu
index dacb7accd4..878a434acb 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -172,11 +172,23 @@ def bench-clean-datadir [datadir: string] {
 # Initialize a database: run `tempo init`, optionally load state bloat
 def bench-init-db [tempo_bin: string, genesis: string, datadir: string, bloat: int, bloat_file: string] {
     print $"Initializing database at ($datadir)..."
-    run-external $tempo_bin "init" "--chain" $genesis "--datadir" $datadir
+    let init_result = (run-external $tempo_bin "init" "--chain" $genesis "--datadir" $datadir | complete)
+    if $init_result.stdout != "" { print $init_result.stdout }
+    if $init_result.stderr != "" { print $init_result.stderr }
+    if $init_result.exit_code != 0 {
+        print $"Error: tempo init failed for ($datadir) with exit code ($init_result.exit_code)"
+        exit $init_result.exit_code
+    }
 
     if $bloat > 0 {
         print $"Loading state bloat into ($datadir)..."
-        run-external $tempo_bin "init-from-binary-dump" "--chain" $genesis "--datadir" $datadir $bloat_file | complete
+        let bloat_result = (run-external $tempo_bin "init-from-binary-dump" "--chain" $genesis "--datadir" $datadir $bloat_file | complete)
+        if $bloat_result.stdout != "" { print $bloat_result.stdout }
+        if $bloat_result.stderr != "" { print $bloat_result.stderr }
+        if $bloat_result.exit_code != 0 {
+            print $"Error: state bloat load failed for ($datadir) with exit code ($bloat_result.exit_code)"
+            exit $bloat_result.exit_code
+        }
     }
 }
 
@@ -1579,7 +1591,7 @@ def restore-system-tuning [tuning_state: record] {
     }
 
     print "Restoring system tuning..."
-    for svc in ["cron"] {
+    for svc in ["cron" "unattended-upgrades"] {
         try { sudo systemctl start $svc } catch { }
     }
     print "System tuning restored."

From 00b6a971b2b05f940bf2d868f0233543db2b63eb Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:10:08 +0100
Subject: [PATCH 05/16] ci(bench): trim legacy e2e helpers

---
 tempo.nu | 701 -------------------------------------------------------
 1 file changed, 701 deletions(-)

diff --git a/tempo.nu b/tempo.nu
index 878a434acb..4400748082 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -1458,32 +1458,6 @@ def build-e2e-consensus-args [node_dir: string, trusted_peers: string, port: int
     ]
 }
 
-def stop-tempo-processes-gracefully [] {
-    let pids = (find-tempo-pids)
-    if ($pids | length) > 0 {
-        print $"Stopping tempo processes: ($pids | str join ', ')"
-    }
-    for pid in $pids {
-        kill -s 2 $pid
-    }
-    for pid in $pids {
-        mut wait = 0
-        while $wait < 30 {
-            if (ps | where pid == $pid | length) == 0 { break }
-            sleep 1sec
-            $wait = $wait + 1
-        }
-        if $wait >= 30 {
-            print $"  Warning: PID ($pid) did not exit, sending SIGKILL"
-            kill -s 9 $pid
-            sleep 1sec
-        }
-    }
-    if ("/tmp/reth.ipc" | path exists) {
-        rm --force /tmp/reth.ipc
-    }
-}
-
 def stop-tracy-capture [] {
     print "  Stopping tracy-capture..."
     let capture_pids = (ps | where name =~ "tracy-capture" | get pid)
@@ -1669,636 +1643,6 @@ def "main bench-init" [
     print $"Virgin snapshot initialized and promoted."
 }
 
-# Initialize the schelk virgin snapshot for the 2-runner e2e consensus bench.
-#
-# This is intended for CI `force-bloat` runs. Both runners call it with the
-# same ordered validator list so they generate identical genesis/trusted-peer
-# data, then each runner copies only its role's validator identity files.
-def "main bench-consensus-init" [
-    --role: string                                      # Runner role: red uses validator 0, blue uses validator 1
-    --validators: string                                # Ordered validator consensus addresses: red_ip:port,blue_ip:port
-    --bench-datadir: string                             # Datadir/snapshot path to initialize and promote
-    --node-dir: string                                  # Validator identity directory for this runner
-    --genesis: string                                   # Destination genesis path for benchmark runs
-    --bloat: int = 0                                    # State bloat size in MiB
-    --accounts: int = 1000                              # Number of benchmark sender accounts
-    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
-    --features: string = $DEFAULT_FEATURES              # Cargo features
-    --seed: int = 42                                    # Deterministic seed shared by both runners
-    --gas-limit: string = ""                            # Optional genesis gas limit override
-] {
-    if $role not-in ["red" "blue"] {
-        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
-        exit 1
-    }
-
-    let validator_list = (
-        $validators
-        | split row ","
-        | each { |v| $v | str trim }
-        | where { |v| $v != "" }
-    )
-    if ($validator_list | length) != 2 {
-        print "Error: --validators must contain exactly two comma-separated consensus addresses ordered as red,blue"
-        exit 1
-    }
-
-    let validator_addr = if $role == "red" {
-        $validator_list | get 0
-    } else {
-        $validator_list | get 1
-    }
-    let validators_arg = ($validator_list | str join ",")
-    let genesis_accounts = ([$accounts 3] | math max) + 1
-    let init_dir = $"($LOCALNET_DIR)/e2e-consensus-init"
-    let generated_genesis = $"($init_dir)/genesis.json"
-    let generated_node_dir = $"($init_dir)/($validator_addr)"
-    let bloat_file = $"($init_dir)/state_bloat.bin"
-    let meta_dir = $"($bench_datadir)/($BENCH_META_SUBDIR)"
-    let generated_trusted_peers = $"($init_dir)/trusted-peers.txt"
-    let gas_limit_args = if $gas_limit != "" { ["--gas-limit" $gas_limit] } else { [] }
-
-    if ($init_dir | path exists) { rm -rf $init_dir }
-    mkdir $init_dir
-
-    build-tempo ["tempo"] $profile $features
-    let tempo_bin = if $profile == "dev" { "./target/debug/tempo" } else { $"./target/($profile)/tempo" }
-
-    print $"Generating e2e localnet config for validators: ($validators_arg)"
-    cargo run -p tempo-xtask --profile $profile -- generate-localnet -o $init_dir --accounts $genesis_accounts --validators $validators_arg --seed $seed --force ...$gas_limit_args
-
-    let trusted_peers = (trusted-peers-from-localnet $init_dir)
-    if $trusted_peers == "" {
-        print "Error: generated localnet did not produce trusted peers"
-        exit 1
-    }
-
-    if $bloat > 0 {
-        print $"Generating e2e state bloat \(($bloat) MiB\)..."
-        let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten)
-        cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args
-    }
-
-    bench-mount
-    bench-clean-datadir $bench_datadir
-    mkdir $bench_datadir
-    mkdir $node_dir
-
-    bench-init-db $tempo_bin $generated_genesis $bench_datadir $bloat $bloat_file
-
-    for file in ["signing.key" "signing.share" "enode.key" "enode.identity"] {
-        cp $"($generated_node_dir)/($file)" $"($node_dir)/($file)"
-    }
-
-    mkdir ($genesis | path dirname)
-    cp $generated_genesis $genesis
-    mkdir $meta_dir
-    $trusted_peers | save -f $generated_trusted_peers
-
-    bench-save-and-promote $bench_datadir $meta_dir {
-        bloat_mib: $bloat
-        accounts: $genesis_accounts
-        bench_datadir: $bench_datadir
-        node_dir: $node_dir
-        validator_role: $role
-        validator_addr: $validator_addr
-        validators: $validators_arg
-        seed: $seed
-        gas_limit: $gas_limit
-        dkg_in_genesis: true
-    } [[$generated_genesis "genesis.json"] [$generated_trusted_peers "trusted-peers.txt"]] 0 ""
-
-    print $"E2E consensus snapshot initialized and promoted for ($role)."
-    print $"Trusted peers: ($trusted_peers)"
-}
-
-# Run one side of a 2-runner e2e consensus benchmark phase.
-def "main bench-consensus-phase" [
-    --role: string                                      # Runner role: red sends load, blue validates only
-    --phase: string                                     # Phase label: baseline-1, feature-1, feature-2, baseline-2
-    --ref: string                                       # Git SHA/ref to build and run
-    --peer-url: string                                  # RPC URL for the validator on the other runner
-    --preset: string = ""                               # Preset: tip20, erc20, swap, order, tempo-mix
-    --tps: int = 10000                                  # Target TPS
-    --duration: int = 300                               # Duration in seconds
-    --accounts: int = 1000                              # Number of accounts
-    --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bench-datadir: string = ""                        # Datadir/snapshot path to recover and pass to the node
-    --bloat: int = 0                                    # State bloat size in MiB; enables bloat mnemonic for the sender
-    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
-    --features: string = $DEFAULT_FEATURES              # Cargo features
-    --samply                                            # Profile this runner's validator with samply
-    --samply-args: list<string> = []                    # Additional samply arguments
-    --tracy: string = "off"                             # Tracy profiling: off, on, full
-    --tracy-filter: string = "debug"                    # Tracy tracing filter level
-    --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
-    --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
-    --node-args: string = ""                            # Additional node args for all phases
-    --baseline-args: string = ""                        # Additional node args for baseline phases
-    --feature-args: string = ""                         # Additional node args for feature phases
-    --bench-args: string = ""                           # Additional tempo-bench args
-    --baseline-env: string = ""                         # Environment vars for baseline node phases
-    --feature-env: string = ""                          # Environment vars for feature node phases
-    --bench-env: string = ""                            # Environment vars for the sender process
-    --benchmark-id: string = ""                         # Shared benchmark identifier
-    --reference-epoch: int = 0                           # Shared timestamp for observability correlation
-    --tune                                              # Apply system tuning
-    --loud                                              # Show node debug logs
-    --node-dir: string                                  # Validator identity directory for this runner
-    --genesis: string                                   # Shared genesis file path
-    --trusted-peers: string                             # Comma-separated enode peers for the network
-    --consensus-port: int = 8000                        # Consensus listen port for this validator
-    --consensus-ip: string = ""                         # Optional consensus listen IP override
-    --results-dir: string = ""                          # Output root directory
-    --gas-limit: string = ""                            # Optional builder gas limit override
-    --tracing-otlp: string = ""                         # OTLP endpoint for tracing
-    --peer-hold-extra: int = 600                         # Extra seconds to wait for phase peer shutdown
-    --wait-peer-offline                                  # Wait for peer RPC to go offline before returning from sender phase
-    --no-cache                                          # Skip binary cache
-] {
-    if $role not-in ["red" "blue"] {
-        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
-        exit 1
-    }
-    if $preset == "" and $bench_args == "" {
-        print "Error: either --preset or --bench-args must be provided"
-        print $"  Available presets: ($PRESETS | columns | str join ', ')"
-        exit 1
-    }
-    if $preset != "" and not ($preset in $PRESETS) {
-        print $"Unknown preset: ($preset). Available: ($PRESETS | columns | str join ', ')"
-        exit 1
-    }
-    if $tracy not-in ["off" "on" "full"] {
-        print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
-        exit 1
-    }
-    if $samply and $tracy != "off" {
-        print "Error: --samply and --tracy are mutually exclusive. Choose one."
-        exit 1
-    }
-    if $tracy != "off" {
-        let has_tracy_capture = (which tracy-capture | length) > 0
-        if not $has_tracy_capture {
-            print "Error: tracy-capture not found. Install tracy and ensure tracy-capture is in PATH."
-            exit 1
-        }
-    }
-    let run_type = if ($phase | str starts-with "baseline") { "baseline" } else { "feature" }
-    let side_args = if $run_type == "baseline" { $baseline_args } else { $feature_args }
-    let side_env = if $run_type == "baseline" { $baseline_env } else { $feature_env }
-    let effective_node_args = ([$node_args $side_args] | where { |a| $a != "" } | str join " ")
-    let extra_args = if $effective_node_args == "" { [] } else { $effective_node_args | split row " " }
-    let weights = if $preset != "" { $PRESETS | get $preset } else { [0.0, 0.0, 0.0, 0.0] }
-    let datadir = if $bench_datadir != "" { $bench_datadir } else { $node_dir }
-    let phase_results_dir = if $results_dir != "" { $results_dir } else { $"($BENCH_RESULTS_DIR)/($phase)" }
-
-    main kill
-    let tuning_state = if $tune { apply-system-tuning } else { { tuned: false } }
-    bench-recover $datadir
-
-    if not ($node_dir | path exists) {
-        print $"Error: node dir does not exist after snapshot recovery: ($node_dir)"
-        exit 1
-    }
-    if not ($genesis | path exists) {
-        print $"Error: genesis file does not exist after snapshot recovery: ($genesis)"
-        exit 1
-    }
-    for required_file in ["signing.key" "signing.share" "enode.key"] {
-        let path = $"($node_dir)/($required_file)"
-        if not ($path | path exists) {
-            print $"Error: missing validator file after snapshot recovery: ($path)"
-            exit 1
-        }
-    }
-
-    let worktree_dir = $"($BENCH_WORKTREES_DIR)/e2e-($role)"
-    git worktree prune
-    mkdir $BENCH_WORKTREES_DIR
-    if ($worktree_dir | path exists) {
-        print $"Removing stale e2e worktree: ($worktree_dir)"
-        try { git worktree remove --force $worktree_dir } catch { rm -rf $worktree_dir }
-    }
-    git worktree add $worktree_dir $ref
-    let tbc = (tracy-build-config $features $tracy)
-    let effective_features = $tbc.features
-    let effective_extra_rustflags = $tbc.extra_rustflags
-    let effective_no_cache = $no_cache or ($tracy != "off")
-    if $effective_no_cache {
-        build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags --bench-features $features $worktree_dir $ref $profile $effective_features $ref
-    } else {
-        build-in-worktree $worktree_dir $ref $profile $effective_features $ref
-    }
-
-    let tempo_bin = (worktree-bin $worktree_dir $profile "tempo")
-    let bench_bin = (worktree-bin $worktree_dir $profile "tempo-bench")
-    if $results_dir == "" and ($phase_results_dir | path exists) { rm -rf $phase_results_dir }
-    mkdir $phase_results_dir
-    for stale in [
-        $"($phase_results_dir)/report-($phase).json"
-        $"($phase_results_dir)/profile-($phase).json.gz"
-        $"($phase_results_dir)/profile-($phase)-($role).json.gz"
-        $"($phase_results_dir)/tracy-profile-($phase).tracy"
-        $"($phase_results_dir)/tracy-profile-($phase)-($role).tracy"
-        $"($phase_results_dir)/logs-($phase)-($role)"
-    ] {
-        if ($stale | path exists) { rm -rf $stale }
-    }
-    if ("report.json" | path exists) { rm report.json }
-
-    let log_dir = $"($LOCALNET_DIR)/logs-e2e-($role)-($phase)"
-    if ($log_dir | path exists) { rm -rf $log_dir }
-    mkdir $log_dir
-
-    let tracing_url = if $tracing_otlp == "" and ($env.GRAFANA_TEMPO? | default "" | str length) > 0 {
-        let base = ($env.GRAFANA_TEMPO | str trim --right --char '/')
-        $"($base)/v1/traces"
-    } else if $tracing_otlp == "" and ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
-        let base = ($env.TEMPO_TELEMETRY_URL | str trim --right --char '/')
-        $"($base)/opentelemetry/v1/traces"
-    } else {
-        $tracing_otlp
-    }
-
-    let node_index = (port-to-node-index $consensus_port)
-    let http_port = 8545 + $node_index
-    let reth_metrics_port = 9001 + $node_index
-    let rpc_url = $"http://localhost:($http_port)"
-
-    let base_args = (build-base-args $genesis $datadir $log_dir "0.0.0.0" $http_port $reth_metrics_port)
-        | append (build-e2e-consensus-args $node_dir $trusted_peers $consensus_port $consensus_ip)
-        | append (log-filter-args $loud)
-        | append (if $gas_limit != "" { ["--builder.gaslimit" $gas_limit] } else { [] })
-        | append (if $tracy != "off" { ["--log.tracy" "--log.tracy.filter" $tracy_filter] } else { [] })
-        | append (if $tracing_url != "" { [$"--tracing-otlp=($tracing_url)"] } else { [] })
-    let args = (dedup-args $base_args $extra_args)
-
-    let tracy_env_prefix = if $tracy == "on" {
-        "TRACY_NO_SYS_TRACE=1 "
-    } else if $tracy == "full" {
-        "TRACY_SAMPLING_HZ=1 "
-    } else { "" }
-    let profile_label = $"($phase)-($role)"
-    let full_samply_args = if $samply {
-        $samply_args | append ["--save-only" "--presymbolicate" "--output" $"($phase_results_dir)/profile-($profile_label).json.gz"]
-    } else { [] }
-    let node_cmd = wrap-samply [$tempo_bin ...$args] $samply $full_samply_args
-    let node_cmd_str = ($node_cmd | str join " ")
-    let otel_attrs = $"OTEL_RESOURCE_ATTRIBUTES=benchmark_id=($benchmark_id),benchmark_run=($phase),runner_role=($role),run_type=($run_type),git_ref=($ref),reference_epoch=($reference_epoch) "
-    let env_prefix = if $side_env != "" { $"($side_env) " } else { "" }
-
-    print $"Starting e2e validator ($role) for ($phase) at ($rpc_url)"
-    job spawn { sh -c $"($env_prefix)($otel_attrs)($tracy_env_prefix)($node_cmd_str) 2>&1" | lines | each { |line| print $"[e2e-($phase)-($role)] ($line)" } }
-
-    sleep 2sec
-    let rpc_timeout = if $bloat > 0 { 600 } else { 300 }
-    wait-for-rpc-online $rpc_url $rpc_timeout
-    wait-for-peers $rpc_url 1 300
-    wait-for-chain-advance $rpc_url 300
-
-    let tracy_output = $"($phase_results_dir)/tracy-profile-($profile_label).tracy"
-    mut tracy_capture_started = false
-    if $tracy != "off" {
-        let seconds_flag = if $tracy_seconds > 0 { $"-s ($tracy_seconds)" } else { "" }
-        let limit_msg = if $tracy_seconds > 0 { $" \(($tracy_seconds)s limit\)" } else { "" }
-        if $tracy_offset > 0 {
-            print $"  Tracy-capture will start in ($tracy_offset)s($limit_msg)..."
-            job spawn { sleep ($"($tracy_offset)sec" | into duration); sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
-        } else {
-            print $"  Starting tracy-capture($limit_msg)..."
-            job spawn { sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" }
-            sleep 500ms
-        }
-        $tracy_capture_started = true
-    }
-
-    mut phase_exit = 0
-    if $role == "red" {
-        wait-for-rpc-online $peer_url 300
-        let bench_cmd = [
-            $bench_bin
-            "run-max-tps"
-            "--tps" $"($tps)"
-            "--duration" $"($duration)"
-            "--accounts" $"($accounts)"
-            "--max-concurrent-requests" $"($max_concurrent_requests)"
-            "--target-urls" $"($rpc_url),($peer_url)"
-            "--faucet"
-            "--clear-txpool"
-        ]
-        | append (if $preset != "" {
-            [
-                "--tip20-weight" $"($weights | get 0)"
-                "--erc20-weight" $"($weights | get 1)"
-                "--swap-weight" $"($weights | get 2)"
-                "--place-order-weight" $"($weights | get 3)"
-            ]
-        } else { [] })
-        | append (if $bloat > 0 {
-            ["--mnemonic" $"'($BLOAT_MNEMONIC)'"]
-        } else { [] })
-        | append (if $bench_args != "" { $bench_args | split row " " } else { [] })
-        | append ["--node-commit-sha" $ref "--build-profile" $profile "--benchmark-mode" "e2e"]
-
-        let bench_env_export = if $bench_env != "" { $"export ($bench_env) && " } else { "" }
-        print $"Running e2e sender: ($bench_cmd | str join ' ')"
-        let bench_result = (bash -c $"($bench_env_export)ulimit -Sn unlimited && ($bench_cmd | str join ' ')" | complete)
-        if $bench_result.stdout != "" { print $bench_result.stdout }
-        if $bench_result.stderr != "" { print $bench_result.stderr }
-        if $bench_result.exit_code != 0 {
-            print $"Sender failed for ($phase) with exit code ($bench_result.exit_code)"
-            $phase_exit = $bench_result.exit_code
-        }
-
-        if ("report.json" | path exists) {
-            cp report.json $"($phase_results_dir)/report-($phase).json"
-            rm report.json
-        } else {
-            print $"ERROR: sender for ($phase) produced no report.json"
-            $phase_exit = 1
-        }
-        if $wait_peer_offline {
-            if $tracy_capture_started {
-                stop-tracy-capture
-                $tracy_capture_started = false
-            }
-            stop-tempo-processes-gracefully
-            if $samply { wait-for-samply-profile }
-            wait-for-rpc-offline $peer_url ($peer_hold_extra + 300)
-        }
-    } else {
-        let hold_seconds = $duration + $peer_hold_extra
-        wait-for-rpc-online $peer_url 300
-        print $"Runner blue is holding validator online until runner red stops \(timeout: ($hold_seconds)s\)..."
-        wait-for-rpc-offline $peer_url $hold_seconds
-    }
-
-    if $tracy_capture_started {
-        stop-tracy-capture
-    }
-    stop-tempo-processes-gracefully
-    if $samply { wait-for-samply-profile }
-    if ($log_dir | path exists) {
-        cp -r $log_dir $"($phase_results_dir)/logs-($phase)-($role)"
-    }
-    try { git worktree remove --force $worktree_dir } catch { }
-    restore-system-tuning $tuning_state
-
-    if $phase_exit != 0 {
-        exit $phase_exit
-    }
-}
-
-# Run the full B-F-F-B e2e sequence on one side of the 2-runner setup.
-def "main bench-consensus" [
-    --role: string                                      # Runner role: red sends load, blue validates only
-    --baseline: string                                  # Baseline git SHA/ref
-    --feature: string                                   # Feature git SHA/ref
-    --peer-url: string                                  # RPC URL for the validator on the other runner
-    --preset: string = ""                               # Preset: tip20, erc20, swap, order, tempo-mix
-    --tps: int = 10000                                  # Target TPS
-    --duration: int = 300                               # Duration in seconds
-    --accounts: int = 1000                              # Number of accounts
-    --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bench-datadir: string = ""                        # Datadir/snapshot path to recover and pass to the node
-    --bloat: int = 0                                    # State bloat size in MiB
-    --profile: string = $DEFAULT_PROFILE                # Cargo build profile
-    --features: string = $DEFAULT_FEATURES              # Cargo features
-    --samply                                            # Profile validators with samply
-    --samply-args: string = ""                          # Additional samply arguments
-    --tracy: string = "off"                             # Tracy profiling: off, on, full
-    --tracy-filter: string = "debug"                    # Tracy tracing filter level
-    --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
-    --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
-    --node-args: string = ""                            # Additional node args for all phases
-    --baseline-args: string = ""                        # Additional node args for baseline phases
-    --feature-args: string = ""                         # Additional node args for feature phases
-    --bench-args: string = ""                           # Additional tempo-bench args
-    --baseline-env: string = ""                         # Environment vars for baseline node phases
-    --feature-env: string = ""                          # Environment vars for feature node phases
-    --bench-env: string = ""                            # Environment vars for the sender process
-    --benchmark-id: string = ""                         # Shared benchmark identifier
-    --reference-epoch: int = 0                           # Shared timestamp for observability correlation
-    --baseline-name: string = ""                         # Baseline display name for summary
-    --feature-name: string = ""                          # Feature display name for summary
-    --tune                                              # Apply system tuning
-    --loud                                              # Show node debug logs
-    --node-dir: string                                  # Validator identity directory for this runner
-    --genesis: string                                   # Shared genesis file path
-    --baseline-genesis: string = ""                     # Baseline genesis for hardfork comparison
-    --feature-genesis: string = ""                      # Feature genesis for hardfork comparison
-    --baseline-node-dir: string = ""                    # Baseline validator identity dir for hardfork comparison
-    --feature-node-dir: string = ""                     # Feature validator identity dir for hardfork comparison
-    --baseline-bench-datadir: string = ""               # Baseline datadir/snapshot path for hardfork comparison
-    --feature-bench-datadir: string = ""                # Feature datadir/snapshot path for hardfork comparison
-    --baseline-hardfork: string = ""                    # Latest active hardfork for baseline phases
-    --feature-hardfork: string = ""                     # Latest active hardfork for feature phases
-    --trusted-peers: string                             # Comma-separated enode peers for the network
-    --consensus-port: int = 8000                        # Consensus listen port for this validator
-    --consensus-ip: string = ""                         # Optional consensus listen IP override
-    --gas-limit: string = ""                            # Optional builder gas limit override
-    --tracing-otlp: string = ""                         # OTLP endpoint for tracing
-    --no-cache                                          # Skip binary cache
-] {
-    if $role not-in ["red" "blue"] {
-        print $"Error: --role must be 'red' or 'blue' \(got '($role)'\)"
-        exit 1
-    }
-
-    let timestamp = (date now | format date "%Y%m%d-%H%M%S")
-    let results_dir = $"($BENCH_RESULTS_DIR)/($timestamp)"
-    mkdir $results_dir
-    print $"BENCH_RESULTS_DIR=($results_dir)"
-
-    let effective_benchmark_id = if $benchmark_id != "" { $benchmark_id } else { $"bench-e2e-($timestamp)" }
-    let effective_reference_epoch = if $reference_epoch != 0 {
-        $reference_epoch
-    } else {
-        ((date now | into int) / 1_000_000_000 | into int)
-    }
-    if $tracy not-in ["off" "on" "full"] {
-        print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
-        exit 1
-    }
-    if $samply and $tracy != "off" {
-        print "Error: --samply and --tracy are mutually exclusive. Choose one."
-        exit 1
-    }
-    if ($baseline_hardfork != "" or $feature_hardfork != "") and ($baseline_hardfork == "" or $feature_hardfork == "") {
-        print "Error: --baseline-hardfork and --feature-hardfork must both be provided"
-        exit 1
-    }
-    if $baseline_hardfork != "" {
-        let valid = ($TEMPO_HARDFORKS | any { |f| $f == ($baseline_hardfork | str upcase) })
-        if not $valid {
-            print $"Error: unknown baseline hardfork '($baseline_hardfork)'. Valid: ($TEMPO_HARDFORKS | str join ', ')"
-            exit 1
-        }
-    }
-    if $feature_hardfork != "" {
-        let valid = ($TEMPO_HARDFORKS | any { |f| $f == ($feature_hardfork | str upcase) })
-        if not $valid {
-            print $"Error: unknown feature hardfork '($feature_hardfork)'. Valid: ($TEMPO_HARDFORKS | str join ', ')"
-            exit 1
-        }
-    }
-    let dual_hardfork = $baseline_hardfork != "" and $feature_hardfork != ""
-    if $tracy == "full" and (^uname | str trim) == "Linux" {
-        print "Configuring system for tracy CPU sampling..."
-        try { sudo sysctl -w kernel.perf_event_paranoid=-1 } catch { }
-        try { sudo mount -t tracefs tracefs /sys/kernel/tracing -o remount,mode=755 } catch { }
-        try { sudo chmod -R a+r /sys/kernel/tracing } catch { }
-    }
-
-    let env_baseline_genesis = ($env.BENCH_E2E_BASELINE_GENESIS? | default "")
-    let env_feature_genesis = ($env.BENCH_E2E_FEATURE_GENESIS? | default "")
-    let env_baseline_node_dir = ($env.BENCH_E2E_BASELINE_NODE_DIR? | default "")
-    let env_feature_node_dir = ($env.BENCH_E2E_FEATURE_NODE_DIR? | default "")
-    let env_baseline_datadir = ($env.BENCH_E2E_BASELINE_DATADIR? | default "")
-    let env_feature_datadir = ($env.BENCH_E2E_FEATURE_DATADIR? | default "")
-    let genesis_dir = ($genesis | path dirname)
-    let baseline_genesis_candidate = $"($genesis_dir)/genesis-baseline.json"
-    let feature_genesis_candidate = $"($genesis_dir)/genesis-feature.json"
-    let effective_baseline_genesis = if $baseline_genesis != "" {
-        $baseline_genesis
-    } else if $env_baseline_genesis != "" {
-        $env_baseline_genesis
-    } else if ($baseline_genesis_candidate | path exists) {
-        $baseline_genesis_candidate
-    } else {
-        $genesis
-    }
-    let effective_feature_genesis = if $feature_genesis != "" {
-        $feature_genesis
-    } else if $env_feature_genesis != "" {
-        $env_feature_genesis
-    } else if ($feature_genesis_candidate | path exists) {
-        $feature_genesis_candidate
-    } else {
-        $genesis
-    }
-    let effective_baseline_node_dir = if $baseline_node_dir != "" {
-        $baseline_node_dir
-    } else if $env_baseline_node_dir != "" {
-        $env_baseline_node_dir
-    } else {
-        $node_dir
-    }
-    let effective_feature_node_dir = if $feature_node_dir != "" {
-        $feature_node_dir
-    } else if $env_feature_node_dir != "" {
-        $env_feature_node_dir
-    } else {
-        $node_dir
-    }
-    if $dual_hardfork {
-        if $effective_baseline_genesis == $genesis or $effective_feature_genesis == $genesis {
-            print "Error: hardfork comparison requires phase-specific e2e genesis files. Set BENCH_E2E_BASELINE_GENESIS and BENCH_E2E_FEATURE_GENESIS, or place genesis-baseline.json and genesis-feature.json next to BENCH_E2E_GENESIS on both runners."
-            exit 1
-        }
-    }
-    let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " }
-    let effective_bench_datadir = if $bench_datadir != "" { $bench_datadir } else { $node_dir }
-    let effective_baseline_datadir = if $baseline_bench_datadir != "" {
-        $baseline_bench_datadir
-    } else if $env_baseline_datadir != "" {
-        $env_baseline_datadir
-    } else {
-        $effective_bench_datadir
-    }
-    let effective_feature_datadir = if $feature_bench_datadir != "" {
-        $feature_bench_datadir
-    } else if $env_feature_datadir != "" {
-        $env_feature_datadir
-    } else {
-        $effective_bench_datadir
-    }
-
-    let runs = [
-        { phase: "baseline-1", ref: $baseline, wait_peer: true, genesis: $effective_baseline_genesis, node_dir: $effective_baseline_node_dir, bench_datadir: $effective_baseline_datadir }
-        { phase: "feature-1", ref: $feature, wait_peer: true, genesis: $effective_feature_genesis, node_dir: $effective_feature_node_dir, bench_datadir: $effective_feature_datadir }
-        { phase: "feature-2", ref: $feature, wait_peer: true, genesis: $effective_feature_genesis, node_dir: $effective_feature_node_dir, bench_datadir: $effective_feature_datadir }
-        { phase: "baseline-2", ref: $baseline, wait_peer: false, genesis: $effective_baseline_genesis, node_dir: $effective_baseline_node_dir, bench_datadir: $effective_baseline_datadir }
-    ]
-
-    for run in $runs {
-        (main bench-consensus-phase
-            --role $role
-            --phase $run.phase
-            --ref $run.ref
-            --peer-url $peer_url
-            --preset $preset
-            --tps $tps
-            --duration $duration
-            --accounts $accounts
-            --max-concurrent-requests $max_concurrent_requests
-            --bench-datadir $run.bench_datadir
-            --bloat $bloat
-            --profile $profile
-            --features $features
-            --samply=$samply
-            --samply-args $samply_args_list
-            --tracy $tracy
-            --tracy-filter $tracy_filter
-            --tracy-seconds $tracy_seconds
-            --tracy-offset $tracy_offset
-            --node-args $node_args
-            --baseline-args $baseline_args
-            --feature-args $feature_args
-            --bench-args $bench_args
-            --baseline-env $baseline_env
-            --feature-env $feature_env
-            --bench-env $bench_env
-            --benchmark-id $effective_benchmark_id
-            --reference-epoch $effective_reference_epoch
-            --tune=$tune
-            --loud=$loud
-            --node-dir $run.node_dir
-            --genesis $run.genesis
-            --trusted-peers $trusted_peers
-            --consensus-port $consensus_port
-            --consensus-ip $consensus_ip
-            --results-dir $results_dir
-            --gas-limit $gas_limit
-            --tracing-otlp $tracing_otlp
-            --peer-hold-extra 600
-            --wait-peer-offline=$run.wait_peer
-            --no-cache=$no_cache)
-    }
-
-    if $samply {
-        print $"\nUploading ($role) samply profiles to Firefox Profiler..."
-        for run in $runs {
-            let profile_label = $"($run.phase)-($role)"
-            let profile = $"($results_dir)/profile-($profile_label).json.gz"
-            let url = (upload-samply-profile $profile)
-            if $url != null {
-                $url | save -f $"($results_dir)/profile-($profile_label)-url.txt"
-            }
-        }
-    }
-    if $tracy != "off" {
-        print $"\nUploading ($role) tracy profiles to R2..."
-        for run in $runs {
-            let profile_label = $"($run.phase)-($role)"
-            let profile = $"($results_dir)/tracy-profile-($profile_label).tracy"
-            let viewer_url = (upload-tracy-profile $profile $profile_label $run.ref)
-            if $viewer_url != null {
-                $viewer_url | save -f $"($results_dir)/tracy-($profile_label)-url.txt"
-            }
-        }
-    }
-
-    if $role == "red" {
-        let baseline_label_base = if $baseline_name != "" { $baseline_name } else { $baseline }
-        let feature_label_base = if $feature_name != "" { $feature_name } else { $feature }
-        let baseline_label = if $dual_hardfork { $"($baseline_label_base) \(($baseline_hardfork | str upcase)\)" } else { $baseline_label_base }
-        let feature_label = if $dual_hardfork { $"($feature_label_base) \(($feature_hardfork | str upcase)\)" } else { $feature_label_base }
-        generate-summary $results_dir $baseline_label $feature_label $bloat $preset $tps $duration --benchmark-id $effective_benchmark_id --reference-epoch $effective_reference_epoch
-    }
-}
-
 # ============================================================================
 # Bench command
 # ============================================================================
@@ -3007,51 +2351,6 @@ def wait-for-rpc-online [url: string, max_attempts: int = 120] {
     }
 }
 
-# Wait for an RPC endpoint to stop answering eth_blockNumber.
-def wait-for-rpc-offline [url: string, max_attempts: int = 120] {
-    mut attempt = 0
-
-    loop {
-        $attempt = $attempt + 1
-        if $attempt > $max_attempts {
-            print $"  Timeout waiting for ($url) to go offline"
-            exit 1
-        }
-        let block = (rpc-block-number $url)
-        if $block == null {
-            print $"  ($url) offline"
-            break
-        }
-        if ($attempt mod 10) == 0 {
-            print $"  Waiting for ($url) to go offline... \(($attempt)s\)"
-        }
-        sleep 1sec
-    }
-}
-
-# Wait for an RPC endpoint to see at least the requested number of peers.
-def wait-for-peers [url: string, min_peers: int = 1, max_attempts: int = 120] {
-    mut attempt = 0
-
-    loop {
-        $attempt = $attempt + 1
-        if $attempt > $max_attempts {
-            print $"  Timeout waiting for ($url) to reach ($min_peers) peer\(s\)"
-            exit 1
-        }
-        let peers = (rpc-peer-count $url)
-        if $peers != null and $peers >= $min_peers {
-            print $"  ($url) has ($peers) peer\(s\)"
-            break
-        }
-        if ($attempt mod 10) == 0 {
-            let current = if $peers == null { "unknown" } else { $"($peers)" }
-            print $"  ($url) peers: ($current)/($min_peers)... \(($attempt)s\)"
-        }
-        sleep 1sec
-    }
-}
-
 # Wait for an RPC endpoint's chain to advance beyond its first observed block.
 def wait-for-chain-advance [url: string, max_attempts: int = 120] {
     mut attempt = 0

From eebef71a9a28218a941d67885d0f3d9ea53447d7 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:14:55 +0100
Subject: [PATCH 06/16] ci(bench): keep e2e helpers local

---
 bench-e2e.nu | 140 +++++++++++++++++++++++++++++++++++-
 tempo.nu     | 200 +++++++--------------------------------------------
 2 files changed, 165 insertions(+), 175 deletions(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index b51bf387c1..e208d0e957 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -127,6 +127,45 @@ def ensure-bloat-space [bloat: int] {
     }
 }
 
+def validator-dirs-in-localnet [localnet_dir: string] {
+    ls $localnet_dir
+    | where type == "dir"
+    | get name
+    | where { |d| ($d | path basename) =~ '^\d+\.\d+\.\d+\.\d+:\d+$' }
+}
+
+def trusted-peers-from-localnet [localnet_dir: string] {
+    validator-dirs-in-localnet $localnet_dir | each { |d|
+        let addr = ($d | path basename)
+        let ip = ($addr | split row ":" | get 0)
+        let port = ($addr | split row ":" | get 1 | into int)
+        let identity = (open $"($d)/enode.identity" | str trim)
+        $"enode://($identity)@($ip):($port + 1)"
+    } | str join ","
+}
+
+def init-e2e-db [tempo_bin: string, genesis: string, datadir: string, bloat: int, bloat_file: string] {
+    print $"Initializing database at ($datadir)..."
+    let init_result = (run-external $tempo_bin "init" "--chain" $genesis "--datadir" $datadir | complete)
+    if $init_result.stdout != "" { print $init_result.stdout }
+    if $init_result.stderr != "" { print $init_result.stderr }
+    if $init_result.exit_code != 0 {
+        print $"Error: tempo init failed for ($datadir) with exit code ($init_result.exit_code)"
+        exit $init_result.exit_code
+    }
+
+    if $bloat > 0 {
+        print $"Loading state bloat into ($datadir)..."
+        let bloat_result = (run-external $tempo_bin "init-from-binary-dump" "--chain" $genesis "--datadir" $datadir $bloat_file | complete)
+        if $bloat_result.stdout != "" { print $bloat_result.stdout }
+        if $bloat_result.stderr != "" { print $bloat_result.stderr }
+        if $bloat_result.exit_code != 0 {
+            print $"Error: state bloat load failed for ($datadir) with exit code ($bloat_result.exit_code)"
+            exit $bloat_result.exit_code
+        }
+    }
+}
+
 def bench-save-e2e-meta [datadir: string, meta_dir: string, marker: record, genesis_files: list] {
     mkdir $meta_dir
     for pair in $genesis_files {
@@ -197,6 +236,40 @@ def start-e2e-local-node [
     }
 }
 
+def build-e2e-consensus-args [node_dir: string, trusted_peers: string, port: int, consensus_ip: string] {
+    let addr = ($node_dir | path basename)
+    let inferred_ip = if ($addr | str contains ":") {
+        $addr | split row ":" | get 0
+    } else {
+        "0.0.0.0"
+    }
+    let ip = if $consensus_ip != "" { $consensus_ip } else { $inferred_ip }
+    let signing_key = $"($node_dir)/signing.key"
+    let signing_share = $"($node_dir)/signing.share"
+    let enode_key = $"($node_dir)/enode.key"
+
+    let execution_p2p_port = $port + 1
+    let metrics_port = $port + 2
+    let authrpc_port = $port + 3
+    let discv5_port = $port + 4
+
+    [
+        "--consensus.signing-key" $signing_key
+        "--consensus.signing-share" $signing_share
+        "--consensus.listen-address" $"($ip):($port)"
+        "--consensus.metrics-address" $"($ip):($metrics_port)"
+        "--trusted-peers" $trusted_peers
+        "--port" $"($execution_p2p_port)"
+        "--discovery.port" $"($execution_p2p_port)"
+        "--discovery.v5.port" $"($discv5_port)"
+        "--p2p-secret-key" $enode_key
+        "--authrpc.port" $"($authrpc_port)"
+        "--consensus.fee-recipient" "0x0000000000000000000000000000000000000000"
+        "--consensus.use-local-defaults"
+        "--consensus.bypass-ip-check"
+    ]
+}
+
 def stop-e2e-processes-gracefully [] {
     let pids = (find-tempo-pids)
     if ($pids | length) > 0 {
@@ -223,6 +296,39 @@ def stop-e2e-processes-gracefully [] {
     }
 }
 
+def stop-tracy-capture [] {
+    print "  Stopping tracy-capture..."
+    let capture_pids = (ps | where name =~ "tracy-capture" | get pid)
+    for pid in $capture_pids {
+        kill -s 2 $pid
+    }
+    mut wait_tracy = 0
+    while $wait_tracy < 30 {
+        if (ps | where name =~ "tracy-capture" | length) == 0 { break }
+        sleep 1sec
+        $wait_tracy = $wait_tracy + 1
+    }
+    if $wait_tracy >= 30 {
+        print "  Warning: tracy-capture did not exit, sending SIGKILL"
+        for pid in (ps | where name =~ "tracy-capture" | get pid) {
+            kill -s 9 $pid
+        }
+    }
+}
+
+def wait-for-samply-profile [] {
+    print "  Waiting for samply to finish saving profile..."
+    mut wait = 0
+    while $wait < 120 {
+        if (ps | where name =~ "samply" | length) == 0 { break }
+        sleep 500ms
+        $wait = $wait + 1
+    }
+    if $wait >= 120 {
+        print "  Warning: samply did not exit in time"
+    }
+}
+
 def stop-local-e2e-systemd-scopes [] {
     if (^uname | str trim) != "Linux" or ((which systemctl | length) == 0) {
         return
@@ -246,6 +352,38 @@ def cleanup-local-e2e-processes [] {
     stop-tracy-capture
 }
 
+def rpc-block-number [url: string] {
+    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' } | complete)
+    if $result.exit_code != 0 {
+        return null
+    }
+    let parsed = (try { $result.stdout | from json } catch { null })
+    if $parsed == null {
+        return null
+    }
+    let hex = ($parsed | get -o result | default "")
+    if $hex == "" {
+        return null
+    }
+    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
+}
+
+def rpc-peer-count [url: string] {
+    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' } | complete)
+    if $result.exit_code != 0 {
+        return null
+    }
+    let parsed = (try { $result.stdout | from json } catch { null })
+    if $parsed == null {
+        return null
+    }
+    let hex = ($parsed | get -o result | default "")
+    if $hex == "" {
+        return null
+    }
+    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
+}
+
 def e2e-wait-for-rpc-online [url: string, max_attempts: int] {
     mut attempt = 0
 
@@ -338,7 +476,7 @@ def init-local-e2e-side [
     mkdir $datadir
     mkdir $node_dir
 
-    bench-init-db $tempo_bin $generated_genesis $datadir $bloat $bloat_file
+    init-e2e-db $tempo_bin $generated_genesis $datadir $bloat $bloat_file
     for file in ["signing.key" "signing.share" "enode.key" "enode.identity"] {
         cp $"($generated_node_dir)/($file)" $"($node_dir)/($file)"
     }
diff --git a/tempo.nu b/tempo.nu
index 4400748082..fb312bd4c7 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -172,23 +172,11 @@ def bench-clean-datadir [datadir: string] {
 # Initialize a database: run `tempo init`, optionally load state bloat
 def bench-init-db [tempo_bin: string, genesis: string, datadir: string, bloat: int, bloat_file: string] {
     print $"Initializing database at ($datadir)..."
-    let init_result = (run-external $tempo_bin "init" "--chain" $genesis "--datadir" $datadir | complete)
-    if $init_result.stdout != "" { print $init_result.stdout }
-    if $init_result.stderr != "" { print $init_result.stderr }
-    if $init_result.exit_code != 0 {
-        print $"Error: tempo init failed for ($datadir) with exit code ($init_result.exit_code)"
-        exit $init_result.exit_code
-    }
+    run-external $tempo_bin "init" "--chain" $genesis "--datadir" $datadir
 
     if $bloat > 0 {
         print $"Loading state bloat into ($datadir)..."
-        let bloat_result = (run-external $tempo_bin "init-from-binary-dump" "--chain" $genesis "--datadir" $datadir $bloat_file | complete)
-        if $bloat_result.stdout != "" { print $bloat_result.stdout }
-        if $bloat_result.stderr != "" { print $bloat_result.stderr }
-        if $bloat_result.exit_code != 0 {
-            print $"Error: state bloat load failed for ($datadir) with exit code ($bloat_result.exit_code)"
-            exit $bloat_result.exit_code
-        }
+        run-external $tempo_bin "init-from-binary-dump" "--chain" $genesis "--datadir" $datadir $bloat_file | complete
     }
 }
 
@@ -287,23 +275,6 @@ def read-bench-marker [datadir: string] {
     }
 }
 
-def validator-dirs-in-localnet [localnet_dir: string] {
-    ls $localnet_dir
-    | where type == "dir"
-    | get name
-    | where { |d| ($d | path basename) =~ '^\d+\.\d+\.\d+\.\d+:\d+$' }
-}
-
-def trusted-peers-from-localnet [localnet_dir: string] {
-    validator-dirs-in-localnet $localnet_dir | each { |d|
-        let addr = ($d | path basename)
-        let ip = ($addr | split row ":" | get 0)
-        let port = ($addr | split row ":" | get 1 | into int)
-        let identity = (open $"($d)/enode.identity" | str trim)
-        $"enode://($identity)@($ip):($port + 1)"
-    } | str join ","
-}
-
 # ============================================================================
 # Comparison mode helpers
 # ============================================================================
@@ -1294,8 +1265,14 @@ def run-consensus-nodes [nodes: int, accounts: int, genesis: string, samply: boo
     let genesis_path = if $genesis != "" { $genesis } else { $"($LOCALNET_DIR)/genesis.json" }
 
     # Build trusted peers from enode.identity files
-    let validator_dirs = (validator-dirs-in-localnet $LOCALNET_DIR)
-    let trusted_peers = (trusted-peers-from-localnet $LOCALNET_DIR)
+    let validator_dirs = (ls $LOCALNET_DIR | where type == "dir" | get name | where { |d| ($d | path basename) =~ '^\d+\.\d+\.\d+\.\d+:\d+$' })
+    let trusted_peers = ($validator_dirs | each { |d|
+        let addr = ($d | path basename)
+        let ip = ($addr | split row ":" | get 0)
+        let port = ($addr | split row ":" | get 1 | into int)
+        let identity = (open $"($d)/enode.identity" | str trim)
+        $"enode://($identity)@($ip):($port + 1)"
+    } | str join ",")
 
     print $"Found ($validator_dirs | length) validator configs"
 
@@ -1423,74 +1400,6 @@ def build-consensus-args [node_dir: string, trusted_peers: string, port: int] {
     ]
 }
 
-# Build consensus args for a single validator running on an isolated e2e runner.
-def build-e2e-consensus-args [node_dir: string, trusted_peers: string, port: int, consensus_ip: string] {
-    let addr = ($node_dir | path basename)
-    let inferred_ip = if ($addr | str contains ":") {
-        $addr | split row ":" | get 0
-    } else {
-        "0.0.0.0"
-    }
-    let ip = if $consensus_ip != "" { $consensus_ip } else { $inferred_ip }
-    let signing_key = $"($node_dir)/signing.key"
-    let signing_share = $"($node_dir)/signing.share"
-    let enode_key = $"($node_dir)/enode.key"
-
-    let execution_p2p_port = $port + 1
-    let metrics_port = $port + 2
-    let authrpc_port = $port + 3
-    let discv5_port = $port + 4
-
-    [
-        "--consensus.signing-key" $signing_key
-        "--consensus.signing-share" $signing_share
-        "--consensus.listen-address" $"($ip):($port)"
-        "--consensus.metrics-address" $"($ip):($metrics_port)"
-        "--trusted-peers" $trusted_peers
-        "--port" $"($execution_p2p_port)"
-        "--discovery.port" $"($execution_p2p_port)"
-        "--discovery.v5.port" $"($discv5_port)"
-        "--p2p-secret-key" $enode_key
-        "--authrpc.port" $"($authrpc_port)"
-        "--consensus.fee-recipient" "0x0000000000000000000000000000000000000000"
-        "--consensus.use-local-defaults"
-        "--consensus.bypass-ip-check"
-    ]
-}
-
-def stop-tracy-capture [] {
-    print "  Stopping tracy-capture..."
-    let capture_pids = (ps | where name =~ "tracy-capture" | get pid)
-    for pid in $capture_pids {
-        kill -s 2 $pid
-    }
-    mut wait_tracy = 0
-    while $wait_tracy < 30 {
-        if (ps | where name =~ "tracy-capture" | length) == 0 { break }
-        sleep 1sec
-        $wait_tracy = $wait_tracy + 1
-    }
-    if $wait_tracy >= 30 {
-        print "  Warning: tracy-capture did not exit, sending SIGKILL"
-        for pid in (ps | where name =~ "tracy-capture" | get pid) {
-            kill -s 9 $pid
-        }
-    }
-}
-
-def wait-for-samply-profile [] {
-    print "  Waiting for samply to finish saving profile..."
-    mut wait = 0
-    while $wait < 120 {
-        if (ps | where name =~ "samply" | length) == 0 { break }
-        sleep 500ms
-        $wait = $wait + 1
-    }
-    if $wait >= 120 {
-        print "  Warning: samply did not exit in time"
-    }
-}
-
 # ============================================================================
 # System tuning for benchmarks
 # ============================================================================
@@ -1565,7 +1474,7 @@ def restore-system-tuning [tuning_state: record] {
     }
 
     print "Restoring system tuning..."
-    for svc in ["cron" "unattended-upgrades"] {
+    for svc in ["cron"] {
         try { sudo systemctl start $svc } catch { }
     }
     print "System tuning restored."
@@ -2294,98 +2203,41 @@ def "main bench" [
     print "Done."
 }
 
-# Fetch the current block number from an RPC endpoint.
-def rpc-block-number [url: string] {
-    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' } | complete)
-    if $result.exit_code != 0 {
-        return null
-    }
-    let parsed = (try { $result.stdout | from json } catch { null })
-    if $parsed == null {
-        return null
-    }
-    let hex = ($parsed | get -o result | default "")
-    if $hex == "" {
-        return null
-    }
-    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
-}
-
-# Fetch the current peer count from an RPC endpoint.
-def rpc-peer-count [url: string] {
-    let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' } | complete)
-    if $result.exit_code != 0 {
-        return null
-    }
-    let parsed = (try { $result.stdout | from json } catch { null })
-    if $parsed == null {
-        return null
-    }
-    let hex = ($parsed | get -o result | default "")
-    if $hex == "" {
-        return null
-    }
-    try { $hex | str replace "0x" "" | into int --radix 16 } catch { null }
-}
-
-# Wait for an RPC endpoint to answer eth_blockNumber.
-def wait-for-rpc-online [url: string, max_attempts: int = 120] {
-    mut attempt = 0
-
-    loop {
-        $attempt = $attempt + 1
-        if $attempt > $max_attempts {
-            print $"  Timeout waiting for ($url)"
-            exit 1
-        }
-        let block = (rpc-block-number $url)
-        if $block != null {
-            print $"  ($url) online \(block ($block)\)"
-            break
-        } else {
-            if ($attempt mod 10) == 0 {
-                print $"  Still waiting for ($url)... \(($attempt)s\)"
-            }
-        }
-        sleep 1sec
-    }
-}
-
-# Wait for an RPC endpoint's chain to advance beyond its first observed block.
-def wait-for-chain-advance [url: string, max_attempts: int = 120] {
+# Wait for an RPC endpoint to be ready and chain advancing
+def wait-for-rpc [url: string, max_attempts: int = 120] {
     mut attempt = 0
     mut start_block: int = -1
 
     loop {
         $attempt = $attempt + 1
         if $attempt > $max_attempts {
-            print $"  Timeout waiting for ($url) chain to advance"
+            print $"  Timeout waiting for ($url)"
             exit 1
         }
-        let block = (rpc-block-number $url)
-        if $block != null {
+        let result = (do { curl -sf $url -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' } | complete)
+        if $result.exit_code == 0 {
+            let hex = ($result.stdout | from json | get result)
+            let block = ($hex | str replace "0x" "" | into int --radix 16)
             if $start_block == -1 {
                 $start_block = $block
                 print $"  ($url) connected \(block ($block)\), waiting for chain to advance..."
             } else if $block > $start_block {
                 print $"  ($url) ready \(block ($start_block) -> ($block)\)"
                 break
-            } else if ($attempt mod 10) == 0 {
-                print $"  ($url) still at block ($block)... \(($attempt)s\)"
+            } else {
+                if ($attempt mod 10) == 0 {
+                    print $"  ($url) still at block ($block)... \(($attempt)s\)"
+                }
+            }
+        } else {
+            if ($attempt mod 10) == 0 {
+                print $"  Still waiting for ($url)... \(($attempt)s\)"
             }
-        } else if ($attempt mod 10) == 0 {
-            print $"  Still waiting for ($url)... \(($attempt)s\)"
         }
         sleep 1sec
     }
 }
 
-# Wait for an RPC endpoint to be ready and chain advancing.
-def wait-for-rpc [url: string, max_attempts: int = 120] {
-    wait-for-rpc-online $url $max_attempts
-    wait-for-chain-advance $url $max_attempts
-}
-
 # ============================================================================
 # Coverage commands
 # ============================================================================

From 11d35f6282beadc31076aa4dfd073c593535b54c Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:15:55 +0100
Subject: [PATCH 07/16] ci(bench): default e2e bloat to 100g

---
 bench-e2e.nu | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index e208d0e957..cd8cb4df2d 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -18,6 +18,8 @@ const E2E_B_MEMORY = ""
 const E2E_GAS_LIMIT = "1000000000000"
 const E2E_BLOAT_TMP_DIR = "/reth-bench-a/.bench-tmp/e2e-local-init"
 const E2E_BLOAT_FREE_MARGIN_MIB = 51200
+const E2E_DEFAULT_BLOAT_MIB = 100000
+const E2E_SUPPORTED_BLOAT_MIB = [1000 10000 100000]
 const E2E_LOCAL_RETH_ARGS = [
     "--ipcdisable"
     "--disable-discovery"
@@ -664,7 +666,7 @@ def "main e2e" [
     --duration: int = 300                               # Duration in seconds
     --accounts: int = 1000                              # Number of accounts
     --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bloat: int = 0                                    # State bloat size in MiB
+    --bloat: int = $E2E_DEFAULT_BLOAT_MIB               # State bloat size in MiB
     --force-bloat                                      # Regenerate and promote both local e2e snapshots
     --init-only                                         # Refresh snapshots and exit without running benchmark phases
     --profile: string = $DEFAULT_PROFILE                # Cargo build profile
@@ -705,6 +707,11 @@ def "main e2e" [
         print "Error: --samply and --tracy are mutually exclusive. Choose one."
         exit 1
     }
+    if $bloat not-in $E2E_SUPPORTED_BLOAT_MIB {
+        let supported = ($E2E_SUPPORTED_BLOAT_MIB | each { |size| $"($size)" } | str join ", ")
+        print $"Error: --bloat must be one of ($supported) MiB \(1G, 10G, 100G snapshots\)"
+        exit 1
+    }
     if $init_only and not $force_bloat {
         print "Error: --init-only requires --force-bloat"
         exit 1

From d9007e30ddf38c1394647c74faed9eb36a4eb46d Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:18:28 +0100
Subject: [PATCH 08/16] ci(bench): use e2e bloat size labels

---
 .github/workflows/bench-e2e.yml |  6 ++---
 .github/workflows/bench.yml     | 20 ++++++++++-----
 bench-e2e.nu                    | 44 +++++++++++++++++++--------------
 3 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index ea234ef910..e3ae356ff2 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -29,10 +29,10 @@ on:
         required: true
         default: "300"
       bloat:
-        description: State bloat size in MiB.
+        description: State bloat snapshot size (1g, 10g, 100g).
         type: string
         required: true
-        default: "100000"
+        default: "100g"
       tps:
         description: Target transactions per second.
         type: string
@@ -383,7 +383,7 @@ jobs:
             const bHf2 = process.env.BENCH_BASELINE_HARDFORK || '';
             const fHf2 = process.env.BENCH_FEATURE_HARDFORK || '';
             const hfNote2 = bHf2 ? `, baseline-hardfork: \`${bHf2}\`, feature-hardfork: \`${fHf2}\`` : '';
-            core.exportVariable('BENCH_CONFIG', `**Config:** mode: \`${mode}\`, preset: \`${preset}\`, duration: \`${duration}s\`, bloat: \`${bloat} MiB\`, tps: \`${tps}\`, baseline: \`${baseline}\`, feature: \`${feature}\`, backend: \`${backend}\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${hfNote2}`);
+            core.exportVariable('BENCH_CONFIG', `**Config:** mode: \`${mode}\`, preset: \`${preset}\`, duration: \`${duration}s\`, bloat: \`${bloat}\`, tps: \`${tps}\`, baseline: \`${baseline}\`, feature: \`${feature}\`, backend: \`${backend}\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${hfNote2}`);
 
             const { buildBody } = require('./.github/scripts/bench-update-status.js');
             await github.rest.issues.updateComment({
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 3124bef00c..8664fd9e71 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -91,11 +91,12 @@ jobs:
             actor = context.payload.comment.user.login;
 
             const body = context.payload.comment.body.trim();
-            const intArgs = new Set(['duration', 'bloat', 'tps', 'tracy-seconds', 'tracy-offset', 'blocks', 'warmup']);
+            const intArgs = new Set(['duration', 'tps', 'tracy-seconds', 'tracy-offset', 'blocks', 'warmup']);
             const refArgs = new Set(['baseline', 'feature', 'txgen-ref']);
             const stringArgs = new Set(['mode', 'preset', 'backend', 'tracy', 'baseline-args', 'feature-args', 'baseline-hardfork', 'feature-hardfork', 'bench-args', 'bench-env', 'baseline-env', 'feature-env', 'chain']);
             const boolArgs = new Set(['samply', 'force-bloat', 'no-slack', 'no-existing-recipients']);
-            const defaults = { mode: 'e2e', preset: 'tip20', duration: '300', bloat: '100', tps: '10000', baseline: '', feature: '', backend: 'tempo-bench', 'txgen-ref': '', samply: 'false', tracy: 'off', 'tracy-seconds': '30', 'tracy-offset': '120', 'baseline-args': '', 'feature-args': '', 'baseline-hardfork': '', 'feature-hardfork': '', 'force-bloat': 'false', 'no-slack': 'false', 'no-existing-recipients': 'false', 'bench-args': '', 'bench-env': '', 'baseline-env': '', 'feature-env': '', blocks: '5000', warmup: '1000', chain: 'mainnet' };
+            const bloatValues = new Set(['1g', '10g', '100g']);
+            const defaults = { mode: 'e2e', preset: 'tip20', duration: '300', bloat: '100g', tps: '10000', baseline: '', feature: '', backend: 'tempo-bench', 'txgen-ref': '', samply: 'false', tracy: 'off', 'tracy-seconds': '30', 'tracy-offset': '120', 'baseline-args': '', 'feature-args': '', 'baseline-hardfork': '', 'feature-hardfork': '', 'force-bloat': 'false', 'no-slack': 'false', 'no-existing-recipients': 'false', 'bench-args': '', 'bench-env': '', 'baseline-env': '', 'feature-env': '', blocks: '5000', warmup: '1000', chain: 'mainnet' };
             const unknown = [];
             const invalid = [];
             const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -140,6 +141,13 @@ jobs:
                 } else {
                   invalid.push(`\`${key}=${value}\` (must be true or false)`);
                 }
+              } else if (key === 'bloat') {
+                const normalized = value.toLowerCase();
+                if (!bloatValues.has(normalized)) {
+                  invalid.push(`\`${key}=${value}\` (must be one of: 1g, 10g, 100g)`);
+                } else {
+                  defaults[key] = normalized;
+                }
               } else if (stringArgs.has(key)) {
                 if (!value) {
                   invalid.push(`\`${key}=\` (must not be empty)`);
@@ -154,7 +162,7 @@ jobs:
             if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
             if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
             if (errors.length) {
-              const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [mode=MODE] [chain=mainnet|testnet] [blocks=N] [warmup=N] [preset=NAME] [duration=N] [bloat=N] [tps=N] [baseline=REF] [feature=REF] [backend=NAME] [txgen-ref=REF] [samply] [force-bloat] [no-slack] [existing-recipients=BOOL] [tracy=MODE] [tracy-seconds=N] [tracy-offset=N] [baseline-args="ARGS"] [feature-args="ARGS"] [baseline-hardfork=FORK] [feature-hardfork=FORK] [bench-args="ARGS"] [bench-env="VARS"] [baseline-env="VARS"] [feature-env="VARS"]\``;
+              const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [mode=MODE] [chain=mainnet|testnet] [blocks=N] [warmup=N] [preset=NAME] [duration=N] [bloat=1g|10g|100g] [tps=N] [baseline=REF] [feature=REF] [backend=NAME] [txgen-ref=REF] [samply] [force-bloat] [no-slack] [existing-recipients=BOOL] [tracy=MODE] [tracy-seconds=N] [tracy-offset=N] [baseline-args="ARGS"] [feature-args="ARGS"] [baseline-hardfork=FORK] [feature-hardfork=FORK] [bench-args="ARGS"] [bench-env="VARS"] [baseline-env="VARS"] [feature-env="VARS"]\``;
               await github.rest.issues.createComment({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
@@ -167,7 +175,7 @@ jobs:
             mode = defaults.mode;
             preset = defaults.preset;
             duration = defaults.duration;
-            bloat = String(parseInt(defaults.bloat, 10) * 1000);
+            bloat = defaults.bloat;
             tps = defaults.tps;
             baseline = defaults.baseline;
             feature = defaults.feature;
@@ -196,7 +204,7 @@ jobs:
             const erFlag = `--existing-recipients=${existingRecipients}`;
             benchArgs = benchArgs ? `${benchArgs} ${erFlag}` : erFlag;
 
-            const usageStr = '**Usage:** `@decofe bench [mode=MODE] [chain=mainnet|testnet] [blocks=N] [warmup=N] [preset=NAME] [duration=N] [bloat=N] [tps=N] [baseline=REF] [feature=REF] [backend=NAME] [txgen-ref=REF] [samply] [force-bloat] [no-slack] [existing-recipients=BOOL] [tracy=MODE] [tracy-seconds=N] [tracy-offset=N] [baseline-args="ARGS"] [feature-args="ARGS"] [baseline-hardfork=FORK] [feature-hardfork=FORK] [bench-args="ARGS"] [bench-env="VARS"] [baseline-env="VARS"] [feature-env="VARS"]`';
+            const usageStr = '**Usage:** `@decofe bench [mode=MODE] [chain=mainnet|testnet] [blocks=N] [warmup=N] [preset=NAME] [duration=N] [bloat=1g|10g|100g] [tps=N] [baseline=REF] [feature=REF] [backend=NAME] [txgen-ref=REF] [samply] [force-bloat] [no-slack] [existing-recipients=BOOL] [tracy=MODE] [tracy-seconds=N] [tracy-offset=N] [baseline-args="ARGS"] [feature-args="ARGS"] [baseline-hardfork=FORK] [feature-hardfork=FORK] [bench-args="ARGS"] [bench-env="VARS"] [baseline-env="VARS"] [feature-env="VARS"]`';
 
             // Validate chain value
             if (!['mainnet', 'testnet'].includes(chain)) {
@@ -410,7 +418,7 @@ jobs:
             const bHf = process.env.ACK_BASELINE_HARDFORK || '';
             const fHf = process.env.ACK_FEATURE_HARDFORK || '';
             const hfNote = bHf ? `, baseline-hardfork: \`${bHf}\`, feature-hardfork: \`${fHf}\`` : '';
-            const config = `**Config:** mode: \`${mode}\`, preset: \`${preset}\`, duration: \`${duration}s\`, bloat: \`${bloat} MiB\`, tps: \`${tps}\`, baseline: \`${baseline}\`, feature: \`${feature}\`, backend: \`${backend}\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${naNote}${hfNote}`;
+            const config = `**Config:** mode: \`${mode}\`, preset: \`${preset}\`, duration: \`${duration}s\`, bloat: \`${bloat}\`, tps: \`${tps}\`, baseline: \`${baseline}\`, feature: \`${feature}\`, backend: \`${backend}\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${naNote}${hfNote}`;
 
             const { data: comment } = await github.rest.issues.createComment({
               owner: context.repo.owner,
diff --git a/bench-e2e.nu b/bench-e2e.nu
index cd8cb4df2d..c7e01cd4e3 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -18,8 +18,7 @@ const E2E_B_MEMORY = ""
 const E2E_GAS_LIMIT = "1000000000000"
 const E2E_BLOAT_TMP_DIR = "/reth-bench-a/.bench-tmp/e2e-local-init"
 const E2E_BLOAT_FREE_MARGIN_MIB = 51200
-const E2E_DEFAULT_BLOAT_MIB = 100000
-const E2E_SUPPORTED_BLOAT_MIB = [1000 10000 100000]
+const E2E_DEFAULT_BLOAT = "100g"
 const E2E_LOCAL_RETH_ARGS = [
     "--ipcdisable"
     "--disable-discovery"
@@ -129,6 +128,16 @@ def ensure-bloat-space [bloat: int] {
     }
 }
 
+def parse-e2e-bloat [bloat: string] {
+    let normalized = ($bloat | str downcase)
+    if $normalized == "1g" { return 1000 }
+    if $normalized == "10g" { return 10000 }
+    if $normalized == "100g" { return 100000 }
+
+    print "Error: --bloat must be one of: 1g, 10g, 100g"
+    exit 1
+}
+
 def validator-dirs-in-localnet [localnet_dir: string] {
     ls $localnet_dir
     | where type == "dir"
@@ -666,7 +675,7 @@ def "main e2e" [
     --duration: int = 300                               # Duration in seconds
     --accounts: int = 1000                              # Number of accounts
     --max-concurrent-requests: int = 100                # Max concurrent requests
-    --bloat: int = $E2E_DEFAULT_BLOAT_MIB               # State bloat size in MiB
+    --bloat: string = $E2E_DEFAULT_BLOAT                # State bloat snapshot size: 1g, 10g, or 100g
     --force-bloat                                      # Regenerate and promote both local e2e snapshots
     --init-only                                         # Refresh snapshots and exit without running benchmark phases
     --profile: string = $DEFAULT_PROFILE                # Cargo build profile
@@ -707,11 +716,7 @@ def "main e2e" [
         print "Error: --samply and --tracy are mutually exclusive. Choose one."
         exit 1
     }
-    if $bloat not-in $E2E_SUPPORTED_BLOAT_MIB {
-        let supported = ($E2E_SUPPORTED_BLOAT_MIB | each { |size| $"($size)" } | str join ", ")
-        print $"Error: --bloat must be one of ($supported) MiB \(1G, 10G, 100G snapshots\)"
-        exit 1
-    }
+    let bloat_mib = (parse-e2e-bloat $bloat)
     if $init_only and not $force_bloat {
         print "Error: --init-only requires --force-bloat"
         exit 1
@@ -737,8 +742,8 @@ def "main e2e" [
     let a_consensus_port = ($a_validator | split row ":" | get 1 | into int)
     let b_ip = ($b_validator | split row ":" | get 0)
     let b_consensus_port = ($b_validator | split row ":" | get 1 | into int)
-    let a_db = $"($E2E_A_MOUNT)/tempo_e2e_($bloat)mb"
-    let b_db = $"($E2E_B_MOUNT)/tempo_e2e_($bloat)mb"
+    let a_db = $"($E2E_A_MOUNT)/tempo_e2e_($bloat_mib)mb"
+    let b_db = $"($E2E_B_MOUNT)/tempo_e2e_($bloat_mib)mb"
     let a_identity = $a_db
     let b_identity = $b_db
     let genesis_path = $"($a_db)/($BENCH_META_SUBDIR)/genesis.json"
@@ -774,15 +779,16 @@ def "main e2e" [
             print "Error: generated localnet did not produce trusted peers"
             exit 1
         }
-        if $bloat > 0 {
-            ensure-bloat-space $bloat
-            print $"Generating local e2e state bloat \(($bloat) MiB\)..."
+        if $bloat_mib > 0 {
+            ensure-bloat-space $bloat_mib
+            print $"Generating local e2e state bloat \(($bloat_mib) MiB\)..."
             let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten)
-            cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args
+            cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat_mib --out $bloat_file ...$token_args
         }
 
         let marker = {
-            bloat_mib: $bloat
+            bloat_mib: $bloat_mib
+            bloat: $bloat
             accounts: $genesis_accounts
             validators: $E2E_VALIDATORS
             seed: $E2E_SEED
@@ -790,8 +796,8 @@ def "main e2e" [
             dkg_in_genesis: true
             topology: "single-runner"
         }
-        init-local-e2e-side a $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db $a_identity $"($init_dir)/($a_validator)" $generated_genesis $trusted_peers $bloat $bloat_file $tempo_bin ($marker | insert bench_datadir $a_db | insert node_dir $a_identity | insert validator_addr $a_validator)
-        init-local-e2e-side b $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db $b_identity $"($init_dir)/($b_validator)" $generated_genesis $trusted_peers $bloat $bloat_file $tempo_bin ($marker | insert bench_datadir $b_db | insert node_dir $b_identity | insert validator_addr $b_validator)
+        init-local-e2e-side a $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db $a_identity $"($init_dir)/($a_validator)" $generated_genesis $trusted_peers $bloat_mib $bloat_file $tempo_bin ($marker | insert bench_datadir $a_db | insert node_dir $a_identity | insert validator_addr $a_validator)
+        init-local-e2e-side b $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db $b_identity $"($init_dir)/($b_validator)" $generated_genesis $trusted_peers $bloat_mib $bloat_file $tempo_bin ($marker | insert bench_datadir $b_db | insert node_dir $b_identity | insert validator_addr $b_validator)
         if ($E2E_BLOAT_TMP_DIR | path exists) {
             rm -rf $E2E_BLOAT_TMP_DIR
         }
@@ -878,7 +884,7 @@ def "main e2e" [
         duration: $duration
         accounts: $accounts
         max_concurrent_requests: $max_concurrent_requests
-        bloat: $bloat
+        bloat: $bloat_mib
         results_dir: $results_dir
         profile: $profile
         samply: $samply
@@ -943,7 +949,7 @@ def "main e2e" [
     let baseline_label = if $baseline_name != "" { $baseline_name } else { $baseline }
     let feature_label = if $feature_name != "" { $feature_name } else { $feature }
     if $e2e_exit == 0 {
-        generate-summary $results_dir $baseline_label $feature_label $bloat $preset $tps $duration --benchmark-id $benchmark_id --reference-epoch $reference_epoch
+        generate-summary $results_dir $baseline_label $feature_label $bloat_mib $preset $tps $duration --benchmark-id $benchmark_id --reference-epoch $reference_epoch
     }
 
     try { git worktree remove --force $baseline_wt } catch { }

From 7a2c3875861cb3108ccb12eda3575cebaef7317f Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 10:59:48 +0100
Subject: [PATCH 09/16] fix(bench): preserve e2e telemetry env

---
 bench-e2e.nu | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index c7e01cd4e3..6fa43f9c11 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -195,10 +195,16 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
 
     let cpu_args = if $cpus != "" { ["-p" $"AllowedCPUs=($cpus)"] } else { [] }
     let memory_args = if $memory != "" { ["-p" $"MemoryMax=($memory)"] } else { [] }
+    let telemetry_env = if ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
+        ["--setenv=TEMPO_TELEMETRY_URL"]
+    } else {
+        []
+    }
     let uid = (id -u | str trim)
     let gid = (id -g | str trim)
     [
         "sudo"
+        "--preserve-env=TEMPO_TELEMETRY_URL"
         "systemd-run"
         "--scope"
         "--quiet"
@@ -207,6 +213,7 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
         "--unit" $unit
         "--uid" $uid
         "--gid" $gid
+        ...$telemetry_env
         "-p" "CPUWeight=100"
         ...$cpu_args
         ...$memory_args

From 871b1eef9e10ebc0b17c3cba504773cbe410b367 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 11:31:34 +0100
Subject: [PATCH 10/16] fix(bench): wire e2e trace telemetry env

---
 bench-e2e.nu | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index 6fa43f9c11..837bc7d411 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -187,6 +187,18 @@ def bench-save-e2e-meta [datadir: string, meta_dir: string, marker: record, gene
     print $"Bench marker written to ($marker_path)"
 }
 
+def derive-tracing-otlp [tracing_otlp: string] {
+    if $tracing_otlp == "" and ($env.GRAFANA_TEMPO? | default "" | str length) > 0 {
+        let base = ($env.GRAFANA_TEMPO | str trim --right --char '/')
+        return $"($base)/v1/traces"
+    }
+    if $tracing_otlp == "" and ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
+        let base = ($env.TEMPO_TELEMETRY_URL | str trim --right --char '/')
+        return $"($base)/opentelemetry/v1/traces"
+    }
+    $tracing_otlp
+}
+
 def systemd-scope-command [unit: string, cpus: string, memory: string, script: string] {
     let can_scope = (^uname | str trim) == "Linux" and ((which systemd-run | length) > 0) and ($cpus != "" or $memory != "")
     if not $can_scope {
@@ -195,16 +207,22 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
 
     let cpu_args = if $cpus != "" { ["-p" $"AllowedCPUs=($cpus)"] } else { [] }
     let memory_args = if $memory != "" { ["-p" $"MemoryMax=($memory)"] } else { [] }
-    let telemetry_env = if ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
-        ["--setenv=TEMPO_TELEMETRY_URL"]
-    } else {
-        []
+    mut telemetry_env_names = []
+    if ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
+        $telemetry_env_names = ($telemetry_env_names | append "TEMPO_TELEMETRY_URL")
+    }
+    if ($env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT? | default "" | str length) > 0 {
+        $telemetry_env_names = ($telemetry_env_names | append "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
     }
+    let preserve_env_args = if ($telemetry_env_names | length) > 0 {
+        [$"--preserve-env=($telemetry_env_names | str join ',')"]
+    } else { [] }
+    let telemetry_env = ($telemetry_env_names | each { |name| $"--setenv=($name)" })
     let uid = (id -u | str trim)
     let gid = (id -g | str trim)
     [
         "sudo"
-        "--preserve-env=TEMPO_TELEMETRY_URL"
+        ...$preserve_env_args
         "systemd-run"
         "--scope"
         "--quiet"
@@ -693,6 +711,7 @@ def "main e2e" [
     --tracy-filter: string = "debug"                    # Tracy tracing filter level
     --tracy-seconds: int = 30                           # Tracy capture duration limit in seconds
     --tracy-offset: int = 120                           # Seconds to wait before starting tracy capture
+    --tracing-otlp: string = ""                         # OTLP endpoint for tracing (auto-derived from GRAFANA_TEMPO/TEMPO_TELEMETRY_URL)
     --node-args: string = ""                            # Additional node args for all phases
     --baseline-args: string = ""                        # Additional node args for baseline phases
     --feature-args: string = ""                         # Additional node args for feature phases
@@ -760,6 +779,10 @@ def "main e2e" [
     let benchmark_id = $"bench-e2e-local-($timestamp)"
     let reference_epoch = (($run_started_at | into int) / 1_000_000_000 | into int)
     let gas_limit_args = if $E2E_GAS_LIMIT != "" { ["--gas-limit" $E2E_GAS_LIMIT] } else { [] }
+    let tracing_otlp = (derive-tracing-otlp $tracing_otlp)
+    if $tracing_otlp != "" {
+        $env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = $tracing_otlp
+    }
 
     validate-schelk-state $E2E_A_STATE_PATH $E2E_B_STATE_PATH
     cleanup-local-e2e-processes

From 89678a7bf8d493f638e42de5b9f00fa33d1c022a Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 12:15:33 +0100
Subject: [PATCH 11/16] ci(bench): make bloat input a choice

---
 .github/workflows/bench-e2e.yml            |  6 +++++-
 .github/workflows/bench-txgen-dispatch.yml | 10 +++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index e3ae356ff2..ed0478bad0 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -30,9 +30,13 @@ on:
         default: "300"
       bloat:
         description: State bloat snapshot size (1g, 10g, 100g).
-        type: string
+        type: choice
         required: true
         default: "100g"
+        options:
+          - "1g"
+          - "10g"
+          - "100g"
       tps:
         description: Target transactions per second.
         type: string
diff --git a/.github/workflows/bench-txgen-dispatch.yml b/.github/workflows/bench-txgen-dispatch.yml
index cfd4160de1..c890433437 100644
--- a/.github/workflows/bench-txgen-dispatch.yml
+++ b/.github/workflows/bench-txgen-dispatch.yml
@@ -35,10 +35,14 @@ on:
         required: true
         default: "30"
       bloat:
-        description: State bloat size in MiB.
-        type: string
+        description: State bloat snapshot size (1g, 10g, 100g).
+        type: choice
         required: true
-        default: "1"
+        default: "100g"
+        options:
+          - "1g"
+          - "10g"
+          - "100g"
       tps:
         description: Target transactions per second.
         type: string

From cfc6dd472e88892ac4101809d88152c0ea241d4b Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 12:17:40 +0100
Subject: [PATCH 12/16] fix(bench): initialize missing e2e snapshots

---
 bench-e2e.nu | 52 +++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index 837bc7d411..57dc77ae05 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -187,6 +187,33 @@ def bench-save-e2e-meta [datadir: string, meta_dir: string, marker: record, gene
     print $"Bench marker written to ($marker_path)"
 }
 
+def e2e-snapshot-required-files [datadir: string] {
+    let meta_dir = $"($datadir)/($BENCH_META_SUBDIR)"
+    [
+        $"($meta_dir)/genesis.json"
+        $"($meta_dir)/trusted-peers.txt"
+        $"($meta_dir)/marker.json"
+        $"($datadir)/signing.key"
+        $"($datadir)/signing.share"
+        $"($datadir)/enode.key"
+        $"($datadir)/enode.identity"
+        $"($datadir)/db"
+        $"($datadir)/static_files"
+    ]
+}
+
+def e2e-snapshot-missing-files [datadir: string] {
+    e2e-snapshot-required-files $datadir | where { |path| not ($path | path exists) }
+}
+
+def e2e-snapshot-ready [datadir: string] {
+    (e2e-snapshot-missing-files $datadir | length) == 0
+}
+
+def e2e-snapshots-ready [a_db: string, b_db: string] {
+    (e2e-snapshot-ready $a_db) and (e2e-snapshot-ready $b_db)
+}
+
 def derive-tracing-otlp [tracing_otlp: string] {
     if $tracing_otlp == "" and ($env.GRAFANA_TEMPO? | default "" | str length) > 0 {
         let base = ($env.GRAFANA_TEMPO | str trim --right --char '/')
@@ -787,14 +814,29 @@ def "main e2e" [
     validate-schelk-state $E2E_A_STATE_PATH $E2E_B_STATE_PATH
     cleanup-local-e2e-processes
 
-    if $force_bloat {
+    bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
+    bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
+
+    let snapshots_ready = (e2e-snapshots-ready $a_db $b_db)
+    let should_init_snapshots = $force_bloat or (not $snapshots_ready)
+    if (not $snapshots_ready) and (not $force_bloat) {
+        print $"Local e2e snapshot ($bloat) is missing required files; initializing it once."
+        let missing_a = (e2e-snapshot-missing-files $a_db)
+        let missing_b = (e2e-snapshot-missing-files $b_db)
+        if ($missing_a | length) > 0 {
+            print $"  Missing from a: ($missing_a | str join ', ')"
+        }
+        if ($missing_b | length) > 0 {
+            print $"  Missing from b: ($missing_b | str join ', ')"
+        }
+    }
+
+    if $should_init_snapshots {
         let init_dir = $"($LOCALNET_DIR)/e2e-local-init"
         let generated_genesis = $"($init_dir)/genesis.json"
         let bloat_file = $"($E2E_BLOAT_TMP_DIR)/state_bloat.bin"
         if ($init_dir | path exists) { rm -rf $init_dir }
         mkdir $init_dir
-        bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
-        bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
         if ($E2E_BLOAT_TMP_DIR | path exists) { rm -rf $E2E_BLOAT_TMP_DIR }
         mkdir $E2E_BLOAT_TMP_DIR
 
@@ -833,10 +875,10 @@ def "main e2e" [
         }
         bench-promote-at $E2E_A_STATE_PATH $a_db
         bench-promote-at $E2E_B_STATE_PATH $b_db
+        bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
+        bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
     }
 
-    bench-restore-at $E2E_A_STATE_PATH $E2E_A_MOUNT $a_db
-    bench-restore-at $E2E_B_STATE_PATH $E2E_B_MOUNT $b_db
     if $init_only {
         cleanup-local-e2e-processes
         return

From 7de25a8f35085436a8c3eb540c1388592acfca9a Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 12:18:33 +0100
Subject: [PATCH 13/16] Revert "ci(bench): make bloat input a choice"

This reverts commit 89678a7bf8d493f638e42de5b9f00fa33d1c022a.
---
 .github/workflows/bench-e2e.yml            |  6 +-----
 .github/workflows/bench-txgen-dispatch.yml | 10 +++-------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index ed0478bad0..e3ae356ff2 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -30,13 +30,9 @@ on:
         default: "300"
       bloat:
         description: State bloat snapshot size (1g, 10g, 100g).
-        type: choice
+        type: string
         required: true
         default: "100g"
-        options:
-          - "1g"
-          - "10g"
-          - "100g"
       tps:
         description: Target transactions per second.
         type: string
diff --git a/.github/workflows/bench-txgen-dispatch.yml b/.github/workflows/bench-txgen-dispatch.yml
index c890433437..cfd4160de1 100644
--- a/.github/workflows/bench-txgen-dispatch.yml
+++ b/.github/workflows/bench-txgen-dispatch.yml
@@ -35,14 +35,10 @@ on:
         required: true
         default: "30"
       bloat:
-        description: State bloat snapshot size (1g, 10g, 100g).
-        type: choice
+        description: State bloat size in MiB.
+        type: string
         required: true
-        default: "100g"
-        options:
-          - "1g"
-          - "10g"
-          - "100g"
+        default: "1"
       tps:
         description: Target transactions per second.
         type: string

From b0491d3bfb249ccda20b0bf996e8c570c53c2ce0 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 15:04:18 +0100
Subject: [PATCH 14/16] ci(bench): wire txgen e2e sender

---
 .github/workflows/bench-e2e.yml          |  16 +-
 .github/workflows/bench.yml              |  20 +-
 bench-e2e.nu                             | 271 ++++++++++++++++++++---
 contrib/bench/txgen/erc20.abi.json       |  26 +++
 contrib/bench/txgen/tip20-template.yaml  |  41 ++++
 contrib/bench/upload-clickhouse-txgen.sh | 168 ++++++++++++++
 tempo.nu                                 |  23 +-
 7 files changed, 504 insertions(+), 61 deletions(-)
 create mode 100644 contrib/bench/txgen/erc20.abi.json
 create mode 100644 contrib/bench/txgen/tip20-template.yaml
 create mode 100755 contrib/bench/upload-clickhouse-txgen.sh

diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml
index e3ae356ff2..79bf351719 100644
--- a/.github/workflows/bench-e2e.yml
+++ b/.github/workflows/bench-e2e.yml
@@ -42,10 +42,10 @@ on:
         description: Benchmark backend.
         type: choice
         required: true
-        default: tempo-bench
+        default: txgen
         options:
-          - tempo-bench
           - txgen
+          - tempo-bench
       txgen-ref:
         description: Optional ref to pin in tempoxyz/txgen.
         type: string
@@ -399,10 +399,6 @@ jobs:
 
       - name: Validate e2e options
         run: |
-          if [ "$BENCH_BACKEND" = "txgen" ]; then
-            echo "::error::mode=e2e currently supports backend=tempo-bench only; backend=txgen will be wired in the txgen bench PR."
-            exit 1
-          fi
           if [ -n "$BENCH_BASELINE_HARDFORK" ] || [ -n "$BENCH_FEATURE_HARDFORK" ]; then
             echo "::error::mode=e2e hardfork comparison is not wired for the single-runner local harness yet."
             exit 1
@@ -537,6 +533,7 @@ jobs:
           cmd+=(
             --preset "$BENCH_PRESET"
             --bloat "$BENCH_BLOAT"
+            --backend "$BENCH_BACKEND"
             --duration "$BENCH_DURATION"
             --tps "$BENCH_TPS"
             --baseline "$BASELINE_REF"
@@ -583,7 +580,12 @@ jobs:
           BENCH_RUN_TYPE: ${{ env.BENCH_RUN_TYPE }}
           BENCH_JOB_URL: ${{ env.BENCH_JOB_URL }}
           BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
-        run: bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
+        run: |
+          if [ "$BENCH_BACKEND" = "txgen" ]; then
+            bash contrib/bench/upload-clickhouse-txgen.sh "$BENCH_RESULTS_DIR"
+          else
+            bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
+          fi
 
       - name: Post results to PR
         if: success()
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 8664fd9e71..47ed4bb257 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -96,7 +96,7 @@ jobs:
             const stringArgs = new Set(['mode', 'preset', 'backend', 'tracy', 'baseline-args', 'feature-args', 'baseline-hardfork', 'feature-hardfork', 'bench-args', 'bench-env', 'baseline-env', 'feature-env', 'chain']);
             const boolArgs = new Set(['samply', 'force-bloat', 'no-slack', 'no-existing-recipients']);
             const bloatValues = new Set(['1g', '10g', '100g']);
-            const defaults = { mode: 'e2e', preset: 'tip20', duration: '300', bloat: '100g', tps: '10000', baseline: '', feature: '', backend: 'tempo-bench', 'txgen-ref': '', samply: 'false', tracy: 'off', 'tracy-seconds': '30', 'tracy-offset': '120', 'baseline-args': '', 'feature-args': '', 'baseline-hardfork': '', 'feature-hardfork': '', 'force-bloat': 'false', 'no-slack': 'false', 'no-existing-recipients': 'false', 'bench-args': '', 'bench-env': '', 'baseline-env': '', 'feature-env': '', blocks: '5000', warmup: '1000', chain: 'mainnet' };
+            const defaults = { mode: 'e2e', preset: 'tip20', duration: '300', bloat: '100g', tps: '10000', baseline: '', feature: '', backend: 'txgen', 'txgen-ref': '', samply: 'false', tracy: 'off', 'tracy-seconds': '30', 'tracy-offset': '120', 'baseline-args': '', 'feature-args': '', 'baseline-hardfork': '', 'feature-hardfork': '', 'force-bloat': 'false', 'no-slack': 'false', 'no-existing-recipients': 'false', 'bench-args': '', 'bench-env': '', 'baseline-env': '', 'feature-env': '', blocks: '5000', warmup: '1000', chain: 'mainnet' };
             const unknown = [];
             const invalid = [];
             const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -245,24 +245,6 @@ jobs:
               return;
             }
 
-            // E2E currently owns the single-runner local consensus harness. The txgen
-            // sender plugs into this path once #3669 lands.
-            if (mode === 'e2e') {
-              const unsupported = [];
-              if (backend !== 'tempo-bench') unsupported.push(`backend=${backend}`);
-              if (unsupported.length > 0) {
-                const msg = `❌ **Invalid bench command**\n\n\`mode=e2e\` does not support ${unsupported.map(s => `\`${s}\``).join(', ')} yet. The e2e harness currently uses \`backend=tempo-bench\`; \`backend=txgen\` will be wired in the txgen bench PR.`;
-                await github.rest.issues.createComment({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  issue_number: context.issue.number,
-                  body: msg,
-                });
-                core.setFailed(msg);
-                return;
-              }
-            }
-
             // Validate tracy value
             if (!['off', 'on', 'full'].includes(tracy)) {
               const msg = `❌ **Invalid bench command**\n\n\`tracy=${tracy}\` is not valid. Must be \`off\`, \`on\`, or \`full\`.\n\n${usageStr}`;
diff --git a/bench-e2e.nu b/bench-e2e.nu
index 57dc77ae05..4e7c5eaaad 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -19,6 +19,12 @@ const E2E_GAS_LIMIT = "1000000000000"
 const E2E_BLOAT_TMP_DIR = "/reth-bench-a/.bench-tmp/e2e-local-init"
 const E2E_BLOAT_FREE_MARGIN_MIB = 51200
 const E2E_DEFAULT_BLOAT = "100g"
+const E2E_DEFAULT_BACKEND = "txgen"
+const TXGEN_DEFAULT_SEED = 99
+const TXGEN_SCRAPE_INTERVAL_MS = 500
+const TXGEN_DRAIN_TIMEOUT_SECS = 300
+const TXGEN_FUND_DRAIN_TIMEOUT_SECS = 120
+const TXGEN_TIP20_TEMPLATE = "contrib/bench/txgen/tip20-template.yaml"
 const E2E_LOCAL_RETH_ARGS = [
     "--ipcdisable"
     "--disable-discovery"
@@ -138,6 +144,124 @@ def parse-e2e-bloat [bloat: string] {
     exit 1
 }
 
+def shell-quote [value: any] {
+    let s = ($value | into string)
+    let escaped = ($s | str replace -a "'" "'\"'\"'")
+    $"'($escaped)'"
+}
+
+def shell-join [args: list<any>] {
+    $args | each { |arg| shell-quote $arg } | str join " "
+}
+
+def resolve-command-path [name: string] {
+    let path = (which $name | get -o 0.path | default "")
+    if $path == "" {
+        error make { msg: $"($name) not found in PATH" }
+    }
+    $path
+}
+
+def resolve-bench-binary [repo_dir: string] {
+    for candidate in [$"($repo_dir)/target/release/bench" $"($repo_dir)/target/release/bench-cli"] {
+        if ($candidate | path exists) {
+            return $candidate
+        }
+    }
+    error make { msg: $"txgen bench binary not found under ($repo_dir)/target/release/" }
+}
+
+def resolve-txgen-paths [] {
+    let repo_dir = ($env.TXGEN_REPO_DIR? | default "")
+    let repo = if $repo_dir != "" { $repo_dir | path expand } else { "" }
+    let generator = if ($env.TXGEN_TEMPO_BIN? | default "") != "" {
+        $env.TXGEN_TEMPO_BIN | path expand
+    } else if $repo != "" and ($"($repo)/target/release/txgen-tempo" | path exists) {
+        $"($repo)/target/release/txgen-tempo"
+    } else {
+        resolve-command-path "txgen-tempo"
+    }
+    let bench = if ($env.TXGEN_BENCH_BIN? | default "") != "" {
+        $env.TXGEN_BENCH_BIN | path expand
+    } else if $repo != "" {
+        resolve-bench-binary $repo
+    } else {
+        resolve-command-path "bench"
+    }
+    if not ($generator | path exists) {
+        error make { msg: $"txgen-tempo binary not found: ($generator)" }
+    }
+    if not ($bench | path exists) {
+        error make { msg: $"txgen bench binary not found: ($bench)" }
+    }
+    { txgen_tempo_bin: $generator, txgen_bench_bin: $bench }
+}
+
+def rpc-call [rpc_url: string, payload: string] {
+    let result = (^curl -sf -X POST -H "Content-Type: application/json" -d $payload $rpc_url | complete)
+    if $result.exit_code != 0 {
+        error make { msg: $"RPC call failed: ($payload)" }
+    }
+    let response = ($result.stdout | from json)
+    if (($response | get -o error) != null) {
+        let rpc_error = ($response | get error)
+        error make { msg: $"RPC error: ($rpc_error | to json -r)" }
+    }
+    $response
+}
+
+def fetch-chain-id [rpc_url: string] {
+    let response = (rpc-call $rpc_url '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}')
+    $response.result | into int
+}
+
+def wait-for-txpool-drain [rpc_url: string, timeout_secs: int] {
+    mut zero_count = 0
+    mut waited = 0
+    while $waited < $timeout_secs {
+        let response = (rpc-call $rpc_url '{"jsonrpc":"2.0","method":"txpool_status","params":[],"id":1}')
+        let pending = ($response.result.pending | into int)
+        if $pending == 0 {
+            $zero_count = $zero_count + 1
+            if $zero_count >= 3 { return }
+        } else {
+            $zero_count = 0
+        }
+        sleep 1sec
+        $waited = $waited + 1
+    }
+    print $"  Warning: txpool drain timeout reached after ($timeout_secs)s"
+}
+
+def fund-txgen-accounts [txgen_bin: string, spec_path: string, rpc_url: string] {
+    let result = (^$txgen_bin addresses -s $spec_path -f shell | complete)
+    if $result.exit_code != 0 {
+        error make { msg: $"failed to list txgen addresses for ($spec_path)" }
+    }
+
+    let addresses = ($result.stdout | str trim | split row " " | where { |addr| $addr != "" })
+    if ($addresses | is-empty) {
+        error make { msg: $"txgen spec produced no addresses: ($spec_path)" }
+    }
+
+    print $"  Funding (($addresses | length)) txgen account\(s\)..."
+    $addresses | par-each { |address|
+        rpc-call $rpc_url $"{\"jsonrpc\":\"2.0\",\"method\":\"tempo_fundAddress\",\"params\":[\"($address)\"],\"id\":1}" | ignore
+    } | ignore
+
+    print "  Waiting for faucet transactions to drain..."
+    wait-for-txpool-drain $rpc_url $TXGEN_FUND_DRAIN_TIMEOUT_SECS
+}
+
+def sanitize-txgen-bench-args [bench_args: string] {
+    if $bench_args == "" {
+        return ""
+    }
+    $bench_args
+        | str replace --all --regex '--existing-recipients=(true|false)' ''
+        | str trim
+}
+
 def validator-dirs-in-localnet [localnet_dir: string] {
     ls $localnet_dir
     | where type == "dir"
@@ -660,43 +784,106 @@ def run-local-e2e-phase [run: record, ctx: record] {
         $tracy_capture_started = true
     }
 
-    let bench_cmd = [
-        $run.bench
-        "run-max-tps"
-        "--tps" $"($ctx.tps)"
-        "--duration" $"($ctx.duration)"
-        "--accounts" $"($ctx.accounts)"
-        "--max-concurrent-requests" $"($ctx.max_concurrent_requests)"
-        "--target-urls" $"($a_rpc),($b_rpc)"
-        "--faucet"
-        "--clear-txpool"
-    ]
-    | append (if $ctx.preset != "" {
-        [
-            "--tip20-weight" $"($weights | get 0)"
-            "--erc20-weight" $"($weights | get 1)"
-            "--swap-weight" $"($weights | get 2)"
-            "--place-order-weight" $"($weights | get 3)"
-        ]
-    } else { [] })
-    | append (if $ctx.bloat > 0 { ["--mnemonic" $"'($BLOAT_MNEMONIC)'"] } else { [] })
-    | append (if $ctx.bench_args != "" { $ctx.bench_args | split row " " } else { [] })
-    | append ["--node-commit-sha" $run.ref "--build-profile" $ctx.profile "--benchmark-mode" "e2e"]
-
     if $phase_exit == 0 {
         let bench_env_export = if $ctx.bench_env != "" { $"export ($ctx.bench_env) && " } else { "" }
-        print $"Running local e2e sender: ($bench_cmd | str join ' ')"
-        let bench_result = (bash -c $"($bench_env_export)ulimit -Sn unlimited && ($bench_cmd | str join ' ')" | complete)
-        if $bench_result.stdout != "" { print $bench_result.stdout }
-        if $bench_result.stderr != "" { print $bench_result.stderr }
-        $phase_exit = $bench_result.exit_code
-
-        if ("report.json" | path exists) {
-            cp report.json $"($ctx.results_dir)/report-($phase).json"
-            rm report.json
+        if $ctx.backend == "txgen" {
+            if $ctx.preset != "tip20" {
+                print $"Error: txgen e2e backend currently supports only preset=tip20 \(got ($ctx.preset)\)"
+                $phase_exit = 1
+            } else {
+                let ignored_bench_args = (sanitize-txgen-bench-args $ctx.bench_args)
+                if $ignored_bench_args != "" {
+                    print $"  Warning: txgen path is ignoring unsupported bench args: ($ignored_bench_args)"
+                }
+                let chain_id = (fetch-chain-id $a_rpc)
+                $env.TXGEN_ACCOUNTS = ($ctx.accounts | into string)
+                let spec_path = ($TXGEN_TIP20_TEMPLATE | path expand)
+                fund-txgen-accounts $ctx.txgen.txgen_tempo_bin $spec_path $a_rpc
+
+                let report_path = $"($ctx.results_dir)/report-($phase).json"
+                let tx_count = [($ctx.tps * $ctx.duration) 1] | math max
+                let txgen_cmd = [
+                    $ctx.txgen.txgen_tempo_bin
+                    "generate"
+                    "-s" $spec_path
+                    "-n" $tx_count
+                    "--seed" $TXGEN_DEFAULT_SEED
+                    "--rpc" $a_rpc
+                ]
+                let bench_cmd = [
+                    $ctx.txgen.txgen_bench_bin
+                    "send"
+                    "--rpc-url" $a_rpc
+                    "--tps" $ctx.tps
+                    "--max-concurrent" $ctx.max_concurrent_requests
+                    "--metrics-url" "http://127.0.0.1:9001/metrics"
+                    "--scrape-interval-ms" $TXGEN_SCRAPE_INTERVAL_MS
+                    "--drain-timeout" $TXGEN_DRAIN_TIMEOUT_SECS
+                    "--report" $"json:($report_path)"
+                    "-m" $"chain_id=($chain_id)"
+                    "-m" $"target_tps=($ctx.tps)"
+                    "-m" $"run_duration_secs=($ctx.duration)"
+                    "-m" $"accounts=($ctx.accounts)"
+                    "-m" $"total_connections=($ctx.max_concurrent_requests)"
+                    "-m" "tip20_weight=1.0"
+                    "-m" "place_order_weight=0.0"
+                    "-m" "swap_weight=0.0"
+                    "-m" "erc20_weight=0.0"
+                    "-m" $"node_commit_sha=($run.ref)"
+                    "-m" $"build_profile=($ctx.profile)"
+                    "-m" "mode=e2e"
+                ]
+                print $"Running local e2e txgen sender: ($txgen_cmd | str join ' ') | ($bench_cmd | str join ' ')"
+                let txgen_cmd_str = (shell-join $txgen_cmd)
+                let bench_cmd_str = (shell-join $bench_cmd)
+                let pipeline = $"set -euo pipefail; ($bench_env_export)ulimit -Sn unlimited && ($txgen_cmd_str) | ($bench_cmd_str)"
+                let bench_result = (bash -lc $pipeline | complete)
+                if $bench_result.stdout != "" { print $bench_result.stdout }
+                if $bench_result.stderr != "" { print $bench_result.stderr }
+                $phase_exit = $bench_result.exit_code
+
+                if not ($report_path | path exists) {
+                    print $"ERROR: txgen sender for ($phase) produced no ($report_path)"
+                    $phase_exit = 1
+                }
+            }
         } else {
-            print $"ERROR: sender for ($phase) produced no report.json"
-            $phase_exit = 1
+            let bench_cmd = [
+                $run.bench
+                "run-max-tps"
+                "--tps" $"($ctx.tps)"
+                "--duration" $"($ctx.duration)"
+                "--accounts" $"($ctx.accounts)"
+                "--max-concurrent-requests" $"($ctx.max_concurrent_requests)"
+                "--target-urls" $"($a_rpc),($b_rpc)"
+                "--faucet"
+                "--clear-txpool"
+            ]
+            | append (if $ctx.preset != "" {
+                [
+                    "--tip20-weight" $"($weights | get 0)"
+                    "--erc20-weight" $"($weights | get 1)"
+                    "--swap-weight" $"($weights | get 2)"
+                    "--place-order-weight" $"($weights | get 3)"
+                ]
+            } else { [] })
+            | append (if $ctx.bloat > 0 { ["--mnemonic" $"'($BLOAT_MNEMONIC)'"] } else { [] })
+            | append (if $ctx.bench_args != "" { $ctx.bench_args | split row " " } else { [] })
+            | append ["--node-commit-sha" $run.ref "--build-profile" $ctx.profile "--benchmark-mode" "e2e"]
+
+            print $"Running local e2e sender: ($bench_cmd | str join ' ')"
+            let bench_result = (bash -c $"($bench_env_export)ulimit -Sn unlimited && ($bench_cmd | str join ' ')" | complete)
+            if $bench_result.stdout != "" { print $bench_result.stdout }
+            if $bench_result.stderr != "" { print $bench_result.stderr }
+            $phase_exit = $bench_result.exit_code
+
+            if ("report.json" | path exists) {
+                cp report.json $"($ctx.results_dir)/report-($phase).json"
+                rm report.json
+            } else {
+                print $"ERROR: sender for ($phase) produced no report.json"
+                $phase_exit = 1
+            }
         }
     } else {
         print $"Skipping local e2e sender for ($phase) because readiness checks failed"
@@ -728,6 +915,7 @@ def "main e2e" [
     --accounts: int = 1000                              # Number of accounts
     --max-concurrent-requests: int = 100                # Max concurrent requests
     --bloat: string = $E2E_DEFAULT_BLOAT                # State bloat snapshot size: 1g, 10g, or 100g
+    --backend: string = $E2E_DEFAULT_BACKEND            # Benchmark backend: txgen or tempo-bench
     --force-bloat                                      # Regenerate and promote both local e2e snapshots
     --init-only                                         # Refresh snapshots and exit without running benchmark phases
     --profile: string = $DEFAULT_PROFILE                # Cargo build profile
@@ -761,6 +949,14 @@ def "main e2e" [
         print $"Unknown preset: ($preset). Available: ($PRESETS | columns | str join ', ')"
         exit 1
     }
+    if $backend not-in ["txgen" "tempo-bench"] {
+        print $"Error: --backend must be one of: txgen, tempo-bench \(got '($backend)'\)"
+        exit 1
+    }
+    if $backend == "txgen" and $preset != "tip20" {
+        print $"Error: --backend txgen currently supports only --preset tip20 \(got '($preset)'\)"
+        exit 1
+    }
     if $tracy not-in ["off" "on" "full"] {
         print $"Error: --tracy must be one of: off, on, full \(got '($tracy)'\)"
         exit 1
@@ -927,6 +1123,11 @@ def "main e2e" [
     let baseline_bench = (worktree-bin $baseline_wt $profile "tempo-bench")
     let feature_tempo = (worktree-bin $feature_wt $profile "tempo")
     let feature_bench = (worktree-bin $feature_wt $profile "tempo-bench")
+    let txgen = if $backend == "txgen" {
+        resolve-txgen-paths
+    } else {
+        { txgen_tempo_bin: "", txgen_bench_bin: "" }
+    }
     let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " }
     let ctx = {
         genesis: $genesis_path
@@ -957,6 +1158,8 @@ def "main e2e" [
         accounts: $accounts
         max_concurrent_requests: $max_concurrent_requests
         bloat: $bloat_mib
+        backend: $backend
+        txgen: $txgen
         results_dir: $results_dir
         profile: $profile
         samply: $samply
diff --git a/contrib/bench/txgen/erc20.abi.json b/contrib/bench/txgen/erc20.abi.json
new file mode 100644
index 0000000000..fbd22efaf3
--- /dev/null
+++ b/contrib/bench/txgen/erc20.abi.json
@@ -0,0 +1,26 @@
+[
+  {
+    "type": "function",
+    "name": "transfer",
+    "inputs": [
+      {
+        "name": "to",
+        "type": "address",
+        "internalType": "address"
+      },
+      {
+        "name": "amount",
+        "type": "uint256",
+        "internalType": "uint256"
+      }
+    ],
+    "outputs": [
+      {
+        "name": "",
+        "type": "bool",
+        "internalType": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable"
+  }
+]
diff --git a/contrib/bench/txgen/tip20-template.yaml b/contrib/bench/txgen/tip20-template.yaml
new file mode 100644
index 0000000000..910e5e7e98
--- /dev/null
+++ b/contrib/bench/txgen/tip20-template.yaml
@@ -0,0 +1,41 @@
+chain_id: 1337
+
+gas:
+  max_fee_per_gas: 100000000000
+  max_priority_fee_per_gas: 100000000000
+
+accounts:
+  users:
+    mnemonic: "test test test test test test test test test test test junk"
+    range:
+      - 0
+      - ${TXGEN_ACCOUNTS}
+
+artifacts:
+  ERC20: erc20.abi.json
+
+templates:
+  tip20_transfer:
+    type: tempo
+    from:
+      pool: users
+      select: random
+    gas_limit: 300000
+    max_fee_per_gas: 100000000000
+    max_priority_fee_per_gas: 100000000000
+    fee_token: "0x20c0000000000000000000000000000000000000"
+    expiring_nonce: true
+    valid_for_secs: 25
+    call:
+      to: "0x20c0000000000000000000000000000000000000"
+      abi: ERC20
+      function: transfer
+      args:
+        - pool:
+            pool: users
+            select: random
+        - 1
+
+mix:
+  - template: tip20_transfer
+    weight: 100
diff --git a/contrib/bench/upload-clickhouse-txgen.sh b/contrib/bench/upload-clickhouse-txgen.sh
new file mode 100755
index 0000000000..f4a4a0baea
--- /dev/null
+++ b/contrib/bench/upload-clickhouse-txgen.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# Upload native txgen bench results to ClickHouse.
+#
+# Reads report-*.json files from the results directory and inserts into:
+#   - tempo_bench_runs   (one row per run)
+#   - tempo_bench_blocks (one row per block per run)
+#
+# Environment:
+#   CLICKHOUSE_URL      - ClickHouse HTTP endpoint (https://host:8443)
+#   CLICKHOUSE_USER     - ClickHouse user
+#   CLICKHOUSE_PASSWORD - ClickHouse password
+#   CLICKHOUSE_DB       - database name (default: "default")
+#
+# Usage: upload-clickhouse-txgen.sh <results-dir>
+
+set -euo pipefail
+
+RESULTS_DIR="$1"
+DB="${CLICKHOUSE_DB:-default}"
+
+if [ -z "${CLICKHOUSE_URL:-}" ] || [ -z "${CLICKHOUSE_USER:-}" ] || [ -z "${CLICKHOUSE_PASSWORD:-}" ]; then
+  echo "Skipping ClickHouse upload: CLICKHOUSE_URL, CLICKHOUSE_USER, or CLICKHOUSE_PASSWORD not set"
+  exit 0
+fi
+
+ch_query() {
+  local query="$1"
+  if ! curl -sf --user "$CLICKHOUSE_USER:$CLICKHOUSE_PASSWORD" \
+    "$CLICKHOUSE_URL/?database=$DB" --data-binary "$query"; then
+    echo "  Warning: ClickHouse query failed" >&2
+    return 1
+  fi
+}
+
+echo "Uploading txgen bench results to ClickHouse..."
+
+for label in baseline-1 feature-1 feature-2 baseline-2; do
+  REPORT="$RESULTS_DIR/report-$label.json"
+  if [ ! -f "$REPORT" ]; then
+    echo "  Warning: $REPORT not found, skipping"
+    continue
+  fi
+
+  echo "  Processing: $label"
+
+  # Generate SQL statements via python (one statement per line, no internal newlines)
+  QUERIES=$(REPORT_PATH="$REPORT" BENCH_RUN_LABEL="$label" python3 << 'PYEOF'
+import json, uuid, os
+
+report = json.load(open(os.environ["REPORT_PATH"]))
+meta = report.get("metadata") or {}
+run_stats = report.get("run_stats") or {}
+
+def as_int(value, default=0):
+    if value is None or value == "":
+        return default
+    return int(value)
+
+def as_float(value, default=0.0):
+    if value is None or value == "":
+        return default
+    return float(value)
+
+def normalized_blocks(report):
+    normalized = []
+    for b in report.get("blocks") or []:
+        tx_count = as_int(b.get("tx_count"))
+        normalized.append({
+            "number": as_int(b.get("number")),
+            "timestamp": as_int(b.get("timestamp", b.get("timestamp_ms"))),
+            "tx_count": tx_count,
+            "ok_count": tx_count,
+            "err_count": 0,
+            "gas_used": as_int(b.get("gas_used")),
+            "latency_ms": b.get("block_time_ms"),
+        })
+    return normalized
+
+blocks = normalized_blocks(report)
+
+run_id = str(uuid.uuid4())
+
+# Compute aggregates
+total_tx = sum(b["tx_count"] for b in blocks)
+total_ok = sum(b["ok_count"] for b in blocks)
+total_err = sum(b["err_count"] for b in blocks)
+total_gas = sum(b["gas_used"] for b in blocks)
+total_blocks = len(blocks)
+
+timestamps = [b["timestamp"] for b in blocks]
+if len(timestamps) > 1:
+    time_span_ms = max(timestamps[-1] - timestamps[0], 1)
+    avg_block_time_ms = time_span_ms / (len(timestamps) - 1)
+    avg_tps = total_tx / (time_span_ms / 1000.0)
+else:
+    avg_block_time_ms = 0.0
+    avg_tps = 0.0
+
+def esc(s):
+    return s.replace("'", "\\'")
+
+sha = esc(meta.get("node_commit_sha") or "")
+profile = esc(meta.get("build_profile") or "")
+mode = esc(meta.get("mode") or "")
+
+run_label = esc(os.environ.get("BENCH_RUN_LABEL", ""))
+pr_number = esc(os.environ.get("BENCH_PR", ""))
+baseline_ref = esc(os.environ.get("BENCH_BASELINE_REF", ""))
+feature_ref = esc(os.environ.get("BENCH_FEATURE_REF", ""))
+triggered_by = esc(os.environ.get("BENCH_ACTOR", ""))
+run_type = esc(os.environ.get("BENCH_RUN_TYPE", "manual"))
+github_run_id_raw = os.environ.get("GITHUB_RUN_ID", "")
+default_run_url = ""
+if github_run_id_raw and os.environ.get("GITHUB_SERVER_URL") and os.environ.get("GITHUB_REPOSITORY"):
+    default_run_url = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{github_run_id_raw}"
+github_run_id = esc(github_run_id_raw)
+github_run_url = esc(os.environ.get("BENCH_JOB_URL") or default_run_url)
+
+print(
+    f"INSERT INTO tempo_bench_runs (run_id, created_at, chain_id, start_block, end_block, "
+    f"target_tps, run_duration_secs, accounts, total_connections, "
+    f"total_blocks, total_transactions, total_successful, total_failed, "
+    f"total_gas_used, avg_block_time_ms, avg_tps, "
+    f"tip20_weight, place_order_weight, swap_weight, erc20_weight, "
+    f"node_commit_sha, build_profile, benchmark_mode, "
+    f"run_label, pr_number, baseline_ref, feature_ref, "
+    f"triggered_by, run_type, github_run_id, github_run_url) VALUES "
+    f"('{run_id}', now64(3), {as_int(meta.get('chain_id'))}, "
+    f"{as_int(run_stats.get('start_block'))}, {as_int(run_stats.get('end_block'))}, "
+    f"{as_int(meta.get('target_tps'))}, {as_int(meta.get('run_duration_secs'))}, "
+    f"{as_int(meta.get('accounts'))}, {as_int(meta.get('total_connections'))}, "
+    f"{total_blocks}, {total_tx}, {total_ok}, {total_err}, "
+    f"{total_gas}, {avg_block_time_ms}, {avg_tps}, "
+    f"{as_float(meta.get('tip20_weight'))}, {as_float(meta.get('place_order_weight'))}, "
+    f"{as_float(meta.get('swap_weight'))}, {as_float(meta.get('erc20_weight'))}, "
+    f"'{sha}', '{profile}', '{mode}', "
+    f"'{run_label}', '{pr_number}', '{baseline_ref}', '{feature_ref}', "
+    f"'{triggered_by}', '{run_type}', '{github_run_id}', '{github_run_url}')"
+)
+
+# Blocks insert (batch all blocks in one statement)
+if blocks:
+    rows = []
+    for b in blocks:
+        lat = b.get("latency_ms")
+        lat_val = lat if lat is not None else 0
+        rows.append(
+            f"('{run_id}', {b['number']}, {b['timestamp']}, "
+            f"{b['tx_count']}, {b['ok_count']}, {b['err_count']}, "
+            f"{b['gas_used']}, 0, {lat_val})"
+        )
+    values = ", ".join(rows)
+    print(
+        f"INSERT INTO tempo_bench_blocks (run_id, block_number, timestamp_ms, "
+        f"tx_count, ok_count, err_count, gas_used, gas_limit, latency_ms) VALUES {values}"
+    )
+PYEOF
+  )
+
+  echo "$QUERIES" | while IFS= read -r query; do
+    [ -z "$query" ] && continue
+    ch_query "$query"
+  done
+
+  echo "  Uploaded: $label"
+done
+
+echo "ClickHouse upload complete."
diff --git a/tempo.nu b/tempo.nu
index fb312bd4c7..80482168aa 100755
--- a/tempo.nu
+++ b/tempo.nu
@@ -793,7 +793,28 @@ def generate-summary [results_dir: string, baseline_ref: string, feature_ref: st
             continue
         }
         let report = (open $report_path)
-        let blocks = ($report | get blocks)
+        let blocks = ($report | get blocks | each { |b|
+            let tx_count = ($b | get tx_count)
+            let timestamp = if (($b | get -o timestamp | default null) != null) {
+                $b | get timestamp
+            } else {
+                $b | get timestamp_ms
+            }
+            let latency_ms = if (($b | get -o latency_ms | default null) != null) {
+                $b | get latency_ms
+            } else {
+                $b | get -o block_time_ms | default null
+            }
+            {
+                number: ($b | get number)
+                timestamp: $timestamp
+                tx_count: $tx_count
+                ok_count: ($b | get -o ok_count | default $tx_count)
+                err_count: ($b | get -o err_count | default 0)
+                gas_used: ($b | get gas_used)
+                latency_ms: $latency_ms
+            }
+        })
         if ($blocks | length) == 0 {
             print $"Warning: ($label) report has no blocks, skipping"
             continue

From edccc01c08d41fbe82b5669c8f5ff4c4cd8bbc2e Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 15:19:26 +0100
Subject: [PATCH 15/16] fix(bench): drop unsupported e2e fee recipient arg

---
 bench-e2e.nu | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index 4e7c5eaaad..124e78d9a7 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -451,7 +451,6 @@ def build-e2e-consensus-args [node_dir: string, trusted_peers: string, port: int
         "--discovery.v5.port" $"($discv5_port)"
         "--p2p-secret-key" $enode_key
         "--authrpc.port" $"($authrpc_port)"
-        "--consensus.fee-recipient" "0x0000000000000000000000000000000000000000"
         "--consensus.use-local-defaults"
         "--consensus.bypass-ip-check"
     ]

From 45d66fda013a4ec09cee1b28d186047aa036ee73 Mon Sep 17 00:00:00 2001
From: joshieDo <93316087+joshieDo@users.noreply.github.com>
Date: Thu, 7 May 2026 16:44:30 +0100
Subject: [PATCH 16/16] fix(bench): pin validators with taskset

---
 bench-e2e.nu | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/bench-e2e.nu b/bench-e2e.nu
index 124e78d9a7..faac79ad70 100644
--- a/bench-e2e.nu
+++ b/bench-e2e.nu
@@ -13,8 +13,8 @@ const E2E_VALIDATORS = "127.0.0.2:8000,127.0.0.3:8100"
 const E2E_SEED = 42
 const E2E_A_CPUS = "0-7,16-23"
 const E2E_B_CPUS = "8-15,24-31"
-const E2E_A_MEMORY = ""
-const E2E_B_MEMORY = ""
+const E2E_A_MEMORY = "60G"
+const E2E_B_MEMORY = "60G"
 const E2E_GAS_LIMIT = "1000000000000"
 const E2E_BLOAT_TMP_DIR = "/reth-bench-a/.bench-tmp/e2e-local-init"
 const E2E_BLOAT_FREE_MARGIN_MIB = 51200
@@ -356,7 +356,6 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
         return ["bash" "-lc" $script]
     }
 
-    let cpu_args = if $cpus != "" { ["-p" $"AllowedCPUs=($cpus)"] } else { [] }
     let memory_args = if $memory != "" { ["-p" $"MemoryMax=($memory)"] } else { [] }
     mut telemetry_env_names = []
     if ($env.TEMPO_TELEMETRY_URL? | default "" | str length) > 0 {
@@ -384,7 +383,6 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
         "--gid" $gid
         ...$telemetry_env
         "-p" "CPUWeight=100"
-        ...$cpu_args
         ...$memory_args
         "bash"
         "-lc"
@@ -392,6 +390,14 @@ def systemd-scope-command [unit: string, cpus: string, memory: string, script: s
     ]
 }
 
+def taskset-command [cmd: list<string>, cpus: string] {
+    if $cpus != "" {
+        ["taskset" "-c" $cpus ...$cmd]
+    } else {
+        $cmd
+    }
+}
+
 def start-e2e-local-node [
     role: string,
     phase: string,
@@ -410,7 +416,8 @@ def start-e2e-local-node [
     let full_samply_args = if $samply {
         $samply_args | append ["--save-only" "--presymbolicate" "--output" $"($results_dir)/profile-($profile_label).json.gz"]
     } else { [] }
-    let node_cmd = wrap-samply [$tempo_bin ...$args] $samply $full_samply_args
+    let pinned_cmd = taskset-command [$tempo_bin ...$args] $cpus
+    let node_cmd = wrap-samply $pinned_cmd $samply $full_samply_args
     let node_cmd_str = ($node_cmd | str join " ")
     let script = $"($env_prefix)($otel_attrs)($tracy_env_prefix)($node_cmd_str) 2>&1"
     let unit_phase = ($phase | str replace -a "_" "-" | str replace -a "." "-")