ci(bench): txgen e2e bench

.github/workflows/bench-e2e.yml

@@ -330,23 +330,6 @@ jobs:
           fetch-depth: 0
           ref: ${{ steps.checkout-ref.outputs.ref }}
 
-      - name: Checkout txgen repository
-        if: env.BENCH_BACKEND == 'txgen'
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          repository: tempoxyz/txgen
-          token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
-          persist-credentials: false
-          path: txgen
-          fetch-depth: 0
-
-      - name: Checkout txgen ref
-        if: env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != ''
-        working-directory: txgen
-        run: |
-          git fetch origin "$BENCH_TXGEN_REF" --quiet
-          git checkout --detach FETCH_HEAD
-
       - name: Resolve job URL and update status
         if: env.BENCH_COMMENT_ID
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -395,14 +378,27 @@ jobs:
       - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
         continue-on-error: true
 
-      - name: Build txgen backend
+      - name: Install txgen backend
         if: env.BENCH_BACKEND == 'txgen'
-        working-directory: txgen
+        env:
+          TXGEN_GIT_TOKEN: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
         run: |
-          cargo build --release --bin txgen-tempo --bin bench
-          echo "TXGEN_REPO_DIR=$GITHUB_WORKSPACE/txgen" >> "$GITHUB_ENV"
-          echo "TXGEN_TEMPO_BIN=$GITHUB_WORKSPACE/txgen/target/release/txgen-tempo" >> "$GITHUB_ENV"
-          echo "TXGEN_BENCH_BIN=$GITHUB_WORKSPACE/txgen/target/release/bench" >> "$GITHUB_ENV"
+          CARGO_BIN_DIR="${CARGO_HOME:-$HOME/.cargo}/bin"
+          echo "$CARGO_BIN_DIR" >> "$GITHUB_PATH"
+          export PATH="$CARGO_BIN_DIR:$PATH"
+
+          TXGEN_GIT_URL="https://x-access-token:${TXGEN_GIT_TOKEN}@github.com/tempoxyz/txgen"
+          install_args=(--git "$TXGEN_GIT_URL" --locked)
+          if [ -n "$BENCH_TXGEN_REF" ]; then
+            TXGEN_REV="$(git ls-remote "$TXGEN_GIT_URL" "$BENCH_TXGEN_REF" "refs/heads/$BENCH_TXGEN_REF" "refs/tags/$BENCH_TXGEN_REF" | awk 'BEGIN { rev = "" } /\^\{\}$/ { rev = $1; exit } rev == "" { rev = $1 } END { if (rev != "") print rev }')"
+            install_args+=(--rev "${TXGEN_REV:-$BENCH_TXGEN_REF}")
+          fi
+
+          cargo install "${install_args[@]}" txgen-tempo bench-cli
+          echo "TXGEN_TEMPO_BIN=$CARGO_BIN_DIR/txgen-tempo" >> "$GITHUB_ENV"
+          echo "TXGEN_BENCH_BIN=$CARGO_BIN_DIR/bench" >> "$GITHUB_ENV"
+          command -v txgen-tempo
+          command -v bench
 
       - name: Resolve PR head branch
         id: pr-info
@@ -577,7 +573,12 @@ jobs:
           BENCH_RUN_TYPE: ${{ env.BENCH_RUN_TYPE }}
           BENCH_JOB_URL: ${{ env.BENCH_JOB_URL }}
           BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
-        run: bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
+        run: |
+          if [ "$BENCH_BACKEND" = "txgen" ]; then
+            bash contrib/bench/upload-clickhouse-txgen.sh "$BENCH_RESULTS_DIR"
+          else
+            bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR"
+          fi
 
       - name: Post results to PR
         if: success()

.github/workflows/bench-txgen-dispatch.yml

@@ -1,8 +1,7 @@
-# Bootstrap stub for the txgen workflow_dispatch entrypoint.
+# Runs the txgen benchmark path directly via workflow_dispatch.
 #
-# GitHub only exposes workflow_dispatch for workflows that exist on the default
-# branch. Keep this file on main so branch-local implementations can replace it
-# at the same path and still be dispatched from the Actions UI or CLI.
+# This is intentionally separate from bench-e2e.yml so the txgen script can be
+# tested in isolation before it is fully integrated into the main benchmark flow.
 
 name: bench-txgen-dispatch
 
@@ -139,27 +138,292 @@ on:
         required: true
         default: false
 
+env:
+  CARGO_TERM_COLOR: always
+  RUSTC_WRAPPER: "sccache"
+
 permissions:
   contents: read
+  issues: write
+  pull-requests: read
 
 jobs:
   bench-txgen-dispatch:
     name: bench-txgen-dispatch
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, X64, bare-metal]
+    timeout-minutes: 300
     steps:
-      - name: Explain bootstrap behavior
+      - name: Clean up previous results
+        run: sudo rm -rf bench-results/ 2>/dev/null || true
+
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Create PR status comment
+        id: pr-comment
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          INPUT_BASELINE: ${{ inputs.baseline }}
+          INPUT_FEATURE: ${{ inputs.feature }}
+          INPUT_TXGEN_REF: ${{ inputs.txgen-ref }}
+          INPUT_PRESET: ${{ inputs.preset }}
+          INPUT_DURATION: ${{ inputs.duration }}
+          INPUT_BLOAT: ${{ inputs.bloat }}
+          INPUT_TPS: ${{ inputs.tps }}
+          INPUT_ACCOUNTS: ${{ inputs.accounts }}
+          INPUT_MAX_CONCURRENT_REQUESTS: ${{ inputs.max-concurrent-requests }}
+          INPUT_SAMPLY: ${{ inputs.samply }}
+          INPUT_TRACY: ${{ inputs.tracy }}
+          INPUT_BASELINE_HARDFORK: ${{ inputs.baseline-hardfork }}
+          INPUT_FEATURE_HARDFORK: ${{ inputs.feature-hardfork }}
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const branch = process.env.GITHUB_REF_NAME;
+            const { data: prs } = await github.rest.pulls.list({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              head: `${context.repo.owner}:${branch}`,
+              state: 'open',
+              per_page: 1,
+            });
+
+            if (!prs.length) {
+              core.info(`No open PR found for branch '${branch}', results will be in job summary`);
+              return;
+            }
+
+            const pr = prs[0];
+            const actor = context.actor;
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            let jobUrl = runUrl;
+            try {
+              const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+              });
+              const job = jobs.jobs.find(j => j.name === 'bench-txgen-dispatch');
+              if (job) jobUrl = job.html_url;
+            } catch (error) {
+              core.info(`Could not resolve job URL: ${error.message}`);
+            }
+
+            const txgenRef = process.env.INPUT_TXGEN_REF || 'default';
+            const samply = process.env.INPUT_SAMPLY === 'true';
+            const samplyNote = samply ? ', samply: `enabled`' : '';
+            const tracy = process.env.INPUT_TRACY || 'off';
+            const tracyNote = tracy !== 'off' ? `, tracy: \`${tracy}\`` : '';
+            const baselineHardfork = process.env.INPUT_BASELINE_HARDFORK || '';
+            const featureHardfork = process.env.INPUT_FEATURE_HARDFORK || '';
+            const hardforkNote = baselineHardfork ? `, baseline-hardfork: \`${baselineHardfork}\`, feature-hardfork: \`${featureHardfork}\`` : '';
+            const config = `**Config:** mode: \`e2e\`, preset: \`${process.env.INPUT_PRESET}\`, duration: \`${process.env.INPUT_DURATION}s\`, bloat: \`${process.env.INPUT_BLOAT} MiB\`, tps: \`${process.env.INPUT_TPS}\`, accounts: \`${process.env.INPUT_ACCOUNTS}\`, max-concurrent-requests: \`${process.env.INPUT_MAX_CONCURRENT_REQUESTS}\`, baseline: \`${process.env.INPUT_BASELINE}\`, feature: \`${process.env.INPUT_FEATURE}\`, backend: \`txgen\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${hardforkNote}`;
+
+            const { data: comment } = await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pr.number,
+              body: `cc @${actor}\n\nTxgen benchmark started. [View job](${jobUrl})\n\nStatus: Setting up runner...\n\n${config}`,
+            });
+
+            core.exportVariable('BENCH_PR', String(pr.number));
+            core.exportVariable('BENCH_ACTOR', actor);
+            core.exportVariable('BENCH_COMMENT_ID', String(comment.id));
+            core.exportVariable('BENCH_JOB_URL', jobUrl);
+            core.exportVariable('BENCH_CONFIG', config);
+            core.setOutput('comment-id', String(comment.id));
+
+      - name: Pre-fetch comparison refs
+        run: |
+          for ref in "${{ inputs.baseline }}" "${{ inputs.feature }}"; do
+            [ -z "$ref" ] && continue
+            [ "$ref" = "local" ] && continue
+            git fetch origin "$ref" --quiet || true
+          done
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: mozilla-actions/sccache-action@v0.0.9
+        continue-on-error: true
+
+      - name: Update status (installing txgen)
+        if: success() && env.BENCH_COMMENT_ID
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const { buildBody } = require('./.github/scripts/bench-update-status.js');
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: parseInt(process.env.BENCH_COMMENT_ID),
+              body: buildBody('Installing txgen backend...'),
+            });
+
+      - name: Install txgen backend
+        env:
+          TXGEN_GIT_TOKEN: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          TXGEN_REF: ${{ inputs.txgen-ref }}
+        run: |
+          CARGO_BIN_DIR="${CARGO_HOME:-$HOME/.cargo}/bin"
+          echo "$CARGO_BIN_DIR" >> "$GITHUB_PATH"
+          export PATH="$CARGO_BIN_DIR:$PATH"
+
+          TXGEN_GIT_URL="https://x-access-token:${TXGEN_GIT_TOKEN}@github.com/tempoxyz/txgen"
+          install_args=(--git "$TXGEN_GIT_URL" --locked)
+          if [ -n "$TXGEN_REF" ]; then
+            TXGEN_REV="$(git ls-remote "$TXGEN_GIT_URL" "$TXGEN_REF" "refs/heads/$TXGEN_REF" "refs/tags/$TXGEN_REF" | awk 'BEGIN { rev = "" } /\^\{\}$/ { rev = $1; exit } rev == "" { rev = $1 } END { if (rev != "") print rev }')"
+            install_args+=(--rev "${TXGEN_REV:-$TXGEN_REF}")
+          fi
+
+          cargo install "${install_args[@]}" txgen-tempo bench-cli
+          echo "TXGEN_TEMPO_BIN=$CARGO_BIN_DIR/txgen-tempo" >> "$GITHUB_ENV"
+          echo "TXGEN_BENCH_BIN=$CARGO_BIN_DIR/bench" >> "$GITHUB_ENV"
+          command -v txgen-tempo
+          command -v bench
+
+      - name: Update status (running benchmark)
+        if: success() && env.BENCH_COMMENT_ID
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const s = require('./.github/scripts/bench-update-status.js');
+            await s({github, context, status: 'Running txgen benchmark...'});
+
+      - name: Run txgen benchmark
+        run: |
+          tracy_mode='${{ inputs.tracy }}'
+
+          cmd=(
+            nu contrib/bench/bench-txgen.nu run
+            --mode e2e
+            --preset "${{ inputs.preset }}"
+            --duration "${{ inputs.duration }}"
+            --bloat "${{ inputs.bloat }}"
+            --tps "${{ inputs.tps }}"
+            --accounts "${{ inputs.accounts }}"
+            --max-concurrent-requests "${{ inputs.max-concurrent-requests }}"
+            --no-infra
+            --baseline "${{ inputs.baseline }}"
+            --feature "${{ inputs.feature }}"
+            --bench-datadir "/reth-bench/tempo_${{ inputs.bloat }}mb"
+            --tune
+            --gas-limit "${{ inputs.gas-limit }}"
+          )
+
+          [ "${{ inputs.samply }}" = 'true' ] && cmd+=(--samply)
+          [ "${{ inputs.no-cache }}" = 'true' ] && cmd+=(--no-cache)
+          [ "${{ inputs.force }}" = 'true' ] && cmd+=(--force)
+          case "$tracy_mode" in
+            ''|off|false)
+              ;;
+            true)
+              cmd+=(--tracy on --tracy-seconds "${{ inputs.tracy-seconds }}" --tracy-offset "${{ inputs.tracy-offset }}")
+              ;;
+            on|full)
+              cmd+=(--tracy "$tracy_mode" --tracy-seconds "${{ inputs.tracy-seconds }}" --tracy-offset "${{ inputs.tracy-offset }}")
+              ;;
+            *)
+              echo "::error::Unsupported tracy input value: $tracy_mode"
+              exit 1
+              ;;
+          esac
+          [ -n "${{ inputs.node-args }}" ] && cmd+=(--node-args="${{ inputs.node-args }}")
+          [ -n "${{ inputs.baseline-args }}" ] && cmd+=(--baseline-args="${{ inputs.baseline-args }}")
+          [ -n "${{ inputs.feature-args }}" ] && cmd+=(--feature-args="${{ inputs.feature-args }}")
+          [ -n "${{ inputs.bench-args }}" ] && cmd+=(--bench-args="${{ inputs.bench-args }}")
+          [ -n "${{ inputs.bench-env }}" ] && cmd+=(--bench-env="${{ inputs.bench-env }}")
+          [ -n "${{ inputs.baseline-env }}" ] && cmd+=(--baseline-env="${{ inputs.baseline-env }}")
+          [ -n "${{ inputs.feature-env }}" ] && cmd+=(--feature-env="${{ inputs.feature-env }}")
+          [ -n "${{ inputs.baseline-hardfork }}" ] && cmd+=(--baseline-hardfork "${{ inputs.baseline-hardfork }}" --feature-hardfork "${{ inputs.feature-hardfork }}")
+
+          printf 'Running command:\n'
+          printf '  %q' "${cmd[@]}"
+          printf '\n'
+
+          "${cmd[@]}"
+
+      - name: Find results directory
+        id: results-dir
+        if: success()
         run: |
-          {
-            echo "### bench-txgen-dispatch bootstrap"
-            echo
-            echo "This default-branch stub exists so GitHub exposes the workflow_dispatch entrypoint."
-            echo "Run this workflow against a branch that contains the txgen benchmark implementation."
-            echo
-            echo "- ref: \`${GITHUB_REF}\`"
-            echo "- sha: \`${GITHUB_SHA}\`"
-          } >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Print next step
+          RESULTS_DIR=$(ls -d bench-results/*/ 2>/dev/null | tail -1 | sed 's:/$::')
+          if [ -z "$RESULTS_DIR" ]; then
+            echo "::error::No results directory found"
+            exit 1
+          fi
+          echo "path=$RESULTS_DIR" >> "$GITHUB_OUTPUT"
+          echo "Results directory: $RESULTS_DIR"
+
+      - name: Add Summary To Job
+        if: success()
         run: |
-          echo "No benchmark runs on this stub."
-          echo "Dispatch the workflow against a branch that replaces .github/workflows/bench-txgen-dispatch.yml."
+          RESULTS_DIR="${{ steps.results-dir.outputs.path }}"
+          if [ -f "$RESULTS_DIR/summary.md" ]; then
+            cat "$RESULTS_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "Benchmark completed, but no summary.md was produced." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Post results to PR
+        if: success() && env.BENCH_COMMENT_ID
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }}
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const fs = require('fs');
+            const resultsDir = process.env.BENCH_RESULTS_DIR;
+
+            let summary = '';
+            try {
+              summary = fs.readFileSync(`${resultsDir}/summary.md`, 'utf8');
+            } catch (e) {
+              summary = 'Benchmark completed but failed to read summary.';
+            }
+
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: parseInt(process.env.BENCH_COMMENT_ID),
+              body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark complete. [View job](${jobUrl})\n\n${summary}`,
+            });
+
+      - name: Upload results
+        if: "!cancelled()"
+        uses: actions/upload-artifact@v4
+        with:
+          name: tempo-bench-txgen-results
+          path: bench-results/
+
+      - name: Update status (failed)
+        if: failure() && env.BENCH_COMMENT_ID
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: parseInt(process.env.BENCH_COMMENT_ID),
+              body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark failed. [View logs](${jobUrl})`,
+            });
+
+      - name: Update status (cancelled)
+        if: cancelled() && env.BENCH_COMMENT_ID
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }}
+          script: |
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: parseInt(process.env.BENCH_COMMENT_ID),
+              body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark cancelled. [View logs](${jobUrl})`,
+            });