diff --git a/.github/workflows/bench-e2e.yml b/.github/workflows/bench-e2e.yml index 5f535cb28c..3874662aab 100644 --- a/.github/workflows/bench-e2e.yml +++ b/.github/workflows/bench-e2e.yml @@ -330,23 +330,6 @@ jobs: fetch-depth: 0 ref: ${{ steps.checkout-ref.outputs.ref }} - - name: Checkout txgen repository - if: env.BENCH_BACKEND == 'txgen' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: tempoxyz/txgen - token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} - persist-credentials: false - path: txgen - fetch-depth: 0 - - - name: Checkout txgen ref - if: env.BENCH_BACKEND == 'txgen' && env.BENCH_TXGEN_REF != '' - working-directory: txgen - run: | - git fetch origin "$BENCH_TXGEN_REF" --quiet - git checkout --detach FETCH_HEAD - - name: Resolve job URL and update status if: env.BENCH_COMMENT_ID uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -395,14 +378,27 @@ jobs: - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 continue-on-error: true - - name: Build txgen backend + - name: Install txgen backend if: env.BENCH_BACKEND == 'txgen' - working-directory: txgen + env: + TXGEN_GIT_TOKEN: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} run: | - cargo build --release --bin txgen-tempo --bin bench - echo "TXGEN_REPO_DIR=$GITHUB_WORKSPACE/txgen" >> "$GITHUB_ENV" - echo "TXGEN_TEMPO_BIN=$GITHUB_WORKSPACE/txgen/target/release/txgen-tempo" >> "$GITHUB_ENV" - echo "TXGEN_BENCH_BIN=$GITHUB_WORKSPACE/txgen/target/release/bench" >> "$GITHUB_ENV" + CARGO_BIN_DIR="${CARGO_HOME:-$HOME/.cargo}/bin" + echo "$CARGO_BIN_DIR" >> "$GITHUB_PATH" + export PATH="$CARGO_BIN_DIR:$PATH" + + TXGEN_GIT_URL="https://x-access-token:${TXGEN_GIT_TOKEN}@github.com/tempoxyz/txgen" + install_args=(--git "$TXGEN_GIT_URL" --locked) + if [ -n "$BENCH_TXGEN_REF" ]; then + TXGEN_REV="$(git ls-remote "$TXGEN_GIT_URL" "$BENCH_TXGEN_REF" "refs/heads/$BENCH_TXGEN_REF" "refs/tags/$BENCH_TXGEN_REF" | awk 'BEGIN { rev = "" } /\^\{\}$/ { rev = $1; exit } rev == "" { rev = $1 } END { if (rev != "") print rev }')" + install_args+=(--rev "${TXGEN_REV:-$BENCH_TXGEN_REF}") + fi + + cargo install "${install_args[@]}" txgen-tempo bench-cli + echo "TXGEN_TEMPO_BIN=$CARGO_BIN_DIR/txgen-tempo" >> "$GITHUB_ENV" + echo "TXGEN_BENCH_BIN=$CARGO_BIN_DIR/bench" >> "$GITHUB_ENV" + command -v txgen-tempo + command -v bench - name: Resolve PR head branch id: pr-info @@ -577,7 +573,12 @@ jobs: BENCH_RUN_TYPE: ${{ env.BENCH_RUN_TYPE }} BENCH_JOB_URL: ${{ env.BENCH_JOB_URL }} BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }} - run: bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR" + run: | + if [ "$BENCH_BACKEND" = "txgen" ]; then + bash contrib/bench/upload-clickhouse-txgen.sh "$BENCH_RESULTS_DIR" + else + bash contrib/bench/upload-clickhouse.sh "$BENCH_RESULTS_DIR" + fi - name: Post results to PR if: success() diff --git a/.github/workflows/bench-txgen-dispatch.yml b/.github/workflows/bench-txgen-dispatch.yml index cfd4160de1..9ffd057cac 100644 --- a/.github/workflows/bench-txgen-dispatch.yml +++ b/.github/workflows/bench-txgen-dispatch.yml @@ -1,8 +1,7 @@ -# Bootstrap stub for the txgen workflow_dispatch entrypoint. +# Runs the txgen benchmark path directly via workflow_dispatch. # -# GitHub only exposes workflow_dispatch for workflows that exist on the default -# branch. Keep this file on main so branch-local implementations can replace it -# at the same path and still be dispatched from the Actions UI or CLI. +# This is intentionally separate from bench-e2e.yml so the txgen script can be +# tested in isolation before it is fully integrated into the main benchmark flow. name: bench-txgen-dispatch @@ -139,27 +138,292 @@ on: required: true default: false +env: + CARGO_TERM_COLOR: always + RUSTC_WRAPPER: "sccache" + permissions: contents: read + issues: write + pull-requests: read jobs: bench-txgen-dispatch: name: bench-txgen-dispatch - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64, bare-metal] + timeout-minutes: 300 steps: - - name: Explain bootstrap behavior + - name: Clean up previous results + run: sudo rm -rf bench-results/ 2>/dev/null || true + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Create PR status comment + id: pr-comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_BASELINE: ${{ inputs.baseline }} + INPUT_FEATURE: ${{ inputs.feature }} + INPUT_TXGEN_REF: ${{ inputs.txgen-ref }} + INPUT_PRESET: ${{ inputs.preset }} + INPUT_DURATION: ${{ inputs.duration }} + INPUT_BLOAT: ${{ inputs.bloat }} + INPUT_TPS: ${{ inputs.tps }} + INPUT_ACCOUNTS: ${{ inputs.accounts }} + INPUT_MAX_CONCURRENT_REQUESTS: ${{ inputs.max-concurrent-requests }} + INPUT_SAMPLY: ${{ inputs.samply }} + INPUT_TRACY: ${{ inputs.tracy }} + INPUT_BASELINE_HARDFORK: ${{ inputs.baseline-hardfork }} + INPUT_FEATURE_HARDFORK: ${{ inputs.feature-hardfork }} + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const branch = process.env.GITHUB_REF_NAME; + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${branch}`, + state: 'open', + per_page: 1, + }); + + if (!prs.length) { + core.info(`No open PR found for branch '${branch}', results will be in job summary`); + return; + } + + const pr = prs[0]; + const actor = context.actor; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + let jobUrl = runUrl; + try { + const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + const job = jobs.jobs.find(j => j.name === 'bench-txgen-dispatch'); + if (job) jobUrl = job.html_url; + } catch (error) { + core.info(`Could not resolve job URL: ${error.message}`); + } + + const txgenRef = process.env.INPUT_TXGEN_REF || 'default'; + const samply = process.env.INPUT_SAMPLY === 'true'; + const samplyNote = samply ? ', samply: `enabled`' : ''; + const tracy = process.env.INPUT_TRACY || 'off'; + const tracyNote = tracy !== 'off' ? `, tracy: \`${tracy}\`` : ''; + const baselineHardfork = process.env.INPUT_BASELINE_HARDFORK || ''; + const featureHardfork = process.env.INPUT_FEATURE_HARDFORK || ''; + const hardforkNote = baselineHardfork ? `, baseline-hardfork: \`${baselineHardfork}\`, feature-hardfork: \`${featureHardfork}\`` : ''; + const config = `**Config:** mode: \`e2e\`, preset: \`${process.env.INPUT_PRESET}\`, duration: \`${process.env.INPUT_DURATION}s\`, bloat: \`${process.env.INPUT_BLOAT} MiB\`, tps: \`${process.env.INPUT_TPS}\`, accounts: \`${process.env.INPUT_ACCOUNTS}\`, max-concurrent-requests: \`${process.env.INPUT_MAX_CONCURRENT_REQUESTS}\`, baseline: \`${process.env.INPUT_BASELINE}\`, feature: \`${process.env.INPUT_FEATURE}\`, backend: \`txgen\`, txgen-ref: \`${txgenRef}\`${samplyNote}${tracyNote}${hardforkNote}`; + + const { data: comment } = await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: `cc @${actor}\n\nTxgen benchmark started. [View job](${jobUrl})\n\nStatus: Setting up runner...\n\n${config}`, + }); + + core.exportVariable('BENCH_PR', String(pr.number)); + core.exportVariable('BENCH_ACTOR', actor); + core.exportVariable('BENCH_COMMENT_ID', String(comment.id)); + core.exportVariable('BENCH_JOB_URL', jobUrl); + core.exportVariable('BENCH_CONFIG', config); + core.setOutput('comment-id', String(comment.id)); + + - name: Pre-fetch comparison refs + run: | + for ref in "${{ inputs.baseline }}" "${{ inputs.feature }}"; do + [ -z "$ref" ] && continue + [ "$ref" = "local" ] && continue + git fetch origin "$ref" --quiet || true + done + + - uses: dtolnay/rust-toolchain@stable + + - uses: mozilla-actions/sccache-action@v0.0.9 + continue-on-error: true + + - name: Update status (installing txgen) + if: success() && env.BENCH_COMMENT_ID + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const { buildBody } = require('./.github/scripts/bench-update-status.js'); + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: parseInt(process.env.BENCH_COMMENT_ID), + body: buildBody('Installing txgen backend...'), + }); + + - name: Install txgen backend + env: + TXGEN_GIT_TOKEN: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + TXGEN_REF: ${{ inputs.txgen-ref }} + run: | + CARGO_BIN_DIR="${CARGO_HOME:-$HOME/.cargo}/bin" + echo "$CARGO_BIN_DIR" >> "$GITHUB_PATH" + export PATH="$CARGO_BIN_DIR:$PATH" + + TXGEN_GIT_URL="https://x-access-token:${TXGEN_GIT_TOKEN}@github.com/tempoxyz/txgen" + install_args=(--git "$TXGEN_GIT_URL" --locked) + if [ -n "$TXGEN_REF" ]; then + TXGEN_REV="$(git ls-remote "$TXGEN_GIT_URL" "$TXGEN_REF" "refs/heads/$TXGEN_REF" "refs/tags/$TXGEN_REF" | awk 'BEGIN { rev = "" } /\^\{\}$/ { rev = $1; exit } rev == "" { rev = $1 } END { if (rev != "") print rev }')" + install_args+=(--rev "${TXGEN_REV:-$TXGEN_REF}") + fi + + cargo install "${install_args[@]}" txgen-tempo bench-cli + echo "TXGEN_TEMPO_BIN=$CARGO_BIN_DIR/txgen-tempo" >> "$GITHUB_ENV" + echo "TXGEN_BENCH_BIN=$CARGO_BIN_DIR/bench" >> "$GITHUB_ENV" + command -v txgen-tempo + command -v bench + + - name: Update status (running benchmark) + if: success() && env.BENCH_COMMENT_ID + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const s = require('./.github/scripts/bench-update-status.js'); + await s({github, context, status: 'Running txgen benchmark...'}); + + - name: Run txgen benchmark + run: | + tracy_mode='${{ inputs.tracy }}' + + cmd=( + nu contrib/bench/bench-txgen.nu run + --mode e2e + --preset "${{ inputs.preset }}" + --duration "${{ inputs.duration }}" + --bloat "${{ inputs.bloat }}" + --tps "${{ inputs.tps }}" + --accounts "${{ inputs.accounts }}" + --max-concurrent-requests "${{ inputs.max-concurrent-requests }}" + --no-infra + --baseline "${{ inputs.baseline }}" + --feature "${{ inputs.feature }}" + --bench-datadir "/reth-bench/tempo_${{ inputs.bloat }}mb" + --tune + --gas-limit "${{ inputs.gas-limit }}" + ) + + [ "${{ inputs.samply }}" = 'true' ] && cmd+=(--samply) + [ "${{ inputs.no-cache }}" = 'true' ] && cmd+=(--no-cache) + [ "${{ inputs.force }}" = 'true' ] && cmd+=(--force) + case "$tracy_mode" in + ''|off|false) + ;; + true) + cmd+=(--tracy on --tracy-seconds "${{ inputs.tracy-seconds }}" --tracy-offset "${{ inputs.tracy-offset }}") + ;; + on|full) + cmd+=(--tracy "$tracy_mode" --tracy-seconds "${{ inputs.tracy-seconds }}" --tracy-offset "${{ inputs.tracy-offset }}") + ;; + *) + echo "::error::Unsupported tracy input value: $tracy_mode" + exit 1 + ;; + esac + [ -n "${{ inputs.node-args }}" ] && cmd+=(--node-args="${{ inputs.node-args }}") + [ -n "${{ inputs.baseline-args }}" ] && cmd+=(--baseline-args="${{ inputs.baseline-args }}") + [ -n "${{ inputs.feature-args }}" ] && cmd+=(--feature-args="${{ inputs.feature-args }}") + [ -n "${{ inputs.bench-args }}" ] && cmd+=(--bench-args="${{ inputs.bench-args }}") + [ -n "${{ inputs.bench-env }}" ] && cmd+=(--bench-env="${{ inputs.bench-env }}") + [ -n "${{ inputs.baseline-env }}" ] && cmd+=(--baseline-env="${{ inputs.baseline-env }}") + [ -n "${{ inputs.feature-env }}" ] && cmd+=(--feature-env="${{ inputs.feature-env }}") + [ -n "${{ inputs.baseline-hardfork }}" ] && cmd+=(--baseline-hardfork "${{ inputs.baseline-hardfork }}" --feature-hardfork "${{ inputs.feature-hardfork }}") + + printf 'Running command:\n' + printf ' %q' "${cmd[@]}" + printf '\n' + + "${cmd[@]}" + + - name: Find results directory + id: results-dir + if: success() run: | - { - echo "### bench-txgen-dispatch bootstrap" - echo - echo "This default-branch stub exists so GitHub exposes the workflow_dispatch entrypoint." - echo "Run this workflow against a branch that contains the txgen benchmark implementation." - echo - echo "- ref: \`${GITHUB_REF}\`" - echo "- sha: \`${GITHUB_SHA}\`" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Print next step + RESULTS_DIR=$(ls -d bench-results/*/ 2>/dev/null | tail -1 | sed 's:/$::') + if [ -z "$RESULTS_DIR" ]; then + echo "::error::No results directory found" + exit 1 + fi + echo "path=$RESULTS_DIR" >> "$GITHUB_OUTPUT" + echo "Results directory: $RESULTS_DIR" + + - name: Add Summary To Job + if: success() run: | - echo "No benchmark runs on this stub." - echo "Dispatch the workflow against a branch that replaces .github/workflows/bench-txgen-dispatch.yml." + RESULTS_DIR="${{ steps.results-dir.outputs.path }}" + if [ -f "$RESULTS_DIR/summary.md" ]; then + cat "$RESULTS_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY" + else + echo "Benchmark completed, but no summary.md was produced." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Post results to PR + if: success() && env.BENCH_COMMENT_ID + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + BENCH_RESULTS_DIR: ${{ steps.results-dir.outputs.path }} + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const fs = require('fs'); + const resultsDir = process.env.BENCH_RESULTS_DIR; + + let summary = ''; + try { + summary = fs.readFileSync(`${resultsDir}/summary.md`, 'utf8'); + } catch (e) { + summary = 'Benchmark completed but failed to read summary.'; + } + + const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: parseInt(process.env.BENCH_COMMENT_ID), + body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark complete. [View job](${jobUrl})\n\n${summary}`, + }); + + - name: Upload results + if: "!cancelled()" + uses: actions/upload-artifact@v4 + with: + name: tempo-bench-txgen-results + path: bench-results/ + + - name: Update status (failed) + if: failure() && env.BENCH_COMMENT_ID + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: parseInt(process.env.BENCH_COMMENT_ID), + body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark failed. [View logs](${jobUrl})`, + }); + + - name: Update status (cancelled) + if: cancelled() && env.BENCH_COMMENT_ID + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.DEREK_BENCH_TOKEN || github.token }} + script: | + const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: parseInt(process.env.BENCH_COMMENT_ID), + body: `cc @${process.env.BENCH_ACTOR}\n\nTxgen benchmark cancelled. [View logs](${jobUrl})`, + }); diff --git a/contrib/bench/bench-txgen.nu b/contrib/bench/bench-txgen.nu index 55c1d32746..339b0918c7 100644 --- a/contrib/bench/bench-txgen.nu +++ b/contrib/bench/bench-txgen.nu @@ -1,44 +1,821 @@ -# Bootstrap stub for the txgen benchmark backend. -# -# This file exists on main so the benchmark workflow can reference a real -# entrypoint before the txgen harness lands on a PR branch. +#!/usr/bin/env nu -def main [] { - print "Tempo txgen benchmark helper" - print "" - print "Usage:" - print " nu contrib/bench/bench-txgen.nu run [flags]" - print "" - print "This is a bootstrap stub. The txgen harness is not implemented on this branch." +source ../../tempo.nu + +const TXGEN_ACCOUNT_MNEMONIC = "test test test test test test test test test test test junk" +const TXGEN_DEFAULT_SEED = 99 +const TXGEN_SCRAPE_INTERVAL_MS = 500 +const TXGEN_DRAIN_TIMEOUT_SECS = 300 +const TXGEN_FUND_DRAIN_TIMEOUT_SECS = 120 +const TXGEN_TIP20_TEMPLATE = "contrib/bench/txgen/tip20-template.yaml" + +def shell-quote [value: any] { + let s = ($value | into string) + let escaped = ($s | str replace -a "'" "'\"'\"'") + $"'($escaped)'" +} + +def shell-join [args: list] { + $args | each { |arg| shell-quote $arg } | str join " " +} + +def resolve-command-path [name: string] { + let path = (which $name | get -o 0.path | default "") + if $path == "" { + error make { msg: $"($name) not found in PATH" } + } + $path +} + +def resolved-runtime-mode [mode: string] { + if $mode == "e2e" { + "dev" + } else { + $mode + } +} + +def sanitize-bench-args [bench_args: string] { + if $bench_args == "" { + return "" + } + + $bench_args + | str replace --all --regex '--existing-recipients=(true|false)' '' + | str trim +} + +def resolve-bench-binary [repo_dir: string] { + let candidates = [ + $"($repo_dir)/target/release/bench" + $"($repo_dir)/target/release/bench-cli" + ] + + for candidate in $candidates { + if ($candidate | path exists) { + return $candidate + } + } + + error make { msg: $"txgen bench binary not found under ($repo_dir)/target/release/" } +} + +def resolve-txgen-paths [repo_dir: string, txgen_tempo_bin: string, txgen_bench_bin: string] { + let env_repo_dir = ($env.TXGEN_REPO_DIR? | default "") + let explicit_repo = $repo_dir != "" or $env_repo_dir != "" + let configured_repo = if $repo_dir != "" { + $repo_dir | path expand + } else if $env_repo_dir != "" { + $env_repo_dir | path expand + } else { + "../txgen" | path expand + } + + if $explicit_repo and not ($configured_repo | path exists) { + error make { msg: $"txgen repo not found: ($configured_repo)" } + } + let repo = if ($configured_repo | path exists) { $configured_repo } else { "" } + + let generator = if $txgen_tempo_bin != "" { + $txgen_tempo_bin | path expand + } else if ($env.TXGEN_TEMPO_BIN? | default "") != "" { + $env.TXGEN_TEMPO_BIN | path expand + } else if $repo != "" and ($"($repo)/target/release/txgen-tempo" | path exists) { + $"($repo)/target/release/txgen-tempo" + } else { + resolve-command-path "txgen-tempo" + } + + let bench = if $txgen_bench_bin != "" { + $txgen_bench_bin | path expand + } else if ($env.TXGEN_BENCH_BIN? | default "") != "" { + $env.TXGEN_BENCH_BIN | path expand + } else if $repo != "" { + resolve-bench-binary $repo + } else { + resolve-command-path "bench" + } + + if not ($generator | path exists) { + error make { msg: $"txgen-tempo binary not found: ($generator)" } + } + if not ($bench | path exists) { + error make { msg: $"txgen bench binary not found: ($bench)" } + } + + { + repo_dir: $repo + txgen_tempo_bin: $generator + txgen_bench_bin: $bench + } +} + +def normalize-tracy-mode [value: any] { + let mode = ($value | into string | str trim | str downcase) + + if $mode in ["" "off" "false"] { + "off" + } else if $mode in ["on" "true"] { + "on" + } else if $mode == "full" { + "full" + } else { + error make { msg: $"--tracy must be one of: off, on, full \(got ($value)\)" } + } +} + +def rpc-call [rpc_url: string, payload: string] { + let result = (^curl -sf -X POST -H "Content-Type: application/json" -d $payload $rpc_url | complete) + if $result.exit_code != 0 { + error make { msg: $"RPC call failed: ($payload)" } + } + let response = ($result.stdout | from json) + if (($response | get -o error) != null) { + let rpc_error = ($response | get error) + error make { msg: $"RPC error: ($rpc_error | to json -r)" } + } + $response +} + +def fetch-chain-id [rpc_url: string] { + let response = (rpc-call $rpc_url '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}') + $response.result | into int +} + +def wait-for-txpool-drain [rpc_url: string, timeout_secs: int] { + mut zero_count = 0 + mut waited = 0 + + while $waited < $timeout_secs { + let response = (rpc-call $rpc_url '{"jsonrpc":"2.0","method":"txpool_status","params":[],"id":1}') + let pending = ($response.result.pending | into int) + + if $pending == 0 { + $zero_count = $zero_count + 1 + if $zero_count >= 3 { + return + } + } else { + $zero_count = 0 + } + + sleep 1sec + $waited = $waited + 1 + } + + print $" Warning: txpool drain timeout reached after ($timeout_secs)s" +} + +def fund-txgen-accounts [txgen_bin: string, spec_path: string, rpc_url: string] { + let result = (^$txgen_bin addresses -s $spec_path -f shell | complete) + if $result.exit_code != 0 { + error make { msg: $"failed to list txgen addresses for ($spec_path)" } + } + + let addresses = ($result.stdout | str trim | split row " " | where { |addr| $addr != "" }) + if ($addresses | is-empty) { + error make { msg: $"txgen spec produced no addresses: ($spec_path)" } + } + + print $" Funding (($addresses | length)) txgen account\(s\)..." + $addresses | par-each { |address| + rpc-call $rpc_url $"{\"jsonrpc\":\"2.0\",\"method\":\"tempo_fundAddress\",\"params\":[\"($address)\"],\"id\":1}" | ignore + } | ignore + + print " Waiting for faucet transactions to drain..." + wait-for-txpool-drain $rpc_url $TXGEN_FUND_DRAIN_TIMEOUT_SECS +} + +def run-txgen-bench-single [ + --tempo-bin: string + --txgen-tempo-bin: string + --txgen-bench-bin: string + --genesis-path: string + --datadir: string + --run-label: string + --results-dir: string + --tps: int + --duration: int + --accounts: int + --max-concurrent-requests: int + --preset: string = "" + --bench-args: string = "" + --loud + --node-args: string = "" + --extra-env: string = "" + --bench-env: string = "" + --bloat: int = 0 + --git-ref: string = "" + --build-profile: string = "" + --benchmark-mode: string = "" + --benchmark-id: string = "" + --reference-epoch: int = 0 + --samply + --samply-args: list = [] + --tracy: any = "off" + --tracy-filter: string = "debug" + --tracy-seconds: int = 0 + --tracy-offset: int = 0 + --tracing-otlp: string = "" +] { + if $preset != "tip20" { + error make { msg: $"txgen benchmark path currently supports only preset=tip20 \(got ($preset)\)" } + } + + let ignored_bench_args = (sanitize-bench-args $bench_args) + if $ignored_bench_args != "" { + print $" Warning: txgen path is ignoring unsupported bench args: ($ignored_bench_args)" + } + + print $"=== Starting txgen run: ($run_label) ===" + + let log_dir = $"($LOCALNET_DIR)/logs-($run_label)" + if ($log_dir | path exists) { + rm -rf $log_dir + } + mkdir $log_dir + + let run_type = if ($run_label | str starts-with "baseline") { "baseline" } else { "feature" } + let run_start_epoch = (date now | into int) / 1_000_000_000 + let labels = { + benchmark_run: $run_label + run_type: $run_type + git_ref: $git_ref + benchmark_id: $benchmark_id + run_start_epoch: $"($run_start_epoch)" + reference_epoch: $"($reference_epoch)" + } + let labels_file = $"($results_dir)/metrics-labels-($run_label).json" + $labels | to json | save -f $labels_file + + let proxy_pid = if ($METRICS_PROXY_SCRIPT | path exists) { + let proxy_job = (job spawn { + python3 $METRICS_PROXY_SCRIPT --upstream "http://127.0.0.1:9001/" --port 9090 --labels $labels_file + }) + sleep 500ms + $proxy_job + } else { + null + } + + let extra_args = if $node_args == "" { [] } else { $node_args | split row " " } + let base_args = (build-base-args $genesis_path $datadir $log_dir "0.0.0.0" 8545 9001) + | append (build-dev-args) + | append (log-filter-args $loud) + | append (if $tracy != "off" { ["--log.tracy" "--log.tracy.filter" $tracy_filter] } else { [] }) + | append (if $tracing_otlp != "" { [$"--tracing-otlp=($tracing_otlp)"] } else { [] }) + let args = (dedup-args $base_args $extra_args) + + let tracy_env_prefix = if $tracy == "on" { + "TRACY_NO_SYS_TRACE=1 " + } else if $tracy == "full" { + "TRACY_SAMPLING_HZ=1 " + } else { "" } + + let otel_attrs = $"OTEL_RESOURCE_ATTRIBUTES=benchmark_id=($benchmark_id),benchmark_run=($run_label),run_type=($run_type),git_ref=($git_ref) " + let full_samply_args = if $samply { + $samply_args | append ["--save-only" "--presymbolicate" "--output" $"($results_dir)/profile-($run_label).json.gz"] + } else { [] } + let node_cmd = wrap-samply [$tempo_bin ...$args] $samply $full_samply_args + let node_cmd_str = ($node_cmd | str join " ") + let profiling_label = if $samply { " (samply)" } else if $tracy != "off" { $" \(tracy=($tracy)\)" } else { "" } + let env_prefix = if $extra_env != "" { $"($extra_env) " } else { "" } + print $" Starting node: ($tempo_bin | path basename)($profiling_label)" + job spawn { sh -c $"($env_prefix)($otel_attrs)($tracy_env_prefix)($node_cmd_str) 2>&1" | lines | each { |line| print $"[($run_label)] ($line)" } } + + sleep 2sec + let rpc_timeout = if $bloat > 0 { 600 } else { 120 } + wait-for-rpc "http://localhost:8545" $rpc_timeout + + let tracy_output = $"($results_dir)/tracy-profile-($run_label).tracy" + let tracy_capture_started = if $tracy != "off" { + let seconds_flag = if $tracy_seconds > 0 { $"-s ($tracy_seconds)" } else { "" } + let limit_msg = if $tracy_seconds > 0 { $" \(($tracy_seconds)s limit\)" } else { "" } + if $tracy_offset > 0 { + print $" Tracy-capture will start in ($tracy_offset)s($limit_msg)..." + job spawn { sleep ($"($tracy_offset)sec" | into duration); sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" } + } else { + print $" Starting tracy-capture($limit_msg)..." + job spawn { sh -c $"tracy-capture -f -o ($tracy_output) ($seconds_flag)" } + sleep 500ms + } + true + } else { false } + + let chain_id = (fetch-chain-id "http://localhost:8545") + $env.TXGEN_ACCOUNTS = ($accounts | into string) + let spec_path = ($TXGEN_TIP20_TEMPLATE | path expand) + fund-txgen-accounts $txgen_tempo_bin $spec_path "http://localhost:8545" + + let report_path = $"($results_dir)/report-($run_label).json" + let tx_count = [($tps * $duration) 1] | math max + let txgen_cmd = [ + $txgen_tempo_bin + "generate" + "-s" $spec_path + "-n" $tx_count + "--seed" $TXGEN_DEFAULT_SEED + "--rpc" "http://localhost:8545" + ] + let bench_cmd = [ + $txgen_bench_bin + "send" + "--rpc-url" "http://localhost:8545" + "--tps" $tps + "--max-concurrent" $max_concurrent_requests + "--metrics-url" "http://127.0.0.1:9090/metrics" + "--scrape-interval-ms" $TXGEN_SCRAPE_INTERVAL_MS + "--drain-timeout" $TXGEN_DRAIN_TIMEOUT_SECS + "--report" $"json:($report_path)" + "-m" $"chain_id=($chain_id)" + "-m" $"target_tps=($tps)" + "-m" $"run_duration_secs=($duration)" + "-m" $"accounts=($accounts)" + "-m" $"total_connections=($max_concurrent_requests)" + "-m" "tip20_weight=1.0" + "-m" "place_order_weight=0.0" + "-m" "swap_weight=0.0" + "-m" "erc20_weight=0.0" + "-m" $"node_commit_sha=($git_ref)" + "-m" $"build_profile=($build_profile)" + "-m" $"mode=($benchmark_mode)" + ] + let bench_env_export = if $bench_env != "" { $"export ($bench_env) && " } else { "" } + let txgen_cmd_str = (shell-join $txgen_cmd) + let bench_cmd_str = (shell-join $bench_cmd) + + print $" Streaming ($tx_count) txgen transaction\(s\) into bench send..." + let pipeline = $"set -euo pipefail; ($bench_env_export)($txgen_cmd_str) | ($bench_cmd_str)" + try { + bash -lc $pipeline + } catch { |e| + print $" txgen benchmark run ($run_label) failed: ($e.msg)" + error make { msg: $"txgen benchmark run ($run_label) failed" } + } + + print $" Report saved: report-($run_label).json" + + if $tracy_capture_started { + print " Stopping tracy-capture..." + let capture_pids = (ps | where name =~ "tracy-capture" | get pid) + for pid in $capture_pids { + kill -s 2 $pid + } + mut wait_tracy = 0 + while $wait_tracy < 30 { + if (ps | where name =~ "tracy-capture" | length) == 0 { break } + sleep 1sec + $wait_tracy = $wait_tracy + 1 + } + if $wait_tracy >= 30 { + print " Warning: tracy-capture did not exit, sending SIGKILL" + for pid in (ps | where name =~ "tracy-capture" | get pid) { + kill -s 9 $pid + } + } + } + + print " Stopping node..." + let pids = (find-tempo-pids) + for pid in $pids { + kill -s 2 $pid + } + for pid in $pids { + mut wait = 0 + while $wait < 30 { + if (ps | where pid == $pid | length) == 0 { break } + sleep 1sec + $wait = $wait + 1 + } + if $wait >= 30 { + print $" Warning: PID ($pid) did not exit, sending SIGKILL" + kill -s 9 $pid + sleep 1sec + } + } + + if $samply { + print " Waiting for samply to finish saving profile..." + mut wait = 0 + while $wait < 120 { + if (ps | where name =~ "samply" | length) == 0 { break } + sleep 500ms + $wait = $wait + 1 + } + if $wait >= 120 { + print " Warning: samply did not exit in time" + } + } + + if $proxy_pid != null { + let proxy_pids = (ps | where name =~ "bench-metrics-proxy" | get pid) + for pid in $proxy_pids { + kill -s 2 $pid + } + } + + if ("/tmp/reth.ipc" | path exists) { + rm --force /tmp/reth.ipc + } + + print $"=== Run ($run_label) complete ===" } def "main run" [ + --mode: string = "e2e" --preset: string = "" - --mode: string = "" - --bloat: string = "" - --duration: string = "" - --tps: string = "" + --tps: int = 10000 + --duration: int = 30 + --accounts: int = 1000 + --max-concurrent-requests: int = 100 + --samply + --samply-args: string = "" + --loud + --profile: string = $DEFAULT_PROFILE + --features: string = $DEFAULT_FEATURES + --node-args: string = "" + --baseline-args: string = "" + --feature-args: string = "" + --bench-args: string = "" + --baseline-env: string = "" + --feature-env: string = "" + --bench-env: string = "" + --bloat: int = 0 --no-infra --baseline: string = "" --feature: string = "" + --force --bench-datadir: string = "" --tune - --gas-limit: string = "" - --samply - --tracy: string = "" - --tracy-seconds: string = "" - --tracy-offset: string = "" - --baseline-args: string = "" - --feature-args: string = "" + --no-cache + --tracy: string = "off" + --tracy-filter: string = "debug" + --tracy-seconds: int = 30 + --tracy-offset: int = 120 + --tracing-otlp: string = "" --baseline-hardfork: string = "" --feature-hardfork: string = "" - --force - --bench-args: string = "" - --bench-env: string = "" - --baseline-env: string = "" - --feature-env: string = "" + --gas-limit: string = "" + --txgen-repo-dir: string = "" + --txgen-tempo-bin: string = "" + --txgen-bench-bin: string = "" ] { - print "txgen benchmark backend is not implemented on this branch." - print "Add the txgen harness in this branch to make `@decofe bench backend=txgen` runnable." - exit 1 + let runtime_mode = (resolved-runtime-mode $mode) + if $runtime_mode != "dev" { + error make { msg: $"txgen benchmark path currently supports only dev/e2e mode \(got ($mode)\)" } + } + if $preset != "tip20" { + error make { msg: $"txgen benchmark path currently supports only preset=tip20 \(got ($preset)\)" } + } + if ($baseline != "" and $feature == "") or ($baseline == "" and $feature != "") { + error make { msg: "--baseline and --feature must both be provided for txgen comparison mode" } + } + if $baseline == "" or $feature == "" { + error make { msg: "txgen benchmark path currently supports comparison mode only" } + } + + let txgen = (resolve-txgen-paths $txgen_repo_dir $txgen_tempo_bin $txgen_bench_bin) + + if $force and ($LOCALNET_DIR | path exists) { + print "Removing existing localnet data (--force)..." + rm -rf $LOCALNET_DIR + } + + main kill + let tuning_state = if $tune { apply-system-tuning } else { { tuned: false } } + + let tracy = (normalize-tracy-mode $tracy) + if $samply and $tracy != "off" { + error make { msg: "--samply and --tracy are mutually exclusive" } + } + if $tracy != "off" and ((which tracy-capture | length) == 0) { + error make { msg: "tracy-capture not found in PATH" } + } + + if ($baseline_hardfork != "" or $feature_hardfork != "") and ($baseline_hardfork == "" or $feature_hardfork == "") { + error make { msg: "--baseline-hardfork and --feature-hardfork must both be provided" } + } + let dual_hardfork = $baseline_hardfork != "" and $feature_hardfork != "" + + let baseline_sha = if $baseline == "local" { "local" } else { resolve-git-ref $baseline } + let feature_sha = if $feature == "local" { "local" } else { resolve-git-ref $feature } + let baseline_label = if $baseline == "local" { "local (working tree)" } else { $"($baseline) → ($baseline_sha)" } + let feature_label = if $feature == "local" { "local (working tree)" } else { $"($feature) → ($feature_sha)" } + print $"Baseline: ($baseline_label)" + print $"Feature: ($feature_label)" + + let timestamp = (date now | format date "%Y%m%d-%H%M%S") + let results_dir = $"($BENCH_RESULTS_DIR)/($timestamp)" + mkdir $results_dir + print $"BENCH_RESULTS_DIR=($results_dir)" + + let baseline_wt = $"($BENCH_WORKTREES_DIR)/baseline" + let feature_wt = $"($BENCH_WORKTREES_DIR)/feature" + git worktree prune + for wt in [$baseline_wt $feature_wt] { + if ($wt | path exists) { + print $"Removing stale worktree: ($wt)" + try { git worktree remove --force $wt } catch { rm -rf $wt } + } + } + + if $baseline != "local" { + git worktree add $baseline_wt $baseline_sha + } + if $feature != "local" { + git worktree add $feature_wt $feature_sha + } + + let tbc = (tracy-build-config $features $tracy) + let effective_features = $tbc.features + let effective_extra_rustflags = $tbc.extra_rustflags + let effective_no_cache = $no_cache or ($tracy != "off") + + if $baseline == "local" or $feature == "local" { + print "Building local tempo binaries..." + build-tempo --extra-rustflags $effective_extra_rustflags ["tempo"] $profile $effective_features + } + if $baseline != "local" { + if $effective_no_cache { + build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags $baseline_wt $baseline $profile $effective_features $baseline_sha + } else { + build-in-worktree $baseline_wt $baseline $profile $effective_features $baseline_sha + } + } + if $feature != "local" { + if $effective_no_cache { + build-in-worktree --no-cache --extra-rustflags $effective_extra_rustflags $feature_wt $feature $profile $effective_features $feature_sha + } else { + build-in-worktree $feature_wt $feature $profile $effective_features $feature_sha + } + } + + let local_bin = { |name: string| if $profile == "dev" { $"./target/debug/($name)" } else { $"./target/($profile)/($name)" } } + let baseline_tempo = if $baseline == "local" { do $local_bin "tempo" } else { worktree-bin $baseline_wt $profile "tempo" } + let feature_tempo = if $feature == "local" { do $local_bin "tempo" } else { worktree-bin $feature_wt $profile "tempo" } + + let abs_localnet = ($LOCALNET_DIR | path expand) + let bloat_file = $"($abs_localnet)/state_bloat.bin" + let datadir = if $bench_datadir != "" { + $bench_datadir + } else if (has-schelk) { + $"/reth-bench/tempo_($bloat)mb" + } else { + $"($abs_localnet)/reth" + } + let meta_dir = $"($datadir)/($BENCH_META_SUBDIR)" + let genesis_accounts = ([$accounts 3] | math max) + 1 + let gas_limit_args = if $gas_limit != "" { ["--gas-limit" $gas_limit] } else { [] } + let txgen_genesis_args = ["--mnemonic" $TXGEN_ACCOUNT_MNEMONIC] + + bench-mount + + if $dual_hardfork { + if not ($abs_localnet | path exists) { mkdir $abs_localnet } + + let baseline_genesis_args = (hardfork-to-genesis-args $baseline_hardfork) + let feature_genesis_args = (hardfork-to-genesis-args $feature_hardfork) + let baseline_genesis_path = $"($abs_localnet)/genesis-baseline.json" + let feature_genesis_path = $"($abs_localnet)/genesis-feature.json" + let baseline_datadir = $"($datadir)/baseline-db" + let feature_datadir = $"($datadir)/feature-db" + + let marker = (read-bench-marker $datadir) + let snapshot_ready = ( + not $force + and $marker != null + and ($marker.bloat_mib | into int) == $bloat + and ($marker.accounts | into int) == $genesis_accounts + and ($marker | get -o txgen_mnemonic | default "") == $TXGEN_ACCOUNT_MNEMONIC + and ($marker | get -o baseline_hardfork | default "") == ($baseline_hardfork | str upcase) + and ($marker | get -o feature_hardfork | default "") == ($feature_hardfork | str upcase) + and ($marker | get -o gas_limit | default "") == $gas_limit + and ($"($baseline_datadir)/db" | path exists) + and ($"($feature_datadir)/db" | path exists) + and ($"($meta_dir)/genesis-baseline.json" | path exists) + and ($"($meta_dir)/genesis-feature.json" | path exists) + ) + + if $snapshot_ready { + cp $"($meta_dir)/genesis-baseline.json" $baseline_genesis_path + cp $"($meta_dir)/genesis-feature.json" $feature_genesis_path + print $"Using cached dual-hardfork snapshot \(initialized ($marker.initialized_at)\)" + } else { + let baseline_genesis_dir = $"($abs_localnet)/genesis-baseline-dir" + if ($baseline_genesis_dir | path exists) { rm -rf $baseline_genesis_dir } + mkdir $baseline_genesis_dir + if $baseline == "local" { + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $baseline_genesis_dir -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$baseline_genesis_args ...$gas_limit_args + } else { + do { + cd $baseline_wt + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $baseline_genesis_dir -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$baseline_genesis_args ...$gas_limit_args + } + } + cp $"($baseline_genesis_dir)/genesis.json" $baseline_genesis_path + rm -rf $baseline_genesis_dir + + let feature_genesis_dir = $"($abs_localnet)/genesis-feature-dir" + if ($feature_genesis_dir | path exists) { rm -rf $feature_genesis_dir } + mkdir $feature_genesis_dir + if $feature == "local" { + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $feature_genesis_dir -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$feature_genesis_args ...$gas_limit_args + } else { + do { + cd $feature_wt + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $feature_genesis_dir -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$feature_genesis_args ...$gas_limit_args + } + } + cp $"($feature_genesis_dir)/genesis.json" $feature_genesis_path + rm -rf $feature_genesis_dir + + if $bloat > 0 and not ($bloat_file | path exists) { + let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten) + if $baseline == "local" { + cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args + } else { + do { + cd $baseline_wt + cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args + } + } + } + + for side in [ + { genesis: $baseline_genesis_path, dd: $baseline_datadir, tempo: $baseline_tempo } + { genesis: $feature_genesis_path, dd: $feature_datadir, tempo: $feature_tempo } + ] { + bench-clean-datadir $side.dd + mkdir $side.dd + bench-init-db $side.tempo $side.genesis $side.dd $bloat $bloat_file + } + + bench-save-and-promote $datadir $meta_dir { + bloat_mib: $bloat + accounts: $genesis_accounts + bench_datadir: $datadir + txgen_mnemonic: $TXGEN_ACCOUNT_MNEMONIC + baseline_hardfork: ($baseline_hardfork | str upcase) + feature_hardfork: ($feature_hardfork | str upcase) + gas_limit: $gas_limit + } [[$baseline_genesis_path "genesis-baseline.json"] [$feature_genesis_path "genesis-feature.json"]] $bloat $bloat_file + } + } else { + let genesis_path_std = $"($abs_localnet)/genesis.json" + let marker = (read-bench-marker $datadir) + let snapshot_ready = ( + not $force + and $marker != null + and ($marker.bloat_mib | into int) == $bloat + and ($marker.accounts | into int) == $genesis_accounts + and ($marker | get -o txgen_mnemonic | default "") == $TXGEN_ACCOUNT_MNEMONIC + and ($marker | get -o gas_limit | default "") == $gas_limit + and ($"($datadir)/db" | path exists) + and ($"($meta_dir)/genesis.json" | path exists) + ) + + if $snapshot_ready { + if not ($abs_localnet | path exists) { mkdir $abs_localnet } + cp $"($meta_dir)/genesis.json" $genesis_path_std + print $"Using cached virgin snapshot \(initialized ($marker.initialized_at)\)" + } else { + if not ($abs_localnet | path exists) { mkdir $abs_localnet } + if ($genesis_path_std | path exists) { rm -f $genesis_path_std } + if $baseline == "local" { + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $abs_localnet -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$gas_limit_args + } else { + do { + cd $baseline_wt + cargo run -p tempo-xtask --profile $profile -- generate-genesis --output $abs_localnet -a $genesis_accounts ...$txgen_genesis_args --no-dkg-in-genesis ...$gas_limit_args + } + } + + if $bloat > 0 and not ($bloat_file | path exists) { + let token_args = ($TIP20_TOKEN_IDS | each { |id| ["--token" $"($id)"] } | flatten) + if $baseline == "local" { + cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args + } else { + do { + cd $baseline_wt + cargo run -p tempo-xtask --profile $profile -- generate-state-bloat --size $bloat --out $bloat_file ...$token_args + } + } + } + + bench-clean-datadir $datadir + bench-init-db $baseline_tempo $genesis_path_std $datadir $bloat $bloat_file + bench-save-and-promote $datadir $meta_dir { + bloat_mib: $bloat + accounts: $genesis_accounts + bench_datadir: $datadir + txgen_mnemonic: $TXGEN_ACCOUNT_MNEMONIC + gas_limit: $gas_limit + } [[$genesis_path_std "genesis.json"]] $bloat $bloat_file + } + } + + let genesis_path = if $dual_hardfork { "" } else { $"($abs_localnet)/genesis.json" } + + if not $no_infra { + docker compose -f $"($BENCH_DIR)/docker-compose.yml" up -d + } + + if $tracy == "full" and (^uname | str trim) == "Linux" { + try { sudo sysctl -w kernel.perf_event_paranoid=-1 } catch { } + try { sudo mount -t tracefs tracefs /sys/kernel/tracing -o remount,mode=755 } catch { } + try { sudo chmod -R a+r /sys/kernel/tracing } catch { } + } + + let benchmark_id = $"bench-($timestamp)" + let reference_epoch = ((date now | into int) / 1_000_000_000 | into int) + let samply_args_list = if $samply_args == "" { [] } else { $samply_args | split row " " } + let runs = if $dual_hardfork { + [ + { label: "baseline-1", tempo: $baseline_tempo, git_ref: $baseline_sha, genesis: $"($abs_localnet)/genesis-baseline.json", datadir: $"($datadir)/baseline-db" } + { label: "feature-1", tempo: $feature_tempo, git_ref: $feature_sha, genesis: $"($abs_localnet)/genesis-feature.json", datadir: $"($datadir)/feature-db" } + { label: "feature-2", tempo: $feature_tempo, git_ref: $feature_sha, genesis: $"($abs_localnet)/genesis-feature.json", datadir: $"($datadir)/feature-db" } + { label: "baseline-2", tempo: $baseline_tempo, git_ref: $baseline_sha, genesis: $"($abs_localnet)/genesis-baseline.json", datadir: $"($datadir)/baseline-db" } + ] + } else { + [ + { label: "baseline-1", tempo: $baseline_tempo, git_ref: $baseline_sha, genesis: $genesis_path, datadir: $datadir } + { label: "feature-1", tempo: $feature_tempo, git_ref: $feature_sha, genesis: $genesis_path, datadir: $datadir } + { label: "feature-2", tempo: $feature_tempo, git_ref: $feature_sha, genesis: $genesis_path, datadir: $datadir } + { label: "baseline-2", tempo: $baseline_tempo, git_ref: $baseline_sha, genesis: $genesis_path, datadir: $datadir } + ] + } + + for run in $runs { + bench-recover $datadir + let run_type = if ($run.label | str starts-with "baseline") { "baseline" } else { "feature" } + let side_args = if $run_type == "baseline" { $baseline_args } else { $feature_args } + let side_env = if $run_type == "baseline" { $baseline_env } else { $feature_env } + let effective_node_args = ([$node_args $side_args] | where { |a| $a != "" } | str join " ") + + (run-txgen-bench-single + --tempo-bin $run.tempo + --txgen-tempo-bin $txgen.txgen_tempo_bin + --txgen-bench-bin $txgen.txgen_bench_bin + --genesis-path $run.genesis + --datadir $run.datadir + --run-label $run.label + --results-dir $results_dir + --tps $tps + --duration $duration + --accounts $accounts + --max-concurrent-requests $max_concurrent_requests + --preset $preset + --bench-args $bench_args + --loud=$loud + --node-args $effective_node_args + --bloat $bloat + --extra-env $side_env + --bench-env $bench_env + --git-ref $run.git_ref + --build-profile $profile + --benchmark-mode $mode + --benchmark-id $benchmark_id + --reference-epoch $reference_epoch + --samply=$samply + --samply-args $samply_args_list + --tracy $tracy + --tracy-filter $tracy_filter + --tracy-seconds $tracy_seconds + --tracy-offset $tracy_offset + --tracing-otlp $tracing_otlp) + } + + let summary_baseline = if $dual_hardfork { $"($baseline) \(($baseline_hardfork | str upcase)\)" } else { $baseline } + let summary_feature = if $dual_hardfork { $"($feature) \(($feature_hardfork | str upcase)\)" } else { $feature } + generate-summary $results_dir $summary_baseline $summary_feature $bloat $preset $tps $duration --benchmark-id $benchmark_id --reference-epoch $reference_epoch + + if $baseline != "local" { try { git worktree remove --force $baseline_wt } catch { } } + if $feature != "local" { try { git worktree remove --force $feature_wt } catch { } } + + if not $no_infra { + docker compose -f $"($BENCH_DIR)/docker-compose.yml" down + } + + if $samply { + for run in $runs { + let profile = $"($results_dir)/profile-($run.label).json.gz" + let url = (upload-samply-profile $profile) + if $url != null { + $url | save -f $"($results_dir)/profile-($run.label)-url.txt" + } + } + } + + if $tracy != "off" { + for run in $runs { + let profile = $"($results_dir)/tracy-profile-($run.label).tracy" + let viewer_url = (upload-tracy-profile $profile $run.label $run.git_ref) + if $viewer_url != null { + $viewer_url | save -f $"($results_dir)/tracy-($run.label)-url.txt" + } + } + } + + restore-system-tuning $tuning_state + print $"Comparison complete! Results: ($results_dir)/" } diff --git a/contrib/bench/txgen/erc20.abi.json b/contrib/bench/txgen/erc20.abi.json new file mode 100644 index 0000000000..fbd22efaf3 --- /dev/null +++ b/contrib/bench/txgen/erc20.abi.json @@ -0,0 +1,26 @@ +[ + { + "type": "function", + "name": "transfer", + "inputs": [ + { + "name": "to", + "type": "address", + "internalType": "address" + }, + { + "name": "amount", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bool", + "internalType": "bool" + } + ], + "stateMutability": "nonpayable" + } +] diff --git a/contrib/bench/txgen/tip20-template.yaml b/contrib/bench/txgen/tip20-template.yaml new file mode 100644 index 0000000000..910e5e7e98 --- /dev/null +++ b/contrib/bench/txgen/tip20-template.yaml @@ -0,0 +1,41 @@ +chain_id: 1337 + +gas: + max_fee_per_gas: 100000000000 + max_priority_fee_per_gas: 100000000000 + +accounts: + users: + mnemonic: "test test test test test test test test test test test junk" + range: + - 0 + - ${TXGEN_ACCOUNTS} + +artifacts: + ERC20: erc20.abi.json + +templates: + tip20_transfer: + type: tempo + from: + pool: users + select: random + gas_limit: 300000 + max_fee_per_gas: 100000000000 + max_priority_fee_per_gas: 100000000000 + fee_token: "0x20c0000000000000000000000000000000000000" + expiring_nonce: true + valid_for_secs: 25 + call: + to: "0x20c0000000000000000000000000000000000000" + abi: ERC20 + function: transfer + args: + - pool: + pool: users + select: random + - 1 + +mix: + - template: tip20_transfer + weight: 100 diff --git a/contrib/bench/upload-clickhouse-txgen.sh b/contrib/bench/upload-clickhouse-txgen.sh new file mode 100755 index 0000000000..f4a4a0baea --- /dev/null +++ b/contrib/bench/upload-clickhouse-txgen.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Upload native txgen bench results to ClickHouse. +# +# Reads report-*.json files from the results directory and inserts into: +# - tempo_bench_runs (one row per run) +# - tempo_bench_blocks (one row per block per run) +# +# Environment: +# CLICKHOUSE_URL - ClickHouse HTTP endpoint (https://host:8443) +# CLICKHOUSE_USER - ClickHouse user +# CLICKHOUSE_PASSWORD - ClickHouse password +# CLICKHOUSE_DB - database name (default: "default") +# +# Usage: upload-clickhouse-txgen.sh + +set -euo pipefail + +RESULTS_DIR="$1" +DB="${CLICKHOUSE_DB:-default}" + +if [ -z "${CLICKHOUSE_URL:-}" ] || [ -z "${CLICKHOUSE_USER:-}" ] || [ -z "${CLICKHOUSE_PASSWORD:-}" ]; then + echo "Skipping ClickHouse upload: CLICKHOUSE_URL, CLICKHOUSE_USER, or CLICKHOUSE_PASSWORD not set" + exit 0 +fi + +ch_query() { + local query="$1" + if ! curl -sf --user "$CLICKHOUSE_USER:$CLICKHOUSE_PASSWORD" \ + "$CLICKHOUSE_URL/?database=$DB" --data-binary "$query"; then + echo " Warning: ClickHouse query failed" >&2 + return 1 + fi +} + +echo "Uploading txgen bench results to ClickHouse..." + +for label in baseline-1 feature-1 feature-2 baseline-2; do + REPORT="$RESULTS_DIR/report-$label.json" + if [ ! -f "$REPORT" ]; then + echo " Warning: $REPORT not found, skipping" + continue + fi + + echo " Processing: $label" + + # Generate SQL statements via python (one statement per line, no internal newlines) + QUERIES=$(REPORT_PATH="$REPORT" BENCH_RUN_LABEL="$label" python3 << 'PYEOF' +import json, uuid, os + +report = json.load(open(os.environ["REPORT_PATH"])) +meta = report.get("metadata") or {} +run_stats = report.get("run_stats") or {} + +def as_int(value, default=0): + if value is None or value == "": + return default + return int(value) + +def as_float(value, default=0.0): + if value is None or value == "": + return default + return float(value) + +def normalized_blocks(report): + normalized = [] + for b in report.get("blocks") or []: + tx_count = as_int(b.get("tx_count")) + normalized.append({ + "number": as_int(b.get("number")), + "timestamp": as_int(b.get("timestamp", b.get("timestamp_ms"))), + "tx_count": tx_count, + "ok_count": tx_count, + "err_count": 0, + "gas_used": as_int(b.get("gas_used")), + "latency_ms": b.get("block_time_ms"), + }) + return normalized + +blocks = normalized_blocks(report) + +run_id = str(uuid.uuid4()) + +# Compute aggregates +total_tx = sum(b["tx_count"] for b in blocks) +total_ok = sum(b["ok_count"] for b in blocks) +total_err = sum(b["err_count"] for b in blocks) +total_gas = sum(b["gas_used"] for b in blocks) +total_blocks = len(blocks) + +timestamps = [b["timestamp"] for b in blocks] +if len(timestamps) > 1: + time_span_ms = max(timestamps[-1] - timestamps[0], 1) + avg_block_time_ms = time_span_ms / (len(timestamps) - 1) + avg_tps = total_tx / (time_span_ms / 1000.0) +else: + avg_block_time_ms = 0.0 + avg_tps = 0.0 + +def esc(s): + return s.replace("'", "\\'") + +sha = esc(meta.get("node_commit_sha") or "") +profile = esc(meta.get("build_profile") or "") +mode = esc(meta.get("mode") or "") + +run_label = esc(os.environ.get("BENCH_RUN_LABEL", "")) +pr_number = esc(os.environ.get("BENCH_PR", "")) +baseline_ref = esc(os.environ.get("BENCH_BASELINE_REF", "")) +feature_ref = esc(os.environ.get("BENCH_FEATURE_REF", "")) +triggered_by = esc(os.environ.get("BENCH_ACTOR", "")) +run_type = esc(os.environ.get("BENCH_RUN_TYPE", "manual")) +github_run_id_raw = os.environ.get("GITHUB_RUN_ID", "") +default_run_url = "" +if github_run_id_raw and os.environ.get("GITHUB_SERVER_URL") and os.environ.get("GITHUB_REPOSITORY"): + default_run_url = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{github_run_id_raw}" +github_run_id = esc(github_run_id_raw) +github_run_url = esc(os.environ.get("BENCH_JOB_URL") or default_run_url) + +print( + f"INSERT INTO tempo_bench_runs (run_id, created_at, chain_id, start_block, end_block, " + f"target_tps, run_duration_secs, accounts, total_connections, " + f"total_blocks, total_transactions, total_successful, total_failed, " + f"total_gas_used, avg_block_time_ms, avg_tps, " + f"tip20_weight, place_order_weight, swap_weight, erc20_weight, " + f"node_commit_sha, build_profile, benchmark_mode, " + f"run_label, pr_number, baseline_ref, feature_ref, " + f"triggered_by, run_type, github_run_id, github_run_url) VALUES " + f"('{run_id}', now64(3), {as_int(meta.get('chain_id'))}, " + f"{as_int(run_stats.get('start_block'))}, {as_int(run_stats.get('end_block'))}, " + f"{as_int(meta.get('target_tps'))}, {as_int(meta.get('run_duration_secs'))}, " + f"{as_int(meta.get('accounts'))}, {as_int(meta.get('total_connections'))}, " + f"{total_blocks}, {total_tx}, {total_ok}, {total_err}, " + f"{total_gas}, {avg_block_time_ms}, {avg_tps}, " + f"{as_float(meta.get('tip20_weight'))}, {as_float(meta.get('place_order_weight'))}, " + f"{as_float(meta.get('swap_weight'))}, {as_float(meta.get('erc20_weight'))}, " + f"'{sha}', '{profile}', '{mode}', " + f"'{run_label}', '{pr_number}', '{baseline_ref}', '{feature_ref}', " + f"'{triggered_by}', '{run_type}', '{github_run_id}', '{github_run_url}')" +) + +# Blocks insert (batch all blocks in one statement) +if blocks: + rows = [] + for b in blocks: + lat = b.get("latency_ms") + lat_val = lat if lat is not None else 0 + rows.append( + f"('{run_id}', {b['number']}, {b['timestamp']}, " + f"{b['tx_count']}, {b['ok_count']}, {b['err_count']}, " + f"{b['gas_used']}, 0, {lat_val})" + ) + values = ", ".join(rows) + print( + f"INSERT INTO tempo_bench_blocks (run_id, block_number, timestamp_ms, " + f"tx_count, ok_count, err_count, gas_used, gas_limit, latency_ms) VALUES {values}" + ) +PYEOF + ) + + echo "$QUERIES" | while IFS= read -r query; do + [ -z "$query" ] && continue + ch_query "$query" + done + + echo " Uploaded: $label" +done + +echo "ClickHouse upload complete." diff --git a/tempo.nu b/tempo.nu index b4c7259923..1aaf0d108b 100755 --- a/tempo.nu +++ b/tempo.nu @@ -793,7 +793,28 @@ def generate-summary [results_dir: string, baseline_ref: string, feature_ref: st continue } let report = (open $report_path) - let blocks = ($report | get blocks) + let blocks = ($report | get blocks | each { |b| + let tx_count = ($b | get tx_count) + let timestamp = if (($b | get -o timestamp | default null) != null) { + $b | get timestamp + } else { + $b | get timestamp_ms + } + let latency_ms = if (($b | get -o latency_ms | default null) != null) { + $b | get latency_ms + } else { + $b | get -o block_time_ms | default null + } + { + number: ($b | get number) + timestamp: $timestamp + tx_count: $tx_count + ok_count: ($b | get -o ok_count | default $tx_count) + err_count: ($b | get -o err_count | default 0) + gas_used: ($b | get gas_used) + latency_ms: $latency_ms + } + }) if ($blocks | length) == 0 { print $"Warning: ($label) report has no blocks, skipping" continue