From 69a94d6c94ae095be8625b20da3fed90a7464470 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 11:17:08 +0530 Subject: [PATCH 01/18] fix: update benchmark workflow to post-results on PRs --- .github/workflows/bench.yml | 23 +++++++++++++++++++++-- test/bench.sh | 12 ++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 5f218f9..9d070ef 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,6 +1,6 @@ name: Benchmark on: - push: + pull_request: workflow_dispatch: jobs: @@ -34,4 +34,23 @@ jobs: - name: Run Benchmark run: | - ./test/bench.sh + output=$(./test/bench.sh 2>&1) + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Benchmark script failed with exit code $exit_code" + echo "$output" + exit $exit_code + fi + echo "$output" | sed -n '/Begin Benchmark/,/End Benchmark/p' > bench_results.txt + + - name: Post Benchmark Results + if: github.event.pull_request.head.repo.full_name == github.repository + run: | + gh pr comment ${{ github.event.pull_request.number }} --body "## Benchmark Results + + \`\`\` + $(cat bench_results.txt) + \`\`\`" --create-if-none --edit-last + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} diff --git a/test/bench.sh b/test/bench.sh index f5d2588..9afb504 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -67,15 +67,23 @@ for ((i = 0; i < files; i++)); do cp /tmp/bench/test.md "/tmp/bench/hugo/content/test_$i.md" done +# begin benchmark +echo "" +echo "Begin Benchmark" +echo "" + # run hyperfine echo "" echo "running benchmark: $files md files and $warm warmup runs" echo "" hyperfine -p 'sync' -w $warm \ "cd /tmp/bench/hugo && hugo" \ - "cd /tmp/bench/anna && ./anna -r \"site/\"" \ - "cd /tmp/bench/anna && ./anna_greentea -r \"site/\"" \ + "cd /tmp/bench/anna && ./anna" \ + "cd /tmp/bench/anna && ./anna_greentea" \ "cd /tmp/bench/saaru && ./saaru --base-path ./docs" \ "cd /tmp/bench/sapling/benchmark && ./../sapling run" echo "" +echo "" +echo "End Benchmark" +echo "" \ No newline at end of file From 9a56aedbac6d79a1c5273e41db1f59e225895f50 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 11:22:50 +0530 Subject: [PATCH 02/18] ci: tee output --- .github/workflows/bench.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9d070ef..8f876b0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -34,14 +34,13 @@ jobs: - name: Run Benchmark run: | - output=$(./test/bench.sh 2>&1) - exit_code=$? + ./test/bench.sh 2>&1 | tee full_output.txt + exit_code=${PIPESTATUS[0]} if [ $exit_code -ne 0 ]; then echo "Benchmark script failed with exit code $exit_code" - echo "$output" exit $exit_code fi - echo "$output" | sed -n '/Begin Benchmark/,/End Benchmark/p' > bench_results.txt + sed -n '/Begin Benchmark/,/End Benchmark/p' full_output.txt > bench_results.txt - name: Post Benchmark Results if: github.event.pull_request.head.repo.full_name == github.repository From 12a344a10ef9f0c9cb12599fd415a0c5f544eef4 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 11:33:51 +0530 Subject: [PATCH 03/18] ci: upload artifacts --- .github/workflows/bench.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8f876b0..74f6d1f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -43,7 +43,7 @@ jobs: sed -n '/Begin Benchmark/,/End Benchmark/p' full_output.txt > bench_results.txt - name: Post Benchmark Results - if: github.event.pull_request.head.repo.full_name == github.repository + if: github.event_name == 'pull_request' run: | gh pr comment ${{ github.event.pull_request.number }} --body "## Benchmark Results @@ -53,3 +53,9 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} + + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: bench_results.txt From 36f9fa1520d4bfd0905439e33e6bd8b8ffc0e9cf Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 11:40:48 +0530 Subject: [PATCH 04/18] ci: fix perms --- .github/workflows/bench.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 74f6d1f..f43a073 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -6,6 +6,10 @@ on: jobs: benchmark: runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + contents: read steps: - name: Checkout Repository @@ -32,6 +36,16 @@ jobs: profile: minimal toolchain: stable + - name: Cache Rust builds + uses: actions/cache@v4 + with: + path: | + /tmp/bench/saaru/target + /tmp/bench/sapling/target + key: ${{ runner.os }}-rust-target-${{ hashFiles('/tmp/bench/saaru/Cargo.lock', '/tmp/bench/sapling/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rust-target- + - name: Run Benchmark run: | ./test/bench.sh 2>&1 | tee full_output.txt @@ -41,19 +55,21 @@ jobs: exit $exit_code fi sed -n '/Begin Benchmark/,/End Benchmark/p' full_output.txt > bench_results.txt + echo "## Benchmark Results" > comment_body.txt + echo "" >> comment_body.txt + echo "\`\`\`" >> comment_body.txt + cat bench_results.txt >> comment_body.txt + echo "" >> comment_body.txt + echo "\`\`\`" >> comment_body.txt - name: Post Benchmark Results if: github.event_name == 'pull_request' run: | - gh pr comment ${{ github.event.pull_request.number }} --body "## Benchmark Results - - \`\`\` - $(cat bench_results.txt) - \`\`\`" --create-if-none --edit-last + gh pr comment ${{ github.event.pull_request.number }} --body-file comment_body.txt --create-if-none --edit-last env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - + - name: Upload Benchmark Results uses: actions/upload-artifact@v4 with: From 81029826d1d0942d4e34e88e4a5d4618ca0087f4 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:06:23 +0530 Subject: [PATCH 05/18] ci: cleanup --- .github/workflows/bench.yml | 35 +++++++++++++++++++---------------- test/bench.sh | 9 ++------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f43a073..4a66a61 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -12,41 +12,43 @@ jobs: contents: read steps: + # Checkout - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Install System Dependencies + # System Dependencies + - name: Dependencies: Install Hyperfine run: | - sudo apt-get update - sudo apt-get install -y hyperfine + sudo apt-get update; sudo apt-get install -y hyperfine - - name: Setup Hugo + # Tool Setup + - name: Tools: Setup Hugo uses: peaceiris/actions-hugo@v3 with: hugo-version: 'latest' - - name: Install Go + - name: Tools: Install Go uses: actions/setup-go@v5 with: go-version-file: "go.mod" - - name: Install Rust + - name: Tools: Install Rust uses: actions-rs/toolchain@v1.0.6 with: profile: minimal toolchain: stable - - name: Cache Rust builds - uses: actions/cache@v4 + # Caching + - name: Cache: Rust Builds + uses: actions/cache@v5 with: path: | /tmp/bench/saaru/target /tmp/bench/sapling/target - key: ${{ runner.os }}-rust-target-${{ hashFiles('/tmp/bench/saaru/Cargo.lock', '/tmp/bench/sapling/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-rust-target- + key: ${{ runner.os }}-rust-target - - name: Run Benchmark + # Execution + - name: Execution: Run Benchmark run: | ./test/bench.sh 2>&1 | tee full_output.txt exit_code=${PIPESTATUS[0]} @@ -54,7 +56,7 @@ jobs: echo "Benchmark script failed with exit code $exit_code" exit $exit_code fi - sed -n '/Begin Benchmark/,/End Benchmark/p' full_output.txt > bench_results.txt + sed -n '/Begin Benchmark/,/End Benchmark/{ /Begin Benchmark/d; /End Benchmark/d; p; }' full_output.txt > bench_results.txt echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt @@ -62,7 +64,8 @@ jobs: echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt - - name: Post Benchmark Results + # Results + - name: Results: Post Benchmark if: github.event_name == 'pull_request' run: | gh pr comment ${{ github.event.pull_request.number }} --body-file comment_body.txt --create-if-none --edit-last @@ -70,7 +73,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - - name: Upload Benchmark Results + - name: Results: Upload Benchmark uses: actions/upload-artifact@v4 with: name: benchmark-results diff --git a/test/bench.sh b/test/bench.sh index 9afb504..0eff717 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -66,14 +66,12 @@ for ((i = 0; i < files; i++)); do cp /tmp/bench/test.md "/tmp/bench/sapling/benchmark/content/blogs/test_$i.md" cp /tmp/bench/test.md "/tmp/bench/hugo/content/test_$i.md" done +echo "" # begin benchmark -echo "" echo "Begin Benchmark" -echo "" # run hyperfine -echo "" echo "running benchmark: $files md files and $warm warmup runs" echo "" hyperfine -p 'sync' -w $warm \ @@ -83,7 +81,4 @@ hyperfine -p 'sync' -w $warm \ "cd /tmp/bench/saaru && ./saaru --base-path ./docs" \ "cd /tmp/bench/sapling/benchmark && ./../sapling run" echo "" - -echo "" -echo "End Benchmark" -echo "" \ No newline at end of file +echo "End Benchmark" \ No newline at end of file From 2763c409ff4e13f554ae9be0db0d7862044d83f9 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:22:09 +0530 Subject: [PATCH 06/18] ci: caches? --- .github/workflows/bench.yml | 22 ++++++------- .gitignore | 1 + test/bench.sh | 66 +++++++++++++++++++++++-------------- 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4a66a61..c6c3e2a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -17,38 +17,36 @@ jobs: uses: actions/checkout@v6 # System Dependencies - - name: Dependencies: Install Hyperfine + - name: Dependencies Install Hyperfine run: | sudo apt-get update; sudo apt-get install -y hyperfine # Tool Setup - - name: Tools: Setup Hugo + - name: Tools Setup Hugo uses: peaceiris/actions-hugo@v3 with: hugo-version: 'latest' - - name: Tools: Install Go + - name: Tools Install Go uses: actions/setup-go@v5 with: go-version-file: "go.mod" - - name: Tools: Install Rust + - name: Tools Install Rust uses: actions-rs/toolchain@v1.0.6 with: profile: minimal toolchain: stable # Caching - - name: Cache: Rust Builds + - name: Cache Rust Builds uses: actions/cache@v5 with: - path: | - /tmp/bench/saaru/target - /tmp/bench/sapling/target - key: ${{ runner.os }}-rust-target + path: tmp/bench + key: ${{ runner.os }}-bench # Execution - - name: Execution: Run Benchmark + - name: Execution Run Benchmark run: | ./test/bench.sh 2>&1 | tee full_output.txt exit_code=${PIPESTATUS[0]} @@ -65,7 +63,7 @@ jobs: echo "\`\`\`" >> comment_body.txt # Results - - name: Results: Post Benchmark + - name: Results Post Benchmark if: github.event_name == 'pull_request' run: | gh pr comment ${{ github.event.pull_request.number }} --body-file comment_body.txt --create-if-none --edit-last @@ -73,7 +71,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - - name: Results: Upload Benchmark + - name: Results Upload Benchmark uses: actions/upload-artifact@v4 with: name: benchmark-results diff --git a/.gitignore b/.gitignore index 42f06b7..5f91039 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ dist/ **/rendered/ test/**/static/ test/**/got_sitemap.xml +tmp/ \ No newline at end of file diff --git a/test/bench.sh b/test/bench.sh index 0eff717..fcc5e81 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -3,10 +3,12 @@ files=1000 warm=10 +BASE_DIR=$(pwd) + # cleanup cleanup() { echo "cleaning up" - rm -rf /tmp/bench + rm -rf $BASE_DIR/tmp/bench } trap cleanup EXIT @@ -25,26 +27,42 @@ fi echo "" echo "clone SSGs" echo "" -git clone --depth=1 https://github.com/anna-ssg/anna /tmp/bench/anna -git clone --depth=1 https://github.com/anirudhRowjee/saaru /tmp/bench/saaru -git clone --depth=1 https://github.com/NavinShrinivas/sapling /tmp/bench/sapling + +clone_or_pull() { + local repo=$1 + local dir=$2 + if [ ! -d "$dir" ]; then + git clone --depth=1 "$repo" "$dir" + else + cd "$dir" && git pull --depth=1 --ff-only + if [ $? -ne 0 ]; then + cd "$(dirname "$dir")" + rm -rf "$dir" + git clone --depth=1 "$repo" "$dir" + fi + fi +} + +clone_or_pull https://github.com/anna-ssg/anna $BASE_DIR/tmp/bench/anna +clone_or_pull https://github.com/anirudhRowjee/saaru $BASE_DIR/tmp/bench/saaru +clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapling # copy benchmark file -cp /tmp/bench/anna/site/content/posts/bench.md /tmp/bench/test.md +cp $BASE_DIR/tmp/bench/anna/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" echo "build SSGs" echo "" -cd /tmp/bench/anna && go build && cd /tmp/bench -cd /tmp/bench/anna && GOEXPERIMENT=greenteagc go build -o anna_greentea && cd /tmp/bench +cd $BASE_DIR/tmp/bench/anna && go build && cd ../.. +cd $BASE_DIR/tmp/bench/anna && GOEXPERIMENT=greenteagc go build -o anna_greentea && cd ../.. # build rust based SSGs (edit this block if they are already installed) -cd /tmp/bench/sapling && cargo build --release && mv target/release/sapling . -cd /tmp/bench/saaru && cargo build --release && mv target/release/saaru . +cd $BASE_DIR/tmp/bench/sapling && cargo build --release && mv target/release/sapling . +cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru . ## setup hugo -hugo new site /tmp/bench/hugo; cd /tmp/bench/hugo -hugo new theme mytheme; echo "theme = 'mytheme'" >> hugo.toml; cd /tmp/bench +hugo new site $BASE_DIR/tmp/bench/hugo; cd $BASE_DIR/tmp/bench/hugo +hugo new theme mytheme; echo "theme = 'mytheme'" >> hugo.toml; cd ../.. ## setup 11ty @@ -52,19 +70,19 @@ hugo new theme mytheme; echo "theme = 'mytheme'" >> hugo.toml; cd /tmp/bench echo "" echo "Cleaning content directories" echo "" -rm -rf /tmp/bench/anna/site/content/posts/* -rm -rf /tmp/bench/saaru/docs/src/* -rm -rf /tmp/bench/sapling/benchmark/content/blog/* -rm -rf /tmp/bench/hugo/content/* +rm -rf $BASE_DIR/tmp/bench/anna/site/content/posts/* +rm -rf $BASE_DIR/tmp/bench/saaru/docs/src/* +rm -rf $BASE_DIR/tmp/bench/sapling/benchmark/content/blog/* +rm -rf $BASE_DIR/tmp/bench/hugo/content/* # create multiple copies of the test file echo "" echo "Spawning $files different markdown files..." for ((i = 0; i < files; i++)); do - cp /tmp/bench/test.md "/tmp/bench/anna/site/content/posts/test_$i.md" - cp /tmp/bench/test.md "/tmp/bench/saaru/docs/src/test_$i.md" - cp /tmp/bench/test.md "/tmp/bench/sapling/benchmark/content/blogs/test_$i.md" - cp /tmp/bench/test.md "/tmp/bench/hugo/content/test_$i.md" + cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/anna/site/content/posts/test_$i.md" + cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/saaru/docs/src/test_$i.md" + cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/sapling/benchmark/content/blogs/test_$i.md" + cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/hugo/content/test_$i.md" done echo "" @@ -75,10 +93,10 @@ echo "Begin Benchmark" echo "running benchmark: $files md files and $warm warmup runs" echo "" hyperfine -p 'sync' -w $warm \ - "cd /tmp/bench/hugo && hugo" \ - "cd /tmp/bench/anna && ./anna" \ - "cd /tmp/bench/anna && ./anna_greentea" \ - "cd /tmp/bench/saaru && ./saaru --base-path ./docs" \ - "cd /tmp/bench/sapling/benchmark && ./../sapling run" + "cd $BASE_DIR/tmp/bench/hugo && hugo" \ + "cd $BASE_DIR/tmp/bench/anna && ./anna" \ + "cd $BASE_DIR/tmp/bench/anna && ./anna_greentea" \ + "cd $BASE_DIR/tmp/bench/saaru && ./saaru --base-path ./docs" \ + "cd $BASE_DIR/tmp/bench/sapling/benchmark && ./../sapling run" echo "" echo "End Benchmark" \ No newline at end of file From 0446f3d1e4ca52881d0c37da5645da43b9aea403 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:33:47 +0530 Subject: [PATCH 07/18] ci: reorganize benchmark --- test/bench.sh | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/test/bench.sh b/test/bench.sh index fcc5e81..e54035e 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -1,24 +1,19 @@ #!/bin/bash -# parameters files=1000 warm=10 - BASE_DIR=$(pwd) -# cleanup cleanup() { echo "cleaning up" rm -rf $BASE_DIR/tmp/bench } trap cleanup EXIT -# check if hyperfine is installed +# deps if ! command -v hyperfine &>/dev/null; then echo "hyperfine is not installed. Please install hyperfine to continue." exit 1 fi - -# check if hugo is installed if ! command -v hugo &>/dev/null; then echo "hugo is not installed. Please install hugo to continue." fi @@ -27,7 +22,6 @@ fi echo "" echo "clone SSGs" echo "" - clone_or_pull() { local repo=$1 local dir=$2 @@ -47,7 +41,6 @@ clone_or_pull https://github.com/anna-ssg/anna $BASE_DIR/tmp/bench/anna clone_or_pull https://github.com/anirudhRowjee/saaru $BASE_DIR/tmp/bench/saaru clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapling -# copy benchmark file cp $BASE_DIR/tmp/bench/anna/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" @@ -55,8 +48,6 @@ echo "build SSGs" echo "" cd $BASE_DIR/tmp/bench/anna && go build && cd ../.. cd $BASE_DIR/tmp/bench/anna && GOEXPERIMENT=greenteagc go build -o anna_greentea && cd ../.. - -# build rust based SSGs (edit this block if they are already installed) cd $BASE_DIR/tmp/bench/sapling && cargo build --release && mv target/release/sapling . cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru . @@ -64,18 +55,12 @@ cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru hugo new site $BASE_DIR/tmp/bench/hugo; cd $BASE_DIR/tmp/bench/hugo hugo new theme mytheme; echo "theme = 'mytheme'" >> hugo.toml; cd ../.. -## setup 11ty - # clean content/* dirs -echo "" -echo "Cleaning content directories" -echo "" rm -rf $BASE_DIR/tmp/bench/anna/site/content/posts/* rm -rf $BASE_DIR/tmp/bench/saaru/docs/src/* rm -rf $BASE_DIR/tmp/bench/sapling/benchmark/content/blog/* rm -rf $BASE_DIR/tmp/bench/hugo/content/* -# create multiple copies of the test file echo "" echo "Spawning $files different markdown files..." for ((i = 0; i < files; i++)); do @@ -86,17 +71,12 @@ for ((i = 0; i < files; i++)); do done echo "" -# begin benchmark echo "Begin Benchmark" - -# run hyperfine -echo "running benchmark: $files md files and $warm warmup runs" -echo "" +echo "running benchmark: $files md files and $warm warmup runs \n" hyperfine -p 'sync' -w $warm \ - "cd $BASE_DIR/tmp/bench/hugo && hugo" \ - "cd $BASE_DIR/tmp/bench/anna && ./anna" \ - "cd $BASE_DIR/tmp/bench/anna && ./anna_greentea" \ "cd $BASE_DIR/tmp/bench/saaru && ./saaru --base-path ./docs" \ - "cd $BASE_DIR/tmp/bench/sapling/benchmark && ./../sapling run" -echo "" -echo "End Benchmark" \ No newline at end of file + "cd $BASE_DIR/tmp/bench/sapling/benchmark && ./../sapling run" \ + "cd $BASE_DIR/tmp/bench/anna && ./anna_greentea" \ + "cd $BASE_DIR/tmp/bench/anna && ./anna" \ + "cd $BASE_DIR/tmp/bench/hugo && hugo" +echo "End Benchmark" From 8a266d3d4bdd71e7a615766e68d7159368a31d0d Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:37:41 +0530 Subject: [PATCH 08/18] trap --- test/bench.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/bench.sh b/test/bench.sh index e54035e..aeff06a 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -3,12 +3,6 @@ files=1000 warm=10 BASE_DIR=$(pwd) -cleanup() { - echo "cleaning up" - rm -rf $BASE_DIR/tmp/bench -} -trap cleanup EXIT - # deps if ! command -v hyperfine &>/dev/null; then echo "hyperfine is not installed. Please install hyperfine to continue." From 818c560cc437a3e94839beafe1fd8a65ab03d873 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:43:20 +0530 Subject: [PATCH 09/18] bench --- .github/workflows/bench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c6c3e2a..f9dac86 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -17,29 +17,29 @@ jobs: uses: actions/checkout@v6 # System Dependencies - - name: Dependencies Install Hyperfine + - name: Dependencies/Hyperfine run: | sudo apt-get update; sudo apt-get install -y hyperfine # Tool Setup - - name: Tools Setup Hugo + - name: Tools/Hugo uses: peaceiris/actions-hugo@v3 with: hugo-version: 'latest' - - name: Tools Install Go + - name: Tools/Go uses: actions/setup-go@v5 with: go-version-file: "go.mod" - - name: Tools Install Rust + - name: Tools/Rust uses: actions-rs/toolchain@v1.0.6 with: profile: minimal toolchain: stable # Caching - - name: Cache Rust Builds + - name: Cache Repos uses: actions/cache@v5 with: path: tmp/bench From 3727e23d0da724b113ad8aa5eba2b9b279ef4e74 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:47:02 +0530 Subject: [PATCH 10/18] benchh --- test/bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bench.sh b/test/bench.sh index aeff06a..c85b544 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -35,7 +35,7 @@ clone_or_pull https://github.com/anna-ssg/anna $BASE_DIR/tmp/bench/anna clone_or_pull https://github.com/anirudhRowjee/saaru $BASE_DIR/tmp/bench/saaru clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapling -cp $BASE_DIR/tmp/bench/anna/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md +cp $BASE_DIR/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" echo "build SSGs" From 5d99be141451b77b6f916bccb5d928367f5e48fb Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:50:20 +0530 Subject: [PATCH 11/18] ok --- test/bench.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/bench.sh b/test/bench.sh index c85b544..6962ef4 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -35,6 +35,14 @@ clone_or_pull https://github.com/anna-ssg/anna $BASE_DIR/tmp/bench/anna clone_or_pull https://github.com/anirudhRowjee/saaru $BASE_DIR/tmp/bench/saaru clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapling +# show commit hashes +echo "" +echo "Commit hashes:" +echo "anna: $(cd $BASE_DIR/tmp/bench/anna && git rev-parse HEAD)" +echo "saaru: $(cd $BASE_DIR/tmp/bench/saaru && git rev-parse HEAD)" +echo "sapling: $(cd $BASE_DIR/tmp/bench/sapling && git rev-parse HEAD)" +echo "" + cp $BASE_DIR/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" @@ -46,8 +54,12 @@ cd $BASE_DIR/tmp/bench/sapling && cargo build --release && mv target/release/sap cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru . ## setup hugo -hugo new site $BASE_DIR/tmp/bench/hugo; cd $BASE_DIR/tmp/bench/hugo -hugo new theme mytheme; echo "theme = 'mytheme'" >> hugo.toml; cd ../.. +rm -rf $BASE_DIR/tmp/bench/hugo +hugo new site $BASE_DIR/tmp/bench/hugo +cd $BASE_DIR/tmp/bench/hugo +hugo new theme mytheme +echo "theme = 'mytheme'" >> hugo.toml +cd ../.. # clean content/* dirs rm -rf $BASE_DIR/tmp/bench/anna/site/content/posts/* From 3faab732837c4748184eb6ebd094022165cddc72 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 12:56:18 +0530 Subject: [PATCH 12/18] fine --- .github/workflows/bench.yml | 17 ++++++++++++++--- test/bench.sh | 18 +++++++++--------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f9dac86..33d9ab1 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -39,11 +39,17 @@ jobs: toolchain: stable # Caching - - name: Cache Repos + - name: Cache Build Artifacts uses: actions/cache@v5 with: - path: tmp/bench - key: ${{ runner.os }}-bench + path: | + tmp/bench/anna/anna + tmp/bench/anna/anna_greentea + tmp/bench/saaru/target + tmp/bench/saaru/saaru + tmp/bench/sapling/target + tmp/bench/sapling/sapling + key: ${{ runner.os }}-bench-artifacts # Execution - name: Execution Run Benchmark @@ -57,6 +63,11 @@ jobs: sed -n '/Begin Benchmark/,/End Benchmark/{ /Begin Benchmark/d; /End Benchmark/d; p; }' full_output.txt > bench_results.txt echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt + echo "### Versions Tested" >> comment_body.txt + echo "- **Anna**: [\`$(cd tmp/bench/anna && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anna-ssg/anna/commit/$(cd tmp/bench/anna && git rev-parse HEAD))" >> comment_body.txt + echo "- **Saaru**: [\`$(cd tmp/bench/saaru && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anirudhRowjee/saaru/commit/$(cd tmp/bench/saaru && git rev-parse HEAD))" >> comment_body.txt + echo "- **Sapling**: [\`$(cd tmp/bench/sapling && git rev-parse HEAD | cut -c1-8)\`](https://github.com/NavinShrinivas/sapling/commit/$(cd tmp/bench/sapling && git rev-parse HEAD))" >> comment_body.txt + echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt cat bench_results.txt >> comment_body.txt echo "" >> comment_body.txt diff --git a/test/bench.sh b/test/bench.sh index 6962ef4..1f7a29c 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -2,6 +2,7 @@ files=1000 warm=10 BASE_DIR=$(pwd) +REPO_ROOT=$(cd $BASE_DIR/.. && pwd) # deps if ! command -v hyperfine &>/dev/null; then @@ -19,15 +20,14 @@ echo "" clone_or_pull() { local repo=$1 local dir=$2 - if [ ! -d "$dir" ]; then - git clone --depth=1 "$repo" "$dir" + if [ "$repo" = "https://github.com/anna-ssg/anna" ]; then + # For Anna, use current source code instead of fresh clone + rm -rf "$dir" + cp -r "$REPO_ROOT" "$dir" else - cd "$dir" && git pull --depth=1 --ff-only - if [ $? -ne 0 ]; then - cd "$(dirname "$dir")" - rm -rf "$dir" - git clone --depth=1 "$repo" "$dir" - fi + # For others, always fresh clone + rm -rf "$dir" + git clone --depth=1 "$repo" "$dir" fi } @@ -43,7 +43,7 @@ echo "saaru: $(cd $BASE_DIR/tmp/bench/saaru && git rev-parse HEAD)" echo "sapling: $(cd $BASE_DIR/tmp/bench/sapling && git rev-parse HEAD)" echo "" -cp $BASE_DIR/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md +cp $REPO_ROOT/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" echo "build SSGs" From f1cb2a832c25b4c8da0f61ab071f5a9d32e709ad Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 13:02:53 +0530 Subject: [PATCH 13/18] refactor: improve benchmark script and workflow for commit hash retrieval --- .github/workflows/bench.yml | 2 +- test/bench.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 33d9ab1..a2208b2 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -64,7 +64,7 @@ jobs: echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt echo "### Versions Tested" >> comment_body.txt - echo "- **Anna**: [\`$(cd tmp/bench/anna && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anna-ssg/anna/commit/$(cd tmp/bench/anna && git rev-parse HEAD))" >> comment_body.txt + echo "- **Anna**: [\`${GITHUB_SHA:0:8}\`](https://github.com/anna-ssg/anna/commit/$GITHUB_SHA)" >> comment_body.txt echo "- **Saaru**: [\`$(cd tmp/bench/saaru && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anirudhRowjee/saaru/commit/$(cd tmp/bench/saaru && git rev-parse HEAD))" >> comment_body.txt echo "- **Sapling**: [\`$(cd tmp/bench/sapling && git rev-parse HEAD | cut -c1-8)\`](https://github.com/NavinShrinivas/sapling/commit/$(cd tmp/bench/sapling && git rev-parse HEAD))" >> comment_body.txt echo "" >> comment_body.txt diff --git a/test/bench.sh b/test/bench.sh index 1f7a29c..b457e53 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -2,7 +2,16 @@ files=1000 warm=10 BASE_DIR=$(pwd) -REPO_ROOT=$(cd $BASE_DIR/.. && pwd) + +# Determine repo root +if [ -d "$BASE_DIR/site" ]; then + REPO_ROOT=$BASE_DIR +else + REPO_ROOT=$(cd $BASE_DIR/.. && pwd) +fi + +# Create bench directory +mkdir -p $BASE_DIR/tmp/bench # deps if ! command -v hyperfine &>/dev/null; then @@ -38,7 +47,7 @@ clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapl # show commit hashes echo "" echo "Commit hashes:" -echo "anna: $(cd $BASE_DIR/tmp/bench/anna && git rev-parse HEAD)" +echo "anna: $(cd $REPO_ROOT && git rev-parse HEAD)" echo "saaru: $(cd $BASE_DIR/tmp/bench/saaru && git rev-parse HEAD)" echo "sapling: $(cd $BASE_DIR/tmp/bench/sapling && git rev-parse HEAD)" echo "" From bcbed6248d92ca3b7fbe5809fade9fd92d2caac1 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 13:10:45 +0530 Subject: [PATCH 14/18] sha --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a2208b2..ee4bf91 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -64,7 +64,7 @@ jobs: echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt echo "### Versions Tested" >> comment_body.txt - echo "- **Anna**: [\`${GITHUB_SHA:0:8}\`](https://github.com/anna-ssg/anna/commit/$GITHUB_SHA)" >> comment_body.txt + echo "- **Anna**: [\`$(git rev-parse HEAD | cut -c1-8)\`](https://github.com/anna-ssg/anna/commit/$(git rev-parse HEAD))" >> comment_body.txt echo "- **Saaru**: [\`$(cd tmp/bench/saaru && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anirudhRowjee/saaru/commit/$(cd tmp/bench/saaru && git rev-parse HEAD))" >> comment_body.txt echo "- **Sapling**: [\`$(cd tmp/bench/sapling && git rev-parse HEAD | cut -c1-8)\`](https://github.com/NavinShrinivas/sapling/commit/$(cd tmp/bench/sapling && git rev-parse HEAD))" >> comment_body.txt echo "" >> comment_body.txt From 083d918f6fbbb06b65652195fe2e79fa9b779cd6 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 13:15:08 +0530 Subject: [PATCH 15/18] refactor: update clone and build process for Anna SSG --- test/bench.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/bench.sh b/test/bench.sh index b457e53..f51ad8d 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -30,9 +30,12 @@ clone_or_pull() { local repo=$1 local dir=$2 if [ "$repo" = "https://github.com/anna-ssg/anna" ]; then - # For Anna, use current source code instead of fresh clone + # For Anna, use current source code rm -rf "$dir" - cp -r "$REPO_ROOT" "$dir" + mkdir -p "$dir" + cp -r "$REPO_ROOT/site" "$dir/" + cd "$REPO_ROOT" && go build -o "$dir/anna" + cd "$REPO_ROOT" && GOEXPERIMENT=greenteagc go build -o "$dir/anna_greentea" else # For others, always fresh clone rm -rf "$dir" @@ -57,8 +60,6 @@ cp $REPO_ROOT/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md echo "" echo "build SSGs" echo "" -cd $BASE_DIR/tmp/bench/anna && go build && cd ../.. -cd $BASE_DIR/tmp/bench/anna && GOEXPERIMENT=greenteagc go build -o anna_greentea && cd ../.. cd $BASE_DIR/tmp/bench/sapling && cargo build --release && mv target/release/sapling . cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru . From 9cf875f09faa65a0ec98c1cc1dbb5c1f6e3d9a91 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 14:12:55 +0530 Subject: [PATCH 16/18] try --- .github/workflows/bench.yml | 28 +-------- test/bench.sh | 113 ++++++++++++++---------------------- 2 files changed, 46 insertions(+), 95 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ee4bf91..f5afce1 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -38,40 +38,18 @@ jobs: profile: minimal toolchain: stable - # Caching - - name: Cache Build Artifacts - uses: actions/cache@v5 - with: - path: | - tmp/bench/anna/anna - tmp/bench/anna/anna_greentea - tmp/bench/saaru/target - tmp/bench/saaru/saaru - tmp/bench/sapling/target - tmp/bench/sapling/sapling - key: ${{ runner.os }}-bench-artifacts - # Execution - name: Execution Run Benchmark run: | - ./test/bench.sh 2>&1 | tee full_output.txt - exit_code=${PIPESTATUS[0]} - if [ $exit_code -ne 0 ]; then - echo "Benchmark script failed with exit code $exit_code" - exit $exit_code - fi - sed -n '/Begin Benchmark/,/End Benchmark/{ /Begin Benchmark/d; /End Benchmark/d; p; }' full_output.txt > bench_results.txt + ./test/bench.sh echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt - echo "### Versions Tested" >> comment_body.txt - echo "- **Anna**: [\`$(git rev-parse HEAD | cut -c1-8)\`](https://github.com/anna-ssg/anna/commit/$(git rev-parse HEAD))" >> comment_body.txt - echo "- **Saaru**: [\`$(cd tmp/bench/saaru && git rev-parse HEAD | cut -c1-8)\`](https://github.com/anirudhRowjee/saaru/commit/$(cd tmp/bench/saaru && git rev-parse HEAD))" >> comment_body.txt - echo "- **Sapling**: [\`$(cd tmp/bench/sapling && git rev-parse HEAD | cut -c1-8)\`](https://github.com/NavinShrinivas/sapling/commit/$(cd tmp/bench/sapling && git rev-parse HEAD))" >> comment_body.txt - echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt cat bench_results.txt >> comment_body.txt echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt + echo " ## Commit Hashes" >> comment_body.txt + cat commit_hashes.txt >> comment_body.txt # Results - name: Results Post Benchmark diff --git a/test/bench.sh b/test/bench.sh index f51ad8d..d7514b3 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -1,98 +1,71 @@ #!/bin/bash files=1000 warm=10 -BASE_DIR=$(pwd) +REPO_ROOT=$(pwd) +BENCH_DIR=$REPO_ROOT/tmp/bench +rm -rf $BENCH_DIR -# Determine repo root -if [ -d "$BASE_DIR/site" ]; then - REPO_ROOT=$BASE_DIR -else - REPO_ROOT=$(cd $BASE_DIR/.. && pwd) -fi - -# Create bench directory -mkdir -p $BASE_DIR/tmp/bench +# build anna +mkdir -p $BENCH_DIR/anna +cp -r site/ $BENCH_DIR/anna/site +go build -o $BENCH_DIR/anna/anna +GOEXPERIMENT=greenteagc go build -o $BENCH_DIR/anna/anna_greentea # deps if ! command -v hyperfine &>/dev/null; then - echo "hyperfine is not installed. Please install hyperfine to continue." - exit 1 + echo "hyperfine is not installed. Please install hyperfine to continue." && exit 1 fi if ! command -v hugo &>/dev/null; then echo "hugo is not installed. Please install hugo to continue." fi # cloning candidates -echo "" -echo "clone SSGs" -echo "" -clone_or_pull() { - local repo=$1 - local dir=$2 - if [ "$repo" = "https://github.com/anna-ssg/anna" ]; then - # For Anna, use current source code - rm -rf "$dir" - mkdir -p "$dir" - cp -r "$REPO_ROOT/site" "$dir/" - cd "$REPO_ROOT" && go build -o "$dir/anna" - cd "$REPO_ROOT" && GOEXPERIMENT=greenteagc go build -o "$dir/anna_greentea" - else - # For others, always fresh clone - rm -rf "$dir" - git clone --depth=1 "$repo" "$dir" - fi -} +echo "\n clone SSGs" -clone_or_pull https://github.com/anna-ssg/anna $BASE_DIR/tmp/bench/anna -clone_or_pull https://github.com/anirudhRowjee/saaru $BASE_DIR/tmp/bench/saaru -clone_or_pull https://github.com/NavinShrinivas/sapling $BASE_DIR/tmp/bench/sapling +# Saaru & Sapling: fresh clones +git clone --depth=1 https://github.com/anirudhRowjee/saaru $BENCH_DIR/saaru +git clone --depth=1 https://github.com/NavinShrinivas/sapling $BENCH_DIR/sapling # show commit hashes -echo "" -echo "Commit hashes:" -echo "anna: $(cd $REPO_ROOT && git rev-parse HEAD)" -echo "saaru: $(cd $BASE_DIR/tmp/bench/saaru && git rev-parse HEAD)" -echo "sapling: $(cd $BASE_DIR/tmp/bench/sapling && git rev-parse HEAD)" -echo "" +echo "\n Commit hashes:" +ANNA_HASH=$(cd $REPO_ROOT && git log --format=%h -1) +echo "anna: [\`$ANNA_HASH\`](https://github.com/anna-ssg/anna/commit/$(cd $REPO_ROOT && git log --format=%H -1))" > $BENCH_DIR/commit_hashes.txt + +SAARU_HASH=$(cd $BENCH_DIR/saaru && git log --format=%h -1) +echo "saaru: [\`$SAARU_HASH\`](https://github.com/anirudhRowjee/saaru/commit/$(cd $BENCH_DIR/saaru && git log --format=%H -1))" >> $BENCH_DIR/commit_hashes.txt + +SAPLING_HASH=$(cd $BENCH_DIR/sapling && git log --format=%h -1) +echo "sapling: [\`$SAPLING_HASH\`](https://github.com/NavinShrinivas/sapling/commit/$(cd $BENCH_DIR/sapling && git log --format=%H -1))" >> $BENCH_DIR/commit_hashes.txt -cp $REPO_ROOT/site/content/posts/bench.md $BASE_DIR/tmp/bench/test.md +cat $BENCH_DIR/commit_hashes.txt -echo "" -echo "build SSGs" -echo "" -cd $BASE_DIR/tmp/bench/sapling && cargo build --release && mv target/release/sapling . -cd $BASE_DIR/tmp/bench/saaru && cargo build --release && mv target/release/saaru . +echo "\n build SSGs" +cargo build --release --manifest-path $BENCH_DIR/sapling/Cargo.toml +cargo build --release --manifest-path $BENCH_DIR/saaru/Cargo.toml ## setup hugo -rm -rf $BASE_DIR/tmp/bench/hugo -hugo new site $BASE_DIR/tmp/bench/hugo -cd $BASE_DIR/tmp/bench/hugo -hugo new theme mytheme +hugo new site $BENCH_DIR/hugo; cd $BENCH_DIR/hugo; hugo new theme mytheme echo "theme = 'mytheme'" >> hugo.toml -cd ../.. # clean content/* dirs -rm -rf $BASE_DIR/tmp/bench/anna/site/content/posts/* -rm -rf $BASE_DIR/tmp/bench/saaru/docs/src/* -rm -rf $BASE_DIR/tmp/bench/sapling/benchmark/content/blog/* -rm -rf $BASE_DIR/tmp/bench/hugo/content/* +rm -rf $BENCH_DIR/anna/site/content && mkdir -p $BENCH_DIR/anna/site/content/posts +rm -rf $BENCH_DIR/saaru/docs/src/* +rm -rf $BENCH_DIR/sapling/benchmark/content/blogs/* +rm -rf $BENCH_DIR/hugo/content/* -echo "" -echo "Spawning $files different markdown files..." +# populate with test files +cp $REPO_ROOT/site/content/posts/bench.md $BENCH_DIR/test.md for ((i = 0; i < files; i++)); do - cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/anna/site/content/posts/test_$i.md" - cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/saaru/docs/src/test_$i.md" - cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/sapling/benchmark/content/blogs/test_$i.md" - cp $BASE_DIR/tmp/bench/test.md "$BASE_DIR/tmp/bench/hugo/content/test_$i.md" + cp $BENCH_DIR/test.md "$BENCH_DIR/anna/site/content/posts/test_$i.md" + cp $BENCH_DIR/test.md "$BENCH_DIR/saaru/docs/src/test_$i.md" + cp $BENCH_DIR/test.md "$BENCH_DIR/sapling/benchmark/content/blogs/test_$i.md" + cp $BENCH_DIR/test.md "$BENCH_DIR/hugo/content/test_$i.md" done -echo "" -echo "Begin Benchmark" -echo "running benchmark: $files md files and $warm warmup runs \n" +echo "running benchmark: $files md files and $warm warmup runs" > $BENCH_DIR/bench_results.txt hyperfine -p 'sync' -w $warm \ - "cd $BASE_DIR/tmp/bench/saaru && ./saaru --base-path ./docs" \ - "cd $BASE_DIR/tmp/bench/sapling/benchmark && ./../sapling run" \ - "cd $BASE_DIR/tmp/bench/anna && ./anna_greentea" \ - "cd $BASE_DIR/tmp/bench/anna && ./anna" \ - "cd $BASE_DIR/tmp/bench/hugo && hugo" -echo "End Benchmark" + "cd $BENCH_DIR/saaru && ./target/release/saaru --base-path ./docs" \ + "cd $BENCH_DIR/sapling/benchmark && ./../target/release/sapling run" \ + "cd $BENCH_DIR/anna && ./anna_greentea" \ + "cd $BENCH_DIR/anna && ./anna" \ + "cd $BENCH_DIR/hugo && hugo" >> $BENCH_DIR/bench_results.txt From c31d5b55ae293edbfe8f31aeda37a8cdbb16769d Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 14:27:57 +0530 Subject: [PATCH 17/18] refactor: enhance benchmark script for improved commit hash retrieval --- .github/workflows/bench.yml | 1 + test/bench.sh | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f5afce1..f24f2cf 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -42,6 +42,7 @@ jobs: - name: Execution Run Benchmark run: | ./test/bench.sh + cd /home/runner/work/anna/anna/tmp/bench echo "## Benchmark Results" > comment_body.txt echo "" >> comment_body.txt echo "\`\`\`" >> comment_body.txt diff --git a/test/bench.sh b/test/bench.sh index d7514b3..b970d8f 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -20,26 +20,31 @@ if ! command -v hugo &>/dev/null; then fi # cloning candidates -echo "\n clone SSGs" +echo "clone SSGs" # Saaru & Sapling: fresh clones git clone --depth=1 https://github.com/anirudhRowjee/saaru $BENCH_DIR/saaru git clone --depth=1 https://github.com/NavinShrinivas/sapling $BENCH_DIR/sapling # show commit hashes -echo "\n Commit hashes:" -ANNA_HASH=$(cd $REPO_ROOT && git log --format=%h -1) -echo "anna: [\`$ANNA_HASH\`](https://github.com/anna-ssg/anna/commit/$(cd $REPO_ROOT && git log --format=%H -1))" > $BENCH_DIR/commit_hashes.txt - -SAARU_HASH=$(cd $BENCH_DIR/saaru && git log --format=%h -1) -echo "saaru: [\`$SAARU_HASH\`](https://github.com/anirudhRowjee/saaru/commit/$(cd $BENCH_DIR/saaru && git log --format=%H -1))" >> $BENCH_DIR/commit_hashes.txt - -SAPLING_HASH=$(cd $BENCH_DIR/sapling && git log --format=%h -1) -echo "sapling: [\`$SAPLING_HASH\`](https://github.com/NavinShrinivas/sapling/commit/$(cd $BENCH_DIR/sapling && git log --format=%H -1))" >> $BENCH_DIR/commit_hashes.txt +get_commit_info() { + local repo_dir=$1 + local name=$2 + local url=$3 + INFO=$(cd $repo_dir && git log --oneline -1) + HASH=$(echo $INFO | cut -d' ' -f1) + MSG=$(echo $INFO | cut -d' ' -f2-) + FULL_HASH=$(cd $repo_dir && git log --format=%H -1) + echo "$name: [\`$HASH\`]($url/commit/$FULL_HASH) $MSG" +} +echo "commit hashes:" +get_commit_info $REPO_ROOT "anna" "https://github.com/anna-ssg/anna" > $BENCH_DIR/commit_hashes.txt +get_commit_info $BENCH_DIR/saaru "saaru" "https://github.com/anirudhRowjee/saaru" >> $BENCH_DIR/commit_hashes.txt +get_commit_info $BENCH_DIR/sapling "sapling" "https://github.com/NavinShrinivas/sapling" >> $BENCH_DIR/commit_hashes.txt cat $BENCH_DIR/commit_hashes.txt -echo "\n build SSGs" +echo "build SSGs" cargo build --release --manifest-path $BENCH_DIR/sapling/Cargo.toml cargo build --release --manifest-path $BENCH_DIR/saaru/Cargo.toml From c6aa4151e5b8f7f34579e0d832446bf07baae1a9 Mon Sep 17 00:00:00 2001 From: Nathan Paul Date: Fri, 26 Dec 2025 14:39:59 +0530 Subject: [PATCH 18/18] final? --- .github/workflows/bench.yml | 18 +++++++++--------- test/bench.sh | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f24f2cf..4d3487b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -43,20 +43,20 @@ jobs: run: | ./test/bench.sh cd /home/runner/work/anna/anna/tmp/bench - echo "## Benchmark Results" > comment_body.txt - echo "" >> comment_body.txt - echo "\`\`\`" >> comment_body.txt - cat bench_results.txt >> comment_body.txt - echo "" >> comment_body.txt - echo "\`\`\`" >> comment_body.txt - echo " ## Commit Hashes" >> comment_body.txt - cat commit_hashes.txt >> comment_body.txt + echo "## Benchmark Results" > ~/comment_body.txt + echo "" >> ~/comment_body.txt + echo "\`\`\`" >> ~/comment_body.txt + cat bench_results.txt >> ~/comment_body.txt + echo "" >> ~/comment_body.txt + echo "\`\`\`" >> ~/comment_body.txt + echo " ## Commit Hashes" >> ~/comment_body.txt + cat commit_hashes.txt >> ~/comment_body.txt # Results - name: Results Post Benchmark if: github.event_name == 'pull_request' run: | - gh pr comment ${{ github.event.pull_request.number }} --body-file comment_body.txt --create-if-none --edit-last + gh pr comment ${{ github.event.pull_request.number }} --body-file ~/comment_body.txt --create-if-none --edit-last env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} diff --git a/test/bench.sh b/test/bench.sh index b970d8f..62818df 100755 --- a/test/bench.sh +++ b/test/bench.sh @@ -67,10 +67,10 @@ for ((i = 0; i < files; i++)); do cp $BENCH_DIR/test.md "$BENCH_DIR/hugo/content/test_$i.md" done -echo "running benchmark: $files md files and $warm warmup runs" > $BENCH_DIR/bench_results.txt +echo "running benchmark: $files md files and $warm warmup runs" | tee $BENCH_DIR/bench_results.txt hyperfine -p 'sync' -w $warm \ "cd $BENCH_DIR/saaru && ./target/release/saaru --base-path ./docs" \ "cd $BENCH_DIR/sapling/benchmark && ./../target/release/sapling run" \ "cd $BENCH_DIR/anna && ./anna_greentea" \ "cd $BENCH_DIR/anna && ./anna" \ - "cd $BENCH_DIR/hugo && hugo" >> $BENCH_DIR/bench_results.txt + "cd $BENCH_DIR/hugo && hugo" | tee -a $BENCH_DIR/bench_results.txt