From d81e22959c4c0e59307b116ba9af612c076073d7 Mon Sep 17 00:00:00 2001 From: Arjun Yogidas Date: Mon, 7 Aug 2023 20:29:18 +0000 Subject: [PATCH 1/3] Add regression check automation This commit adds benchmark_regression_test.yml workflow that runs a regression test by comparing the benchmark results of the current branch with the previous benchmark results of the code in main branch. Signed-off-by: Arjun Raja Yogidas --- .../workflows/benchmark_regression_test.yml | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/benchmark_regression_test.yml diff --git a/.github/workflows/benchmark_regression_test.yml b/.github/workflows/benchmark_regression_test.yml new file mode 100644 index 000000000..379dee060 --- /dev/null +++ b/.github/workflows/benchmark_regression_test.yml @@ -0,0 +1,68 @@ +name: Benchmark Regression Check + +on: + pull_request: + branches: [ main ] + paths: + - '**.go' + - 'go.*' + - 'cmd/go.*' + - 'Makefile' + - 'Dockerfile' + - 'integration/**' + - 'scripts/**' + - '.github/workflows/**' + +jobs: + benchmark-and-fetch-previous-results_and_compare: + name: Run Benchmark on current Branch + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.18.10' + - run: make + - name: Run benchmark + run: make benchmarks-perf-test + - name: Make previous directory + run: mkdir -v ${{ github.workspace }}/previous + - name: Download previous run artifact + id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + name: benchmark-result-artifact + name_is_regexp: true + path: ${{ github.workspace }}/previous + repo: ${{ github.repository }} + check_artifacts: false + search_artifacts: true + skip_unpack: false + if_no_artifact_found: fail + workflow: benchmark_visualization.yml + - name: Perform Comparison and log results + id: run-compare + run: | + sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/benchmark-result-artifact/results.json ${{github.workspace}}/benchmark/performanceTest/output/results.json; then + echo "Comparison successful. All P90 values are within the acceptable range." + else + echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." + echo "regression-detected=true" >> $GITHUB_OUTPUT + fi + - name: Stop the workflow if regression is detected + if: steps.run-compare.outputs.regression-detected == 'true' + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comment = ` + :warning: **Regression Detected** :warning: + + The benchmark comparison indicates that there has been a performance regression. + Please investigate and address the issue. + To Investigate check logs of the previous job above. + `; + + core.setFailed(comment); \ No newline at end of file From ca3d3bc6361f4e02f8b6972929d19af910102d56 Mon Sep 17 00:00:00 2001 From: Arjun Yogidas Date: Tue, 15 Aug 2023 18:23:08 +0000 Subject: [PATCH 2/3] Update workflow to run two benchmarks This commit is a test to check if running both the benchmarks in the same workflow would make any difference to the results Signed-off-by: Arjun Raja Yogidas --- .../workflows/benchmark_regression_test.yml | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark_regression_test.yml b/.github/workflows/benchmark_regression_test.yml index 379dee060..d3a6067b0 100644 --- a/.github/workflows/benchmark_regression_test.yml +++ b/.github/workflows/benchmark_regression_test.yml @@ -12,40 +12,42 @@ on: - 'integration/**' - 'scripts/**' - '.github/workflows/**' - + jobs: - benchmark-and-fetch-previous-results_and_compare: - name: Run Benchmark on current Branch + test-twice: runs-on: ubuntu-20.04 + steps: - - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: go-version: '1.18.10' + - name: Checkout main + uses: actions/checkout@v3 + with: + ref: main - run: make - name: Run benchmark - run: make benchmarks-perf-test + run: make benchmarks-perf-test - name: Make previous directory run: mkdir -v ${{ github.workspace }}/previous - - name: Download previous run artifact - id: download-artifact - uses: dawidd6/action-download-artifact@v2 + - name: Copy results to previous directory + run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/previous + - name: Check out PR + uses: actions/checkout@v3 with: - github_token: ${{secrets.GITHUB_TOKEN}} - name: benchmark-result-artifact - name_is_regexp: true - path: ${{ github.workspace }}/previous - repo: ${{ github.repository }} - check_artifacts: false - search_artifacts: true - skip_unpack: false - if_no_artifact_found: fail - workflow: benchmark_visualization.yml + ref: ${{ github.event.pull_request.head.sha }} + - run: make + - name: Run benchmark + run: make benchmarks-perf-test + - name: Make current directory + run: mkdir -v ${{ github.workspace }}/current + - name: Copy results to current directory + run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/current - name: Perform Comparison and log results id: run-compare run: | sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh - if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/benchmark-result-artifact/results.json ${{github.workspace}}/benchmark/performanceTest/output/results.json; then + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/current/results.json; then echo "Comparison successful. All P90 values are within the acceptable range." else echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." From 0b7f06e60b68da6cde1b9fd8b5ac7ac35933db57 Mon Sep 17 00:00:00 2001 From: Arjun Yogidas Date: Thu, 17 Aug 2023 01:21:07 +0000 Subject: [PATCH 3/3] Update workflow to use upload artifact action Updated the workflow to use upload artifact action Signed-off-by: Arjun Raja Yogidas --- .../workflows/benchmark_regression_test.yml | 103 ++++++++++++------ 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/.github/workflows/benchmark_regression_test.yml b/.github/workflows/benchmark_regression_test.yml index d3a6067b0..b4a480ed5 100644 --- a/.github/workflows/benchmark_regression_test.yml +++ b/.github/workflows/benchmark_regression_test.yml @@ -1,6 +1,7 @@ name: Benchmark Regression Check on: + workflow_dispatch: pull_request: branches: [ main ] paths: @@ -14,13 +15,12 @@ on: - '.github/workflows/**' jobs: - test-twice: + run_benchmark_twice: runs-on: ubuntu-20.04 - steps: - uses: actions/setup-go@v4 with: - go-version: '1.18.10' + go-version: '1.20.6' - name: Checkout main uses: actions/checkout@v3 with: @@ -28,10 +28,16 @@ jobs: - run: make - name: Run benchmark run: make benchmarks-perf-test - - name: Make previous directory - run: mkdir -v ${{ github.workspace }}/previous - - name: Copy results to previous directory - run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/previous + - name: Upload latest benchmark result + uses: actions/upload-artifact@v3 + with: + name: benchmark-result-artifact-main + path: ${{github.workspace}}/benchmark/performanceTest/output/results.json + - name: remove output directory + run: sudo rm -rf ${{ github.workspace }}/benchmark/performanceTest/output + - name: Stash uncommitted changes + run: git stash push --keep-index --include-untracked -m "Stashing changes for tests" + - name: Check out PR uses: actions/checkout@v3 with: @@ -39,32 +45,63 @@ jobs: - run: make - name: Run benchmark run: make benchmarks-perf-test - - name: Make current directory - run: mkdir -v ${{ github.workspace }}/current - - name: Copy results to current directory - run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/current - - name: Perform Comparison and log results - id: run-compare - run: | - sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh - if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/current/results.json; then - echo "Comparison successful. All P90 values are within the acceptable range." - else - echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." - echo "regression-detected=true" >> $GITHUB_OUTPUT - fi - - name: Stop the workflow if regression is detected - if: steps.run-compare.outputs.regression-detected == 'true' - uses: actions/github-script@v6 + - name: Upload latest benchmark result + uses: actions/upload-artifact@v3 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const comment = ` - :warning: **Regression Detected** :warning: + name: benchmark-result-artifact-pr + path: ${{github.workspace}}/benchmark/performanceTest/output/results.json + + download_and_perform_comparison: + runs-on: ubuntu-20.04 + needs: run_benchmark_twice + steps: + - uses: actions/setup-go@v4 + with: + go-version: '1.18.10' + - name: Checkout main + uses: actions/checkout@v3 + with: + ref: main + - run: make + - name: Install basic calculator + run: sudo apt-get install bc + + - name: Create previous directory + run: mkdir -v ${{ github.workspace }}/previous + - name: Create current directory + run: mkdir -v ${{ github.workspace }}/current + - name: Download previous benchmark result + uses: actions/download-artifact@v3 + with: + name: benchmark-result-artifact-main + path: ${{github.workspace}}/previous + - name: Download current benchmark result + uses: actions/download-artifact@v3 + with: + name: benchmark-result-artifact-pr + path: ${{github.workspace}}/current + - name: Perform Comparison and log results + id: run-compare + run: | + sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/current/results.json; then + echo "Comparison successful. All P90 values are within the acceptable range." + else + echo "Comparison failed. Current P90 values exceed 150% threshold of the corresponding past values." + echo "regression-detected=true" >> $GITHUB_OUTPUT + fi + - name: Stop the workflow if regression is detected + if: steps.run-compare.outputs.regression-detected == 'true' + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comment = ` + :warning: **Regression Detected** :warning: - The benchmark comparison indicates that there has been a performance regression. - Please investigate and address the issue. - To Investigate check logs of the previous job above. - `; + The benchmark comparison indicates that there has been a performance regression. + Please investigate and address the issue. + To Investigate check logs of the previous job above. + `; - core.setFailed(comment); \ No newline at end of file + core.setFailed(comment); \ No newline at end of file