From b6489b6640fab915b4218e307dbdd84d48301325 Mon Sep 17 00:00:00 2001 From: Arjun Yogidas Date: Mon, 7 Aug 2023 20:29:18 +0000 Subject: [PATCH 1/2] Add regression check automation This commit adds benchmark_regression_test.yml workflow that runs a regression test by comparing the benchmark results of the current branch with the previous benchmark results of the code in main branch. Signed-off-by: Arjun Raja Yogidas --- .../workflows/benchmark_regression_test.yml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/benchmark_regression_test.yml diff --git a/.github/workflows/benchmark_regression_test.yml b/.github/workflows/benchmark_regression_test.yml new file mode 100644 index 000000000..52a46bb10 --- /dev/null +++ b/.github/workflows/benchmark_regression_test.yml @@ -0,0 +1,78 @@ +name: Benchmark Regression Check + +on: + pull_request: + branches: [ main ] + paths: + - '**.go' + - 'go.*' + - 'cmd/go.*' + - 'Makefile' + - 'Dockerfile' + - 'integration/**' + - 'scripts/**' + - '.github/workflows/**' + +jobs: + benchmark: + name: Run Benchmark on current Branch + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.18.10' + - run: make + - name: Run benchmark + run: make benchmarks-perf-test + + fetch-previous-results_and_compare: + name: Fetch results from previous run + needs: benchmark + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.18.10' + - run: make + - name: Download artifact + id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + name: benchmark-result-artifact + name_is_regexp: true + path: ${{ github.workspace }} + repo: ${{ github.repository }} + check_artifacts: false + search_artifacts: true + skip_unpack: false + if_no_artifact_found: fail + workflow: benchmark_visualization.yml + - name: Perform Comparison and log results + id: run-compare + run: | + sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/benchmark-result.json ${{ github.workspace }}/benchmark-result-current.json; then + echo "Comparison successful. All P90 values are within the acceptable range." + else + echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." + echo "regression-detected=true" >> $GITHUB_OUTPUT + fi + + - name: Stop the workflow if regression is detected + if: steps.run-compare.outputs.regression-detected == 'true' + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comment = ` + :warning: **Regression Detected** :warning: + + The benchmark comparison indicates that there has been a performance regression. + Please investigate and address the issue. + To Investigate check logs of the previous job above. + `; + + core.setFailed(comment); \ No newline at end of file From bd37c952d7a0407312bff001b9d4b9f730295ee6 Mon Sep 17 00:00:00 2001 From: Arjun Yogidas Date: Mon, 14 Aug 2023 16:05:32 +0000 Subject: [PATCH 2/2] Add baseline results comparison This commit adds baseline comparison logic to the benchmark_regression_test workflow. The baseline artifact is uploaded via the benchmark_baseline workflow once every 20 days. The baseline artifact comparison will be skipped if the baseline result artifact is not available for download, this is to handle the first 20 days after merge. Signed-off-by: Arjun Raja Yogidas --- .../workflows/benchmark_regression_test.yml | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark_regression_test.yml b/.github/workflows/benchmark_regression_test.yml index 52a46bb10..8bc31c39e 100644 --- a/.github/workflows/benchmark_regression_test.yml +++ b/.github/workflows/benchmark_regression_test.yml @@ -35,34 +35,65 @@ jobs: - uses: actions/setup-go@v4 with: go-version: '1.18.10' - - run: make - - name: Download artifact + - name: Make previous directory + run: mkdir -v ${{ github.workspace }}/previous + - name: Download previous run artifact id: download-artifact uses: dawidd6/action-download-artifact@v2 with: github_token: ${{secrets.GITHUB_TOKEN}} name: benchmark-result-artifact name_is_regexp: true - path: ${{ github.workspace }} + path: ${{ github.workspace }}/previous repo: ${{ github.repository }} check_artifacts: false search_artifacts: true skip_unpack: false if_no_artifact_found: fail workflow: benchmark_visualization.yml + - name: Make baseline directory + run: mkdir -v ${{ github.workspace }}/baseline + - name: Download previous run artifact + id: download-artifact-baseline + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + name: benchmark-baseline-artifact + name_is_regexp: true + path: ${{ github.workspace }}/baseline + repo: ${{ github.repository }} + check_artifacts: false + search_artifacts: true + skip_unpack: false + if_no_artifact_found: warn + workflow: benchmark_baseline.yml - name: Perform Comparison and log results id: run-compare run: | sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh - if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/benchmark-result.json ${{ github.workspace }}/benchmark-result-current.json; then + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/benchmark/performanceTest/output/results.json; then echo "Comparison successful. All P90 values are within the acceptable range." else echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." echo "regression-detected=true" >> $GITHUB_OUTPUT fi - + - name: Perform baseline comparison and log results + id: run-compare-baseline + run: | + sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh + if [ -f "${{ github.workspace }}/baseline/results.json" ]; then + if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/baseline/results.json ${{github.workspace}}/benchmark/performanceTest/output/results.json; then + echo "Comparison successful. All P90 values are within the acceptable range." + else + echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values." + echo "regression-detected=true" >> $GITHUB_OUTPUT + fi + else + echo "Baseline comparison skipped. No baseline results found." + exit 0 # Exit the workflow gracefully + fi - name: Stop the workflow if regression is detected - if: steps.run-compare.outputs.regression-detected == 'true' + if: steps.run-compare.outputs.regression-detected == 'true' || steps.run-compare-baseline.outputs.regression-detected == 'true' uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }}