From 8d4637667a68bfc7dcd4a7afed0b590d7ff911c1 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:13:16 -0600 Subject: [PATCH 01/16] Copy over changes to ci.yml --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63b74fde..0d50822b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,10 +56,35 @@ jobs: git config --global user.name oll-bot git config --global user.email developers@openlawlib.org + + - name: test pull request reading + if: contains(github.event.pull_request.body, 'test') + run: | + echo body: ${{ github.event.pull_request.body }} + - name: Run pre-commit and test with pytest run: | pre-commit run --all-files - pytest taf/tests + pytest taf/tests --benchmark-skip + + + - name: Download artifact + id: download-artifact + uses: dawidd6/action-download-artifact@v8 + with: + branch: CalShucha/automated-benchmarking #change to "master" when testing is done + name: benchmark-results${{ matrix.python-version }} + path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + workflow_conclusion: success + + - name: Run benchmark + run: | + python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + - name: Archive benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results${{ matrix.python-version }} + path: 0001_output.json build_and_upload_wheel: runs-on: ubuntu-latest From 153067aa36dff91be3bb31d8942888b2f4f99d73 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:28:03 -0600 Subject: [PATCH 02/16] Temporarily disable artifact downloading for test --- .github/workflows/ci.yml | 28 +++++++++---------- .../test_updater/test_update_benchmarking.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d50822b..41c6d159 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,20 +62,20 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - - name: Run pre-commit and test with pytest - run: | - pre-commit run --all-files - pytest taf/tests --benchmark-skip - - - - name: Download artifact - id: download-artifact - uses: dawidd6/action-download-artifact@v8 - with: - branch: CalShucha/automated-benchmarking #change to "master" when testing is done - name: benchmark-results${{ matrix.python-version }} - path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - workflow_conclusion: success + # - name: Run pre-commit and test with pytest + # run: | + # pre-commit run --all-files + # pytest taf/tests --benchmark-skip + + + # - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available + # id: download-artifact + # uses: dawidd6/action-download-artifact@v8 + # with: + # branch: CalShucha/automated-benchmarking #change to "master" when testing is done + # name: benchmark-results${{ matrix.python-version }} + # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + # workflow_conclusion: success - name: Run benchmark run: | diff --git a/taf/tests/test_updater/test_update_benchmarking.py b/taf/tests/test_updater/test_update_benchmarking.py index a43cf53b..aa0f3e0a 100644 --- a/taf/tests/test_updater/test_update_benchmarking.py +++ b/taf/tests/test_updater/test_update_benchmarking.py @@ -4,7 +4,7 @@ ) -@pytest.mark.skip(reason="benchmarking disabled for time being") + @pytest.mark.parametrize( "origin_auth_repo", [ From 03ac0f6316e8ff34ab1a8753d7522211082a9d53 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:34:57 -0600 Subject: [PATCH 03/16] Disable comparison for testing --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41c6d159..c6d21fba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + python -m pytest --benchmark-only --benchmark-json=0001_output.json - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From 9995440b89c5637fccfef91ccc1bf8732f1ace5d Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:38:06 -0600 Subject: [PATCH 04/16] Re-enable everything --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6d21fba..08d6dbb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,24 +62,24 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - # - name: Run pre-commit and test with pytest - # run: | - # pre-commit run --all-files - # pytest taf/tests --benchmark-skip - - - # - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available - # id: download-artifact - # uses: dawidd6/action-download-artifact@v8 - # with: - # branch: CalShucha/automated-benchmarking #change to "master" when testing is done - # name: benchmark-results${{ matrix.python-version }} - # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - # workflow_conclusion: success + - name: Run pre-commit and test with pytest + run: | + pre-commit run --all-files + pytest taf/tests --benchmark-skip + + + - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available + id: download-artifact + uses: dawidd6/action-download-artifact@v8 + with: + branch: CalShucha/automated-benchmarking #change to "master" when testing is done + name: benchmark-results${{ matrix.python-version }} + path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + workflow_conclusion: success - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json + python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From 4fce15c5aaf548dd38f9aaf56074a9d717b2dfb1 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:42:54 -0600 Subject: [PATCH 05/16] Format with black --- taf/tests/test_updater/test_update_benchmarking.py | 1 - 1 file changed, 1 deletion(-) diff --git a/taf/tests/test_updater/test_update_benchmarking.py b/taf/tests/test_updater/test_update_benchmarking.py index aa0f3e0a..5286f3d5 100644 --- a/taf/tests/test_updater/test_update_benchmarking.py +++ b/taf/tests/test_updater/test_update_benchmarking.py @@ -4,7 +4,6 @@ ) - @pytest.mark.parametrize( "origin_auth_repo", [ From 86cbb34b1369b2e2edc747f96498c664022d7c36 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 14:57:11 -0600 Subject: [PATCH 06/16] Re-disable comparison for testing --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08d6dbb0..c6d21fba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,24 +62,24 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - - name: Run pre-commit and test with pytest - run: | - pre-commit run --all-files - pytest taf/tests --benchmark-skip - - - - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available - id: download-artifact - uses: dawidd6/action-download-artifact@v8 - with: - branch: CalShucha/automated-benchmarking #change to "master" when testing is done - name: benchmark-results${{ matrix.python-version }} - path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - workflow_conclusion: success + # - name: Run pre-commit and test with pytest + # run: | + # pre-commit run --all-files + # pytest taf/tests --benchmark-skip + + + # - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available + # id: download-artifact + # uses: dawidd6/action-download-artifact@v8 + # with: + # branch: CalShucha/automated-benchmarking #change to "master" when testing is done + # name: benchmark-results${{ matrix.python-version }} + # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + # workflow_conclusion: success - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + python -m pytest --benchmark-only --benchmark-json=0001_output.json - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From 04eaa06ecaf7cd72109de741d1cd0207a2338f0b Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 15:01:27 -0600 Subject: [PATCH 07/16] Re-re-enable everything --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6d21fba..b8d25eac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,24 +62,24 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - # - name: Run pre-commit and test with pytest - # run: | - # pre-commit run --all-files - # pytest taf/tests --benchmark-skip - - - # - name: Download artifact # Known issue: this will fail the whole workflow if there is no artifact available - # id: download-artifact - # uses: dawidd6/action-download-artifact@v8 - # with: - # branch: CalShucha/automated-benchmarking #change to "master" when testing is done - # name: benchmark-results${{ matrix.python-version }} - # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - # workflow_conclusion: success + - name: Run pre-commit and test with pytest + run: | + pre-commit run --all-files + pytest taf/tests --benchmark-skip + + + - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired + id: download-artifact + uses: dawidd6/action-download-artifact@v8 + with: + branch: CalShucha/automated-benchmarking #change to "master" when testing is done + name: benchmark-results${{ matrix.python-version }} + path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + workflow_conclusion: success - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json + python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From 4fa2d3e6905ab941c349d2611655419e23a7185d Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 16:01:39 -0600 Subject: [PATCH 08/16] Disable parts of workflow for testing --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8d25eac..b3dad937 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,24 +62,24 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - - name: Run pre-commit and test with pytest - run: | - pre-commit run --all-files - pytest taf/tests --benchmark-skip - - - - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired - id: download-artifact - uses: dawidd6/action-download-artifact@v8 - with: - branch: CalShucha/automated-benchmarking #change to "master" when testing is done - name: benchmark-results${{ matrix.python-version }} - path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - workflow_conclusion: success + # - name: Run pre-commit and test with pytest + # run: | + # pre-commit run --all-files + # pytest taf/tests --benchmark-skip + + + # - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired + # id: download-artifact + # uses: dawidd6/action-download-artifact@v8 + # with: + # branch: CalShucha/automated-benchmarking #change to "master" when testing is done + # name: benchmark-results${{ matrix.python-version }} + # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + # workflow_conclusion: success - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + python -m pytest --benchmark-only --benchmark-json=0001_output.json - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From c90d312841e49a338e3bc7de88a2c8f790106d17 Mon Sep 17 00:00:00 2001 From: CalShucha Date: Tue, 4 Feb 2025 16:04:38 -0600 Subject: [PATCH 09/16] Re-enable parts of workflow for testing --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3dad937..b8d25eac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,24 +62,24 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - # - name: Run pre-commit and test with pytest - # run: | - # pre-commit run --all-files - # pytest taf/tests --benchmark-skip - - - # - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired - # id: download-artifact - # uses: dawidd6/action-download-artifact@v8 - # with: - # branch: CalShucha/automated-benchmarking #change to "master" when testing is done - # name: benchmark-results${{ matrix.python-version }} - # path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - # workflow_conclusion: success + - name: Run pre-commit and test with pytest + run: | + pre-commit run --all-files + pytest taf/tests --benchmark-skip + + + - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired + id: download-artifact + uses: dawidd6/action-download-artifact@v8 + with: + branch: CalShucha/automated-benchmarking #change to "master" when testing is done + name: benchmark-results${{ matrix.python-version }} + path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit + workflow_conclusion: success - name: Run benchmark run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json + python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% - name: Archive benchmark results uses: actions/upload-artifact@v4 with: From 846c8423f47fc937610226cea6264898f08868d5 Mon Sep 17 00:00:00 2001 From: Renata Date: Tue, 4 Mar 2025 20:00:49 -0500 Subject: [PATCH 10/16] devops: download artifacts without using an external, unofficial script --- .github/workflows/ci.yml | 104 +++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8d25eac..691e1ce3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,9 @@ name: CI +permissions: + contents: read + actions: read + on: push: {} pull_request: @@ -31,6 +35,13 @@ jobs: strategy: matrix: python-version: ${{ fromJSON(needs.set_python_versions.outputs.all_versions) }} + + env: + ARTIFACT_NAME: benchmark-results${{ matrix.python-version }} + BRANCH_NAME: 'renatav/performance-tests' + BENCHMARKS_DIR: '/home/runner/work/taf/taf/.benchmarks/' + ARTIFACT_PATH: '/home/runner/work/taf/taf/.benchmarks/archive.zip' + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #@v4 @@ -50,6 +61,7 @@ jobs: mkdir -p ~/bin/ && ln -s /usr/bin/swig3.0 ~/bin/swig && export PATH=~/bin/:$PATH pip install wheel # Ensure wheel is installed pip install -e .[ci,test,yubikey] + npm install axios - name: Setup GitHub user run: | @@ -62,28 +74,92 @@ jobs: run: | echo body: ${{ github.event.pull_request.body }} - - name: Run pre-commit and test with pytest - run: | - pre-commit run --all-files - pytest taf/tests --benchmark-skip - + # - name: Run pre-commit and test with pytest + # run: | + # pre-commit run --all-files + # pytest taf/tests --benchmark-skip - - name: Download artifact # Known issue: this will fail if the previous run failed before saving artifacts, or if those artifacts have expired - id: download-artifact - uses: dawidd6/action-download-artifact@v8 + - name: Fetch the latest artifact by name and branch + id: get_artifact + uses: actions/github-script@v6 with: - branch: CalShucha/automated-benchmarking #change to "master" when testing is done - name: benchmark-results${{ matrix.python-version }} - path: /home/runner/work/taf/taf/.benchmarks/${{ runner.os }}-CPython-${{ matrix.python-version }}-64bit - workflow_conclusion: success + script: | + const artifacts = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + const filteredArtifacts = artifacts.data.artifacts.filter(artifact => artifact.name === process.env.ARTIFACT_NAME); + let latestArtifact = null; + + for (const artifact of filteredArtifacts) { + const run = await github.rest.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: artifact.workflow_run.id, + }); + + if (run.data.head_branch === process.env.BRANCH_NAME) { + if (!latestArtifact || new Date(artifact.created_at) > new Date(latestArtifact.created_at)) { + latestArtifact = artifact; + } + } + } + + if (latestArtifact) { + console.log(`Found latest artifact: ${latestArtifact.id}`); + core.setOutput('artifact_id', latestArtifact.id.toString()); + } else { + console.log('No matching artifacts found.'); + core.setFailed('No matching artifacts found.'); + } + + - name: Download the artifact by ID + uses: actions/github-script@v6 + if: steps.get_artifact.outputs.artifact_id + with: + script: | + const fs = require('fs'); + const path = require('path'); + const artifactId = '${{ steps.get_artifact.outputs.artifact_id }}'; + console.log(`Downloading artifact ID: ${artifactId}`); + const benchmarksDir = process.env.BENCHMARKS_DIR + const artifactPath = process.env.ARTIFACT_PATH + + // First, get the URL to download the artifact + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifactId, + archive_format: 'zip', + }); + + if (!fs.existsSync(benchmarksDir)) { + fs.mkdirSync(benchmarksDir, { recursive: true }); + } + fs.writeFileSync(artifactPath, Buffer.from(download.data)); + console.log('Artifact downloaded:', artifactPath); + + - name: 'Unzip artifact' + if: steps.get_artifact.outputs.artifact_id + run: unzip $ARTIFACT_PATH -d $BENCHMARKS_DIR + + - name: List extracted files + run: ls -l $BENCHMARKS_DIR - - name: Run benchmark + - name: Run benchmark with comparison + if: steps.get_artifact.outputs.artifact_id run: | python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + + - name: Run benchmark without comparison + if: steps.get_artifact.outputs.artifact_id == '' || steps.get_artifact.outputs.artifact_id == null + run: | + python -m pytest --benchmark-only --benchmark-json=0001_output.json + - name: Archive benchmark results uses: actions/upload-artifact@v4 with: - name: benchmark-results${{ matrix.python-version }} + name: ${{ env.ARTIFACT_NAME }} path: 0001_output.json build_and_upload_wheel: From d7af1ff466933b1e406a937bf38f7c012530e1ca Mon Sep 17 00:00:00 2001 From: Renata Date: Wed, 5 Mar 2025 02:00:26 -0500 Subject: [PATCH 11/16] devops: implement a way of redefining default threshold --- .github/workflows/ci.yml | 22 +++++++++++++--------- benchmark_thresholds.json | 3 +++ 2 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 benchmark_thresholds.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 691e1ce3..1a25471f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,16 +68,20 @@ jobs: git config --global user.name oll-bot git config --global user.email developers@openlawlib.org - - - name: test pull request reading - if: contains(github.event.pull_request.body, 'test') + - name: Set Benchmark Threshold from File run: | - echo body: ${{ github.event.pull_request.body }} + BRANCH_NAME=${GITHUB_REF#refs/heads/} + + THRESHOLD=$(jq --arg branch "$BRANCH_NAME" '.[$branch]' ./benchmark_thresholds.json) + + if [ "$THRESHOLD" == "null" ]; then + THRESHOLD=10 # Default threshold if not specified + fi + echo "BENCHMARK_THRESHOLD=$THRESHOLD" >> $GITHUB_ENV + echo "Using benchmark threshold: $THRESHOLD" - # - name: Run pre-commit and test with pytest - # run: | - # pre-commit run --all-files - # pytest taf/tests --benchmark-skip + - name: Display Benchmark Threshold + run: echo "Benchmark Threshold is $BENCHMARK_THRESHOLD" - name: Fetch the latest artifact by name and branch id: get_artifact @@ -149,7 +153,7 @@ jobs: - name: Run benchmark with comparison if: steps.get_artifact.outputs.artifact_id run: | - python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:20% + python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:${BENCHMARK_THRESHOLD}% - name: Run benchmark without comparison if: steps.get_artifact.outputs.artifact_id == '' || steps.get_artifact.outputs.artifact_id == null diff --git a/benchmark_thresholds.json b/benchmark_thresholds.json new file mode 100644 index 00000000..4a828827 --- /dev/null +++ b/benchmark_thresholds.json @@ -0,0 +1,3 @@ +{ + "renatav/performance-tests": 15 +} \ No newline at end of file From c364b9414cd9ce3843746a0ebe88ce6bd9983f9c Mon Sep 17 00:00:00 2001 From: Renata Date: Wed, 5 Mar 2025 02:16:49 -0500 Subject: [PATCH 12/16] docs: performance testing docs --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 2 + docs/developers/benchmark-testing.md | 3 +- .../performance-regression-tests.md | 39 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 docs/developers/performance-regression-tests.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a25471f..adb07caa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: env: ARTIFACT_NAME: benchmark-results${{ matrix.python-version }} - BRANCH_NAME: 'renatav/performance-tests' + BRANCH_NAME: 'renatav/performance-tests' # needs to be changed to master before merging BENCHMARKS_DIR: '/home/runner/work/taf/taf/.benchmarks/' ARTIFACT_PATH: '/home/runner/work/taf/taf/.benchmarks/archive.zip' diff --git a/CHANGELOG.md b/CHANGELOG.md index c1c0c11c..6848b38d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning][semver]. ### Added +- Integrate benchmark tests into CI ([597]) - Add Commitish model ([596]) - Aadd tests for part of the git module ([596]) - Implement adaptive timeout for Git clone operations ([592]) @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning][semver]. - Ensure that every target role has a delegations attribute [(595)] - Fix wrong default branch assignment for target repos [(593)] +[597]: https://github.com/openlawlibrary/taf/pull/597 [596]: https://github.com/openlawlibrary/taf/pull/596 [595]: https://github.com/openlawlibrary/taf/pull/595 [593]: https://github.com/openlawlibrary/taf/pull/593 diff --git a/docs/developers/benchmark-testing.md b/docs/developers/benchmark-testing.md index fef29a2b..b88e6d91 100644 --- a/docs/developers/benchmark-testing.md +++ b/docs/developers/benchmark-testing.md @@ -20,5 +20,4 @@ If you want to cause failure in the case of regression, also add "–-benchmark- To compare two already stored results, the standalone command "pytest-benchmark compare" followed by the paths of the files you wish to compare can do that. Unlike the previous method, these files do not need to be stored in any specific folder. - -For further options, see https://pytest-benchmark.readthedocs.io/en/latest/usage.html \ No newline at end of file +For further options, see https://pytest-benchmark.readthedocs.io/en/latest/usage.html diff --git a/docs/developers/performance-regression-tests.md b/docs/developers/performance-regression-tests.md new file mode 100644 index 00000000..a28d2745 --- /dev/null +++ b/docs/developers/performance-regression-tests.md @@ -0,0 +1,39 @@ +# CI Setup and Benchmark Testing Documentation + +## Overview + +This document outlines the Continuous Integration (CI) process implemented to facilitate dynamic benchmark testing based on configurable thresholds. The process involves a JSON configuration file that allows branch-specific benchmark thresholds to be set, enabling more precise control over performance testing. + +## CI Process Description + +The CI workflow is designed to perform benchmark tests on direct pushes to branches. It uses `pytest` with benchmark plugins to assess performance regressions or improvements. To accommodate varying performance expectations across features or branches, a JSON file named `benchmark_thresholds.json` is used to specify custom benchmark thresholds. + +## How Benchmark Testing Works + +Benchmark tests are run using `pytest` configured to only execute benchmark-related tests: + +- **Flag `--benchmark-only`**: Restricts `pytest` to run only benchmark tests. +- **Flag `--benchmark-json`**: Outputs the results to a JSON file, allowing for subsequent comparisons. +- **Flag `--benchmark-compare`**: Compares current benchmarks against the last saved benchmarks. +- **Flag `--benchmark-compare-fail`**: Specifies a threshold for test failure; if performance degrades beyond this percentage, the test fails. + +These benchmarks help ensure that new code does not introduce significant performance regressions. + +## Setting Benchmark Thresholds + +The benchmark threshold can be defined for each branch to accommodate specific performance criteria associated with different types of work (e.g., feature development, bug fixes). This threshold is set in the `benchmark_thresholds.json` file. If not set, threshold is set to 10 by default. + +### Example `benchmark_thresholds.json`: + +```json +{ + "feature/cool-feature": 20, + "bugfix/urgent-fix": 15 +} +``` + +## Artifact Handling + +- **Uploading Artifacts**: After benchmarks are run, the resulting `0001_output.json` file is uploaded as an artifact to GitHub Actions, allowing it to be used in future benchmark comparisons. +- **Downloading Artifacts**: In subsequent CI runs, the previously saved benchmark artifact (`0001_output.json`) for the master branch is downloaded before tests are run. + From 75bbfeb0c0ff74c119e8118a30e92ba627e99417 Mon Sep 17 00:00:00 2001 From: Renata Date: Wed, 5 Mar 2025 02:24:52 -0500 Subject: [PATCH 13/16] devops: regenerate artifacts every 90 days --- .github/workflows/scheduled.yml | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/scheduled.yml diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml new file mode 100644 index 00000000..73e6555c --- /dev/null +++ b/.github/workflows/scheduled.yml @@ -0,0 +1,38 @@ +name: Scheduled Benchmark Tests + +on: + schedule: + # Runs at 00:00 UTC every 90 days + - cron: '0 0 */90 * *' + +jobs: + run_tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, 3.10, 3.11, 3.12] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-benchmark + # Add any other dependencies your project requires + + - name: Run Benchmark Tests + run: | + python -m pytest --benchmark-only --benchmark-json=0001_output.json + + - name: Upload Benchmark Results + uses: actions/upload-artifact@v2 + with: + name: benchmark-results-${{ matrix.python-version }} + path: 0001_output.json From a648792cbae11f5ad342852c040a3cfcd477503e Mon Sep 17 00:00:00 2001 From: Renata Date: Wed, 5 Mar 2025 07:37:54 -0500 Subject: [PATCH 14/16] fix, chore: fix the scheduled build script, cleanup --- .github/workflows/ci.yml | 31 +++++++++++++++++++------------ .github/workflows/scheduled.yml | 27 ++++++++++++++++++--------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adb07caa..46add00d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,5 @@ name: CI -permissions: - contents: read - actions: read - on: push: {} pull_request: @@ -38,7 +34,7 @@ jobs: env: ARTIFACT_NAME: benchmark-results${{ matrix.python-version }} - BRANCH_NAME: 'renatav/performance-tests' # needs to be changed to master before merging + BRANCH_NAME: 'master' # needs to be changed to master before merging BENCHMARKS_DIR: '/home/runner/work/taf/taf/.benchmarks/' ARTIFACT_PATH: '/home/runner/work/taf/taf/.benchmarks/archive.zip' @@ -61,18 +57,22 @@ jobs: mkdir -p ~/bin/ && ln -s /usr/bin/swig3.0 ~/bin/swig && export PATH=~/bin/:$PATH pip install wheel # Ensure wheel is installed pip install -e .[ci,test,yubikey] - npm install axios - name: Setup GitHub user run: | git config --global user.name oll-bot git config --global user.email developers@openlawlib.org + - name: Run pre-commit and test with pytest + run: | + pre-commit run --all-files + pytest taf/tests + - name: Set Benchmark Threshold from File run: | - BRANCH_NAME=${GITHUB_REF#refs/heads/} + CURRENT_BRANCH=${GITHUB_REF#refs/heads/} - THRESHOLD=$(jq --arg branch "$BRANCH_NAME" '.[$branch]' ./benchmark_thresholds.json) + THRESHOLD=$(jq --arg branch "$CURRENT_BRANCH" '.[$branch]' ./benchmark_thresholds.json) if [ "$THRESHOLD" == "null" ]; then THRESHOLD=10 # Default threshold if not specified @@ -85,7 +85,7 @@ jobs: - name: Fetch the latest artifact by name and branch id: get_artifact - uses: actions/github-script@v6 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #@v7.01 with: script: | const artifacts = await github.rest.actions.listArtifactsForRepo({ @@ -114,11 +114,11 @@ jobs: core.setOutput('artifact_id', latestArtifact.id.toString()); } else { console.log('No matching artifacts found.'); - core.setFailed('No matching artifacts found.'); + core.setOutput('artifact_id', ''); } - name: Download the artifact by ID - uses: actions/github-script@v6 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #@v7.01 if: steps.get_artifact.outputs.artifact_id with: script: | @@ -148,6 +148,7 @@ jobs: run: unzip $ARTIFACT_PATH -d $BENCHMARKS_DIR - name: List extracted files + if: steps.get_artifact.outputs.artifact_id run: ls -l $BENCHMARKS_DIR - name: Run benchmark with comparison @@ -160,8 +161,14 @@ jobs: run: | python -m pytest --benchmark-only --benchmark-json=0001_output.json + # Use a step to extract the branch name from github.ref + - name: Extract Branch Name + id: extract_branch + run: echo "::set-output name=branch::$(echo ${GITHUB_REF#refs/heads/})" + - name: Archive benchmark results - uses: actions/upload-artifact@v4 + if: steps.extract_branch.outputs.branch == env.BRANCH_NAME + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 #@v4.6.1 with: name: ${{ env.ARTIFACT_NAME }} path: 0001_output.json diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 73e6555c..93ae6164 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -6,33 +6,42 @@ on: - cron: '0 0 */90 * *' jobs: - run_tests: + upload_benchmark_artifacts: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, 3.10, 3.11, 3.12] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - - name: Checkout code - uses: actions/checkout@v3 + - name: Checkout default branch + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #@v5 with: python-version: ${{ matrix.python-version }} + - name: Upgrade pip, setuptools, and wheel + run: | + pip install --upgrade pip setuptools wheel + - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytest pytest-benchmark - # Add any other dependencies your project requires + sudo apt-get update + pip install wheel # Ensure wheel is installed + pip install -e .[ci,test] + + - name: Setup GitHub user + run: | + git config --global user.name oll-bot + git config --global user.email developers@openlawlib.org - name: Run Benchmark Tests run: | python -m pytest --benchmark-only --benchmark-json=0001_output.json - name: Upload Benchmark Results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 #@v4.6.1 with: name: benchmark-results-${{ matrix.python-version }} path: 0001_output.json From a7f863fdceb5dd2ee0abf04c7bfedaa52f1363b3 Mon Sep 17 00:00:00 2001 From: Renata Date: Wed, 12 Mar 2025 01:28:52 -0400 Subject: [PATCH 15/16] devops, docs: move all docs to existing document, move fetch and download code to a separate js file --- .github/workflows/artifactsActions.js | 57 ++++++++++++++++ .github/workflows/ci.yml | 66 +++---------------- docs/developers/benchmark-testing.md | 34 ++++++++++ .../performance-regression-tests.md | 39 ----------- 4 files changed, 101 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/artifactsActions.js delete mode 100644 docs/developers/performance-regression-tests.md diff --git a/.github/workflows/artifactsActions.js b/.github/workflows/artifactsActions.js new file mode 100644 index 00000000..c5f93d0f --- /dev/null +++ b/.github/workflows/artifactsActions.js @@ -0,0 +1,57 @@ +const fs = require('fs'); + +async function fetchArtifact(core, github, repo) { + const artifacts = await github.rest.actions.listArtifactsForRepo({ + owner: repo.owner, + repo: repo.repo, + }); + const filteredArtifacts = artifacts.data.artifacts.filter(artifact => artifact.name === process.env.ARTIFACT_NAME); + let latestArtifact = null; + + for (const artifact of filteredArtifacts) { + const run = await github.rest.actions.getWorkflowRun({ + owner: repo.owner, + repo: repo.repo, + run_id: artifact.workflow_run.id, + }); + + if (run.data.head_branch === process.env.BRANCH_NAME) { + if (!latestArtifact || new Date(artifact.created_at) > new Date(latestArtifact.created_at)) { + latestArtifact = artifact; + } + } + } + + if (latestArtifact) { + console.log(`Found latest artifact: ${latestArtifact.id}`); + core.setOutput('artifact_id', latestArtifact.id.toString()); + return { artifactId: latestArtifact.id.toString()}; + } else { + console.log('No matching artifacts found.'); + core.setOutput('artifact_id', ''); + return { artifactId: '' }; + } +} + +async function downloadArtifact(github, repo, artifactId) { + const benchmarksDir = process.env.BENCHMARKS_DIR; + const artifactPath = process.env.ARTIFACT_PATH; + + const download = await github.rest.actions.downloadArtifact({ + owner: repo.owner, + repo: repo.repo, + artifact_id: artifactId, + archive_format: 'zip', + }); + + if (!fs.existsSync(benchmarksDir)) { + fs.mkdirSync(benchmarksDir, { recursive: true }); + } + fs.writeFileSync(artifactPath, Buffer.from(download.data)); + console.log('Artifact downloaded:', artifactPath); +} + +module.exports = { + fetchArtifact, + downloadArtifact +}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46add00d..fbc53bfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: env: ARTIFACT_NAME: benchmark-results${{ matrix.python-version }} - BRANCH_NAME: 'master' # needs to be changed to master before merging + BRANCH_NAME: 'master' BENCHMARKS_DIR: '/home/runner/work/taf/taf/.benchmarks/' ARTIFACT_PATH: '/home/runner/work/taf/taf/.benchmarks/archive.zip' @@ -63,10 +63,10 @@ jobs: git config --global user.name oll-bot git config --global user.email developers@openlawlib.org - - name: Run pre-commit and test with pytest - run: | - pre-commit run --all-files - pytest taf/tests + # - name: Run pre-commit and test with pytest + # run: | + # pre-commit run --all-files + # pytest taf/tests - name: Set Benchmark Threshold from File run: | @@ -88,60 +88,14 @@ jobs: uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #@v7.01 with: script: | - const artifacts = await github.rest.actions.listArtifactsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - }); - const filteredArtifacts = artifacts.data.artifacts.filter(artifact => artifact.name === process.env.ARTIFACT_NAME); - let latestArtifact = null; - - for (const artifact of filteredArtifacts) { - const run = await github.rest.actions.getWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: artifact.workflow_run.id, - }); - - if (run.data.head_branch === process.env.BRANCH_NAME) { - if (!latestArtifact || new Date(artifact.created_at) > new Date(latestArtifact.created_at)) { - latestArtifact = artifact; - } - } - } - - if (latestArtifact) { - console.log(`Found latest artifact: ${latestArtifact.id}`); - core.setOutput('artifact_id', latestArtifact.id.toString()); + const { fetchArtifact, downloadArtifact } = require('./.github/workflows/artifactsActions.js'); + const result = await fetchArtifact(core, github, context.repo); + if (result && result.artifactId) { + downloadArtifact(github, context.repo, result.artifactId); } else { - console.log('No matching artifacts found.'); - core.setOutput('artifact_id', ''); + console.log('No artifact ID available for download'); } - - name: Download the artifact by ID - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #@v7.01 - if: steps.get_artifact.outputs.artifact_id - with: - script: | - const fs = require('fs'); - const path = require('path'); - const artifactId = '${{ steps.get_artifact.outputs.artifact_id }}'; - console.log(`Downloading artifact ID: ${artifactId}`); - const benchmarksDir = process.env.BENCHMARKS_DIR - const artifactPath = process.env.ARTIFACT_PATH - - // First, get the URL to download the artifact - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifactId, - archive_format: 'zip', - }); - - if (!fs.existsSync(benchmarksDir)) { - fs.mkdirSync(benchmarksDir, { recursive: true }); - } - fs.writeFileSync(artifactPath, Buffer.from(download.data)); - console.log('Artifact downloaded:', artifactPath); - name: 'Unzip artifact' if: steps.get_artifact.outputs.artifact_id diff --git a/docs/developers/benchmark-testing.md b/docs/developers/benchmark-testing.md index b88e6d91..56dffe3f 100644 --- a/docs/developers/benchmark-testing.md +++ b/docs/developers/benchmark-testing.md @@ -21,3 +21,37 @@ If you want to cause failure in the case of regression, also add "–-benchmark- To compare two already stored results, the standalone command "pytest-benchmark compare" followed by the paths of the files you wish to compare can do that. Unlike the previous method, these files do not need to be stored in any specific folder. For further options, see https://pytest-benchmark.readthedocs.io/en/latest/usage.html + + +## CI Integration + +The CI workflow is designed to perform benchmark tests on direct pushes to branches. It uses `pytest` with benchmark plugins to assess performance regressions or improvements. To accommodate varying performance expectations across features or branches, a JSON file named `benchmark_thresholds.json` is used to specify custom benchmark thresholds. + +## How Benchmark Testing Works + +Benchmark tests are run using `pytest` configured to only execute benchmark-related tests: + +- **Flag `--benchmark-only`**: Restricts `pytest` to run only benchmark tests. +- **Flag `--benchmark-json`**: Outputs the results to a JSON file, allowing for subsequent comparisons. +- **Flag `--benchmark-compare`**: Compares current benchmarks against the last saved benchmarks. +- **Flag `--benchmark-compare-fail`**: Specifies a threshold for test failure; if performance degrades beyond this percentage, the test fails. + +These benchmarks help ensure that new code does not introduce significant performance regressions. + +## Setting Benchmark Thresholds + +The benchmark threshold can be defined for each branch to accommodate specific performance criteria associated with different types of work (e.g., feature development, bug fixes). This threshold is set in the `benchmark_thresholds.json` file. If not set, threshold is set to 10 (10%) by default. + +### Example `benchmark_thresholds.json`: + +```json +{ + "feature/cool-feature": 20, + "bugfix/urgent-fix": 15 +} +``` + +## Artifact Handling + +- **Uploading Artifacts**: After benchmarks are run, the resulting `0001_output.json` file is uploaded as an artifact to GitHub Actions, allowing it to be used in future benchmark comparisons. +- **Downloading Artifacts**: In subsequent CI runs, the previously saved benchmark artifact (`0001_output.json`) for the master branch is downloaded before tests are run. diff --git a/docs/developers/performance-regression-tests.md b/docs/developers/performance-regression-tests.md deleted file mode 100644 index a28d2745..00000000 --- a/docs/developers/performance-regression-tests.md +++ /dev/null @@ -1,39 +0,0 @@ -# CI Setup and Benchmark Testing Documentation - -## Overview - -This document outlines the Continuous Integration (CI) process implemented to facilitate dynamic benchmark testing based on configurable thresholds. The process involves a JSON configuration file that allows branch-specific benchmark thresholds to be set, enabling more precise control over performance testing. - -## CI Process Description - -The CI workflow is designed to perform benchmark tests on direct pushes to branches. It uses `pytest` with benchmark plugins to assess performance regressions or improvements. To accommodate varying performance expectations across features or branches, a JSON file named `benchmark_thresholds.json` is used to specify custom benchmark thresholds. - -## How Benchmark Testing Works - -Benchmark tests are run using `pytest` configured to only execute benchmark-related tests: - -- **Flag `--benchmark-only`**: Restricts `pytest` to run only benchmark tests. -- **Flag `--benchmark-json`**: Outputs the results to a JSON file, allowing for subsequent comparisons. -- **Flag `--benchmark-compare`**: Compares current benchmarks against the last saved benchmarks. -- **Flag `--benchmark-compare-fail`**: Specifies a threshold for test failure; if performance degrades beyond this percentage, the test fails. - -These benchmarks help ensure that new code does not introduce significant performance regressions. - -## Setting Benchmark Thresholds - -The benchmark threshold can be defined for each branch to accommodate specific performance criteria associated with different types of work (e.g., feature development, bug fixes). This threshold is set in the `benchmark_thresholds.json` file. If not set, threshold is set to 10 by default. - -### Example `benchmark_thresholds.json`: - -```json -{ - "feature/cool-feature": 20, - "bugfix/urgent-fix": 15 -} -``` - -## Artifact Handling - -- **Uploading Artifacts**: After benchmarks are run, the resulting `0001_output.json` file is uploaded as an artifact to GitHub Actions, allowing it to be used in future benchmark comparisons. -- **Downloading Artifacts**: In subsequent CI runs, the previously saved benchmark artifact (`0001_output.json`) for the master branch is downloaded before tests are run. - From abb7d3a8a36bf9d78bc723d6850190489428cb35 Mon Sep 17 00:00:00 2001 From: Renata Vaderna Date: Thu, 13 Mar 2025 09:08:39 -0400 Subject: [PATCH 16/16] chore: uncomment running tests step, previously commented out to speed up development --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbc53bfb..6d70ae2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,10 +63,10 @@ jobs: git config --global user.name oll-bot git config --global user.email developers@openlawlib.org - # - name: Run pre-commit and test with pytest - # run: | - # pre-commit run --all-files - # pytest taf/tests + - name: Run pre-commit and test with pytest + run: | + pre-commit run --all-files + pytest taf/tests - name: Set Benchmark Threshold from File run: |