Performance tests

.github/workflows/artifactsActions.js

@@ -0,0 +1,57 @@
+const fs = require('fs');
+
+async function fetchArtifact(core, github, repo) {
+  const artifacts = await github.rest.actions.listArtifactsForRepo({
+    owner: repo.owner,
+    repo: repo.repo,
+  });
+  const filteredArtifacts = artifacts.data.artifacts.filter(artifact => artifact.name === process.env.ARTIFACT_NAME);
+  let latestArtifact = null;
+
+  for (const artifact of filteredArtifacts) {
+    const run = await github.rest.actions.getWorkflowRun({
+      owner: repo.owner,
+      repo: repo.repo,
+      run_id: artifact.workflow_run.id,
+    });
+
+    if (run.data.head_branch === process.env.BRANCH_NAME) {
+      if (!latestArtifact || new Date(artifact.created_at) > new Date(latestArtifact.created_at)) {
+        latestArtifact = artifact;
+      }
+    }
+  }
+
+  if (latestArtifact) {
+    console.log(`Found latest artifact: ${latestArtifact.id}`);
+    core.setOutput('artifact_id', latestArtifact.id.toString());
+    return { artifactId: latestArtifact.id.toString()};
+  } else {
+    console.log('No matching artifacts found.');
+    core.setOutput('artifact_id', '');
+    return { artifactId: '' };
+  }
+}
+
+async function downloadArtifact(github, repo, artifactId) {
+  const benchmarksDir = process.env.BENCHMARKS_DIR;
+  const artifactPath = process.env.ARTIFACT_PATH;
+
+  const download = await github.rest.actions.downloadArtifact({
+    owner: repo.owner,
+    repo: repo.repo,
+    artifact_id: artifactId,
+    archive_format: 'zip',
+  });
+
+  if (!fs.existsSync(benchmarksDir)) {
+    fs.mkdirSync(benchmarksDir, { recursive: true });
+  }
+  fs.writeFileSync(artifactPath, Buffer.from(download.data));
+  console.log('Artifact downloaded:', artifactPath);
+}
+
+module.exports = {
+    fetchArtifact,
+    downloadArtifact
+};

.github/workflows/ci.yml

@@ -31,6 +31,13 @@ jobs:
     strategy:
       matrix:
         python-version: ${{ fromJSON(needs.set_python_versions.outputs.all_versions) }}
+
+    env:
+      ARTIFACT_NAME: benchmark-results${{ matrix.python-version }}
+      BRANCH_NAME: 'master'
+      BENCHMARKS_DIR: '/home/runner/work/taf/taf/.benchmarks/'
+      ARTIFACT_PATH: '/home/runner/work/taf/taf/.benchmarks/archive.zip'
+
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #@v4
 
@@ -61,6 +68,65 @@ jobs:
           pre-commit run --all-files
           pytest taf/tests
 
+      - name: Set Benchmark Threshold from File
+        run: |
+          CURRENT_BRANCH=${GITHUB_REF#refs/heads/}
+
+          THRESHOLD=$(jq --arg branch "$CURRENT_BRANCH" '.[$branch]' ./benchmark_thresholds.json)
+
+          if [ "$THRESHOLD" == "null" ]; then
+            THRESHOLD=10  # Default threshold if not specified
+          fi
+          echo "BENCHMARK_THRESHOLD=$THRESHOLD" >> $GITHUB_ENV
+          echo "Using benchmark threshold: $THRESHOLD"
+
+      - name: Display Benchmark Threshold
+        run: echo "Benchmark Threshold is $BENCHMARK_THRESHOLD"
+
+      - name: Fetch the latest artifact by name and branch
+        id: get_artifact
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #@v7.01
+        with:
+          script: |
+            const { fetchArtifact, downloadArtifact } = require('./.github/workflows/artifactsActions.js');
+            const result = await fetchArtifact(core, github, context.repo);
+            if (result && result.artifactId) {
+              downloadArtifact(github, context.repo, result.artifactId);
+            } else {
+              console.log('No artifact ID available for download');
+            }
+
+
+      - name: 'Unzip artifact'
+        if: steps.get_artifact.outputs.artifact_id
+        run: unzip $ARTIFACT_PATH -d $BENCHMARKS_DIR
+
+      - name: List extracted files
+        if: steps.get_artifact.outputs.artifact_id
+        run: ls -l $BENCHMARKS_DIR
+
+      - name: Run benchmark with comparison
+        if: steps.get_artifact.outputs.artifact_id
+        run: |
+          python -m pytest --benchmark-only --benchmark-json=0001_output.json --benchmark-compare --benchmark-compare-fail=mean:${BENCHMARK_THRESHOLD}%
+
+      - name: Run benchmark without comparison
+        if: steps.get_artifact.outputs.artifact_id == '' || steps.get_artifact.outputs.artifact_id == null
+        run: |
+          python -m pytest --benchmark-only --benchmark-json=0001_output.json
+
+      # Use a step to extract the branch name from github.ref
+      - name: Extract Branch Name
+        id: extract_branch
+        run: echo "::set-output name=branch::$(echo ${GITHUB_REF#refs/heads/})"
+
+      - name: Archive benchmark results
+        if: steps.extract_branch.outputs.branch == env.BRANCH_NAME
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 #@v4.6.1
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: 0001_output.json
+
   build_and_upload_wheel:
     runs-on: ubuntu-latest
     needs: [set_python_versions, run_tests]

.github/workflows/scheduled.yml

@@ -0,0 +1,47 @@
+name: Scheduled Benchmark Tests
+
+on:
+  schedule:
+    # Runs at 00:00 UTC every 90 days
+    - cron: '0 0 */90 * *'
+
+jobs:
+  upload_benchmark_artifacts:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:  ['3.8', '3.9', '3.10', '3.11', '3.12']
+
+    steps:
+      - name: Checkout default branch
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip, setuptools, and wheel
+        run: |
+          pip install --upgrade pip setuptools wheel
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          pip install wheel  # Ensure wheel is installed
+          pip install -e .[ci,test]
+
+      - name: Setup GitHub user
+        run: |
+          git config --global user.name oll-bot
+          git config --global user.email developers@openlawlib.org
+
+      - name: Run Benchmark Tests
+        run: |
+          python -m pytest --benchmark-only --benchmark-json=0001_output.json
+
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 #@v4.6.1
+        with:
+          name: benchmark-results-${{ matrix.python-version }}
+          path: 0001_output.json

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning][semver].
 ### Added
 
 - Implement get all auth repos logic ([599])
+- Integrate benchmark tests into CI ([597])
 - Add Commitish model ([596])
 - Aadd tests for part of the git module ([596])
 - Implement adaptive timeout for Git clone operations ([592])
@@ -30,9 +31,9 @@ and this project adheres to [Semantic Versioning][semver].
 - Ensure that every target role has a delegations attribute [(595)]
 - Fix wrong default branch assignment for target repos [(593)]
 
-
 [603]: https://github.com/openlawlibrary/taf/pull/603
 [599]: https://github.com/openlawlibrary/taf/pull/599
+[597]: https://github.com/openlawlibrary/taf/pull/597
 [596]: https://github.com/openlawlibrary/taf/pull/596
 [595]: https://github.com/openlawlibrary/taf/pull/595
 [593]: https://github.com/openlawlibrary/taf/pull/593

benchmark_thresholds.json

@@ -0,0 +1,3 @@
+{
+    "renatav/performance-tests": 15
+}

docs/developers/benchmark-testing.md

@@ -20,5 +20,38 @@ If you want to cause failure in the case of regression, also add "–-benchmark-
 
 To compare two already stored results, the standalone command "pytest-benchmark compare" followed by the paths of the files you wish to compare can do that. Unlike the previous method, these files do not need to be stored in any specific folder.
 
+For further options, see https://pytest-benchmark.readthedocs.io/en/latest/usage.html
 
-For further options, see https://pytest-benchmark.readthedocs.io/en/latest/usage.html
+
+## CI Integration
+
+The CI workflow is designed to perform benchmark tests on direct pushes to branches. It uses `pytest` with benchmark plugins to assess performance regressions or improvements. To accommodate varying performance expectations across features or branches, a JSON file named `benchmark_thresholds.json` is used to specify custom benchmark thresholds.
+
+## How Benchmark Testing Works
+
+Benchmark tests are run using `pytest` configured to only execute benchmark-related tests:
+
+- **Flag `--benchmark-only`**: Restricts `pytest` to run only benchmark tests.
+- **Flag `--benchmark-json`**: Outputs the results to a JSON file, allowing for subsequent comparisons.
+- **Flag `--benchmark-compare`**: Compares current benchmarks against the last saved benchmarks.
+- **Flag `--benchmark-compare-fail`**: Specifies a threshold for test failure; if performance degrades beyond this percentage, the test fails.
+
+These benchmarks help ensure that new code does not introduce significant performance regressions.
+
+## Setting Benchmark Thresholds
+
+The benchmark threshold can be defined for each branch to accommodate specific performance criteria associated with different types of work (e.g., feature development, bug fixes). This threshold is set in the `benchmark_thresholds.json` file. If not set, threshold is set to 10 (10%) by default.
+
+### Example `benchmark_thresholds.json`:
+
+```json
+{
+  "feature/cool-feature": 20,
+  "bugfix/urgent-fix": 15
+}
+```
+
+## Artifact Handling
+
+- **Uploading Artifacts**: After benchmarks are run, the resulting `0001_output.json` file is uploaded as an artifact to GitHub Actions, allowing it to be used in future benchmark comparisons.
+- **Downloading Artifacts**: In subsequent CI runs, the previously saved benchmark artifact (`0001_output.json`) for the master branch is downloaded before tests are run.

taf/tests/test_updater/test_update_benchmarking.py

@@ -4,7 +4,6 @@
 )
 
 
-@pytest.mark.skip(reason="benchmarking disabled for time being")
 @pytest.mark.parametrize(
     "origin_auth_repo",
     [