From b365b1cf9fbf00847d0d896a9e3eb116d737f4e3 Mon Sep 17 00:00:00 2001 From: Matthias Goerens Date: Fri, 21 Mar 2025 15:04:59 +0100 Subject: [PATCH] Cache python virtualenv Rely on GitHub cache to save/restore virtualenv instead of installing the python dependencies and scripts at the start of each job. Signed-off-by: Matthias Goerens --- .github/actions/install-ci-scripts/action.yml | 33 ++++++++++ .github/actions/setup-python/action.yml | 5 ++ .github/workflows/behave.yml | 14 ++-- .github/workflows/build.yml | 65 +++++++------------ .github/workflows/check-contributor.yml | 18 ++--- .../check-locks-on-owners-submission.yml | 13 ++-- .github/workflows/lock-sanity-check.yml | 13 ++-- .github/workflows/mercury_bot.yml | 13 ++-- .github/workflows/metrics.yml | 12 ++-- .github/workflows/owners.yml | 13 ++-- .github/workflows/python-test.yml | 13 ++-- .github/workflows/release.yml | 46 +++++-------- .github/workflows/test.yml | 13 ++-- 13 files changed, 127 insertions(+), 144 deletions(-) create mode 100644 .github/actions/install-ci-scripts/action.yml diff --git a/.github/actions/install-ci-scripts/action.yml b/.github/actions/install-ci-scripts/action.yml new file mode 100644 index 00000000..148e61fd --- /dev/null +++ b/.github/actions/install-ci-scripts/action.yml @@ -0,0 +1,33 @@ +name: Install our CI scripts +description: | + Handles installation and caching/restore of the CI scripts in a virtualenv. +inputs: + working-directory: + description: Basedir in which to save / restore cache and install venv. + default: './' + required: false + python-version: + description: Installed Python version + required: true +runs: + using: composite + steps: + # This attempts to restore the virtualenv from a GitHub cache with the corresponding cache key. + # On cache-miss, this automatically adds a post step that creates a cache from this path and using this cache key. + - name: Cache virtualenv + id: cache + uses: actions/cache@v4 + with: + key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('scripts/requirements.txt') }} + path: ${{ inputs.working-directory }}/ve1 + + - name: Setup venv on cache-miss + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + # set up python + python3 -m venv ve1 + cd scripts + ../ve1/bin/pip3 install -r requirements.txt + ../ve1/bin/pip3 install . diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml index 0b312a53..ca4aef6f 100644 --- a/.github/actions/setup-python/action.yml +++ b/.github/actions/setup-python/action.yml @@ -5,9 +5,14 @@ description: | Serves as the single place to update python versions over time across the project. +outputs: + python-version: + description: Version of Python installed + value: ${{ steps.setup-python.outputs.python-version }} runs: using: composite steps: - uses: actions/setup-python@v5 + id: setup-python with: python-version: '3.10' diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index 285cf3cf..6f12dd54 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -116,17 +116,13 @@ jobs: fetch-depth: ${{ inputs.checkout-fetch-depth }} - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up CI scripts - run: | - # set up python scripts - echo "set up python script in $PWD" - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} # Pull request numbers are included in generated chart names in E2E, so it's included # as an environment variable which E2E consumes. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95b364ee..0640235d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,16 +29,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Check for CI changes id: check_ci_changes @@ -128,16 +125,13 @@ jobs: path: "pr-branch" - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Extract PR information id: validate-submission @@ -217,16 +211,13 @@ jobs: path: "pr-branch" - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Download submission information uses: actions/download-artifact@v4 @@ -410,16 +401,13 @@ jobs: path: "pr-branch" - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} # Submission information should always be present - name: Download submission information @@ -537,16 +525,13 @@ jobs: path: "pr-branch" - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: install chart verifier for action uses: redhat-actions/openshift-tools-installer@v1 diff --git a/.github/workflows/check-contributor.yml b/.github/workflows/check-contributor.yml index fe4b78cf..d8d51a69 100644 --- a/.github/workflows/check-contributor.yml +++ b/.github/workflows/check-contributor.yml @@ -43,18 +43,14 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - - name: Install CI Scripts - run: | - # set up python scripts - echo "set up python script in $PWD" - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. - + + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} + - name: Check contributor # The return code from this script is what's important in this workflow. id: check-contributor diff --git a/.github/workflows/check-locks-on-owners-submission.yml b/.github/workflows/check-locks-on-owners-submission.yml index b7e928a3..7f94c3f0 100644 --- a/.github/workflows/check-locks-on-owners-submission.yml +++ b/.github/workflows/check-locks-on-owners-submission.yml @@ -30,16 +30,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: get files changed id: get_files_changed diff --git a/.github/workflows/lock-sanity-check.yml b/.github/workflows/lock-sanity-check.yml index 5d499df3..96d2a6f6 100644 --- a/.github/workflows/lock-sanity-check.yml +++ b/.github/workflows/lock-sanity-check.yml @@ -19,15 +19,12 @@ jobs: - name: checkout uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: generate chart locks id: generate uses: ./.github/actions/generate-chart-locks diff --git a/.github/workflows/mercury_bot.yml b/.github/workflows/mercury_bot.yml index 38f14b79..b7085f9a 100644 --- a/.github/workflows/mercury_bot.yml +++ b/.github/workflows/mercury_bot.yml @@ -25,16 +25,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: get files changed id: get_files_changed diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index d9fdf687..fdb928be 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -24,15 +24,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Release Metrics id: release_metrics diff --git a/.github/workflows/owners.yml b/.github/workflows/owners.yml index 0c9a7adf..c43bdf7d 100644 --- a/.github/workflows/owners.yml +++ b/.github/workflows/owners.yml @@ -17,16 +17,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - uses: jitterbit/get-changed-files@v1 id: filesChangedOrModified diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 8d0c5c52..bedd5552 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,15 +13,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Run pytest run: | ./ve1/bin/pytest scripts/src/submission/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 481dd38e..da41142e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,18 +43,14 @@ jobs: path: "pr-branch" - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python scripts in pr-branch - working-directory: ./pr-branch - run: | - # set up python scripts - echo "set up python script in $PWD" - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} + working-directory: ./pr-branch - name: Check if dev repo is targetted, only release file is in PR and user is authorized id: check_only_version_in_PR_and_authorized @@ -174,18 +170,14 @@ jobs: token: ${{ secrets.BOT_TOKEN }} path: "stage-repo" - - name: Set up Python 3.x Part 1 + - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up CI scripts - run: | - # set up python scripts - echo "set up python script in $PWD" - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Check if version updated id: check_version_updated @@ -282,17 +274,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up CI scripts - run: | - # set up python scripts - echo "set up python script in $PWD" - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Approve PR id: approve_pr diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63c76799..76b4142e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,16 +37,13 @@ jobs: fetch-depth: 0 - name: Set up Python + id: setup-python uses: ./.github/actions/setup-python - - name: Set up Python 3.x Part 2 - run: | - # set up python requirements and scripts - python3 -m venv ve1 - cd scripts - ../ve1/bin/pip3 install -r requirements.txt - ../ve1/bin/pip3 install . - cd .. + - name: Install CI scripts + uses: ./.github/actions/install-ci-scripts + with: + python-version: ${{ steps.setup-python.outputs.python-version }} - name: Check Request id: check_request