move cache uploading to separate workflow

.github/actions/checkout-gcc-hash/action.yaml

@@ -0,0 +1,33 @@
+name: "Checkout gcc hash"
+description: "Checks out gcc hash with retries"
+inputs:
+  hash:
+    description: 'hash to checkout to'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+      - name: Checkout hash
+        shell: bash
+        working-directory: ./riscv-gnu-toolchain
+        id: pull
+        run: |
+          cd gcc
+          git checkout master
+          git pull
+          git checkout ${{ inputs.hash }}
+        continue-on-error: true
+
+      - name: Sleep and retry
+        shell: bash
+        working-directory: ./riscv-gnu-toolchain
+        if: ${{ steps.pull.outcome == 'failure' }}
+        run: |
+          echo "Failed to checkout and pull gcc. Retrying in 1 min"
+          sleep 60
+          cd gcc
+          git checkout master
+          git pull
+          git checkout ${{ inputs.hash }}
+

.github/actions/common/download-comparison-artifacts/action.yaml

@@ -25,7 +25,11 @@ runs:
       if: ${{ !cancelled() && hashFiles(format('./current_logs/{0}', inputs.report-artifact-name)) == '' }}
       shell: bash
       run: |
-        pip install pygithub requests
+        python -m venv venv
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+        python -m pip install -U pyopenssl cryptography
+        python -m pip install pygithub==1.59.1 requests
         cd riscv-gnu-toolchain
         python ./scripts/download_artifact.py -name ${{ inputs.report-artifact-name }} -repo ${{ inputs.repo }} -token ${{ inputs.github-token }} -outdir current_logs
       continue-on-error: true
@@ -42,6 +46,7 @@ runs:
       if: ${{ !cancelled() && hashFiles(format('./current_logs/{0}', inputs.report-artifact-name)) == '' && hashFiles(format('./temp/{0}', inputs.binary-artifact-name)) == '' }}
       shell: bash
       run: |
+        source venv/bin/activate
         cd riscv-gnu-toolchain
         python ./scripts/download_artifact.py -name ${{ inputs.binary-artifact-name }} -repo ${{ inputs.repo }} -token ${{ inputs.github-token }} -outdir temp
       continue-on-error: true

.github/workflows/call-patchworks-api.yaml

@@ -20,14 +20,6 @@ jobs:
         with:
           free_up_space: false
 
-      - name: Bump pyopenssl and crypto
-        run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
-          pip install pygithub==1.59.1 requests
-
       - name: Get patch-name
         id: patch-name
         run: |
@@ -50,9 +42,18 @@ jobs:
           echo "check=$CHECK" >> $GITHUB_OUTPUT
           echo "context=$CONTEXT" >> $GITHUB_OUTPUT
 
+      - name: Setup python env
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
+
       - name: Download patch
         run: |
           mkdir temp
+          source venv/bin/activate
           python ./scripts/download_artifact.py -name ${{ steps.patch-name.outputs.patch_name }}-downloaded-patches -repo "ewlu/gcc-precommit-ci" -token ${{ secrets.GITHUB_TOKEN }} -outdir ./
 
       - name: Extract patch
@@ -64,7 +65,8 @@ jobs:
         id: patch-id
         run: |
           export PATCH_ID=$(tail -n 1 ./patches/${{ steps.patch-name.outputs.patch_name }})
-          echo $PATCH_ID
+          echo $PATCH_ID1.59.1 requests
+
           echo "patch_id=$PATCH_ID" >> $GITHUB_OUTPUT
 
     outputs:
@@ -89,13 +91,13 @@ jobs:
         with:
           free_up_space: false
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
-          pip install requests
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Report pass
         run: |
@@ -119,16 +121,17 @@ jobs:
         with:
           free_up_space: false
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
-          pip install requests
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Report Warning
         run: |
+          source venv/bin/activate
           python scripts/post_check_to_patchworks.py -event ${{ github.event_name }} -repo ewlu/gcc-precommit-ci -pid ${{ needs.get-patch-info.outputs.patch_id }} -desc '${{ needs.get-patch-info.outputs.check }} warning' -iid ${{ github.event.issue.number }} -state 'warning' -context '${{ needs.get-patch-info.outputs.context }}' -token ${{ secrets.PATCHWORK_API }}
 
 
@@ -148,13 +151,14 @@ jobs:
         with:
           free_up_space: false
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
-          pip install requests
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
+
 
       - name: Report Error
         run: |

.github/workflows/create-cache.yaml

@@ -0,0 +1,114 @@
+name: Create-Cache
+
+on:
+  schedule:
+    # Run at 1:07 on sundays
+    # https://stackoverflow.com/questions/59560214/github-action-works-on-push-but-not-scheduled
+    - cron: 7 1 * * 0
+
+jobs:
+  init-submodules:
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        working-directory: riscv-gnu-toolchain
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup env
+        uses: ./.github/actions/common/setup-env
+        with:
+          free_up_space: false
+
+      - name: Retrieve cache
+        id: retrieve-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            riscv-gnu-toolchain/.git
+            riscv-gnu-toolchain/binutils
+            riscv-gnu-toolchain/dejagnu
+            riscv-gnu-toolchain/gcc
+            riscv-gnu-toolchain/gdb
+            riscv-gnu-toolchain/glibc
+            riscv-gnu-toolchain/newlib
+            riscv-gnu-toolchain/qemu
+          key: submodules-archive-10 # Numbered archive to allow for easy transition when bumping submodules
+
+      - name: Initalize submodules cache
+        id: cache-init
+        if: steps.retrieve-cache.outputs.cache-hit != 'true'
+        run: |
+          git submodule update --init --recursive --depth 1 binutils
+          git submodule update --init --recursive --depth 1 dejagnu
+          git submodule update --init --recursive --depth 1 gdb
+          git submodule update --init --recursive --depth 1 glibc
+          git submodule update --init --recursive --depth 1 newlib
+          git submodule update --init --recursive --depth 1 qemu
+        continue-on-error: true
+
+      - name: Initalize submodules cache
+        if: steps.cache-init.outcome == 'failure'
+        run: |
+          echo "Failed to initialize cache submodules. Retrying in 1 min"
+          sleep 60
+          git submodule update --init --recursive --depth 1 binutils
+          git submodule update --init --recursive --depth 1 dejagnu
+          git submodule update --init --recursive --depth 1 gdb
+          git submodule update --init --recursive --depth 1 glibc
+          git submodule update --init --recursive --depth 1 newlib
+          git submodule update --init --recursive --depth 1 qemu
+
+      - name: Initialize gcc
+        if: steps.retrieve-cache.outputs.cache-hit != 'true'
+        id: gcc-cache
+        uses: ./.github/actions/common/init-and-pull-gcc
+        with:
+          init: true
+
+        # Does not remove and reclone gcc if we hit cache
+      - name: Checkout GCC
+        if: steps.gcc-cache.outcome == 'skipped'
+        uses: ./.github/actions/common/init-and-pull-gcc
+        with:
+          init: false
+
+      - name: Apply newlib fixups
+        if: steps.retrieve-cache.outputs.cache-hit != 'true'
+        run: |
+          cd newlib
+          git config --global user.email "github-bot@example.com"
+          git config --global user.name "Github Bot"
+          git am ../fixups/newlib/*.patch
+
+      - name: Cache submodules
+        if: steps.retrieve-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            riscv-gnu-toolchain/.git
+            riscv-gnu-toolchain/binutils
+            riscv-gnu-toolchain/dejagnu
+            riscv-gnu-toolchain/gcc
+            riscv-gnu-toolchain/gdb
+            riscv-gnu-toolchain/glibc
+            riscv-gnu-toolchain/newlib
+            riscv-gnu-toolchain/qemu
+          key: submodules-archive-10
+
+      - name: Make cache zip
+        run: |
+          zip -r cache.zip .git binutils dejagnu gcc gdb glibc newlib qemu
+
+      # Use artifact rather than cache since cache downloads are flaky/hang.
+      # Artifacts are reliable but ~30 min slower to set up.
+      # Setup is done on one runner, so this isn't a show stopper.
+      - name: Upload git cache
+        uses: actions/upload-artifact@v3
+        with:
+          name: gcc-sources
+          path: |
+            riscv-gnu-toolchain/cache.zip
+          retention-days: 9
+

.github/workflows/downtime-runner.yaml

@@ -26,18 +26,19 @@ jobs:
           sudo apt install python-is-python3 -y
           sudo apt install zip -y
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Download last successful run artifact
         run: |
-          pip install pygithub requests
           mkdir result
           mkdir temp
+          source venv/bin/activate
           python ./scripts/download_artifact.py -name "downtime_runner_success" -repo ewlu/gcc-precommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir result || true
           ls result
 
@@ -96,12 +97,13 @@ jobs:
           sudo apt install python-is-python3 -y
           sudo apt install zip -y
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip -y
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Get Timestamps
         run: |

.github/workflows/generate-precommit-summary.yaml

@@ -73,17 +73,18 @@ jobs:
           mkdir temp
           mkdir summaries
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
-          pip install pygithub==1.59.1 requests
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Download baseline sum files
         run: |
           # Download linux
+          source venv/bin/activate
           python ./scripts/download_artifact.py -name gcc-linux-rv64gcv-lp64d-${{ inputs.baseline_hash }}-multilib-sum-files -repo patrick-rivos/gcc-postcommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./temp
           python ./scripts/download_artifact.py -name gcc-linux-rv64gc_zba_zbb_zbc_zbs-lp64d-${{ inputs.baseline_hash }}-non-multilib-sum-files -repo patrick-rivos/gcc-postcommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./temp
           python ./scripts/download_artifact.py -name gcc-linux-rv32gc_zba_zbb_zbc_zbs-ilp32d-${{ inputs.baseline_hash }}-non-multilib-sum-files -repo patrick-rivos/gcc-postcommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./temp
@@ -246,14 +247,18 @@ jobs:
           name: ${{ needs.compare-artifacts.outputs.baseline_hash }}-previous-logs
           path: ./riscv-gnu-toolchain
 
+      - name: Setup python env
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
+
       - name: Download patches artifact
         if: ${{ needs.compare-artifacts.outputs.workflow_dispatch == 'true' }}
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography pygithub==1.59.1 requests
-          mkdir temp
+          source venv/bin/activate
           python ./scripts/download_artifact.py -name ${{ inputs.patch_name }}-downloaded-patches -repo ewlu/gcc-precommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./temp -repo ewlu/gcc-precommit-ci
           ls temp
           unzip temp/${{ inputs.patch_name }}-downloaded-patches

.github/workflows/issue-closer.yaml

@@ -28,12 +28,13 @@ jobs:
           sudo apt install python-is-python3 -y
           sudo apt install zip -y
 
-      - name: Bump pyopenssl and crypto
+      - name: Setup python env
         run: |
-          sudo apt remove python3-pip
-          wget https://bootstrap.pypa.io/get-pip.py
-          sudo python3 get-pip.py
-          pip install -U pyopenssl cryptography
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install -U pyopenssl cryptography
+          python -m pip install pygithub==1.59.1 requests
 
       - name: Close issues
         run: |