Skip to content

Commit b01df0f

Browse files
ewluewlu
committed
move cache uploading to separate workflow
1 parent cdcf2c9 commit b01df0f

File tree

7 files changed

+306
-46
lines changed

7 files changed

+306
-46
lines changed

‎.github/actions/checkout-gcc-hash/action.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: "Checkout gcc hash"
2+
description: "Checks out gcc hash with retries"
3+
inputs:
4+
hash:
5+
description: 'hash to checkout to'
6+
required: true
7+
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Checkout hash
12+
shell: bash
13+
working-directory: ./riscv-gnu-toolchain
14+
id: pull
15+
run: |
16+
cd gcc
17+
git checkout master
18+
git pull
19+
git checkout ${{ inputs.hash }}
20+
continue-on-error: true
21+
22+
- name: Sleep and retry
23+
shell: bash
24+
working-directory: ./riscv-gnu-toolchain
25+
if: ${{ steps.pull.outcome == 'failure' }}
26+
run: |
27+
echo "Failed to checkout and pull gcc. Retrying in 1 min"
28+
sleep 60
29+
cd gcc
30+
git checkout master
31+
git pull
32+
git checkout ${{ inputs.hash }}
33+

‎.github/workflows/create-cache.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Create-Cache
2+
3+
on:
4+
schedule:
5+
# Run at 1:07 on mondays and fridays
6+
# https://stackoverflow.com/questions/59560214/github-action-works-on-push-but-not-scheduled
7+
- cron: 7 1 * * 1,5
8+
push:
9+
branches:
10+
- main
11+
12+
jobs:
13+
init-submodules:
14+
runs-on: ubuntu-20.04
15+
defaults:
16+
run:
17+
working-directory: riscv-gnu-toolchain
18+
19+
steps:
20+
- uses: actions/checkout@v3
21+
22+
- name: Setup env
23+
uses: ./.github/actions/common/setup-env
24+
with:
25+
free_up_space: false
26+
27+
- name: Retrieve cache
28+
id: retrieve-cache
29+
uses: actions/cache@v3
30+
with:
31+
path: |
32+
riscv-gnu-toolchain/.git
33+
riscv-gnu-toolchain/binutils
34+
riscv-gnu-toolchain/dejagnu
35+
riscv-gnu-toolchain/gcc
36+
riscv-gnu-toolchain/gdb
37+
riscv-gnu-toolchain/glibc
38+
riscv-gnu-toolchain/newlib
39+
riscv-gnu-toolchain/qemu
40+
key: submodules-archive-9 # Numbered archive to allow for easy transition when bumping submodules
41+
42+
- name: Initalize submodules cache
43+
id: cache-init
44+
if: steps.retrieve-cache.outputs.cache-hit != 'true'
45+
run: |
46+
git submodule update --init --recursive --depth 1 binutils
47+
git submodule update --init --recursive --depth 1 dejagnu
48+
git submodule update --init --recursive --depth 1 gdb
49+
git submodule update --init --recursive --depth 1 glibc
50+
git submodule update --init --recursive --depth 1 newlib
51+
git submodule update --init --recursive --depth 1 qemu
52+
continue-on-error: true
53+
54+
- name: Initalize submodules cache
55+
if: steps.cache-init.outcome == 'failure'
56+
run: |
57+
echo "Failed to initialize cache submodules. Retrying in 1 min"
58+
sleep 60
59+
git submodule update --init --recursive --depth 1 binutils
60+
git submodule update --init --recursive --depth 1 dejagnu
61+
git submodule update --init --recursive --depth 1 gdb
62+
git submodule update --init --recursive --depth 1 glibc
63+
git submodule update --init --recursive --depth 1 newlib
64+
git submodule update --init --recursive --depth 1 qemu
65+
66+
- name: Initialize gcc
67+
if: steps.retrieve-cache.outputs.cache-hit != 'true'
68+
id: gcc-cache
69+
uses: ./.github/actions/common/init-and-pull-gcc
70+
with:
71+
init: true
72+
73+
# Does not remove and reclone gcc if we hit cache
74+
- name: Checkout GCC
75+
if: steps.gcc-cache.outcome == 'skipped'
76+
uses: ./.github/actions/common/init-and-pull-gcc
77+
with:
78+
init: false
79+
80+
- name: Apply newlib fixups
81+
if: steps.retrieve-cache.outputs.cache-hit != 'true'
82+
run: |
83+
cd newlib
84+
git config --global user.email "github-bot@example.com"
85+
git config --global user.name "Github Bot"
86+
git am ../fixups/newlib/*.patch
87+
88+
- name: Cache submodules
89+
if: steps.retrieve-cache.outputs.cache-hit != 'true'
90+
uses: actions/cache/save@v3
91+
with:
92+
path: |
93+
riscv-gnu-toolchain/.git
94+
riscv-gnu-toolchain/binutils
95+
riscv-gnu-toolchain/dejagnu
96+
riscv-gnu-toolchain/gcc
97+
riscv-gnu-toolchain/gdb
98+
riscv-gnu-toolchain/glibc
99+
riscv-gnu-toolchain/newlib
100+
riscv-gnu-toolchain/qemu
101+
key: submodules-archive-9
102+
103+
- name: Make cache zip
104+
run: |
105+
zip -r cache.zip .git binutils dejagnu gcc gdb glibc newlib qemu
106+
107+
# Use artifact rather than cache since cache downloads are flaky/hang.
108+
# Artifacts are reliable but ~30 min slower to set up.
109+
# Setup is done on one runner, so this isn't a show stopper.
110+
- name: Upload git cache
111+
uses: actions/upload-artifact@v3
112+
with:
113+
name: gcc-sources
114+
path: |
115+
riscv-gnu-toolchain/cache.zip
116+
retention-days: 5
117+

‎.github/workflows/lint.yaml‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,22 @@ jobs:
3636
with:
3737
free_up_space: false
3838

39+
- name: Bump pyopenssl and crypto
40+
run: |
41+
sudo apt remove python3-pip
42+
wget https://bootstrap.pypa.io/get-pip.py
43+
sudo python3 get-pip.py
44+
pip install -U pyopenssl cryptography
45+
pip install pygithub==1.59.1 requests
46+
3947
- name: Restore submodules from cache
40-
uses: actions/download-artifact@v3
41-
with:
42-
name: gcc-sources-${{ inputs.baseline_hash }}
43-
path: ./riscv-gnu-toolchain
48+
run: |
49+
mkdir temp
50+
python ./scripts/download_artifact.py -name gcc-sources -repo ewlu/gcc-precommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./
51+
# using download_artifact.py the downloaded file is zipped twice for whatever reason
52+
# remove the first layer of zip files
53+
unzip gcc-sources
54+
rm -rf gcc-sources
4455
4556
- name: Restore submodules
4657
run: |

‎.github/workflows/patchworks.yaml‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,65 @@ jobs:
134134
outputs:
135135
list_of_patch_names: ${{ steps.list_patches.outputs.patch_list }}
136136

137+
137138
init-submodules:
138139
needs: [fetch_patches]
139140
if: ${{ needs.fetch_patches.outputs.list_of_patch_names != '[]' }}
140-
uses: ./.github/workflows/init-submodules.yaml
141-
with:
142-
baseline_hash: ${{ inputs.baseline_gcc_hash }}
143-
tot_hash: ${{ inputs.tip_of_tree_hash }}
141+
runs-on: ubuntu-20.04
142+
defaults:
143+
run:
144+
working-directory: riscv-gnu-toolchain
145+
146+
steps:
147+
- uses: actions/checkout@v3
148+
149+
- name: Setup env
150+
uses: ./.github/actions/common/setup-env
151+
with:
152+
free_up_space: false
153+
154+
- name: Determine baseline
155+
id: baseline-hash
156+
run: |
157+
python ./scripts/get_baseline_hash.py -token ${{ secrets.GITHUB_TOKEN }}
158+
159+
if [ '${{ inputs.baseline_hash }}' == '' ] ; then export BASELINE_HASH=$(cat baseline.txt); else export BASELINE_HASH=${{ inputs.baseline_hash }}; fi
160+
echo $BASELINE_HASH
161+
echo "baseline_hash=$BASELINE_HASH" >> $GITHUB_OUTPUT
162+
163+
# https://stackoverflow.com/questions/24750215/getting-the-last-commit-hash-from-a-remote-repo-without-cloning
164+
- name: Determine tip-of-tree hash
165+
id: tot-hash
166+
run: |
167+
export TOT_HASH=$(git ls-remote https://gcc.gnu.org/git/gcc.git HEAD | awk '{ print $1 }')
168+
echo $TOT_HASH
169+
echo "tot_hash=$TOT_HASH" >> $GITHUB_OUTPUT
170+
continue-on-error: true
171+
172+
- name: Determine tip-of-tree-hash retry
173+
id: tot-retry
174+
if: ${{ steps.tot-hash.outcome == 'failure' }}
175+
run: |
176+
echo "retrying to get tip of tree hash due to git error"
177+
sleep 60
178+
export TOT_HASH=$(git ls-remote https://gcc.gnu.org/git/gcc.git HEAD | awk '{ print $1 }')
179+
echo $TOT_HASH
180+
echo "tot_hash=$TOT_HASH" >> $GITHUB_OUTPUT
181+
182+
- name: Effective tip-of-tree-hash
183+
id: etot-hash
184+
run:
185+
if [ '${{ steps.tot-hash.outputs.tot_hash }}' == '' ];
186+
then
187+
echo "tot_hash=${{ steps.tot-retry.outputs.tot_hash }}" >> $GITHUB_OUTPUT
188+
else
189+
echo "tot_hash=${{ steps.tot-hash.outputs.tot_hash }}" >> $GITHUB_OUTPUT
190+
fi
191+
192+
outputs:
193+
baseline_hash: ${{ steps.baseline-hash.outputs.baseline_hash }}
194+
tot_hash: ${{ steps.etot-hash.outputs.tot_hash }}
195+
144196

145197
patch_matrix:
146198
needs: [fetch_patches, init-submodules]

‎.github/workflows/run-checks.yaml‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,6 @@ jobs:
2727
with:
2828
free_up_space: false
2929

30-
- name: Restore submodules from cache
31-
uses: actions/download-artifact@v3
32-
with:
33-
name: gcc-sources-${{ inputs.baseline_hash }}
34-
path: ./riscv-gnu-toolchain
35-
36-
- name: Restore submodules
37-
run: |
38-
rm -rf .git binutils dejagnu gcc gdb glibc newlib qemu
39-
unzip cache.zip
40-
rm -rf cache.zip
41-
4230
- name: Download patch urls artifact
4331
uses: actions/download-artifact@v3
4432
with:
@@ -227,18 +215,34 @@ jobs:
227215
with:
228216
free_up_space: false
229217

218+
- name: Bump pyopenssl and crypto
219+
run: |
220+
sudo apt remove python3-pip
221+
wget https://bootstrap.pypa.io/get-pip.py
222+
sudo python3 get-pip.py
223+
pip install -U pyopenssl cryptography
224+
pip install pygithub==1.59.1 requests
225+
230226
- name: Restore submodules from cache
231-
uses: actions/download-artifact@v3
232-
with:
233-
name: gcc-sources-${{ inputs.baseline_hash }}
234-
path: ./riscv-gnu-toolchain
227+
run: |
228+
mkdir temp
229+
python ./scripts/download_artifact.py -name gcc-sources -repo ewlu/gcc-precommit-ci -token ${{ secrets.GITHUB_TOKEN }} -outdir ./
230+
# using download_artifact.py the downloaded file is zipped twice for whatever reason
231+
# remove the first layer of zip files
232+
unzip gcc-sources
233+
rm -rf gcc-sources
235234
236235
- name: Restore submodules
237236
run: |
238237
rm -rf .git binutils dejagnu gcc gdb glibc newlib qemu
239238
unzip cache.zip
240239
rm -rf cache.zip
241240
241+
- name: Checkout gcc hash
242+
uses: ./.github/actions/checkout-gcc-hash
243+
with:
244+
hash: ${{ inputs.baseline_hash }}
245+
242246
- name: Download patches artifact
243247
uses: actions/download-artifact@v3
244248
with:
@@ -249,11 +253,6 @@ jobs:
249253
run: |
250254
unzip ${{ inputs.patch_name }}-downloaded-patches.zip
251255
252-
- name: Checkout gcc hash
253-
run: |
254-
cd gcc
255-
git checkout ${{ inputs.baseline_hash }}
256-
257256
- name: Apply patches to baseline
258257
id: apply-baseline
259258
run: |

0 commit comments

Comments
 (0)