Skip to content

Commit b6665c3

Browse files
evertlammertsevertlammerts
committed
Create a PR to bump submodule if nightly is stale
1 parent e35a2ce commit b6665c3

File tree

2 files changed

+119
-16
lines changed

2 files changed

+119
-16
lines changed

‎.github/workflows/release.yml‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
options:
2323
- test
2424
- prod
25+
nightly-stale-after-days:
26+
type: string
27+
description: After how many days should nightlies be considered stale
28+
required: true
29+
default: 4
2530
store-s3:
2631
type: boolean
2732
description: Also store test packages in S3 (always true for prod)
@@ -51,23 +56,32 @@ jobs:
5156
runs-on: ubuntu-latest
5257
steps:
5358
- id: index_check
54-
name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
59+
name: Check version on PyPI
5560
run: |
56-
set -eu
57-
# Check PyPI whether the release we're building is already present
61+
set -ex
5862
pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
59-
pkg_version=${{ needs.build_sdist.outputs.package-version }}
60-
url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
61-
http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
62-
if [[ $http_status == "200" ]]; then
63-
echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
64-
pypi_state=VERSION_FOUND
65-
elif [[ $http_status == 000* ]]; then
66-
echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
67-
pypi_state=UNKNOWN
68-
else
69-
echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
63+
# install duckdb
64+
curl https://install.duckdb.org | sh
65+
# query pypi
66+
result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
67+
---- Output lines
68+
.mode line
69+
---- Query that fetches the given version's age, if the version already exists
70+
SELECT
71+
today() - (file.value->>'upload_time_iso_8601')::DATE AS age,
72+
FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
73+
CROSS JOIN json_each(jd.releases) AS rel(key, value)
74+
CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value)
75+
WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
76+
LIMIT 1;
77+
EOF
78+
)
79+
if [ -z "$result" ]; then
7080
pypi_state=VERSION_NOT_FOUND
81+
elif [ "${result#age = }" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
82+
pypi_state=VERSION_STALE
83+
else
84+
pypi_state=VERSION_FOUND
7185
fi
7286
echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
7387
@@ -96,7 +110,7 @@ jobs:
96110
echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
97111
exit 0
98112
fi
99-
if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
113+
if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
100114
echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
101115
exit 0
102116
fi
@@ -107,10 +121,22 @@ jobs:
107121
echo "::notice::Generated S3 URL: ${s3_url}"
108122
echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT
109123
124+
submodule_pr:
125+
name: Create PR to bump submodule to given SHA if version on PyPI is stale
126+
needs: workflow_state
127+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_STALE' }}
128+
uses: ./.github/workflows/submodule_auto_pr.yml
129+
with:
130+
duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
131+
duckdb-sha: ${{ inputs.duckdb-sha }}
132+
secrets:
133+
# reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
134+
DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
135+
110136
build_wheels:
111137
name: Build and test releases
112138
needs: workflow_state
113-
if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
139+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
114140
uses: ./.github/workflows/packaging_wheels.yml
115141
with:
116142
minimal: false

‎.github/workflows/submodule_auto_pr.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Submodule Auto PR
2+
on:
3+
workflow_call:
4+
inputs:
5+
duckdb-python-sha:
6+
type: string
7+
description: The commit to build against (defaults to latest commit of current ref)
8+
required: false
9+
duckdb-sha:
10+
type: string
11+
description: The DuckDB submodule commit or ref to build against
12+
required: true
13+
secrets:
14+
DUCKDBLABS_BOT_TOKEN:
15+
description: Github token of the DuckDBLabs bot
16+
required: true
17+
18+
defaults:
19+
run:
20+
shell: bash
21+
22+
jobs:
23+
create_pr:
24+
name: Create PR to bump duckdb submodule to given SHA
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: Checkout DuckDB Python
28+
uses: actions/checkout@v4
29+
with:
30+
ref: ${{ inputs.duckdb-python-sha }}
31+
fetch-depth: 0
32+
submodules: true
33+
34+
- name: Checkout or Create Needed Branch
35+
run: |
36+
git fetch --all
37+
head_sha=${{ inputs.duckdb-python-sha }}
38+
branch_name="vendoring-${{ github.ref_name }}"
39+
if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
40+
# branch exists
41+
git checkout ${branch_name}
42+
else
43+
# new branch
44+
git checkout -b ${branch_name}
45+
fi
46+
[[ ${head_sha} ]] && git reset --hard ${head_sha} || true
47+
48+
- name: Checkout DuckDB at Given SHA
49+
run: |
50+
cd external/duckdb
51+
git fetch origin
52+
git checkout ${{ inputs.duckdb-sha }}
53+
54+
- name: Set Git User
55+
run: |
56+
git config --global user.email "github_bot@duckdblabs.com"
57+
git config --global user.name "DuckDB Labs GitHub Bot"
58+
59+
- name: Create PR to Bump DuckDB Submodule
60+
env:
61+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
62+
run: |
63+
# First commit and push
64+
git add external/duckdb
65+
git commit -m "Bump submodule"
66+
git push --force origin vendoring-${{ github.ref_name }}
67+
# create PR msg
68+
echo "Bump duckdb submodule:" > body.txt
69+
echo " Target branch: ${{ github.ref_name }}" >> body.txt
70+
echo " Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
71+
echo " DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
72+
echo " Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
73+
gh pr create \
74+
--head "vendoring-${{ github.ref_name }}" \
75+
--base ${{ github.ref_name }} \
76+
--title "[duckdb-labs bot] Bump DuckDB submodule" \
77+
--body-file body.txt

0 commit comments

Comments
 (0)