Skip to content

Commit 870a501

Browse files
evertlammertsevertlammerts
committed
Create a PR to bump submodule if nightly is stale
1 parent e35a2ce commit 870a501

File tree

2 files changed

+170
-16
lines changed

2 files changed

+170
-16
lines changed

‎.github/workflows/release.yml‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
options:
2323
- test
2424
- prod
25+
nightly-stale-after-days:
26+
type: string
27+
description: After how many days should nightlies be considered stale
28+
required: true
29+
default: 4
2530
store-s3:
2631
type: boolean
2732
description: Also store test packages in S3 (always true for prod)
@@ -51,23 +56,32 @@ jobs:
5156
runs-on: ubuntu-latest
5257
steps:
5358
- id: index_check
54-
name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
59+
name: Check version on PyPI
5560
run: |
56-
set -eu
57-
# Check PyPI whether the release we're building is already present
61+
set -ex
5862
pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
59-
pkg_version=${{ needs.build_sdist.outputs.package-version }}
60-
url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
61-
http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
62-
if [[ $http_status == "200" ]]; then
63-
echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
64-
pypi_state=VERSION_FOUND
65-
elif [[ $http_status == 000* ]]; then
66-
echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
67-
pypi_state=UNKNOWN
68-
else
69-
echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
63+
# install duckdb
64+
curl https://install.duckdb.org | sh
65+
# query pypi
66+
result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
67+
---- Output lines
68+
.mode line
69+
---- Query that fetches the given version's age, if the version already exists
70+
SELECT
71+
today() - (file.value->>'upload_time_iso_8601')::DATE AS age,
72+
FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
73+
CROSS JOIN json_each(jd.releases) AS rel(key, value)
74+
CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value)
75+
WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
76+
LIMIT 1;
77+
EOF
78+
)
79+
if [ -z "$result" ]; then
7080
pypi_state=VERSION_NOT_FOUND
81+
elif [ "${result#age = }" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
82+
pypi_state=VERSION_STALE
83+
else
84+
pypi_state=VERSION_FOUND
7185
fi
7286
echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
7387
@@ -96,7 +110,7 @@ jobs:
96110
echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
97111
exit 0
98112
fi
99-
if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
113+
if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
100114
echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
101115
exit 0
102116
fi
@@ -107,10 +121,22 @@ jobs:
107121
echo "::notice::Generated S3 URL: ${s3_url}"
108122
echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT
109123
124+
submodule_pr:
125+
name: Create PR to bump submodule to given SHA if version on PyPI is stale
126+
needs: workflow_state
127+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_STALE' }}
128+
uses: ./.github/workflows/submodule_auto_pr.yml
129+
with:
130+
duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
131+
duckdb-sha: ${{ inputs.duckdb-sha }}
132+
secrets:
133+
# reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
134+
DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
135+
110136
build_wheels:
111137
name: Build and test releases
112138
needs: workflow_state
113-
if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
139+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
114140
uses: ./.github/workflows/packaging_wheels.yml
115141
with:
116142
minimal: false

‎.github/workflows/submodule_auto_pr.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: Submodule Auto PR
2+
on:
3+
workflow_call:
4+
inputs:
5+
duckdb-python-sha:
6+
type: string
7+
description: The commit to build against (defaults to latest commit of current ref)
8+
required: false
9+
duckdb-sha:
10+
type: string
11+
description: The DuckDB submodule commit or ref to build against
12+
required: true
13+
auto-land:
14+
type: boolean
15+
description: Immediately merge the PR (placeholder - doesn't work)
16+
default: false
17+
secrets:
18+
DUCKDBLABS_BOT_TOKEN:
19+
description: Github token of the DuckDBLabs bot
20+
required: true
21+
22+
defaults:
23+
run:
24+
shell: bash
25+
26+
jobs:
27+
create_pr:
28+
name: Create PR to bump duckdb submodule to given SHA
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout DuckDB Python
32+
uses: actions/checkout@v4
33+
with:
34+
ref: ${{ inputs.duckdb-python-sha }}
35+
fetch-depth: 0
36+
submodules: true
37+
38+
- name: Checkout or Create Needed Branch
39+
run: |
40+
git fetch --all
41+
head_sha=${{ inputs.duckdb-python-sha }}
42+
branch_name="vendoring-${{ github.ref_name }}"
43+
if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
44+
# branch exists
45+
git checkout ${branch_name}
46+
else
47+
# new branch
48+
git checkout -b ${branch_name}
49+
fi
50+
[[ ${head_sha} ]] && git reset --hard ${head_sha} || true
51+
52+
- name: Checkout DuckDB at Given SHA
53+
run: |
54+
cd external/duckdb
55+
git fetch origin
56+
git checkout ${{ inputs.duckdb-sha }}
57+
58+
- name: Determine GH PR Command
59+
id: gh_pr_command
60+
env:
61+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
62+
run: |
63+
pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' )
64+
if [[ $pr_url ]]; then
65+
echo "::notice::Found existing pr, will edit (${pr_url})"
66+
gh_command="edit ${pr_url}"
67+
else
68+
echo "::notice::No existing PR, will create new"
69+
gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}"
70+
fi
71+
echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT
72+
73+
- name: Set Git User
74+
run: |
75+
git config --global user.email "github_bot@duckdblabs.com"
76+
git config --global user.name "DuckDB Labs GitHub Bot"
77+
78+
- name: Create PR to Bump DuckDB Submodule
79+
env:
80+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
81+
run: |
82+
# First commit and push
83+
git add external/duckdb
84+
git commit -m "Bump submodule"
85+
git push --force origin vendoring-${{ github.ref_name }}
86+
# create PR msg
87+
echo "Bump duckdb submodule:" > body.txt
88+
echo "- Target branch: ${{ github.ref_name }}" >> body.txt
89+
echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
90+
echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
91+
echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
92+
subcommand="${{ steps.gh_pr_command.outputs.subcommand }}"
93+
gh pr ${subcommand} \
94+
--title "[duckdb-labs bot] Bump DuckDB submodule" \
95+
--body-file body.txt > output.txt 2>&1
96+
success=$?
97+
# Show summary
98+
url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true )
99+
echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY
100+
if [[ $success ]]; then
101+
prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" )
102+
echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY
103+
else
104+
echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY
105+
fi
106+
echo '```' >> $GITHUB_STEP_SUMMARY
107+
cat output.txt >> $GITHUB_STEP_SUMMARY
108+
echo '```' >> $GITHUB_STEP_SUMMARY
109+
[[ $success ]] || exit 1
110+
111+
- name: Automerge PR
112+
if: ${{ inputs.auto-land }}
113+
env:
114+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
115+
run: |
116+
# PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS
117+
set -ex
118+
gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt
119+
success=$?
120+
# Show summary
121+
if [[ $success ]]; then
122+
echo "### PR merged" >> $GITHUB_STEP_SUMMARY
123+
else
124+
echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY
125+
fi
126+
echo '```' >> $GITHUB_STEP_SUMMARY
127+
cat output.txt >> $GITHUB_STEP_SUMMARY
128+
echo '```' >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)