Skip to content

Commit c9e9314

Browse files
elpranselprans
committed
ci: Switch to staged release process
Uploading release artifacts directly to PyPI from the build matrix is bad, as this may result in an incomplete or broken release if any of the platforms fail to build. To fix the process, implement a "staged" build process whereby artifacts are built from a dedicated "releases" branch and are uploaded to an S3 bucket. Once all artifacts have been successfully built, the release tag may be pushed to master, and the previously built artifacts will be uploaded to PyPI as a single transaction.
1 parent 0185586 commit c9e9314

16 files changed

+376
-111
lines changed

‎.ci/appveyor.yml‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ services:
44
environment:
55
global:
66
PGINSTALLATION: C:\\Program Files\\PostgreSQL\\9.5\\bin
7-
TWINE_USERNAME: magicstack-ci
8-
TWINE_PASSWORD:
9-
secure: lJ+XbDBgnR3SA1hyBl7kZ+fOJYQYVlZ2A27Vt9p+TNJ6Fsv41Yx7O0fb9bTdiVV+
7+
S3_UPLOAD_USERNAME: oss-ci-bot
8+
S3_UPLOAD_BUCKET: magicstack-oss-releases
9+
S3_UPLOAD_ACCESSKEY:
10+
secure: 1vmOqSXq5zDN8UdezZ3H4l0A9LUJiTr7Wuy9whCdffE=
11+
S3_UPLOAD_SECRET:
12+
secure: XudOvV6WtY9yRoqKahXMswFth8SF1UTnSXws4UBjeqzQUjOx2V2VRvIdpPfiqUKt
1013

1114
matrix:
1215
- PYTHON: "C:\\Python35\\python.exe"
@@ -29,4 +32,15 @@ artifacts:
2932
- path: dist\*
3033

3134
deploy_script:
32-
- cmd: "if %APPVEYOR_REPO_TAG%==true %PYTHON% -m twine upload dist\\*.whl"
35+
- ps: |
36+
if ($env:appveyor_repo_branch -eq 'releases') {
37+
$PACKAGE_VERSION = & "$env:PYTHON" ".ci/package-version.py"
38+
$PYPI_VERSION = & "$env:PYTHON" ".ci/pypi-check.py" "asyncpg"
39+
40+
if ($PACKAGE_VERSION -eq $PYPI_VERSION) {
41+
Write-Error "asyncpg-$PACKAGE_VERSION is already published on PyPI"
42+
exit 1
43+
}
44+
45+
& "$env:PYTHON" ".ci/s3-upload.py" dist\*.whl
46+
}

‎.ci/build-manylinux-wheels.sh‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,14 @@
22

33
set -e -x
44

5-
PYTHON_VERSIONS="cp35-cp35m"
6-
75
# Compile wheels
8-
for PYTHON_VERSION in ${PYTHON_VERSIONS}; do
9-
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
10-
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
11-
${PIP} install --upgrade pip wheel
12-
${PIP} install --upgrade setuptools
13-
${PIP} install -r /io/.ci/requirements.txt
14-
make -C /io/ PYTHON="${PYTHON}"
15-
${PIP} wheel /io/ -w /io/dist/
16-
done
6+
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
7+
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
8+
${PIP} install --upgrade pip wheel
9+
${PIP} install --upgrade setuptools
10+
${PIP} install -r /io/.ci/requirements.txt
11+
make -C /io/ PYTHON="${PYTHON}"
12+
${PIP} wheel /io/ -w /io/dist/
1713

1814
# Bundle external shared libraries into the wheels.
1915
for whl in /io/dist/*.whl; do
@@ -25,11 +21,9 @@ done
2521
export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq)
2622
export PGUSER="postgres"
2723

28-
for PYTHON_VERSION in ${PYTHON_VERSIONS}; do
29-
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
30-
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
31-
${PIP} install ${PYMODULE} --no-index -f file:///io/dist
32-
rm -rf /io/tests/__pycache__
33-
make -C /io/ PYTHON="${PYTHON}" test
34-
rm -rf /io/tests/__pycache__
35-
done
24+
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
25+
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
26+
${PIP} install ${PYMODULE} --no-index -f file:///io/dist
27+
rm -rf /io/tests/__pycache__
28+
make -C /io/ PYTHON="${PYTHON}" test
29+
rm -rf /io/tests/__pycache__

‎.ci/package-version.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import os.path
5+
import sys
6+
7+
8+
def main():
9+
setup_py = os.path.join(os.path.dirname(os.path.dirname(__file__)),
10+
'setup.py')
11+
12+
with open(setup_py, 'r') as f:
13+
for line in f:
14+
if line.startswith('VERSION ='):
15+
_, _, version = line.partition('=')
16+
print(version.strip(" \n'\""))
17+
return 0
18+
19+
print('could not find package version in setup.py', file=sys.stderr)
20+
return 1
21+
22+
23+
if __name__ == '__main__':
24+
sys.exit(main())

‎.ci/pypi-check.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import sys
6+
import xmlrpc.client
7+
8+
9+
def main():
10+
parser = argparse.ArgumentParser(description='PyPI package checker')
11+
parser.add_argument('package_name', metavar='PACKAGE-NAME')
12+
13+
parser.add_argument(
14+
'--pypi-index-url',
15+
help=('PyPI index URL.'),
16+
default='https://pypi.python.org/pypi')
17+
18+
args = parser.parse_args()
19+
20+
pypi = xmlrpc.client.ServerProxy(args.pypi_index_url)
21+
releases = pypi.package_releases(args.package_name)
22+
23+
if releases:
24+
print(next(iter(sorted(releases, reverse=True))))
25+
26+
return 0
27+
28+
29+
if __name__ == '__main__':
30+
sys.exit(main())

‎.ci/requirements-win.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
cython>=0.24
2-
twine
2+
tinys3

‎.ci/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
cython>=0.24
22
uvloop>=0.5.0
3+
tinys3
34
twine

‎.ci/s3-download-release.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import os
6+
import os.path
7+
import sys
8+
import urllib.request
9+
10+
import tinys3
11+
12+
13+
def main():
14+
parser = argparse.ArgumentParser(description='S3 File Uploader')
15+
parser.add_argument(
16+
'--s3-bucket',
17+
help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
18+
default=os.environ.get('S3_UPLOAD_BUCKET'))
19+
parser.add_argument(
20+
'--s3-region',
21+
help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
22+
default=os.environ.get('S3_UPLOAD_REGION'))
23+
parser.add_argument(
24+
'--s3-username',
25+
help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
26+
default=os.environ.get('S3_UPLOAD_USERNAME'))
27+
parser.add_argument(
28+
'--s3-key',
29+
help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
30+
default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
31+
parser.add_argument(
32+
'--s3-secret',
33+
help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
34+
default=os.environ.get('S3_UPLOAD_SECRET'))
35+
parser.add_argument(
36+
'--destdir',
37+
help='Destination directory.')
38+
parser.add_argument(
39+
'package', metavar='PACKAGE',
40+
help='Package name and version to download.')
41+
42+
args = parser.parse_args()
43+
44+
if args.s3_region:
45+
endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
46+
else:
47+
endpoint = 's3.amazonaws.com'
48+
49+
conn = tinys3.Connection(
50+
access_key=args.s3_key,
51+
secret_key=args.s3_secret,
52+
default_bucket=args.s3_bucket,
53+
tls=True,
54+
endpoint=endpoint,
55+
)
56+
57+
files = []
58+
59+
for entry in conn.list(args.package):
60+
files.append(entry['key'])
61+
62+
destdir = args.destdir or os.getpwd()
63+
64+
for file in files:
65+
print('Downloading {}...'.format(file))
66+
url = 'https://{}/{}/{}'.format(endpoint, args.s3_bucket, file)
67+
target = os.path.join(destdir, file)
68+
urllib.request.urlretrieve(url, target)
69+
70+
return 0
71+
72+
73+
if __name__ == '__main__':
74+
sys.exit(main())

‎.ci/s3-upload.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import glob
6+
import os
7+
import os.path
8+
import sys
9+
10+
import tinys3
11+
12+
13+
def main():
14+
parser = argparse.ArgumentParser(description='S3 File Uploader')
15+
parser.add_argument(
16+
'--s3-bucket',
17+
help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
18+
default=os.environ.get('S3_UPLOAD_BUCKET'))
19+
parser.add_argument(
20+
'--s3-region',
21+
help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
22+
default=os.environ.get('S3_UPLOAD_REGION'))
23+
parser.add_argument(
24+
'--s3-username',
25+
help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
26+
default=os.environ.get('S3_UPLOAD_USERNAME'))
27+
parser.add_argument(
28+
'--s3-key',
29+
help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
30+
default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
31+
parser.add_argument(
32+
'--s3-secret',
33+
help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
34+
default=os.environ.get('S3_UPLOAD_SECRET'))
35+
parser.add_argument(
36+
'files', nargs='+', metavar='FILE', help='Files to upload')
37+
38+
args = parser.parse_args()
39+
40+
if args.s3_region:
41+
endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
42+
else:
43+
endpoint = 's3.amazonaws.com'
44+
45+
conn = tinys3.Connection(
46+
access_key=args.s3_key,
47+
secret_key=args.s3_secret,
48+
default_bucket=args.s3_bucket,
49+
tls=True,
50+
endpoint=endpoint,
51+
)
52+
53+
for pattern in args.files:
54+
for fn in glob.iglob(pattern):
55+
with open(fn, 'rb') as f:
56+
conn.upload(os.path.basename(fn), f)
57+
58+
return 0
59+
60+
61+
if __name__ == '__main__':
62+
sys.exit(main())

‎.ci/travis-build-and-upload.sh‎

Lines changed: 0 additions & 67 deletions
This file was deleted.

0 commit comments

Comments
 (0)