From b63acfe54cf10d7995f015f7639a1e11c7e68aed Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 15 Apr 2024 13:56:27 +0100 Subject: [PATCH 01/10] Add CI for automatically publishing to PyPI --- .github/workflows/pypi.yaml | 136 ++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 .github/workflows/pypi.yaml diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml new file mode 100644 index 0000000..3a7f0b9 --- /dev/null +++ b/.github/workflows/pypi.yaml @@ -0,0 +1,136 @@ +# This file is autogenerated by maturin v1.5.1 +# To update, run +# +# maturin generate-ci github +# +name: CI + +on: + push: + branches: + - main + - master + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + - runner: ubuntu-latest + target: s390x + - runner: ubuntu-latest + target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + - runner: windows-latest + target: x86 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + architecture: ${{ matrix.platform.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-latest + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v4 + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* From 7d36d2173e34ac3c5c580f590c4ff86d7326a334 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 15 Apr 2024 14:11:35 +0100 Subject: [PATCH 02/10] install OpenSSL on linux runners taking into account CentOS vs. Debian-based manylinux images. Solution taken from: https://github.com/sfackler/rust-openssl/issues/2036#issuecomment-1724324145 --- .github/workflows/pypi.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 3a7f0b9..aa6a77b 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -48,6 +48,20 @@ jobs: args: --release --out dist --find-interpreter sccache: 'true' manylinux: auto + before-script-linux: | + # If we're running on rhel centos, install needed packages. + if command -v yum &> /dev/null; then + yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic + + # If we're running on i686 we need to symlink libatomic + # in order to build openssl with -latomic flag. + if [[ ! -d "/usr/lib64" ]]; then + ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so + fi + else + # If we're running on debian-based system. + apt update -y && apt-get install -y libssl-dev openssl pkg-config + fi - name: Upload wheels uses: actions/upload-artifact@v4 with: From aabc5faee8f2a7a12099a3e916a61cd97abda6b0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 15 Apr 2024 14:41:51 +0100 Subject: [PATCH 03/10] Statically link openssl from a vendored copy --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 96ccd5b..33b6795 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ required-features = ["clap"] [dependencies] indicatif = "0.17.6" -openssl = "0.10.60" +openssl = { version = "0.10.60", features = ["vendored"] } postgres = "0.19.7" postgres-openssl = "0.5.0" rand = "0.8.5" From ad8dccc35c5272c4527a7a7327915543d0c22685 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 16 Apr 2024 15:41:06 +0100 Subject: [PATCH 04/10] Add comments --- .github/workflows/pypi.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index aa6a77b..87b1e07 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -1,8 +1,11 @@ -# This file is autogenerated by maturin v1.5.1 +# This file is autogenerated by maturin v1.5.1, and minimally edited by hand. +# See comments below for the edited sections. +# # To update, run # # maturin generate-ci github # +# and re-add the edits. name: CI on: @@ -48,6 +51,10 @@ jobs: args: --release --out dist --find-interpreter sccache: 'true' manylinux: auto + # BEGIN EDITED SECTION # + # Install OpenSSL development headers into the manylinux docker container + # used to build the wheels. + # Note: libatomic is necessary for the build to succeed. before-script-linux: | # If we're running on rhel centos, install needed packages. if command -v yum &> /dev/null; then @@ -62,6 +69,7 @@ jobs: # If we're running on debian-based system. apt update -y && apt-get install -y libssl-dev openssl pkg-config fi + # END EDITED SECTION # - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -145,6 +153,10 @@ jobs: - uses: actions/download-artifact@v4 - name: Publish to PyPI uses: PyO3/maturin-action@v1 + # BEGIN EDITED SECTION + # The `MATURIN_PYPI_TOKEN` env var was removed in order to authenticate + # to PyPI using the Trusted Publishers feature instead. + # END EDITED SECTION. with: command: upload args: --non-interactive --skip-existing wheels-*/* From a77f3b2afe4a53c624b78dba4b4860059c6dd23a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 29 Apr 2024 15:36:55 +0100 Subject: [PATCH 05/10] Revert "Statically link openssl from a vendored copy" This reverts commit aabc5faee8f2a7a12099a3e916a61cd97abda6b0. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 33b6795..96ccd5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ required-features = ["clap"] [dependencies] indicatif = "0.17.6" -openssl = { version = "0.10.60", features = ["vendored"] } +openssl = "0.10.60" postgres = "0.19.7" postgres-openssl = "0.5.0" rand = "0.8.5" From 096f6bab7d021aec6e164f29cd9a42d528bad682 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 29 Apr 2024 15:38:26 +0100 Subject: [PATCH 06/10] Remove support for building windows wheels --- .github/workflows/pypi.yaml | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 87b1e07..afdb66b 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -75,33 +75,10 @@ jobs: with: name: wheels-linux-${{ matrix.platform.target }} path: dist - - windows: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - - runner: windows-latest - target: x86 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - architecture: ${{ matrix.platform.target }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: 'true' - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist + + # BEGIN EDITED SECTION # + # The `windows` job has been removed as it is not a supported platform. + # END EDITED SECTION # macos: runs-on: ${{ matrix.platform.runner }} @@ -148,7 +125,10 @@ jobs: name: Release runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" - needs: [linux, windows, macos, sdist] + # BEGIN EDITED SECTION # + # Note: The `windows` job was removed from this list. + needs: [linux, macos, sdist] + # END EDITED SECTION # steps: - uses: actions/download-artifact@v4 - name: Publish to PyPI From 6221bab9211d8f72f87bcb96277d0a32252be623 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 29 Apr 2024 15:40:51 +0100 Subject: [PATCH 07/10] Remove `s390x` and `ppc64le` arch support To save on CI runner time. --- .github/workflows/pypi.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index afdb66b..89f9580 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -35,10 +35,9 @@ jobs: target: aarch64 - runner: ubuntu-latest target: armv7 - - runner: ubuntu-latest - target: s390x - - runner: ubuntu-latest - target: ppc64le + # BEGIN EDITED SECTION # + # The `s390x` and `ppc64le` architectures have been removed as they are not supported. + # END EDITED SECTION # steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 024b2037aa86405da2ef9869cfa23ebb8777c573 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 2 May 2024 10:25:58 +0100 Subject: [PATCH 08/10] Debug linux finding openssl headers --- .github/workflows/pypi.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 89f9580..5f62086 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -55,6 +55,8 @@ jobs: # used to build the wheels. # Note: libatomic is necessary for the build to succeed. before-script-linux: | + set -ex + # If we're running on rhel centos, install needed packages. if command -v yum &> /dev/null; then yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic @@ -64,10 +66,26 @@ jobs: if [[ ! -d "/usr/lib64" ]]; then ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so fi + + OPENSSL_LIB_DIR=/usr/lib else # If we're running on debian-based system. apt update -y && apt-get install -y libssl-dev openssl pkg-config + + OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu fi + + export OPENSSL_INCLUDE_DIR=/usr/include/openssl + + ls $OPENSSL_LIB_DIR + ls $OPENSSL_INCLUDE_DIR + + + echo "OpenSSL Lib Dir: $OPENSSL_LIB_DIR" + echo "OpenSSL Include Dir: $OPENSSL_INCLUDE_DIR" + + pkg-config --list-all + pkg-config --libs --cflags openssl # END EDITED SECTION # - name: Upload wheels uses: actions/upload-artifact@v4 From 3fcdc3ef1caf833074422b2b35d95326e68b643b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 2 May 2024 14:53:09 +0100 Subject: [PATCH 09/10] OPENSSL_NO_VENDOR=1 --- .github/workflows/pypi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 5f62086..af5eda6 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -76,6 +76,7 @@ jobs: fi export OPENSSL_INCLUDE_DIR=/usr/include/openssl + export OPENSSL_NO_VENDOR=1 ls $OPENSSL_LIB_DIR ls $OPENSSL_INCLUDE_DIR From 21cbd6eb42716207051aea54648bdbf85e8a3f12 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 2 May 2024 14:59:40 +0100 Subject: [PATCH 10/10] Set OPENSSL_DIR --- .github/workflows/pypi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index af5eda6..455baff 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -75,6 +75,7 @@ jobs: OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu fi + export OPENSSL_DIR="/usr/lib/ssl" export OPENSSL_INCLUDE_DIR=/usr/include/openssl export OPENSSL_NO_VENDOR=1