From c4aad893663fe9c305d398ca11e5fc38b9700652 Mon Sep 17 00:00:00 2001 From: behnazh Date: Fri, 11 Apr 2025 12:27:01 +1000 Subject: [PATCH 1/8] test: test ubuntu-24.04-arm Signed-off-by: behnazh --- .github/workflows/_build.yaml | 2 +- .pre-commit-config.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 39bbbaae..9d7528f7 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -50,7 +50,7 @@ jobs: matrix: # It is recommended to pin a Runner version specifically: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] python: ['3.10', '3.11', '3.12', '3.13'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 399991af..5b665ebb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,10 +155,10 @@ repos: # args: [--autofix] # Check GitHub Actions workflow files. -- repo: https://github.com/Mateusz-Grzelinski/actionlint-py - rev: v1.7.4.18 - hooks: - - id: actionlint +# - repo: https://github.com/Mateusz-Grzelinski/actionlint-py +# rev: v1.7.4.18 +# hooks: +# - id: actionlint # On push to the remote, run the unit tests. - repo: local From d2e66f44b3436a953c5d0df01c15c3ad3d4a22a3 Mon Sep 17 00:00:00 2001 From: behnazh Date: Fri, 11 Apr 2025 13:15:10 +1000 Subject: [PATCH 2/8] test: upload artifacts for all OSes --- .github/workflows/_build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 9d7528f7..66b761e2 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -114,7 +114,7 @@ jobs: # the SHA digest for all the release files and encode them using Base64, and export it # from this job. - name: Compute package hash - if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + if: matrix.python == env.ARTIFACT_PYTHON id: compute-hash shell: bash run: | @@ -133,7 +133,7 @@ jobs: # For now only generate artifacts for the specified OS and Python version in env variables. # Currently reusable workflows do not support setting strategy property from the caller workflow. - name: Upload the package artifact for debugging and release - if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + if: matrix.python == env.ARTIFACT_PYTHON uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: artifact-${{ matrix.os }}-python-${{ matrix.python }} From ef6f1b4d0a579abbaeb849b3dad5d39daabf6f3a Mon Sep 17 00:00:00 2001 From: behnazh Date: Fri, 11 Apr 2025 14:13:14 +1000 Subject: [PATCH 3/8] test: sha256sum --- .github/workflows/_build.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 66b761e2..90965f87 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -50,8 +50,10 @@ jobs: matrix: # It is recommended to pin a Runner version specifically: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners - os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] - python: ['3.10', '3.11', '3.12', '3.13'] + # os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + # python: ['3.10', '3.11', '3.12', '3.13'] + os: [ubuntu-24.04-arm] + python: ['3.13'] steps: - name: Harden Runner @@ -126,6 +128,7 @@ jobs: HTML_DOCS_PATH=$(find dist/ -type f -name "*-docs-html.zip") MARKDOWN_DOCS_PATH=$(find dist/ -type f -name "*-docs-md.zip") BUILD_EPOCH_PATH=$(find dist/ -type f -name "*-build-epoch.txt") + sha256sum --version DIGEST=$(sha256sum "$TARBALL_PATH" "$WHEEL_PATH" "$REQUIREMENTS_PATH" "$SBOM_PATH" "$HTML_DOCS_PATH" "$MARKDOWN_DOCS_PATH" "$BUILD_EPOCH_PATH" | base64 -w0) echo "Digest of artifacts is $DIGEST." echo "artifacts-sha256=$DIGEST" >> "$GITHUB_OUTPUT" From 6e22e6b08388a9e6a6e2c4d51712fc64c27c8065 Mon Sep 17 00:00:00 2001 From: behnazh Date: Fri, 11 Apr 2025 14:54:57 +1000 Subject: [PATCH 4/8] test: rename artifacts --- .github/workflows/_build.yaml | 12 ++++----- .github/workflows/pr-change-set.yaml | 30 ++++++++++++++++++++++ Makefile | 37 +++++++++++++++++++--------- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 90965f87..9c739b65 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -50,10 +50,10 @@ jobs: matrix: # It is recommended to pin a Runner version specifically: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners - # os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] - # python: ['3.10', '3.11', '3.12', '3.13'] - os: [ubuntu-24.04-arm] - python: ['3.13'] + # os: [ubuntu-24.04, ubuntu-24.04-arm, macos-latest, windows-latest] + os: [ubuntu-24.04, ubuntu-24.04-arm] + python: ['3.10', '3.11', '3.12', '3.13'] + steps: - name: Harden Runner @@ -101,13 +101,13 @@ jobs: # Generate the requirements.txt that contains the hash digests of the dependencies and # generate the SBOM using CyclonDX SBOM generator. - name: Generate requirements.txt and SBOM - if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + if: matrix.python == env.ARTIFACT_PYTHON run: make requirements sbom # Remove the old requirements.txt file (which includes _all_ packages) and generate a # new one for the package and its actual and required dependencies only. - name: Prune packages and generate required requirements.txt - if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + if: matrix.python == env.ARTIFACT_PYTHON run: | rm requirements.txt make prune requirements diff --git a/.github/workflows/pr-change-set.yaml b/.github/workflows/pr-change-set.yaml index 8c09ad72..33e02617 100644 --- a/.github/workflows/pr-change-set.yaml +++ b/.github/workflows/pr-change-set.yaml @@ -21,3 +21,33 @@ jobs: contents: read with: disable-pip-audit: ${{ vars.DISABLE_PIP_AUDIT == 'true' }} + + test: + needs: [build] + name: test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Download artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: dist + + # Verify hashes by first computing hashes for the artifacts and then comparing them + # against the hashes computed by the build job. + - name: Verify the artifact hash + env: + ARTIFACT_HASH: ${{ needs.build.outputs.artifacts-sha256 }} + run: | + set -euo pipefail + echo "Hash of package should be $ARTIFACT_HASH." + echo "Decoding the artifact hash:" + echo "$ARTIFACT_HASH" | base64 --decode + echo "$ARTIFACT_HASH" | base64 --decode | sha256sum --strict --check --status || exit 1 diff --git a/Makefile b/Makefile index 061534ff..cc72a8b2 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,18 @@ SHELL := bash # Set the package's name and version for use throughout the Makefile. PACKAGE_NAME := package -PACKAGE_VERSION := $(shell python -c $$'try: import $(PACKAGE_NAME); print($(PACKAGE_NAME).__version__);\nexcept: print("unknown");') +PACKAGE_VERSION := $(shell python -c $$'try: import $(PACKAGE_NAME); print($(PACKAGE_NAME).__version__, end="");\nexcept: print("unknown");') + +OS_NAME := "$(shell uname)" +ifeq ($(OS_NAME), "Darwin") + OS := darwin +else + ifeq ($(OS_NAME), "Linux") + OS := linux + endif +endif + +ARCH := $(shell echo `uname -m` | xargs)# E.g., arm64 or x86_64. # This variable contains the first goal that matches any of the listed goals # here, else it contains an empty string. The net effect is to filter out @@ -107,7 +118,7 @@ upgrade-quiet: # Generate a Software Bill of Materials (SBOM). .PHONY: sbom sbom: requirements - cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-sbom.json + cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-sbom.json # Generate a requirements.txt file containing version and integrity hashes for all # packages currently installed in the virtual environment. There's no easy way to @@ -129,14 +140,14 @@ requirements.txt: pyproject.toml [[ $$pkg =~ (.*)==(.*) ]] && curl -s https://pypi.org/pypi/$${BASH_REMATCH[1]}/$${BASH_REMATCH[2]}/json | python -c "import json, sys; print(''.join(f''' \\\\\n --hash=sha256:{pkg['digests']['sha256']}''' for pkg in json.load(sys.stdin)['urls']));" >> requirements.txt; \ done echo -e -n "$(PACKAGE_NAME)==$(PACKAGE_VERSION)" >> requirements.txt - if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz ]; then \ - echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ + if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ fi - if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl ]; then \ - echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl | grep '^\-\-hash')" >> requirements.txt; \ + if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl | grep '^\-\-hash')" >> requirements.txt; \ fi echo "" >> requirements.txt - cp requirements.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-requirements.txt + cp requirements.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-requirements.txt # Audit the currently installed packages. Skip packages that are installed in # editable mode (like the one in development here) because they may not have @@ -175,17 +186,19 @@ test: # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH # set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html .PHONY: dist -dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test +dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format wheel -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test + mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format sdist + mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs-html python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html/ dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip: docs-md python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip docs/_build/markdown/ -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt: - echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt: + echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt # Build the HTML and Markdown documentation from the package's source. DOCS_SOURCE := $(shell git ls-files docs/source) From 62267e2172b4052ddbc2ae20830a3bc0a59d8f5f Mon Sep 17 00:00:00 2001 From: behnazh Date: Sat, 12 Apr 2025 05:42:36 +1000 Subject: [PATCH 5/8] test: merge the download dest paths --- .github/workflows/pr-change-set.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-change-set.yaml b/.github/workflows/pr-change-set.yaml index 33e02617..8e4fdd51 100644 --- a/.github/workflows/pr-change-set.yaml +++ b/.github/workflows/pr-change-set.yaml @@ -39,6 +39,7 @@ jobs: uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: path: dist + merge-multiple: true # Verify hashes by first computing hashes for the artifacts and then comparing them # against the hashes computed by the build job. @@ -49,5 +50,6 @@ jobs: set -euo pipefail echo "Hash of package should be $ARTIFACT_HASH." echo "Decoding the artifact hash:" + ls dist echo "$ARTIFACT_HASH" | base64 --decode echo "$ARTIFACT_HASH" | base64 --decode | sha256sum --strict --check --status || exit 1 From cbcea5646eedf32d9cc4942e9c608a7d7fdde91c Mon Sep 17 00:00:00 2001 From: behnazh Date: Sat, 12 Apr 2025 06:25:16 +1000 Subject: [PATCH 6/8] test: artifact hashes --- .github/workflows/_build.yaml | 78 ++++++++++++++++++++-------- .github/workflows/pr-change-set.yaml | 30 +++++++---- 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 9c739b65..195ae1a8 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -27,22 +27,17 @@ on: type: boolean required: true description: Enable or disable running pip_audit to check installed packages for vulnerabilities - outputs: - artifacts-sha256: - value: ${{ jobs.build.outputs.artifacts-sha256 }} - description: The hash of the artifacts permissions: contents: read env: - ARTIFACT_OS: ubuntu-latest # The default OS for release. - ARTIFACT_PYTHON: '3.13' # The default Python version for release. + RELEASE_OS_X86_64: ubuntu-24.04 # Default OS for x86_64-compatible release artifacts. + RELEASE_OS_ARM64: ubuntu-24.04-arm # Default OS for ARM64-compatible release artifacts. + RELEASE_PYTHON_VERSION: '3.13' # Default Python version used for release artifacts. jobs: build: # Uncomment the following to disable checks and tests for Draft pull requests. # if: github.event.pull_request.draft == false - outputs: - artifacts-sha256: ${{ steps.compute-hash.outputs.artifacts-sha256 }} name: Build Python ${{ matrix.python }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -50,12 +45,31 @@ jobs: matrix: # It is recommended to pin a Runner version specifically: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners - # os: [ubuntu-24.04, ubuntu-24.04-arm, macos-latest, windows-latest] - os: [ubuntu-24.04, ubuntu-24.04-arm] + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-latest, windows-latest] python: ['3.10', '3.11', '3.12', '3.13'] + outputs: + arch-env: ${{ steps.set-arch-env.outputs.arch_env }} + steps: + # Create a GitHub Actions environment variable that maps a matrix.os value to a more descriptive environment + # value (e.g., ubuntu-x86-64 or ubuntu-arm64). + - name: Determine architecture label + id: set-arch-env + shell: bash + run: | + if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then + echo "arch_env=ubuntu-x86-64" >> "$GITHUB_OUTPUT" + elif [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then + echo "arch_env=ubuntu-arm64" >> "$GITHUB_OUTPUT" + else + echo "arch_env=unknown" >> "$GITHUB_OUTPUT" + fi + + - name: Test the env variable + run: echo "Architecture-specific value ${{ steps.set-arch-env.outputs.arch_env }}" + - name: Harden Runner uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: @@ -99,24 +113,33 @@ jobs: HYPOTHESIS_PROFILE: github # Generate the requirements.txt that contains the hash digests of the dependencies and - # generate the SBOM using CyclonDX SBOM generator. + # generate the SBOM using CyclonDX SBOM generator for the release Python version and + # supported release OS targets. - name: Generate requirements.txt and SBOM - if: matrix.python == env.ARTIFACT_PYTHON + if: > + matrix.python == env.RELEASE_PYTHON_VERSION && + (matrix.os == env.RELEASE_OS_X86_64 || matrix.os == env.RELEASE_OS_ARM64) run: make requirements sbom # Remove the old requirements.txt file (which includes _all_ packages) and generate a - # new one for the package and its actual and required dependencies only. + # new one for the package and its actual and required dependencies only. Run this step + # for the release Python version and supported release OS targets only. - name: Prune packages and generate required requirements.txt - if: matrix.python == env.ARTIFACT_PYTHON + if: > + matrix.python == env.RELEASE_PYTHON_VERSION && + (matrix.os == env.RELEASE_OS_X86_64 || matrix.os == env.RELEASE_OS_ARM64) run: | rm requirements.txt make prune requirements # Find the paths to the artifact files that will be included in the release, compute # the SHA digest for all the release files and encode them using Base64, and export it - # from this job. + # from this job. Run this step for the release Python version and supported release + # OS targets only. - name: Compute package hash - if: matrix.python == env.ARTIFACT_PYTHON + if: > + matrix.python == env.RELEASE_PYTHON_VERSION && + (matrix.os == env.RELEASE_OS_X86_64 || matrix.os == env.RELEASE_OS_ARM64) id: compute-hash shell: bash run: | @@ -131,15 +154,28 @@ jobs: sha256sum --version DIGEST=$(sha256sum "$TARBALL_PATH" "$WHEEL_PATH" "$REQUIREMENTS_PATH" "$SBOM_PATH" "$HTML_DOCS_PATH" "$MARKDOWN_DOCS_PATH" "$BUILD_EPOCH_PATH" | base64 -w0) echo "Digest of artifacts is $DIGEST." - echo "artifacts-sha256=$DIGEST" >> "$GITHUB_OUTPUT" + echo "$DIGEST" > artifacts-sha256-file-${{ steps.set-arch-env.outputs.arch_env }} - # For now only generate artifacts for the specified OS and Python version in env variables. # Currently reusable workflows do not support setting strategy property from the caller workflow. + # Run this step for the release Python version and supported release OS targets only. - name: Upload the package artifact for debugging and release - if: matrix.python == env.ARTIFACT_PYTHON + if: > + matrix.python == env.RELEASE_PYTHON_VERSION && + (matrix.os == env.RELEASE_OS_X86_64 || matrix.os == env.RELEASE_OS_ARM64) uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: - name: artifact-${{ matrix.os }}-python-${{ matrix.python }} - path: dist + name: artifacts-${{ steps.set-arch-env.outputs.arch_env }} + path: ./dist*/ if-no-files-found: error retention-days: 7 + + # Run this step for the release Python version and supported release OS targets only. + - name: Upload artifacts-sha256 + if: > + matrix.python == env.RELEASE_PYTHON_VERSION && + (matrix.os == env.RELEASE_OS_X86_64 || matrix.os == env.RELEASE_OS_ARM64) + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: artifacts-sha256-file-${{ steps.set-arch-env.outputs.arch_env }} + path: artifacts-sha256-file-${{ steps.set-arch-env.outputs.arch_env }} + retention-days: 7 diff --git a/.github/workflows/pr-change-set.yaml b/.github/workflows/pr-change-set.yaml index 8e4fdd51..cfc6f9f5 100644 --- a/.github/workflows/pr-change-set.yaml +++ b/.github/workflows/pr-change-set.yaml @@ -38,18 +38,30 @@ jobs: - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - path: dist - merge-multiple: true + path: downloads # Verify hashes by first computing hashes for the artifacts and then comparing them # against the hashes computed by the build job. - name: Verify the artifact hash - env: - ARTIFACT_HASH: ${{ needs.build.outputs.artifacts-sha256 }} run: | set -euo pipefail - echo "Hash of package should be $ARTIFACT_HASH." - echo "Decoding the artifact hash:" - ls dist - echo "$ARTIFACT_HASH" | base64 --decode - echo "$ARTIFACT_HASH" | base64 --decode | sha256sum --strict --check --status || exit 1 + cd downloads + for ARCH in "ubuntu-x86-64" "ubuntu-arm64"; do + HASH_DIR="artifacts-sha256-file-${ARCH}" + ARTIFACT_DIR="artifacts-${ARCH}" + HASH_FILE="${HASH_DIR}/artifacts-sha256-file-${ARCH}" + + echo "Verifying artifacts for ${ARCH}" + echo "Decoding expected SHA256 digest:" + DECODED_HASH=$(base64 --decode "${HASH_FILE}") + echo "$DECODED_HASH" + + cd "${ARTIFACT_DIR}" + echo "$DECODED_HASH" | sha256sum --strict --check --status || { + echo "Hash verification failed for ${ARCH}!" + exit 1 + } + cd - > /dev/null + + echo "Hash verified successfully for ${ARCH}" + done From 4cb47708e17e6a9611f2ddfa5b5e62cc222d5f11 Mon Sep 17 00:00:00 2001 From: behnazh Date: Tue, 15 Apr 2025 11:46:47 +1000 Subject: [PATCH 7/8] test: clean up and improve the workflow & Makefile --- .github/workflows/pr-change-set.yaml | 8 +++++-- Makefile | 31 +++++++++++++--------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr-change-set.yaml b/.github/workflows/pr-change-set.yaml index cfc6f9f5..82e1cbb0 100644 --- a/.github/workflows/pr-change-set.yaml +++ b/.github/workflows/pr-change-set.yaml @@ -22,9 +22,9 @@ jobs: with: disable-pip-audit: ${{ vars.DISABLE_PIP_AUDIT == 'true' }} - test: + verify_artifacts: needs: [build] - name: test + name: verify_artifacts runs-on: ubuntu-latest permissions: contents: read @@ -35,6 +35,10 @@ jobs: with: fetch-depth: 0 + # Download all uploaded artifacts in the build job into the 'downloads' directory. + # This includes built package distributions and SHA256 hash files from all matrix jobs. + # The `path` input ensures all artifacts are placed under the 'downloads/' folder while + # maintaining their respective artifact subdirectory structure. - name: Download artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: diff --git a/Makefile b/Makefile index cc72a8b2..b90787a3 100644 --- a/Makefile +++ b/Makefile @@ -7,16 +7,13 @@ SHELL := bash PACKAGE_NAME := package PACKAGE_VERSION := $(shell python -c $$'try: import $(PACKAGE_NAME); print($(PACKAGE_NAME).__version__, end="");\nexcept: print("unknown");') -OS_NAME := "$(shell uname)" -ifeq ($(OS_NAME), "Darwin") - OS := darwin -else - ifeq ($(OS_NAME), "Linux") - OS := linux - endif -endif +# Determine the OS and architecture. +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') + +ARCH := $(shell uname -m) -ARCH := $(shell echo `uname -m` | xargs)# E.g., arm64 or x86_64. +# Construct full package identifier. +PACKAGE_FULL_NAME := $(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH) # This variable contains the first goal that matches any of the listed goals # here, else it contains an empty string. The net effect is to filter out @@ -118,7 +115,7 @@ upgrade-quiet: # Generate a Software Bill of Materials (SBOM). .PHONY: sbom sbom: requirements - cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-sbom.json + cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_FULL_NAME)-sbom.json # Generate a requirements.txt file containing version and integrity hashes for all # packages currently installed in the virtual environment. There's no easy way to @@ -140,14 +137,14 @@ requirements.txt: pyproject.toml [[ $$pkg =~ (.*)==(.*) ]] && curl -s https://pypi.org/pypi/$${BASH_REMATCH[1]}/$${BASH_REMATCH[2]}/json | python -c "import json, sys; print(''.join(f''' \\\\\n --hash=sha256:{pkg['digests']['sha256']}''' for pkg in json.load(sys.stdin)['urls']));" >> requirements.txt; \ done echo -e -n "$(PACKAGE_NAME)==$(PACKAGE_VERSION)" >> requirements.txt - if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz ]; then \ - echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ + if [ -f dist/$(PACKAGE_FULL_NAME).tar.gz ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_FULL_NAME).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ fi if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl ]; then \ echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl | grep '^\-\-hash')" >> requirements.txt; \ fi echo "" >> requirements.txt - cp requirements.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-requirements.txt + cp requirements.txt dist/$(PACKAGE_FULL_NAME)-requirements.txt # Audit the currently installed packages. Skip packages that are installed in # editable mode (like the one in development here) because they may not have @@ -186,19 +183,19 @@ test: # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH # set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html .PHONY: dist -dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt +dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_FULL_NAME)-build-epoch.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format wheel mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format sdist - mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH).tar.gz + mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_FULL_NAME).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs-html python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html/ dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip: docs-md python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip docs/_build/markdown/ -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt: - echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH)-build-epoch.txt +dist/$(PACKAGE_FULL_NAME)-build-epoch.txt: + echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_FULL_NAME)-build-epoch.txt # Build the HTML and Markdown documentation from the package's source. DOCS_SOURCE := $(shell git ls-files docs/source) From ac005a5faec4b56e1d864b207b7ffc5d48b92058 Mon Sep 17 00:00:00 2001 From: behnazh Date: Tue, 15 Apr 2025 13:34:44 +1000 Subject: [PATCH 8/8] build: follow pypi conventions --- Makefile | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index b90787a3..33b23760 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,22 @@ PACKAGE_NAME := package PACKAGE_VERSION := $(shell python -c $$'try: import $(PACKAGE_NAME); print($(PACKAGE_NAME).__version__, end="");\nexcept: print("unknown");') # Determine the OS and architecture. -OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +OS := $(shell uname -s) +ifeq ($(OS),Darwin) + PLATFORM_NAME := macosx +else + ifeq ($(OS),Linux) + PLATFORM_NAME := manylinux + endif +endif ARCH := $(shell uname -m) +# Construct short package identifier. +PACKAGE_SDIST_NAME := $(PACKAGE_NAME)-$(PACKAGE_VERSION) + # Construct full package identifier. -PACKAGE_FULL_NAME := $(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(OS)-$(ARCH) +PACKAGE_WHEEL_DIST_NAME := $(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-cp3-abi3-$(PLATFORM_NAME)_$(ARCH) # This variable contains the first goal that matches any of the listed goals # here, else it contains an empty string. The net effect is to filter out @@ -115,7 +125,7 @@ upgrade-quiet: # Generate a Software Bill of Materials (SBOM). .PHONY: sbom sbom: requirements - cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_FULL_NAME)-sbom.json + cyclonedx-py requirements --output-format json --outfile dist/$(PACKAGE_WHEEL_DIST_NAME)-sbom.json # Generate a requirements.txt file containing version and integrity hashes for all # packages currently installed in the virtual environment. There's no easy way to @@ -137,14 +147,14 @@ requirements.txt: pyproject.toml [[ $$pkg =~ (.*)==(.*) ]] && curl -s https://pypi.org/pypi/$${BASH_REMATCH[1]}/$${BASH_REMATCH[2]}/json | python -c "import json, sys; print(''.join(f''' \\\\\n --hash=sha256:{pkg['digests']['sha256']}''' for pkg in json.load(sys.stdin)['urls']));" >> requirements.txt; \ done echo -e -n "$(PACKAGE_NAME)==$(PACKAGE_VERSION)" >> requirements.txt - if [ -f dist/$(PACKAGE_FULL_NAME).tar.gz ]; then \ - echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_FULL_NAME).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ + if [ -f dist/$(PACKAGE_SDIST_NAME).tar.gz ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_SDIST_NAME).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ fi - if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl ]; then \ - echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl | grep '^\-\-hash')" >> requirements.txt; \ + if [ -f dist/$(PACKAGE_WHEEL_DIST_NAME).whl ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_WHEEL_DIST_NAME).whl | grep '^\-\-hash')" >> requirements.txt; \ fi echo "" >> requirements.txt - cp requirements.txt dist/$(PACKAGE_FULL_NAME)-requirements.txt + cp requirements.txt dist/$(PACKAGE_WHEEL_DIST_NAME)-requirements.txt # Audit the currently installed packages. Skip packages that are installed in # editable mode (like the one in development here) because they may not have @@ -183,19 +193,18 @@ test: # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH # set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html .PHONY: dist -dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_FULL_NAME)-build-epoch.txt +dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_SDIST_NAME).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_WHEEL_DIST_NAME)-build-epoch.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format wheel - mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-$(OS)-$(ARCH).whl -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: + mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_WHEEL_DIST_NAME).whl +dist/$(PACKAGE_SDIST_NAME).tar.gz: SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format sdist - mv dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_FULL_NAME).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs-html python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html/ dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip: docs-md python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip docs/_build/markdown/ -dist/$(PACKAGE_FULL_NAME)-build-epoch.txt: - echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_FULL_NAME)-build-epoch.txt +dist/$(PACKAGE_WHEEL_DIST_NAME)-build-epoch.txt: + echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_WHEEL_DIST_NAME)-build-epoch.txt # Build the HTML and Markdown documentation from the package's source. DOCS_SOURCE := $(shell git ls-files docs/source)