From ede7b21f45a0a1e945a8e03c5aca72a515bcc9a9 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Mon, 26 Jan 2026 10:34:25 +0100 Subject: [PATCH 1/4] feat: Added test workflow to install builded wheels before the upload --- .github/workflows/build-wheels-defined.yml | 79 ++++- .github/workflows/build-wheels-platforms.yml | 10 +- .github/workflows/test-wheels-install.yml | 111 +++++++ .github/workflows/unit-tests.yml | 38 +++ .github/workflows/wheels-repair.yml | 32 +- test/test_build_wheels.py | 325 +++++++++++++++++++ test/test_wheels_install.py | 137 ++++++++ test_build_wheels.py | 165 ---------- 8 files changed, 715 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/test-wheels-install.yml create mode 100644 .github/workflows/unit-tests.yml create mode 100644 test/test_build_wheels.py create mode 100644 test/test_wheels_install.py delete mode 100644 test_build_wheels.py diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index 0c43309..12fc0b6 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -40,6 +40,11 @@ on: type: boolean required: false default: true + os_linux_armv7_legacy: + description: Build on linux armv7 legacy (bullseye, glibc 2.31) + type: boolean + required: false + default: false env: GH_TOKEN : ${{ secrets.GITHUB_TOKEN }} @@ -86,7 +91,7 @@ jobs: - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-ubuntu-${{ matrix.python-version }} + name: wheels-download-directory-linux-x86_64-${{ matrix.python-version }} path: ./downloaded_wheels @@ -123,7 +128,7 @@ jobs: - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-windows-${{ matrix.python-version }} + name: wheels-download-directory-windows-x86_64-${{ matrix.python-version }} path: ./downloaded_wheels @@ -165,7 +170,7 @@ jobs: - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-macos-x86-${{ matrix.python-version }} + name: wheels-download-directory-macos-x86_64-${{ matrix.python-version }} path: ./downloaded_wheels @@ -246,6 +251,7 @@ jobs: -w /work \ -e GH_TOKEN="${GH_TOKEN}" \ -e PIP_NO_CACHE_DIR=1 \ + -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \ python:${{ matrix.python-version }}-bookworm \ bash -c " set -e @@ -262,7 +268,7 @@ jobs: - name: Upload artifacts of downloaded_wheels directory uses: actions/upload-artifact@v4 with: - name: wheels-download-directory-linux-arm7-${{ matrix.python-version }} + name: wheels-download-directory-linux-armv7-${{ matrix.python-version }} path: ./downloaded_wheels @@ -301,17 +307,76 @@ jobs: name: wheels-download-directory-linux-arm64-${{ matrix.python-version }} path: ./downloaded_wheels + + linux-armv7-legacy: + needs: get-supported-versions + name: linux aarch32 (armv7 legacy) + if: ${{ inputs.os_linux_armv7_legacy }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + exclude: + # Python 3.14 doesn't have bullseye images for ARM + - python-version: '3.14' + steps: + - name: Set up QEMU for ARMv7 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build wheels - ARMv7 Legacy (in Docker) + # Build on Bullseye (glibc 2.31) for compatibility with older systems + run: | + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + -e GH_TOKEN="${GH_TOKEN}" \ + -e PIP_NO_CACHE_DIR=1 \ + -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \ + python:${{ matrix.python-version }}-bullseye \ + bash -c " + set -e + python --version + # Install pip packages without cache to reduce memory usage + python -m pip install --no-cache-dir --upgrade pip + python -m pip install --no-cache-dir -r build_requirements.txt + bash os_dependencies/linux_arm.sh + # Source Rust environment after installation + . \$HOME/.cargo/env + python build_wheels_from_file.py --requirements '${{ inputs.packages }}' + " + + - name: Upload artifacts of downloaded_wheels directory + uses: actions/upload-artifact@v4 + with: + name: wheels-download-directory-linux-armv7legacy-${{ matrix.python-version }} + path: ./downloaded_wheels + # Repair wheels for dynamically linked libraries on all platforms # https://github.com/espressif/idf-python-wheels/blob/main/README.md#universal-wheel-tag---linking-of-dynamic-libraries repair-wheels: if: ${{ always() }} - needs: [get-supported-versions, ubuntu-latest, windows-latest, macos-latest, macos-m1, linux-armv7, linux-arm64] + needs: [get-supported-versions, ubuntu-latest, windows-latest, macos-latest, macos-m1, linux-armv7, linux-arm64, linux-armv7-legacy] name: Repair wheels uses: ./.github/workflows/wheels-repair.yml + # Test that all wheels can be installed on all supported platforms + test-wheels: + if: ${{ always() }} + needs: [get-supported-versions, repair-wheels] + name: Test wheels installation + uses: ./.github/workflows/test-wheels-install.yml + with: + supported_python_versions: ${{ needs.get-supported-versions.outputs.supported_python }} + upload-python-wheels: if: ${{ always() }} - needs: [repair-wheels] + needs: [test-wheels] name: Upload Python wheels - uses: espressif/idf-python-wheels/.github/workflows/upload-python-wheels.yml@main + uses: ./.github/workflows/upload-python-wheels.yml secrets: inherit diff --git a/.github/workflows/build-wheels-platforms.yml b/.github/workflows/build-wheels-platforms.yml index 313962a..cfbf869 100644 --- a/.github/workflows/build-wheels-platforms.yml +++ b/.github/workflows/build-wheels-platforms.yml @@ -213,8 +213,16 @@ jobs: name: Repair wheels uses: ./.github/workflows/wheels-repair.yml + # Test that all wheels can be installed on all supported platforms + test-wheels: + needs: [get-supported-versions, repair-wheels] + name: Test wheels installation + uses: ./.github/workflows/test-wheels-install.yml + with: + supported_python_versions: ${{ needs.get-supported-versions.outputs.supported_python }} + upload-python-wheels: - needs: [repair-wheels] + needs: [test-wheels] name: Upload Python wheels uses: ./.github/workflows/upload-python-wheels.yml secrets: inherit diff --git a/.github/workflows/test-wheels-install.yml b/.github/workflows/test-wheels-install.yml new file mode 100644 index 0000000..6de4c1c --- /dev/null +++ b/.github/workflows/test-wheels-install.yml @@ -0,0 +1,111 @@ +name: Test wheels installation + +# Test that all built wheels are valid and platform-compatible +# This workflow runs after repair-wheels and before upload to catch any issues +# Uses test_wheels_install.py for consistent testing across all platforms + +on: + workflow_call: + inputs: + supported_python_versions: + description: 'JSON array of supported Python versions' + required: true + type: string + +jobs: + test-install: + name: Test ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + os: + - Windows + - Linux x86_64 + - macOS Intel + - macOS ARM + - Linux ARM64 + - Linux ARMv7 + - Linux ARMv7 Legacy + include: + - os: Windows + runner: windows-latest + arch: windows-x86_64 + - os: Linux x86_64 + runner: ubuntu-latest + arch: linux-x86_64 + - os: macOS Intel + runner: macos-15-intel + arch: macos-x86_64 + - os: macOS ARM + runner: macos-latest + arch: macos-arm64 + - os: Linux ARM64 + runner: ubuntu-24.04-arm + arch: linux-arm64 + - os: Linux ARMv7 + runner: ubuntu-latest + arch: linux-armv7 + - os: Linux ARMv7 Legacy + runner: ubuntu-latest + arch: linux-armv7legacy + python-version: ${{ fromJson(inputs.supported_python_versions) }} + exclude: + # Python 3.14 doesn't have bullseye images for ARM + - python-version: '3.14' + os: Linux ARMv7 Legacy + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU for ARMv7 + if: matrix.os == 'Linux ARMv7' || matrix.os == 'Linux ARMv7 Legacy' + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + + - name: Download repaired wheels + uses: actions/download-artifact@v4 + with: + name: wheels-repaired-${{ matrix.arch }} + path: ./downloaded_wheels + + - name: Setup Python + if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy' + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Test wheel installation + if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy' + run: | + python --version + python -m pip install --upgrade pip + python test/test_wheels_install.py + + - name: Test wheel installation - ARMv7 (in Docker) + if: matrix.os == 'Linux ARMv7' + run: | + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + python:${{ matrix.python-version }}-bookworm \ + bash -c " + python --version + python -m pip install --upgrade pip + python test/test_wheels_install.py + " + + - name: Test wheel installation - ARMv7 Legacy (in Docker) + if: matrix.os == 'Linux ARMv7 Legacy' + run: | + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + python:${{ matrix.python-version }}-bullseye \ + bash -c " + python --version + python -m pip install --upgrade pip + python test/test_wheels_install.py + " diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..9e1f5d4 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,38 @@ +name: Unit Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + get-supported-versions: + name: Get Supported Versions + uses: ./.github/workflows/get-supported-versions.yml + secrets: inherit + + unit-tests: + name: Unit Tests (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install packaging pyyaml colorama requests + + - name: Run unit tests + run: python -m unittest discover -s test -v diff --git a/.github/workflows/wheels-repair.yml b/.github/workflows/wheels-repair.yml index ea0376a..c58f407 100644 --- a/.github/workflows/wheels-repair.yml +++ b/.github/workflows/wheels-repair.yml @@ -82,37 +82,50 @@ jobs: path: ./downloaded_wheels merge-multiple: true + - name: Check for wheels + id: check-wheels + run: | + if [ -d "./downloaded_wheels" ] && [ -n "$(find ./downloaded_wheels -name '*.whl' 2>/dev/null)" ]; then + echo "has_wheels=true" >> $GITHUB_OUTPUT + echo "Found wheels to repair" + else + echo "has_wheels=false" >> $GITHUB_OUTPUT + echo "No wheels found for ${{ matrix.platform }} - skipping repair" + fi + shell: bash + - name: Setup Python + if: steps.check-wheels.outputs.has_wheels == 'true' uses: actions/setup-python@v5 with: python-version: '3.12' - name: Set up QEMU - if: matrix.setup_qemu + if: matrix.setup_qemu && steps.check-wheels.outputs.has_wheels == 'true' uses: docker/setup-qemu-action@v3 with: platforms: ${{ matrix.qemu_platform }} - name: Install repair tool - Windows - if: matrix.tool == 'delvewheel' + if: matrix.tool == 'delvewheel' && steps.check-wheels.outputs.has_wheels == 'true' run: python -m pip install delvewheel - name: Install repair tool - macOS - if: matrix.tool == 'delocate' + if: matrix.tool == 'delocate' && steps.check-wheels.outputs.has_wheels == 'true' run: python -m pip install delocate - name: Install OS dependencies - macOS - if: matrix.tool == 'delocate' + if: matrix.tool == 'delocate' && steps.check-wheels.outputs.has_wheels == 'true' run: bash os_dependencies/macos.sh - name: Install dependencies and repair - Windows/macOS - if: matrix.tool != 'auditwheel' + if: matrix.tool != 'auditwheel' && steps.check-wheels.outputs.has_wheels == 'true' run: | python -m pip install -r build_requirements.txt python repair_wheels.py - name: Repair Linux x86_64 wheels in manylinux container - if: matrix.platform == 'Linux x86_64' + if: matrix.platform == 'Linux x86_64' && steps.check-wheels.outputs.has_wheels == 'true' run: | docker run --rm \ -v $(pwd):/work \ @@ -127,7 +140,7 @@ jobs: " - name: Repair Linux ARM64 wheels in manylinux container - if: matrix.platform == 'Linux ARM64' + if: matrix.platform == 'Linux ARM64' && steps.check-wheels.outputs.has_wheels == 'true' run: | docker run --rm \ --platform ${{ matrix.docker_platform }} \ @@ -144,7 +157,7 @@ jobs: - name: Repair Linux ARMv7 wheels in manylinux container # Using --break-system-packages and --ignore-installed to avoid conflicts with system packages - if: matrix.platform == 'Linux ARMv7' + if: matrix.platform == 'Linux ARMv7' && steps.check-wheels.outputs.has_wheels == 'true' run: | docker run --rm \ --platform ${{ matrix.docker_platform }} \ @@ -160,7 +173,7 @@ jobs: - name: Repair Linux ARMv7 Legacy wheels in manylinux container # Using --break-system-packages and --ignore-installed to avoid conflicts with system packages - if: matrix.platform == 'Linux ARMv7 Legacy' + if: matrix.platform == 'Linux ARMv7 Legacy' && steps.check-wheels.outputs.has_wheels == 'true' run: | docker run --rm \ --platform ${{ matrix.docker_platform }} \ @@ -175,6 +188,7 @@ jobs: " - name: Re-upload artifacts with repaired wheels + if: steps.check-wheels.outputs.has_wheels == 'true' uses: actions/upload-artifact@v4 with: name: wheels-repaired-${{ matrix.arch }} diff --git a/test/test_build_wheels.py b/test/test_build_wheels.py new file mode 100644 index 0000000..27021d0 --- /dev/null +++ b/test/test_build_wheels.py @@ -0,0 +1,325 @@ +# ruff: noqa: E501 +# line too long skip in ruff for whole file (formatting would be worst than long lines) +# +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +import sys +import unittest + +from pathlib import Path +from unittest.mock import patch + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from packaging.requirements import Requirement + +from _helper_functions import get_no_binary_args +from _helper_functions import merge_requirements +from build_wheels import _add_into_requirements +from build_wheels import get_used_idf_branches +from yaml_list_adapter import YAMLListAdapter + + +class TestChangeSpecifierLogic(unittest.TestCase): + """Test the _change_specifier_logic method.""" + + def setUp(self): + """Create a YAMLListAdapter instance for testing.""" + # Create instance with a minimal valid YAML file + self.adapter = YAMLListAdapter.__new__(YAMLListAdapter) + self.adapter._yaml_list = [] + self.adapter.exclude = False + self.adapter.requirements = set() + + def test_change_specifier_logic(self): + """Test that specifier logic is correctly inverted (logical negation).""" + # The function performs logical negation: + # > becomes <= (not greater means less or equal) + # < becomes >= (not less means greater or equal) + # >= becomes < (not greater-or-equal means less) + # <= becomes > (not less-or-equal means greater) + test_cases = ( + (">0.9.0.2", "<=0.9.0.2"), + ("<0.9.0.2", ">=0.9.0.2"), + ("==0.9.0.2", "!=0.9.0.2"), + (">=0.9.0.2", "<0.9.0.2"), + ("<=0.9.0.2", ">0.9.0.2"), + ("!=0.9.0.2", "==0.9.0.2"), + ("===0.9.0.2", "===0.9.0.2"), + ) + + for original, expected in test_cases: + with self.subTest(original=original): + new_spec, ver, _ = self.adapter._change_specifier_logic(original) + result = f"{new_spec}{ver}" + self.assertEqual(result, expected) + + +class TestYAMLtoRequirement(unittest.TestCase): + """Test the _yaml_to_requirement method.""" + + def setUp(self): + """Create a YAMLListAdapter instance for testing.""" + self.adapter = YAMLListAdapter.__new__(YAMLListAdapter) + self.adapter._yaml_list = [] + self.adapter.exclude = False + self.adapter.requirements = set() + + def test_simple_package(self): + """Test conversion of a simple package without markers.""" + yaml_list = [{"package_name": "numpy"}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("numpy")}) + + def test_package_with_version(self): + """Test conversion of a package with version specifier.""" + yaml_list = [{"package_name": "numpy", "version": "<1.20"}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("numpy<1.20")}) + + def test_package_with_multiple_versions(self): + """Test conversion of a package with multiple version specifiers.""" + yaml_list = [{"package_name": "numpy", "version": ["<1.20", ">=1.10"]}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("numpy<1.20,>=1.10")}) + + def test_package_with_platform(self): + """Test conversion of a package with platform marker.""" + yaml_list = [{"package_name": "pywin32", "platform": "win32"}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("pywin32; sys_platform == 'win32'")}) + + def test_package_with_multiple_platforms(self): + """Test conversion of a package with multiple platform markers.""" + yaml_list = [{"package_name": "pkg", "platform": ["win32", "linux"]}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("pkg; sys_platform == 'win32' or sys_platform == 'linux'")}) + + def test_package_with_python_version(self): + """Test conversion of a package with python version marker.""" + yaml_list = [{"package_name": "pkg", "python": ">=3.8"}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("pkg; python_version >= '3.8'")}) + + def test_package_with_version_and_platform(self): + """Test conversion of a package with version and platform.""" + yaml_list = [{"package_name": "numpy", "version": "<=1.20", "platform": "win32"}] + result = self.adapter._yaml_to_requirement(yaml_list) + self.assertEqual(result, {Requirement("numpy<=1.20; sys_platform == 'win32'")}) + + def test_exclude_simple_platform(self): + """Test exclude mode with platform marker.""" + yaml_list = [{"package_name": "pkg", "platform": "win32"}] + result = self.adapter._yaml_to_requirement(yaml_list, exclude=True) + self.assertEqual(result, {Requirement("pkg; sys_platform != 'win32'")}) + + def test_exclude_version(self): + """Test exclude mode with version specifier.""" + yaml_list = [{"package_name": "numpy", "version": "<1.20"}] + result = self.adapter._yaml_to_requirement(yaml_list, exclude=True) + self.assertEqual(result, {Requirement("numpy>=1.20")}) + + +class TestYAMLListAdapterIntegration(unittest.TestCase): + """Integration tests using actual YAML files.""" + + def test_load_include_list(self): + """Test loading the include_list.yaml file.""" + try: + adapter = YAMLListAdapter("include_list.yaml") + self.assertIsInstance(adapter.requirements, set) + except FileNotFoundError: + self.skipTest("include_list.yaml not found") + + def test_load_exclude_list(self): + """Test loading the exclude_list.yaml file.""" + try: + adapter = YAMLListAdapter("exclude_list.yaml", exclude=True) + self.assertIsInstance(adapter.requirements, set) + except FileNotFoundError: + self.skipTest("exclude_list.yaml not found") + + +class TestWheelCompatibility(unittest.TestCase): + """Test the is_wheel_compatible function from test_wheels_install.py.""" + + def setUp(self): + """Import the function to test.""" + sys.path.insert(0, str(Path(__file__).parent)) + from test_wheels_install import is_wheel_compatible + + self.is_wheel_compatible = is_wheel_compatible + + def test_exact_python_version_match(self): + """Test that cpXY wheels match the exact Python version.""" + self.assertTrue(self.is_wheel_compatible("numpy-1.0.0-cp311-cp311-linux_x86_64.whl", "311")) + self.assertFalse(self.is_wheel_compatible("numpy-1.0.0-cp310-cp310-linux_x86_64.whl", "311")) + + def test_universal_py3_wheel(self): + """Test that py3 wheels are compatible with any Python 3.""" + self.assertTrue(self.is_wheel_compatible("six-1.0.0-py3-none-any.whl", "311")) + self.assertTrue(self.is_wheel_compatible("six-1.0.0-py3-none-any.whl", "39")) + + def test_universal_py2_py3_wheel(self): + """Test that py2.py3 wheels are compatible with any Python.""" + self.assertTrue(self.is_wheel_compatible("six-1.0.0-py2.py3-none-any.whl", "311")) + self.assertTrue(self.is_wheel_compatible("six-1.0.0-py2.py3-none-any.whl", "39")) + + def test_abi3_wheel(self): + """Test that abi3 wheels are compatible.""" + self.assertTrue(self.is_wheel_compatible("cryptography-41.0.0-cp39-abi3-linux_x86_64.whl", "311")) + self.assertTrue(self.is_wheel_compatible("cryptography-41.0.0-cp39-abi3-linux_x86_64.whl", "39")) + + +class TestGetUsedIdfBranches(unittest.TestCase): + """Test the get_used_idf_branches function.""" + + @patch("build_wheels.MIN_IDF_MAJOR_VERSION", 5) + @patch("build_wheels.MIN_IDF_MINOR_VERSION", 0) + def test_filters_old_branches(self): + """Test that branches older than minimum version are filtered out.""" + branches = [ + "release/v4.4", + "release/v5.0", + "release/v5.1", + "release/v5.2", + "master", + ] + result = get_used_idf_branches(branches) + self.assertIn("release/v5.0", result) + self.assertIn("release/v5.1", result) + self.assertIn("release/v5.2", result) + self.assertIn("master", result) + self.assertNotIn("release/v4.4", result) + + @patch("build_wheels.MIN_IDF_MAJOR_VERSION", 5) + @patch("build_wheels.MIN_IDF_MINOR_VERSION", 1) + def test_filters_by_minor_version(self): + """Test that filtering works correctly with minor version.""" + branches = [ + "release/v5.0", + "release/v5.1", + "release/v5.2", + ] + result = get_used_idf_branches(branches) + self.assertNotIn("release/v5.0", result) + self.assertIn("release/v5.1", result) + self.assertIn("release/v5.2", result) + + def test_ignores_non_release_branches(self): + """Test that non-release branches (except master) are ignored.""" + branches = [ + "feature/test", + "bugfix/something", + "release/v5.0", + ] + result = get_used_idf_branches(branches) + self.assertNotIn("feature/test", result) + self.assertNotIn("bugfix/something", result) + self.assertIn("master", result) + + +class TestAddIntoRequirements(unittest.TestCase): + """Test the _add_into_requirements function.""" + + def test_parses_simple_requirements(self): + """Test parsing simple requirement lines.""" + lines = ["numpy", "pandas>=1.0", "requests==2.28.0"] + result = _add_into_requirements(lines) + self.assertEqual(len(result), 3) + names = {r.name for r in result} + self.assertIn("numpy", names) + self.assertIn("pandas", names) + self.assertIn("requests", names) + + def test_ignores_comments(self): + """Test that comment lines are ignored.""" + lines = [ + "# This is a comment", + "numpy", + "pandas # inline comment", + ] + result = _add_into_requirements(lines) + self.assertEqual(len(result), 2) + + def test_ignores_empty_lines(self): + """Test that empty lines are ignored.""" + lines = ["numpy", "", " ", "pandas"] + result = _add_into_requirements(lines) + self.assertEqual(len(result), 2) + + def test_handles_whitespace(self): + """Test that leading/trailing whitespace is handled.""" + lines = [" numpy ", "\tpandas\t"] + result = _add_into_requirements(lines) + self.assertEqual(len(result), 2) + + +class TestMergeRequirements(unittest.TestCase): + """Test the merge_requirements function.""" + + def test_merge_specifiers(self): + """Test merging two requirements with version specifiers.""" + req1 = Requirement("numpy>=1.0") + req2 = Requirement("numpy<2.0") + result = merge_requirements(req1, req2) + self.assertEqual(result.name, "numpy") + self.assertIn(">=1.0", str(result.specifier)) + self.assertIn("<2.0", str(result.specifier)) + + def test_merge_markers(self): + """Test merging two requirements with markers.""" + req1 = Requirement("numpy; sys_platform == 'win32'") + req2 = Requirement("numpy; python_version >= '3.8'") + result = merge_requirements(req1, req2) + self.assertEqual(result.name, "numpy") + self.assertIn("sys_platform", str(result.marker)) + self.assertIn("python_version", str(result.marker)) + + def test_merge_preserves_name(self): + """Test that package name is preserved after merge.""" + req1 = Requirement("requests>=2.0") + req2 = Requirement("requests; sys_platform == 'linux'") + result = merge_requirements(req1, req2) + self.assertEqual(result.name, "requests") + + +class TestGetNoBinaryArgs(unittest.TestCase): + """Test the get_no_binary_args function.""" + + @patch("_helper_functions.platform.system", return_value="Linux") + def test_returns_args_for_source_build_packages_on_linux(self, mock_system): + """Test that --no-binary args are returned for specified packages on Linux.""" + result = get_no_binary_args("cffi") + self.assertEqual(result, ["--no-binary", "cffi"]) + + @patch("_helper_functions.platform.system", return_value="Linux") + def test_handles_requirement_with_version(self, mock_system): + """Test that package name is extracted from requirement string.""" + result = get_no_binary_args("cffi>=1.0") + self.assertEqual(result, ["--no-binary", "cffi"]) + + @patch("_helper_functions.platform.system", return_value="Windows") + def test_returns_empty_on_windows(self, mock_system): + """Test that empty list is returned on Windows.""" + result = get_no_binary_args("cffi") + self.assertEqual(result, []) + + @patch("_helper_functions.platform.system", return_value="Darwin") + def test_returns_empty_on_macos(self, mock_system): + """Test that empty list is returned on macOS.""" + result = get_no_binary_args("cffi") + self.assertEqual(result, []) + + @patch("_helper_functions.platform.system", return_value="Linux") + def test_returns_empty_for_non_source_build_package(self, mock_system): + """Test that empty list is returned for packages not in source build list.""" + result = get_no_binary_args("requests") + self.assertEqual(result, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_wheels_install.py b/test/test_wheels_install.py new file mode 100644 index 0000000..0d3a68b --- /dev/null +++ b/test/test_wheels_install.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +""" +Test wheel installation script for CI workflows. + +This script finds and installs wheels compatible with the current Python version, +verifying that wheel files are valid and platform-compatible. +""" + +from __future__ import annotations + +import re +import subprocess +import sys + +from pathlib import Path + +WHEELS_DIR = Path("./downloaded_wheels") + + +def get_python_version_tag() -> str: + """Get the Python version tag (e.g., '311' for Python 3.11).""" + return f"{sys.version_info.major}{sys.version_info.minor}" + + +def is_wheel_compatible(wheel_name: str, python_version: str) -> bool: + """ + Check if a wheel is compatible with the given Python version. + + Compatible wheels are: + - cpXY: exact Python version match (e.g., cp311 for Python 3.11) + - py3: universal Python 3 wheels + - py2.py3: universal Python 2/3 wheels + - abi3: stable ABI wheels (compatible with Python >= base version) + """ + patterns = [ + rf"-cp{python_version}-", # Exact version match + r"-py3-", # Universal Python 3 + r"-py2\.py3-", # Universal Python 2/3 + r"-abi3-", # Stable ABI + ] + return any(re.search(pattern, wheel_name) for pattern in patterns) + + +def find_compatible_wheels(python_version: str) -> list[Path]: + """Find all wheel files compatible with the given Python version.""" + if not WHEELS_DIR.exists(): + return [] + + wheels = [] + for wheel_path in WHEELS_DIR.glob("*.whl"): + if is_wheel_compatible(wheel_path.name, python_version): + wheels.append(wheel_path) + + return sorted(wheels) + + +def install_wheel(wheel_path: Path) -> tuple[bool, str]: + """ + Install a wheel with --no-deps to verify wheel validity. + + Returns: + tuple: (success: bool, error_message: str) + """ + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--no-deps", + "--no-index", + "--find-links", + str(WHEELS_DIR), + str(wheel_path), + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + return True, "" + + return False, (result.stderr or result.stdout).strip() + + +def main() -> int: + python_version = get_python_version_tag() + + # Find compatible wheels + wheels = find_compatible_wheels(python_version) + print(f"Found {len(wheels)} compatible wheels to test\n") + + if not wheels: + print("No compatible wheels found!") + return 1 + + # Install each wheel + installed = 0 + failed = 0 + failed_wheels = [] + + print("Installing wheels (--no-deps to test wheel validity only)...") + print("-" * 60) + + for wheel_path in wheels: + success, error_message = install_wheel(wheel_path) + + if success: + installed += 1 + else: + failed += 1 + failed_wheels.append((wheel_path.name, error_message)) + print() + print(f"ERROR: Failed to install {wheel_path.name}") + if error_message: + for line in error_message.split("\n"): + print(f" {line}") + print() + + print("-" * 60) + print(f"Results: {installed} installed successfully, {failed} failed\n") + + # Print summary of failures + if failed_wheels: + print("Failed wheels:") + for wheel_name, _ in failed_wheels: + print(f" - {wheel_name}") + print() + return 1 + + print("All wheels installed successfully!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_build_wheels.py b/test_build_wheels.py deleted file mode 100644 index 11ef7c9..0000000 --- a/test_build_wheels.py +++ /dev/null @@ -1,165 +0,0 @@ -# ruff: noqa: E501 -# line too long skip in ruff for whole file (formatting would be worst than long lines) -# -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# -# SPDX-License-Identifier: Apache-2.0 -# -import unittest - -from packaging.requirements import Requirement - -from yaml_list_adapter import YAMLListAdapter - - -class TestYAMLtoRequirement(unittest.TestCase): - def test_change_specifier_logic(self): - version_with_specifier = ( - (">0.9.0.2", "<0.9.0.2"), - ("<0.9.0.2", ">0.9.0.2"), - ("==0.9.0.2", "!=0.9.0.2"), - (">=0.9.0.2", "<=0.9.0.2"), - ("<=0.9.0.2", ">=0.9.0.2"), - ("!=0.9.0.2", "==0.9.0.2"), - ("===0.9.0.2", "===0.9.0.2"), - ) - - for case in version_with_specifier: - self.assertEqual( - f"{YAMLListAdapter._change_specifier_logic(case[0])[0]}{YAMLListAdapter._change_specifier_logic(case[0])[1]}", - case[1], - ) - - def test_yaml_to_requirement(self): - test_requirements = { - Requirement("platform;sys_platform == 'win32'"), - Requirement("platform;sys_platform == 'win32' or sys_platform == 'linux'"), - Requirement("version<42"), - Requirement("version<42,>50"), - Requirement("python;python_version > '3.10'"), - Requirement("python;python_version > '3.10' and python_version != '3.8'"), - Requirement("version-platform<=0.9.0.2;sys_platform == 'win32'"), - Requirement("version-platform<=0.9.0.2,>0.9.1;sys_platform == 'win32'"), - Requirement("version-platform<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux'"), - Requirement("version-platform<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux'"), - Requirement("version-python<=0.9.0.2;python_version < '3.8'"), - Requirement("version-python<=0.9.0.2,>0.9.1;python_version < '3.8'"), - Requirement("version-python<=0.9.0.2;python_version < '3.8' and python_version > '3.11'"), - Requirement("version-python<=0.9.0.2,>0.9.1;python_version < '3.8' and python_version > '3.11'"), - Requirement("platform-python;sys_platform == 'win32' and python_version < '3.8'"), - Requirement( - "platform-python;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'" - ), - Requirement( - "platform-python;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "platform-python;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement("version-platform-python<=0.9.0.2;sys_platform == 'win32' and python_version < '3.8'"), - Requirement("version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' and python_version < '3.8'"), - Requirement( - "version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'" - ), - Requirement( - "version-platform-python<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'" - ), - Requirement( - "version-platform-python<=0.9.0.2;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python<=0.9.0.2,>0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python<=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'" - ), - } - - self.assertEqual(YAMLListAdapter._yaml_to_requirement("test/test_list.yaml"), test_requirements) - - def test_yaml_to_requirement_exclude(self): - test_requirements_exclude = { - Requirement("platform;sys_platform != 'win32'"), - Requirement("platform;sys_platform != 'win32' or sys_platform != 'linux'"), - Requirement("version>42"), - Requirement("version>42,<50"), - Requirement("python;python_version < '3.10'"), - Requirement("python;python_version < '3.10' and python_version == '3.8'"), - Requirement("version-platform>=0.9.0.2;sys_platform == 'win32'"), - Requirement("version-platform;sys_platform != 'win32'"), - Requirement("version-platform>=0.9.0.2,<0.9.1;sys_platform == 'win32'"), - Requirement("version-platform;sys_platform != 'win32'"), - Requirement("version-platform>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux'"), - Requirement("version-platform;sys_platform != 'win32' or sys_platform != 'linux'"), - Requirement("version-platform>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux'"), - Requirement("version-platform;sys_platform != 'win32' or sys_platform != 'linux'"), - Requirement("version-python>=0.9.0.2;python_version < '3.8'"), - Requirement("version-python;python_version > '3.8'"), - Requirement("version-python>=0.9.0.2,<0.9.1;python_version < '3.8'"), - Requirement("version-python;python_version > '3.8'"), - Requirement("version-python>=0.9.0.2;python_version < '3.8' and python_version > '3.11'"), - Requirement("version-python;python_version > '3.8' and python_version < '3.11'"), - Requirement("version-python>=0.9.0.2,<0.9.1;python_version < '3.8' and python_version > '3.11'"), - Requirement("version-python;python_version > '3.8' and python_version < '3.11'"), - Requirement("platform-python;sys_platform != 'win32' and python_version > '3.8'"), - Requirement( - "platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'" - ), - Requirement( - "platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'" - ), - Requirement( - "platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'" - ), - Requirement("version-platform-python>=0.9.0.2;sys_platform == 'win32' and python_version < '3.8'"), - Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8'"), - Requirement("version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' and python_version < '3.8'"), - Requirement("version-platform-python;sys_platform != 'win32' and python_version > '3.8'"), - Requirement( - "version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'" - ), - Requirement( - "version-platform-python>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8'" - ), - Requirement( - "version-platform-python>=0.9.0.2;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'" - ), - Requirement( - "version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' and python_version > '3.8' and python_version < '3.11'" - ), - Requirement( - "version-platform-python>=0.9.0.2,<0.9.1;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'" - ), - Requirement( - "version-platform-python>=0.9.0.2;sys_platform == 'win32' or sys_platform == 'linux' and python_version < '3.8' and python_version > '3.11'" - ), - Requirement( - "version-platform-python;sys_platform != 'win32' or sys_platform != 'linux' and python_version > '3.8' and python_version < '3.11'" - ), - } - - self.assertEqual( - YAMLListAdapter._yaml_to_requirement("test/test_list.yaml", exclude=True), test_requirements_exclude - ) - - -if __name__ == "__main__": - unittest.main() From acbb166ac857b777bc7b6931f2e52f894442f67c Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Fri, 30 Jan 2026 11:41:02 +0100 Subject: [PATCH 2/4] feat: Added additional exclude mechanism for forbidden wheels --- .github/workflows/build-wheels-defined.yml | 1 - .github/workflows/test-wheels-install.yml | 14 +- .github/workflows/upload-python-wheels.yml | 4 +- .github/workflows/wheels-repair.yml | 24 +++ test/test_build_wheels.py | 84 +++++++++ test/test_wheels_install.py | 189 ++++++++++++++++++--- upload_wheels.py | 47 ++++- 7 files changed, 332 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index 12fc0b6..c992c21 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -375,7 +375,6 @@ jobs: supported_python_versions: ${{ needs.get-supported-versions.outputs.supported_python }} upload-python-wheels: - if: ${{ always() }} needs: [test-wheels] name: Upload Python wheels uses: ./.github/workflows/upload-python-wheels.yml diff --git a/.github/workflows/test-wheels-install.yml b/.github/workflows/test-wheels-install.yml index 6de4c1c..fc8a9bd 100644 --- a/.github/workflows/test-wheels-install.yml +++ b/.github/workflows/test-wheels-install.yml @@ -65,10 +65,10 @@ jobs: with: platforms: linux/arm/v7 - - name: Download repaired wheels + - name: Download all repaired wheels uses: actions/download-artifact@v4 with: - name: wheels-repaired-${{ matrix.arch }} + name: wheels-repaired-all path: ./downloaded_wheels - name: Setup Python @@ -82,6 +82,7 @@ jobs: run: | python --version python -m pip install --upgrade pip + pip install -r build_requirements.txt python test/test_wheels_install.py - name: Test wheel installation - ARMv7 (in Docker) @@ -94,6 +95,7 @@ jobs: bash -c " python --version python -m pip install --upgrade pip + pip install -r build_requirements.txt python test/test_wheels_install.py " @@ -107,5 +109,13 @@ jobs: bash -c " python --version python -m pip install --upgrade pip + pip install -r build_requirements.txt python test/test_wheels_install.py " + + - name: Upload tested wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-tested-${{ matrix.arch }}-${{ matrix.python-version }} + path: ./downloaded_wheels/*.whl + if-no-files-found: warn diff --git a/.github/workflows/upload-python-wheels.yml b/.github/workflows/upload-python-wheels.yml index 170c3b0..9b2764e 100644 --- a/.github/workflows/upload-python-wheels.yml +++ b/.github/workflows/upload-python-wheels.yml @@ -22,10 +22,12 @@ jobs: - name: Install dependencies run: python -m pip install -r build_requirements.txt - - name: Download artifacts + - name: Download tested wheels uses: actions/download-artifact@v4 with: + pattern: wheels-tested-* path: ./downloaded_wheels + merge-multiple: true - name: Upload release asset to S3 bucket run: | diff --git a/.github/workflows/wheels-repair.yml b/.github/workflows/wheels-repair.yml index c58f407..cf936f7 100644 --- a/.github/workflows/wheels-repair.yml +++ b/.github/workflows/wheels-repair.yml @@ -195,3 +195,27 @@ jobs: path: ./downloaded_wheels overwrite: true retention-days: 1 + + merge-all-wheels: + name: Merge all repaired wheels + needs: repair-wheels + runs-on: ubuntu-latest + steps: + - name: Download all repaired wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-repaired-* + path: ./all_wheels + merge-multiple: true + + - name: List merged wheels + run: | + echo "Total merged wheels:" + find ./all_wheels -name "*.whl" | wc -l + + - name: Upload merged wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-repaired-all + path: ./all_wheels + retention-days: 1 diff --git a/test/test_build_wheels.py b/test/test_build_wheels.py index 27021d0..9e1c97e 100644 --- a/test/test_build_wheels.py +++ b/test/test_build_wheels.py @@ -174,6 +174,90 @@ def test_abi3_wheel(self): self.assertTrue(self.is_wheel_compatible("cryptography-41.0.0-cp39-abi3-linux_x86_64.whl", "39")) +class TestParseWheelName(unittest.TestCase): + """Test the parse_wheel_name function from test_wheels_install.py.""" + + def setUp(self): + """Import the function to test.""" + sys.path.insert(0, str(Path(__file__).parent)) + from test_wheels_install import parse_wheel_name + + self.parse_wheel_name = parse_wheel_name + + def test_parse_simple_wheel(self): + """Test parsing a simple wheel name.""" + result = self.parse_wheel_name("numpy-1.24.0-cp311-cp311-linux_x86_64.whl") + self.assertEqual(result, ("numpy", "1.24.0")) + + def test_parse_wheel_with_underscores(self): + """Test parsing wheel name with underscores (preserved, canonicalization done later).""" + result = self.parse_wheel_name("ruamel_yaml_clib-0.2.8-cp311-cp311-linux_x86_64.whl") + self.assertEqual(result, ("ruamel_yaml_clib", "0.2.8")) + + def test_parse_wheel_with_pre_release(self): + """Test parsing wheel name with pre-release version.""" + result = self.parse_wheel_name("package-1.0.0a1-py3-none-any.whl") + self.assertEqual(result, ("package", "1.0.0a1")) + + def test_parse_universal_wheel(self): + """Test parsing universal wheel name.""" + result = self.parse_wheel_name("six-1.16.0-py2.py3-none-any.whl") + self.assertEqual(result, ("six", "1.16.0")) + + +class TestShouldExcludeWheel(unittest.TestCase): + """Test the should_exclude_wheel function from test_wheels_install.py. + + Note: The function expects requirements created with exclude=True from YAMLListAdapter, + which inverts the logic (e.g., ==1.5.0 becomes !=1.5.0). + """ + + def setUp(self): + """Import the function to test.""" + sys.path.insert(0, str(Path(__file__).parent)) + from test_wheels_install import should_exclude_wheel + + self.should_exclude_wheel = should_exclude_wheel + + def test_exclude_by_package_name_only(self): + """Test excluding a package by name only (no inversion needed).""" + # Package name only - same for both exclude=True and exclude=False + exclude_requirements = {Requirement("esptool")} + result, reason = self.should_exclude_wheel("esptool-4.0.0-py3-none-any.whl", exclude_requirements) + self.assertTrue(result) + self.assertIn("esptool", reason) + + def test_exclude_by_version(self): + """Test excluding a package by version constraint (inverted specifier).""" + # With exclude=True, ==1.5.0 becomes !=1.5.0 + # So version 1.5.0 is NOT in !=1.5.0 -> should EXCLUDE + # And version 2.0.0 IS in !=1.5.0 -> should KEEP + exclude_requirements = {Requirement("gevent!=1.5.0")} + # Should exclude 1.5.0 (not in !=1.5.0) + result, _ = self.should_exclude_wheel("gevent-1.5.0-cp311-cp311-linux_x86_64.whl", exclude_requirements) + self.assertTrue(result) + # Should not exclude 2.0.0 (is in !=1.5.0) + result, _ = self.should_exclude_wheel("gevent-2.0.0-cp311-cp311-linux_x86_64.whl", exclude_requirements) + self.assertFalse(result) + + def test_no_match_returns_false(self): + """Test that non-matching packages return False.""" + exclude_requirements = {Requirement("esptool")} + result, _ = self.should_exclude_wheel("numpy-1.24.0-cp311-cp311-linux_x86_64.whl", exclude_requirements) + self.assertFalse(result) + + def test_exclude_with_version_range(self): + """Test excluding a package with version range (inverted specifier).""" + # With exclude=True, ==9.5.0 becomes !=9.5.0 + exclude_requirements = {Requirement("pillow!=9.5.0")} + # Should exclude 9.5.0 (not in !=9.5.0) + result, _ = self.should_exclude_wheel("Pillow-9.5.0-cp311-cp311-linux_x86_64.whl", exclude_requirements) + self.assertTrue(result) + # Should not exclude 10.0.0 (is in !=9.5.0) + result, _ = self.should_exclude_wheel("Pillow-10.0.0-cp311-cp311-linux_x86_64.whl", exclude_requirements) + self.assertFalse(result) + + class TestGetUsedIdfBranches(unittest.TestCase): """Test the get_used_idf_branches function.""" diff --git a/test/test_wheels_install.py b/test/test_wheels_install.py index 0d3a68b..c1242ed 100644 --- a/test/test_wheels_install.py +++ b/test/test_wheels_install.py @@ -7,6 +7,7 @@ This script finds and installs wheels compatible with the current Python version, verifying that wheel files are valid and platform-compatible. +It also checks wheels against exclude_list.yaml and removes incompatible ones. """ from __future__ import annotations @@ -17,7 +18,15 @@ from pathlib import Path +from colorama import Fore +from packaging.utils import canonicalize_name +from packaging.version import Version + +from _helper_functions import print_color +from yaml_list_adapter import YAMLListAdapter + WHEELS_DIR = Path("./downloaded_wheels") +EXCLUDE_LIST_PATH = Path("exclude_list.yaml") def get_python_version_tag() -> str: @@ -25,23 +34,112 @@ def get_python_version_tag() -> str: return f"{sys.version_info.major}{sys.version_info.minor}" +def parse_wheel_name(wheel_name: str) -> tuple[str, str] | None: + """ + Parse wheel filename to extract package name and version. + + Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + """ + pattern = re.compile(r"^([A-Za-z0-9_.-]+)-(\d+(?:\.\d+)*(?:[a-zA-Z0-9.]+)?)-") + match = pattern.match(wheel_name) + if match: + pkg_name = match.group(1) + version = match.group(2) + return pkg_name, version + return None + + +def load_exclude_requirements() -> set: + """Load exclude_list.yaml using YAMLListAdapter and return requirements set.""" + adapter = YAMLListAdapter(str(EXCLUDE_LIST_PATH), exclude=True) + return adapter.requirements + + +def should_exclude_wheel(wheel_name: str, exclude_requirements: set) -> tuple[bool, str]: + """ + Check if a wheel should be excluded based on exclude_list.yaml rules. + + Uses YAMLListAdapter with exclude=True, so the logic is inverted: + - If marker evaluates to True -> wheel satisfies "keep" condition, skip + - If version is in the (inverted) specifier -> wheel satisfies "keep" condition, skip + - Otherwise -> wheel should be excluded + + Returns: + tuple: (should_exclude: bool, reason: str) + """ + parsed = parse_wheel_name(wheel_name) + if not parsed: + return False, "" + + pkg_name, wheel_version = parsed + canonical_name = canonicalize_name(pkg_name) + + for req in exclude_requirements: + # Check if package name matches (using canonical names) + if canonicalize_name(req.name) != canonical_name: + continue + + # With exclude=True, if marker evaluates to True -> KEEP the wheel + if req.marker and req.marker.evaluate(): + continue + + # With exclude=True, if version is in the (inverted) specifier -> KEEP the wheel + if req.specifier: + try: + if Version(wheel_version) in req.specifier: + continue + except Exception: + pass + + # Name matches, and marker is False (or absent), and version not in specifier (or absent) + # -> EXCLUDE the wheel + return True, f"matches exclude rule: {req}" + + return False, "" + + +def get_platform_patterns() -> list[str]: + """Get regex patterns for wheels compatible with current platform.""" + platform = sys.platform + if platform == "win32": + return [r"-win_amd64\.whl$", r"-win32\.whl$", r"-any\.whl$"] + elif platform == "darwin": + return [r"-macosx_.*\.whl$", r"-any\.whl$"] + elif platform == "linux": + return [r"-manylinux.*\.whl$", r"-linux.*\.whl$", r"-any\.whl$"] + else: + # Unknown platform, only match universal wheels + return [r"-any\.whl$"] + + def is_wheel_compatible(wheel_name: str, python_version: str) -> bool: """ - Check if a wheel is compatible with the given Python version. + Check if a wheel is compatible with the given Python version AND current platform. - Compatible wheels are: + Python version compatibility: - cpXY: exact Python version match (e.g., cp311 for Python 3.11) - py3: universal Python 3 wheels - py2.py3: universal Python 2/3 wheels - abi3: stable ABI wheels (compatible with Python >= base version) + + Platform compatibility: + - Windows: win32, win_amd64, any + - macOS: macosx_*, any + - Linux: manylinux*, linux*, any """ - patterns = [ + # Check Python version compatibility + python_patterns = [ rf"-cp{python_version}-", # Exact version match r"-py3-", # Universal Python 3 r"-py2\.py3-", # Universal Python 2/3 r"-abi3-", # Stable ABI ] - return any(re.search(pattern, wheel_name) for pattern in patterns) + if not any(re.search(pattern, wheel_name) for pattern in python_patterns): + return False + + # Check platform compatibility + platform_patterns = get_platform_patterns() + return any(re.search(pattern, wheel_name) for pattern in platform_patterns) def find_compatible_wheels(python_version: str) -> list[Path]: @@ -84,52 +182,99 @@ def install_wheel(wheel_path: Path) -> tuple[bool, str]: return False, (result.stderr or result.stdout).strip() +def is_python_version_error(error_message: str) -> bool: + """Check if the error is due to Python version constraints in package metadata.""" + return "requires a different Python" in error_message + + def main() -> int: - python_version = get_python_version_tag() + python_version_tag = get_python_version_tag() + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + + print_color(f"---------- TEST WHEELS INSTALL (Python {python_version}) ----------") + print(f"Platform: {sys.platform}\n") + + # Load exclude list using YAMLListAdapter + exclude_requirements = load_exclude_requirements() + print(f"Loaded {len(exclude_requirements)} exclude requirements from {EXCLUDE_LIST_PATH}\n") # Find compatible wheels - wheels = find_compatible_wheels(python_version) + wheels = find_compatible_wheels(python_version_tag) print(f"Found {len(wheels)} compatible wheels to test\n") if not wheels: - print("No compatible wheels found!") + print_color("No compatible wheels found!", Fore.RED) return 1 - # Install each wheel + # First pass: Check wheels against exclude_list and remove excluded ones + excluded = 0 + excluded_wheels = [] + + print_color("---------- EXCLUDE LIST CHECK ----------") + + wheels_to_install = [] + for wheel_path in wheels: + should_exclude, reason = should_exclude_wheel(wheel_path.name, exclude_requirements) + if should_exclude: + excluded += 1 + excluded_wheels.append((wheel_path.name, reason)) + wheel_path.unlink() + print_color(f"-- {wheel_path.name}", Fore.RED) + print(f" Reason: {reason}") + else: + wheels_to_install.append(wheel_path) + + print_color("---------- END EXCLUDE LIST CHECK ----------") + print(f"Excluded {excluded} wheels\n") + + # Second pass: Install remaining wheels installed = 0 failed = 0 + deleted = 0 failed_wheels = [] + deleted_wheels = [] - print("Installing wheels (--no-deps to test wheel validity only)...") - print("-" * 60) + print_color("---------- INSTALL WHEELS ----------") - for wheel_path in wheels: + for wheel_path in wheels_to_install: success, error_message = install_wheel(wheel_path) if success: installed += 1 + elif is_python_version_error(error_message): + # Wheel is valid but has Python version constraints in metadata + # Delete it as it's incompatible with this Python version + deleted += 1 + deleted_wheels.append(wheel_path.name) + wheel_path.unlink() + print_color(f"-- {wheel_path.name} (Python version constraint)", Fore.YELLOW) else: failed += 1 failed_wheels.append((wheel_path.name, error_message)) - print() - print(f"ERROR: Failed to install {wheel_path.name}") + print_color(f"-- {wheel_path.name}", Fore.RED) if error_message: - for line in error_message.split("\n"): - print(f" {line}") - print() + for line in error_message.split("\n")[:3]: + print(f" {line}") + + print_color("---------- END INSTALL WHEELS ----------") - print("-" * 60) - print(f"Results: {installed} installed successfully, {failed} failed\n") + # Print statistics + print_color("---------- STATISTICS ----------") + print_color(f"Installed {installed} wheels", Fore.GREEN) + if excluded > 0: + print_color(f"Excluded {excluded} wheels (exclude_list.yaml)", Fore.YELLOW) + if deleted > 0: + print_color(f"Deleted {deleted} wheels (Python version constraint)", Fore.YELLOW) + if failed > 0: + print_color(f"Failed {failed} wheels", Fore.RED) - # Print summary of failures if failed_wheels: - print("Failed wheels:") + print_color("\nFailed wheels:", Fore.RED) for wheel_name, _ in failed_wheels: print(f" - {wheel_name}") - print() return 1 - print("All wheels installed successfully!") + print_color("\nAll compatible wheels processed successfully!", Fore.GREEN) return 0 diff --git a/upload_wheels.py b/upload_wheels.py index 3a601d6..a21a5bd 100644 --- a/upload_wheels.py +++ b/upload_wheels.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # @@ -13,6 +13,10 @@ import boto3 +from colorama import Fore + +from _helper_functions import print_color + s3 = boto3.resource("s3") try: BUCKET = s3.Bucket(sys.argv[1]) @@ -23,13 +27,31 @@ if not os.path.exists(WHEELS_DIR): raise SystemExit(f"Error: The wheels directory {WHEELS_DIR} not found.") -wheels_subdirs = os.listdir(WHEELS_DIR) - def normalize(name): return re.sub(r"[-_.]+", "-", name).lower() +def get_existing_wheels(): + """Get set of S3 keys for wheels currently on server.""" + existing = set() + for obj in BUCKET.objects.filter(Prefix="pypi/"): + if obj.key.endswith(".whl"): + existing.add(obj.key) + return existing + + +print_color("---------- UPLOAD WHEELS TO S3 ----------") + +existing_wheels = get_existing_wheels() +print(f"Found {len(existing_wheels)} existing wheels on S3\n") + +print_color("---------- UPLOADING WHEELS ----------") + +wheels_subdirs = os.listdir(WHEELS_DIR) +new_wheels = 0 +existing_count = 0 + for subdir in wheels_subdirs: wheel_files = os.listdir(f"{WHEELS_DIR}{os.sep}{subdir}") @@ -38,8 +60,23 @@ def normalize(name): match = pattern.search(wheel) if match: wheel_name = match.group(1) - wheel_name = normalize(wheel_name) + is_new = f"pypi/{wheel_name}/{wheel}" not in existing_wheels + BUCKET.upload_file(f"{WHEELS_DIR}{os.sep}{subdir}{os.sep}{wheel}", f"pypi/{wheel_name}/{wheel}") - print(f"Uploaded {wheel_name}/{wheel}") + + if is_new: + new_wheels += 1 + print_color(f"++ {wheel_name}/{wheel}", Fore.GREEN) + else: + existing_count += 1 + print(f" {wheel_name}/{wheel}") + +print_color("---------- END UPLOADING ----------") + +print_color("---------- STATISTICS ----------") +print_color(f"New wheels: {new_wheels}", Fore.GREEN) +print(f"Existing wheels (re-uploaded): {existing_count}") +print(f"Total uploaded: {new_wheels + existing_count}") +print_color("---------- END STATISTICS ----------") From 9d18e21a73e0028afc3a36b104e826e4be138890 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Fri, 30 Jan 2026 13:51:27 +0100 Subject: [PATCH 3/4] feat: Added cross-verification tests for S3 and exclude_list in PR --- .github/workflows/build-wheels-defined.yml | 30 +++ .github/workflows/build-wheels-platforms.yml | 30 +++ .github/workflows/unit-tests.yml | 261 +++++++++++++++++++ _helper_functions.py | 140 ++++++++++ extract_exclude_packages.py | 52 ++++ test/test_build_wheels.py | 10 +- test/test_wheels_install.py | 73 +----- verify_s3_wheels.py | 119 +++++++++ 8 files changed, 640 insertions(+), 75 deletions(-) create mode 100644 extract_exclude_packages.py create mode 100644 verify_s3_wheels.py diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index c992c21..ef53aef 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -379,3 +379,33 @@ jobs: name: Upload Python wheels uses: ./.github/workflows/upload-python-wheels.yml secrets: inherit + + verify-s3-wheels: + needs: [get-supported-versions, upload-python-wheels] + name: Verify S3 wheels against exclude list + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get oldest Python version + id: python-version + run: | + echo "version=${{ needs.get-supported-versions.outputs.oldest_supported_python }}" >> $GITHUB_OUTPUT + + - name: Setup Python ${{ steps.python-version.outputs.version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ steps.python-version.outputs.version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Verify S3 wheels + run: python verify_s3_wheels.py ${{ secrets.DL_BUCKET }} ${{ needs.get-supported-versions.outputs.oldest_supported_python }} diff --git a/.github/workflows/build-wheels-platforms.yml b/.github/workflows/build-wheels-platforms.yml index cfbf869..7d858e6 100644 --- a/.github/workflows/build-wheels-platforms.yml +++ b/.github/workflows/build-wheels-platforms.yml @@ -226,3 +226,33 @@ jobs: name: Upload Python wheels uses: ./.github/workflows/upload-python-wheels.yml secrets: inherit + + verify-s3-wheels: + needs: [get-supported-versions, upload-python-wheels] + name: Verify S3 wheels against exclude list + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get oldest Python version + id: python-version + run: | + echo "version=${{ needs.get-supported-versions.outputs.oldest_supported_python }}" >> $GITHUB_OUTPUT + + - name: Setup Python ${{ steps.python-version.outputs.version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ steps.python-version.outputs.version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Verify S3 wheels + run: python verify_s3_wheels.py ${{ secrets.DL_BUCKET }} ${{ needs.get-supported-versions.outputs.oldest_supported_python }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9e1f5d4..a01ee0e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -36,3 +36,264 @@ jobs: - name: Run unit tests run: python -m unittest discover -s test -v + + verify-s3-wheels: + name: Verify S3 wheels against exclude list + needs: get-supported-versions + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get latest Python version + id: python-version + run: | + VERSIONS='${{ needs.get-supported-versions.outputs.supported_python }}' + LATEST=$(echo "$VERSIONS" | jq -r '.[0]') + echo "version=$LATEST" >> $GITHUB_OUTPUT + + - name: Setup Python ${{ steps.python-version.outputs.version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ steps.python-version.outputs.version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Verify S3 wheels + run: python verify_s3_wheels.py ${{ secrets.DL_BUCKET }} ${{ needs.get-supported-versions.outputs.oldest_supported_python }} + + # Test building excluded packages on Linux + test-exclude-linux: + name: Test exclude builds - Linux (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Install OS dependencies + run: os_dependencies/ubuntu.sh + + - name: Get packages to test + id: packages + run: | + PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + echo "packages=$PACKAGES" >> $GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages + if: steps.packages.outputs.packages != '' + run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + + # Test building excluded packages on Windows + test-exclude-windows: + name: Test exclude builds - Windows (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: windows-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Get packages to test + id: packages + run: | + $PACKAGES = python extract_exclude_packages.py windows ${{ matrix.python-version }} + echo "packages=$PACKAGES" >> $env:GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages + if: steps.packages.outputs.packages != '' + run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + + # Test building excluded packages on macOS + test-exclude-macos: + name: Test exclude builds - macOS (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: macos-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Install OS dependencies + run: os_dependencies/macos.sh + + - name: Get packages to test + id: packages + run: | + PACKAGES=$(python extract_exclude_packages.py macos ${{ matrix.python-version }}) + echo "packages=$PACKAGES" >> $GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages + if: steps.packages.outputs.packages != '' + run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + + # Test building excluded packages on macOS ARM64 (Apple Silicon) + test-exclude-macos-arm64: + name: Test exclude builds - macOS arm64 (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: macos-latest-xlarge + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Install OS dependencies + run: os_dependencies/macos.sh + + - name: Get packages to test + id: packages + run: | + PACKAGES=$(python extract_exclude_packages.py macos_arm64 ${{ matrix.python-version }}) + echo "packages=$PACKAGES" >> $GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages + if: steps.packages.outputs.packages != '' + run: python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} + + # Test building excluded packages on Linux ARM64 + test-exclude-linux-arm64: + name: Test exclude builds - Linux ARM64 (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: ubuntu-24.04-arm + container: python:${{ matrix.python-version }}-bookworm + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r build_requirements.txt + + - name: Install OS dependencies + run: os_dependencies/linux_arm.sh + + - name: Get packages to test + id: packages + run: | + PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + echo "packages=$PACKAGES" >> $GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages + if: steps.packages.outputs.packages != '' + run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + + # Test building excluded packages on Linux ARMv7 + test-exclude-linux-armv7: + name: Test exclude builds - Linux ARMv7 (Python ${{ matrix.python-version }}) + needs: get-supported-versions + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }} + steps: + - name: Set up QEMU for ARMv7 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get packages to test + id: packages + run: | + pip install pyyaml + PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + echo "packages=$PACKAGES" >> $GITHUB_OUTPUT + echo "Packages to test: $PACKAGES" + + - name: Test building excluded packages (in Docker) + if: steps.packages.outputs.packages != '' + run: | + docker run --rm --platform linux/arm/v7 \ + -v $(pwd):/work \ + -w /work \ + -e PIP_NO_CACHE_DIR=1 \ + -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \ + python:${{ matrix.python-version }}-bookworm \ + bash -c " + set -e + python --version + python -m pip install --no-cache-dir --upgrade pip + python -m pip install --no-cache-dir -r build_requirements.txt + bash os_dependencies/linux_arm.sh + . \$HOME/.cargo/env 2>/dev/null || true + python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + " diff --git a/_helper_functions.py b/_helper_functions.py index 32f305e..81f3a4c 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -3,12 +3,16 @@ # # SPDX-License-Identifier: Apache-2.0 # +from __future__ import annotations + import platform import re from colorama import Fore from colorama import Style from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version # Packages that should be built from source on Linux to ensure correct library linking # These packages often have pre-built wheels on PyPI that link against different library versions @@ -24,6 +28,8 @@ "bitarray", ] +EXCLUDE_LIST_PATH = "exclude_list.yaml" + def get_no_binary_args(requirement_name: str) -> list: """Get --no-binary arguments if this package should be built from source. @@ -89,3 +95,137 @@ def merge_requirements(requirement: Requirement, another_req: Requirement) -> Re ) return new_requirement + + +def parse_wheel_name(wheel_name: str) -> tuple[str, str] | None: + """ + Parse wheel filename to extract package name and version. + + Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + + Returns: + tuple: (package_name, version) or None if parsing fails + """ + pattern = re.compile(r"^([A-Za-z0-9_.-]+)-(\d+(?:\.\d+)*(?:[a-zA-Z0-9.]+)?)-") + match = pattern.match(wheel_name) + if match: + return match.group(1), match.group(2) + return None + + +def should_exclude_wheel(wheel_name: str, exclude_requirements: set) -> tuple[bool, str]: + """ + Check if a wheel should be excluded based on exclude_list.yaml rules. + + Evaluates markers against the CURRENT running Python environment. + + Uses YAMLListAdapter with exclude=True, so the logic is inverted: + - If marker evaluates to True -> wheel satisfies "keep" condition, skip + - If version is in the (inverted) specifier -> wheel satisfies "keep" condition, skip + - Otherwise -> wheel should be excluded + + Args: + wheel_name: The wheel filename (e.g., "requests-2.31.0-py3-none-any.whl") + exclude_requirements: Set of Requirement objects from YAMLListAdapter + + Returns: + tuple: (should_exclude: bool, reason: str) + """ + parsed = parse_wheel_name(wheel_name) + if not parsed: + return False, "" + + pkg_name, wheel_version = parsed + canonical_name = canonicalize_name(pkg_name) + + for req in exclude_requirements: + # Check if package name matches (using canonical names) + if canonicalize_name(req.name) != canonical_name: + continue + + # With exclude=True, if marker evaluates to True -> KEEP the wheel + if req.marker and req.marker.evaluate(): + continue + + # With exclude=True, if version is in the (inverted) specifier -> KEEP the wheel + if req.specifier and wheel_version: + try: + if Version(wheel_version) in req.specifier: + continue + except Exception: + pass + + # Name matches, and marker is False (or absent), and version not in specifier (or absent) + # -> EXCLUDE the wheel + return True, f"matches exclude rule: {req}" + + return False, "" + + +def get_wheel_python_version(wheel_name: str) -> str | None: + """ + Extract Python version from wheel filename. + + Examples: + - "pkg-1.0-cp311-cp311-linux.whl" -> "3.11" + - "pkg-1.0-py3-none-any.whl" -> None (universal) + """ + match = re.search(r"-cp(\d)(\d+)-", wheel_name) + if match: + return f"{match.group(1)}.{match.group(2)}" + return None + + +def should_exclude_wheel_s3(wheel_name: str, exclude_requirements: set) -> tuple[bool, str]: + """ + Check if a wheel should be excluded for S3 verification. + + Uses DIRECT exclusion logic (not inverted): + - If marker is True → exclusion applies → EXCLUDE + - If marker is False → exclusion doesn't apply → KEEP + - If version matches specifier → EXCLUDE + + Skips sys_platform markers (can't evaluate cross-platform). + + Args: + wheel_name: The wheel filename + exclude_requirements: Set of Requirement objects from YAMLListAdapter (exclude=False) + + Returns: + tuple: (should_exclude: bool, reason: str) + """ + parsed = parse_wheel_name(wheel_name) + if not parsed: + return False, "" + + pkg_name, wheel_version = parsed + canonical_name = canonicalize_name(pkg_name) + wheel_python = get_wheel_python_version(wheel_name) + + for req in exclude_requirements: + if canonicalize_name(req.name) != canonical_name: + continue + + # Skip rules with sys_platform - can't evaluate cross-platform + if req.marker and "sys_platform" in str(req.marker): + continue + + # Evaluate python_version markers with wheel's target Python + # If marker is False → exclusion doesn't apply → KEEP (continue) + if req.marker: + env = {"python_version": wheel_python} if wheel_python else {} + if not req.marker.evaluate(environment=env if env else None): + continue # Exclusion condition not met → keep + + # If we get here, marker is True (or no marker) + # Check version specifier - if version matches, EXCLUDE + if req.specifier and wheel_version: + try: + if Version(wheel_version) not in req.specifier: + continue # Version doesn't match exclusion → keep + except Exception: + pass + + return True, f"matches exclude rule: {req}" + + return False, "" diff --git a/extract_exclude_packages.py b/extract_exclude_packages.py new file mode 100644 index 0000000..b728596 --- /dev/null +++ b/extract_exclude_packages.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Extract excluded packages for a platform/python combination. + +Usage: python extract_exclude_packages.py [platform] [python_version] +""" + +import sys + +import yaml + +from packaging.specifiers import SpecifierSet + +PLATFORM_MAP = {"win32": "windows", "linux": "linux", "darwin": "macos"} +ALL_PLATFORMS = ["linux", "windows", "macos"] + + +def get_excluded_packages(platform=None, python_version=None): + """Get packages excluded for the given platform/python combination.""" + packages = set() + + with open("exclude_list.yaml") as f: + data = yaml.safe_load(f) + + for entry in data: + platforms = entry.get("platform", []) + platforms = [platforms] if isinstance(platforms, str) else platforms + platforms = [PLATFORM_MAP.get(p, p) for p in platforms] or ALL_PLATFORMS + + if platform and platform not in platforms: + continue + + pythons = entry.get("python", []) + pythons = [pythons] if isinstance(pythons, str) else pythons + + if python_version and pythons: + if not any(python_version in SpecifierSet(c) for c in pythons): + continue + + packages.add(entry["package_name"]) + + return sorted(packages) + + +if __name__ == "__main__": + platform = sys.argv[1] if len(sys.argv) > 1 else None + python_ver = sys.argv[2] if len(sys.argv) > 2 else None + print(" ".join(get_excluded_packages(platform, python_ver))) diff --git a/test/test_build_wheels.py b/test/test_build_wheels.py index 9e1c97e..d353894 100644 --- a/test/test_build_wheels.py +++ b/test/test_build_wheels.py @@ -175,12 +175,11 @@ def test_abi3_wheel(self): class TestParseWheelName(unittest.TestCase): - """Test the parse_wheel_name function from test_wheels_install.py.""" + """Test the parse_wheel_name function from _helper_functions.py.""" def setUp(self): """Import the function to test.""" - sys.path.insert(0, str(Path(__file__).parent)) - from test_wheels_install import parse_wheel_name + from _helper_functions import parse_wheel_name self.parse_wheel_name = parse_wheel_name @@ -206,7 +205,7 @@ def test_parse_universal_wheel(self): class TestShouldExcludeWheel(unittest.TestCase): - """Test the should_exclude_wheel function from test_wheels_install.py. + """Test the should_exclude_wheel function from _helper_functions.py. Note: The function expects requirements created with exclude=True from YAMLListAdapter, which inverts the logic (e.g., ==1.5.0 becomes !=1.5.0). @@ -214,8 +213,7 @@ class TestShouldExcludeWheel(unittest.TestCase): def setUp(self): """Import the function to test.""" - sys.path.insert(0, str(Path(__file__).parent)) - from test_wheels_install import should_exclude_wheel + from _helper_functions import should_exclude_wheel self.should_exclude_wheel = should_exclude_wheel diff --git a/test/test_wheels_install.py b/test/test_wheels_install.py index c1242ed..5a94708 100644 --- a/test/test_wheels_install.py +++ b/test/test_wheels_install.py @@ -19,14 +19,13 @@ from pathlib import Path from colorama import Fore -from packaging.utils import canonicalize_name -from packaging.version import Version +from _helper_functions import EXCLUDE_LIST_PATH from _helper_functions import print_color +from _helper_functions import should_exclude_wheel from yaml_list_adapter import YAMLListAdapter WHEELS_DIR = Path("./downloaded_wheels") -EXCLUDE_LIST_PATH = Path("exclude_list.yaml") def get_python_version_tag() -> str: @@ -34,70 +33,6 @@ def get_python_version_tag() -> str: return f"{sys.version_info.major}{sys.version_info.minor}" -def parse_wheel_name(wheel_name: str) -> tuple[str, str] | None: - """ - Parse wheel filename to extract package name and version. - - Wheel format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl - """ - pattern = re.compile(r"^([A-Za-z0-9_.-]+)-(\d+(?:\.\d+)*(?:[a-zA-Z0-9.]+)?)-") - match = pattern.match(wheel_name) - if match: - pkg_name = match.group(1) - version = match.group(2) - return pkg_name, version - return None - - -def load_exclude_requirements() -> set: - """Load exclude_list.yaml using YAMLListAdapter and return requirements set.""" - adapter = YAMLListAdapter(str(EXCLUDE_LIST_PATH), exclude=True) - return adapter.requirements - - -def should_exclude_wheel(wheel_name: str, exclude_requirements: set) -> tuple[bool, str]: - """ - Check if a wheel should be excluded based on exclude_list.yaml rules. - - Uses YAMLListAdapter with exclude=True, so the logic is inverted: - - If marker evaluates to True -> wheel satisfies "keep" condition, skip - - If version is in the (inverted) specifier -> wheel satisfies "keep" condition, skip - - Otherwise -> wheel should be excluded - - Returns: - tuple: (should_exclude: bool, reason: str) - """ - parsed = parse_wheel_name(wheel_name) - if not parsed: - return False, "" - - pkg_name, wheel_version = parsed - canonical_name = canonicalize_name(pkg_name) - - for req in exclude_requirements: - # Check if package name matches (using canonical names) - if canonicalize_name(req.name) != canonical_name: - continue - - # With exclude=True, if marker evaluates to True -> KEEP the wheel - if req.marker and req.marker.evaluate(): - continue - - # With exclude=True, if version is in the (inverted) specifier -> KEEP the wheel - if req.specifier: - try: - if Version(wheel_version) in req.specifier: - continue - except Exception: - pass - - # Name matches, and marker is False (or absent), and version not in specifier (or absent) - # -> EXCLUDE the wheel - return True, f"matches exclude rule: {req}" - - return False, "" - - def get_platform_patterns() -> list[str]: """Get regex patterns for wheels compatible with current platform.""" platform = sys.platform @@ -194,8 +129,8 @@ def main() -> int: print_color(f"---------- TEST WHEELS INSTALL (Python {python_version}) ----------") print(f"Platform: {sys.platform}\n") - # Load exclude list using YAMLListAdapter - exclude_requirements = load_exclude_requirements() + # Load exclude list using YAMLListAdapter (exclude=True for runtime filtering) + exclude_requirements = YAMLListAdapter(EXCLUDE_LIST_PATH, exclude=True).requirements print(f"Loaded {len(exclude_requirements)} exclude requirements from {EXCLUDE_LIST_PATH}\n") # Find compatible wheels diff --git a/verify_s3_wheels.py b/verify_s3_wheels.py new file mode 100644 index 0000000..9cae093 --- /dev/null +++ b/verify_s3_wheels.py @@ -0,0 +1,119 @@ +# +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Verify S3 wheels against exclude_list.yaml. + +Checks all wheels on S3, extracting Python version from wheel filename +to evaluate python_version markers correctly. +""" + +import re +import sys + +import boto3 + +from colorama import Fore + +from _helper_functions import EXCLUDE_LIST_PATH +from _helper_functions import get_wheel_python_version +from _helper_functions import print_color +from _helper_functions import should_exclude_wheel_s3 +from yaml_list_adapter import YAMLListAdapter + + +def is_unsupported_python(wheel_name: str, oldest_supported: str) -> tuple[bool, str]: + """Check if wheel is for Python 2 or older than oldest_supported_python.""" + # Check for Python 2 only wheels (not py2.py3 which supports Python 3) + if re.search(r"-cp2\d-|-py2-|-py2\.", wheel_name) and "-py2.py3-" not in wheel_name: + return True, "Python 2 wheel" + + # Get wheel's Python version + wheel_python = get_wheel_python_version(wheel_name) + if not wheel_python: + return False, "" # Universal wheel, skip + + # Parse versions for comparison + try: + wheel_parts = [int(x) for x in wheel_python.split(".")] + oldest_parts = [int(x) for x in oldest_supported.split(".")] + + # Compare major.minor + if wheel_parts < oldest_parts: + return True, f"Python {wheel_python} < oldest supported {oldest_supported}" + except ValueError: + pass + + return False, "" + + +def main(): + if len(sys.argv) < 3: + raise SystemExit("Usage: verify_s3_wheels.py ") + + bucket_name = sys.argv[1] + oldest_supported_python = sys.argv[2] + + print_color("---------- VERIFY S3 WHEELS AGAINST EXCLUDE LIST ----------") + print(f"Oldest supported Python: {oldest_supported_python}\n") + + # Connect to S3 + s3 = boto3.resource("s3") + bucket = s3.Bucket(bucket_name) + + # Load exclude requirements (direct logic, no inversion) + exclude_requirements = YAMLListAdapter(EXCLUDE_LIST_PATH, exclude=False).requirements + print(f"Loaded {len(exclude_requirements)} exclude rules\n") + + # Get all wheels from S3 + print_color("---------- SCANNING S3 WHEELS ----------") + wheels = [] + for obj in bucket.objects.filter(Prefix="pypi/"): + if obj.key.endswith(".whl"): + wheel_name = obj.key.split("/")[-1] + wheels.append(wheel_name) + + print(f"Found {len(wheels)} wheels on S3\n") + + # Check each wheel + print_color("---------- CHECKING WHEELS ----------") + violations = [] + old_python_wheels = [] + + for wheel in wheels: + # Check for unsupported Python versions (warning only, not a violation) + is_old, reason = is_unsupported_python(wheel, oldest_supported_python) + if is_old: + old_python_wheels.append((wheel, reason)) + continue + + # Check against exclude_list (actual violations) + should_exclude, reason = should_exclude_wheel_s3(wheel, exclude_requirements) + if should_exclude: + violations.append((wheel, reason)) + print_color(f"-- {wheel}", Fore.RED) + print(f" {reason}") + + print_color("---------- END CHECKING ----------") + + # Statistics + print_color("---------- STATISTICS ----------") + print(f"Checked: {len(wheels)} wheels") + if old_python_wheels: + print_color(f"Old Python wheels: {len(old_python_wheels)} (warning only)", Fore.YELLOW) + if violations: + print_color(f"Violations: {len(violations)}", Fore.RED) + print_color("\nWheels that should be deleted:", Fore.RED) + for wheel, _ in violations: + print(f" - {wheel}") + print_color("---------- END STATISTICS ----------") + return 1 + else: + print_color("Violations: 0", Fore.GREEN) + print_color("---------- END STATISTICS ----------") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 988982bd26bbbd385395089b6dfd2aabbb36eafe Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Fri, 30 Jan 2026 16:27:46 +0100 Subject: [PATCH 4/4] feat: Added architecture resolve mechanism for requirements in exclude_list --- .github/workflows/build-wheels-defined.yml | 26 ++++++++-- .../build-wheels-python-dependent.yml | 4 ++ .github/workflows/unit-tests.yml | 29 +++++------ _helper_functions.py | 48 +++++++++++++++++- build_wheels.py | 5 +- build_wheels_from_file.py | 17 ++++++- exclude_list.yaml | 49 +++++++++++++------ extract_exclude_packages.py | 15 +++--- os_dependencies/linux_arm.sh | 46 +++++++++-------- repair_wheels.py | 5 +- test/test_wheels_install.py | 7 ++- yaml_list_adapter.py | 38 ++++++++++++-- 12 files changed, 217 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build-wheels-defined.yml b/.github/workflows/build-wheels-defined.yml index ef53aef..75b5d84 100644 --- a/.github/workflows/build-wheels-defined.yml +++ b/.github/workflows/build-wheels-defined.yml @@ -10,6 +10,21 @@ on: type: string required: true + platform: + description: Build only for this architecture (or "all" to use checkboxes below) + type: choice + required: false + default: all + options: + - all + - linux_x86_64 + - windows + - macos_x86_64 + - macos_arm64 + - linux_arm64 + - linux_armv7 + - linux_armv7_legacy + os_ubuntu_latest: description: Build on ubuntu-latest(x86_64) type: boolean @@ -58,7 +73,7 @@ jobs: ubuntu-latest: needs: get-supported-versions name: linux x86_64 - if: ${{ inputs.os_ubuntu_latest }} + if: ${{ inputs.platform == 'all' && inputs.os_ubuntu_latest || inputs.platform == 'linux_x86_64' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -121,6 +136,9 @@ jobs: - name: Install build dependencies run: python -m pip install -r build_requirements.txt + - name: Install additional OS dependencies - Windows + run: powershell -ExecutionPolicy Bypass -File os_dependencies/windows.ps1 + - name: Build wheels run: | python build_wheels_from_file.py --requirements ${{ inputs.packages }} @@ -135,7 +153,7 @@ jobs: macos-latest: needs: get-supported-versions name: macos x86_64 - if: ${{ inputs.os_macos_latest }} + if: ${{ inputs.platform == 'all' && inputs.os_macos_latest || inputs.platform == 'macos_x86_64' }} runs-on: macos-latest strategy: fail-fast: false @@ -229,7 +247,7 @@ jobs: linux-armv7: needs: get-supported-versions name: linux aarch32 (armv7) - if: ${{ inputs.os_linux_armv7 }} + if: ${{ inputs.platform == 'all' && inputs.os_linux_armv7 || inputs.platform == 'linux_armv7' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -311,7 +329,7 @@ jobs: linux-armv7-legacy: needs: get-supported-versions name: linux aarch32 (armv7 legacy) - if: ${{ inputs.os_linux_armv7_legacy }} + if: ${{ inputs.platform == 'all' && inputs.os_linux_armv7_legacy || inputs.platform == 'linux_armv7_legacy' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/build-wheels-python-dependent.yml b/.github/workflows/build-wheels-python-dependent.yml index 4c019b8..df567ec 100644 --- a/.github/workflows/build-wheels-python-dependent.yml +++ b/.github/workflows/build-wheels-python-dependent.yml @@ -111,6 +111,10 @@ jobs: if: matrix.os == 'Linux ARM64' run: os_dependencies/linux_arm.sh + - name: Install additional OS dependencies - Windows + if: matrix.os == 'Windows' + run: powershell -ExecutionPolicy Bypass -File os_dependencies/windows.ps1 + - name: Download artifacts uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a01ee0e..f475bad 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -69,9 +69,9 @@ jobs: - name: Verify S3 wheels run: python verify_s3_wheels.py ${{ secrets.DL_BUCKET }} ${{ needs.get-supported-versions.outputs.oldest_supported_python }} - # Test building excluded packages on Linux + # Test building excluded packages on Linux x86_64 test-exclude-linux: - name: Test exclude builds - Linux (Python ${{ matrix.python-version }}) + name: Test exclude builds - Linux x86_64 (Python ${{ matrix.python-version }}) needs: get-supported-versions runs-on: ubuntu-latest continue-on-error: true @@ -99,13 +99,13 @@ jobs: - name: Get packages to test id: packages run: | - PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + PACKAGES=$(python extract_exclude_packages.py linux_x86_64 ${{ matrix.python-version }}) echo "packages=$PACKAGES" >> $GITHUB_OUTPUT echo "Packages to test: $PACKAGES" - name: Test building excluded packages if: steps.packages.outputs.packages != '' - run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + run: python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} # Test building excluded packages on Windows test-exclude-windows: @@ -140,7 +140,7 @@ jobs: - name: Test building excluded packages if: steps.packages.outputs.packages != '' - run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + run: python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} # Test building excluded packages on macOS test-exclude-macos: @@ -172,13 +172,14 @@ jobs: - name: Get packages to test id: packages run: | - PACKAGES=$(python extract_exclude_packages.py macos ${{ matrix.python-version }}) + PLATFORM=$(python -c "from _helper_functions import get_current_platform; print(get_current_platform())") + PACKAGES=$(python extract_exclude_packages.py "$PLATFORM" ${{ matrix.python-version }}) echo "packages=$PACKAGES" >> $GITHUB_OUTPUT echo "Packages to test: $PACKAGES" - name: Test building excluded packages if: steps.packages.outputs.packages != '' - run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + run: python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} # Test building excluded packages on macOS ARM64 (Apple Silicon) test-exclude-macos-arm64: @@ -220,7 +221,7 @@ jobs: # Test building excluded packages on Linux ARM64 test-exclude-linux-arm64: - name: Test exclude builds - Linux ARM64 (Python ${{ matrix.python-version }}) + name: Test exclude builds - Linux arm64 (Python ${{ matrix.python-version }}) needs: get-supported-versions runs-on: ubuntu-24.04-arm container: python:${{ matrix.python-version }}-bookworm @@ -244,17 +245,17 @@ jobs: - name: Get packages to test id: packages run: | - PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + PACKAGES=$(python extract_exclude_packages.py linux_arm64 ${{ matrix.python-version }}) echo "packages=$PACKAGES" >> $GITHUB_OUTPUT echo "Packages to test: $PACKAGES" - name: Test building excluded packages if: steps.packages.outputs.packages != '' - run: python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + run: python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} # Test building excluded packages on Linux ARMv7 test-exclude-linux-armv7: - name: Test exclude builds - Linux ARMv7 (Python ${{ matrix.python-version }}) + name: Test exclude builds - Linux armv7 (Python ${{ matrix.python-version }}) needs: get-supported-versions runs-on: ubuntu-latest continue-on-error: true @@ -274,8 +275,8 @@ jobs: - name: Get packages to test id: packages run: | - pip install pyyaml - PACKAGES=$(python extract_exclude_packages.py linux ${{ matrix.python-version }}) + pip install pyyaml packaging + PACKAGES=$(python extract_exclude_packages.py linux_armv7 ${{ matrix.python-version }}) echo "packages=$PACKAGES" >> $GITHUB_OUTPUT echo "Packages to test: $PACKAGES" @@ -295,5 +296,5 @@ jobs: python -m pip install --no-cache-dir -r build_requirements.txt bash os_dependencies/linux_arm.sh . \$HOME/.cargo/env 2>/dev/null || true - python build_wheels_from_file.py --requirements ${{ steps.packages.outputs.packages }} + python build_wheels_from_file.py --ci-tests --requirements ${{ steps.packages.outputs.packages }} " diff --git a/_helper_functions.py b/_helper_functions.py index 81f3a4c..bcff289 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -7,6 +7,7 @@ import platform import re +import sys from colorama import Fore from colorama import Style @@ -30,6 +31,51 @@ EXCLUDE_LIST_PATH = "exclude_list.yaml" +# Platform names for exclude_list.yaml (YAML -> runner name) +PLATFORM_MAP = {"win32": "windows", "linux": "linux", "darwin": "macos"} +ALL_PLATFORMS = ["linux", "windows", "macos"] +LINUX_ARCHS = ["linux_x86_64", "linux_arm64", "linux_armv7"] +MACOS_ARCHS = ["macos_x86_64", "macos_arm64"] + + +def get_current_platform() -> str: + """Return current runner platform: + windows, macos, linux, linux_x86_64, linux_arm64, linux_armv7, macos_x86_64, macos_arm64 + """ + system = platform.system().lower() + machine = platform.machine().lower() + if system == "linux": + if machine in ("x86_64", "amd64"): + return "linux_x86_64" + if machine == "aarch64": + return "linux_arm64" + if machine == "armv7l": + return "linux_armv7" + return "linux" + if system == "darwin": + if machine in ("x86_64", "amd64"): + return "macos_x86_64" + if machine == "arm64": + return "macos_arm64" + return "macos" + if system == "windows": + return "windows" + return sys.platform + + +def exclude_entry_applies_to_platform(entry: dict, current_platform: str) -> bool: + """True if this exclude_list entry applies to current_platform (so we should exclude from build).""" + platforms = entry.get("platform", []) + platforms = [platforms] if isinstance(platforms, str) else platforms + platforms = [PLATFORM_MAP.get(p, p) for p in platforms] or ALL_PLATFORMS + if current_platform in platforms: + return True + if current_platform in LINUX_ARCHS and "linux" in platforms: + return True + if current_platform in MACOS_ARCHS and "macos" in platforms: + return True + return False + def get_no_binary_args(requirement_name: str) -> list: """Get --no-binary arguments if this package should be built from source. @@ -211,7 +257,7 @@ def should_exclude_wheel_s3(wheel_name: str, exclude_requirements: set) -> tuple continue # Evaluate python_version markers with wheel's target Python - # If marker is False → exclusion doesn't apply → KEEP (continue) + # If marker is False -> exclusion doesn't apply -> KEEP (continue) if req.marker: env = {"python_version": wheel_python} if wheel_python else {} if not req.marker.evaluate(environment=env if env else None): diff --git a/build_wheels.py b/build_wheels.py index abfdb60..861537a 100644 --- a/build_wheels.py +++ b/build_wheels.py @@ -20,6 +20,7 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement +from _helper_functions import get_current_platform from _helper_functions import get_no_binary_args from _helper_functions import merge_requirements from _helper_functions import print_color @@ -389,7 +390,9 @@ def main() -> int: requirements = assemble_requirements(idf_branches, idf_constraints, True) - exclude_list = YAMLListAdapter("exclude_list.yaml", exclude=True).requirements + exclude_list = YAMLListAdapter( + "exclude_list.yaml", exclude=True, current_platform=get_current_platform() + ).requirements after_exclude_requirements = exclude_from_requirements(requirements, exclude_list) diff --git a/build_wheels_from_file.py b/build_wheels_from_file.py index ce7b783..c4fcb7c 100644 --- a/build_wheels_from_file.py +++ b/build_wheels_from_file.py @@ -31,6 +31,11 @@ nargs="*", help="requirement(s) to be build wheel(s) for", ) +parser.add_argument( + "--ci-tests", + action="store_true", + help="CI exclude-tests mode: fail if all wheels succeed (expect some to fail, e.g. excluded packages)", +) args = parser.parse_args() @@ -83,7 +88,11 @@ print_color(f"Succeeded {succeeded_wheels} wheels", Fore.GREEN) print_color(f"Failed {failed_wheels} wheels", Fore.RED) - if failed_wheels != 0: + if args.ci_tests: + total = succeeded_wheels + failed_wheels + if total > 0 and failed_wheels == 0: + raise SystemExit("CI: expected some builds to fail (excluded packages)") + elif failed_wheels != 0: raise SystemExit("One or more wheels failed to build") # Build wheels from passed requirements @@ -122,5 +131,9 @@ print_color(f"Succeeded {succeeded_wheels} wheels", Fore.GREEN) print_color(f"Failed {failed_wheels} wheels", Fore.RED) - if failed_wheels != 0: + if args.ci_tests: + total = succeeded_wheels + failed_wheels + if total > 0 and failed_wheels == 0: + raise SystemExit("CI: expected some builds to fail (excluded packages)") + elif failed_wheels != 0: raise SystemExit("One or more wheels failed to build") diff --git a/exclude_list.yaml b/exclude_list.yaml index 04bdb87..906f70b 100644 --- a/exclude_list.yaml +++ b/exclude_list.yaml @@ -2,18 +2,19 @@ #"From assembled
exclude with on for version". # exclude_list template +# Omitted platform = excluded on all platforms. Omitted python = excluded for all Python versions. #- package_name: '' # version: '' / ['', ''] # optional -# platform: '' / ['', '', ''] # optional +# platform: win32 | darwin | linux | linux_x86_64 | linux_arm64 | linux_armv7 | macos_x86_64 | macos_arm64 (or list) # optional # python: '' / ['', '', ''] # optional -# dbus-python can not be build on Windows +# dbus-python 1.4.0 leads to patchelf dependency and cannot be built on Windows - package_name: 'dbus-python' platform: ['win32'] -# dbus-python has persistent build issues on Linux ARM due to missing dbus-1.pc files in containers +# dbus-python 1.4.0 build issues on Linux (x86_64, ARM64, ARMv7) and macOS with Python 3.8 - package_name: 'dbus-python' - platform: ['linux'] + platform: ['linux', 'darwin'] python: '==3.8' # dbus-python can not be build with Python > 3.11 on MacOS @@ -21,34 +22,49 @@ platform: 'darwin' python: '>3.11' + +- package_name: 'pygobject' + python: '==3.8' + # PyGObject is difficult to build from source on Windows due to MSYS2/pkg-config issues - package_name: 'pygobject' platform: ['win32'] - python: '==3.8' # PyGObject has persistent girepository dependency issues on ARMv7 - package_name: 'pygobject' - platform: ['linux'] + platform: ['linux_armv7'] -# gevent==1.5.0 can not be build with Python > 3.8 +# gevent==1.5.0 can not be build with Python > 3.8 on Windows - package_name: 'gevent' version: '==1.5.0' + platform: ['win32'] python: '>3.8' -# gdbgui==0.13.2.0 leads to installation of gevent 1.5.0 which can not be build +# gdbgui==0.13.2.0 requires gevent<2.0 which fails to build: +# - Windows: Cython long type issue in gevent 1.5.0 +# - Python 3.8/3.9: general compatibility issues +# Only works on Linux/macOS with Python 3.10+ - package_name: 'gdbgui' version: '==0.13.2.0' + platform: ['win32'] -# Pillow 9.5.0 is problematic on ARMv7 and Python 3.13 +- package_name: 'gdbgui' + version: '==0.13.2.0' + platform: ['linux', 'darwin'] + python: ['==3.8', '==3.9'] + +# Pillow 9.5.0 is incompatible with Python 3.13+ (setup.py __version__ extraction fails) - package_name: 'Pillow' version: '==9.5.0' + python: '>=3.13' -# some versions of greenlet are not supported by Python 3.13 and newer +# greenlet <=3.0.0 uses internal Python APIs (pycore_frame.h) incompatible with Python 3.13+ +# Affects all platforms - requires greenlet >= 3.1.0 for Python 3.13+ - package_name: 'greenlet' - version: '<3.0' + version: '<=3.0.0' python: '>=3.13' -# Python 3.13 and newer does not support ruamel.yaml.clib 0.2.8 (0.2.12+ supports Python 3.13 and newer) +# ruamel.yaml.clib 0.2.8 not supported on Python 3.13+ on Windows/macOS (works on Linux 3.10+) - package_name: 'ruamel.yaml.clib' version: '==0.2.8' python: '>=3.13' @@ -65,11 +81,11 @@ # ARMv7 Linux can't build tree-sitter-c with Python 3.8 - package_name: 'tree-sitter-c' python: '==3.8' - platform: ['linux'] + platform: ['linux_armv7'] # ARMv7 Linux can't build cryptography with Python 3.8 - package_name: 'cryptography' - platform: ['linux'] + platform: ['linux_armv7'] python: '==3.8' # Esptool wheels many times are faulty. Mostly because it installs "esptool.py" which collides with the package name. @@ -97,12 +113,13 @@ platform: ['win32'] python: '>=3.14' -# pydantic supports Python 3.14 from version >= 2.35.0 +# pydantic_core supports Python 3.14 from version >= 2.35.0 (pyo3 compatibility) # https://pypi.org/project/pydantic_core/#history - package_name: 'pydantic_core' version: '<2.35.0' python: '>=3.14' -# rdps_py supports Python 3.14 from version >= 0.26.0 + +# rpds_py supports Python 3.14 from version >= 0.26.0 (pyo3 compatibility) # https://pypi.org/project/rpds-py/#history - package_name: 'rpds_py' version: '<0.26.0' diff --git a/extract_exclude_packages.py b/extract_exclude_packages.py index b728596..86a904d 100644 --- a/extract_exclude_packages.py +++ b/extract_exclude_packages.py @@ -7,6 +7,8 @@ """Extract excluded packages for a platform/python combination. Usage: python extract_exclude_packages.py [platform] [python_version] + +Platform can be: windows, macos, linux, linux_x86_64, linux_arm64, linux_armv7, macos_x86_64, macos_arm64 """ import sys @@ -15,8 +17,7 @@ from packaging.specifiers import SpecifierSet -PLATFORM_MAP = {"win32": "windows", "linux": "linux", "darwin": "macos"} -ALL_PLATFORMS = ["linux", "windows", "macos"] +from _helper_functions import exclude_entry_applies_to_platform def get_excluded_packages(platform=None, python_version=None): @@ -27,16 +28,16 @@ def get_excluded_packages(platform=None, python_version=None): data = yaml.safe_load(f) for entry in data: + # Skip packages excluded everywhere (no platform/python) - don't test build those platforms = entry.get("platform", []) - platforms = [platforms] if isinstance(platforms, str) else platforms - platforms = [PLATFORM_MAP.get(p, p) for p in platforms] or ALL_PLATFORMS + pythons = entry.get("python", []) + if not platforms and not pythons: + continue - if platform and platform not in platforms: + if platform and not exclude_entry_applies_to_platform(entry, platform): continue - pythons = entry.get("python", []) pythons = [pythons] if isinstance(pythons, str) else pythons - if python_version and pythons: if not any(python_version in SpecifierSet(c) for c in pythons): continue diff --git a/os_dependencies/linux_arm.sh b/os_dependencies/linux_arm.sh index 44ac06f..4f34820 100755 --- a/os_dependencies/linux_arm.sh +++ b/os_dependencies/linux_arm.sh @@ -1,25 +1,31 @@ #!/bin/bash +# Use sudo if available (not present in some CI containers where we run as root) +SUDO="" +if command -v sudo >/dev/null 2>&1; then + SUDO="sudo" +fi + arch=$(uname -m) -sudo apt-get update +$SUDO apt-get update # AWS -sudo apt-get install -y -q --no-install-recommends awscli +$SUDO apt-get install -y -q --no-install-recommends awscli -sudo apt-get install -y cmake build-essential +$SUDO apt-get install -y cmake build-essential # PyGObject needs build dependecies https://pygobject.readthedocs.io/en/latest/getting_started.html -sudo apt-get install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-4.0 libglib2.0-dev +$SUDO apt-get install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev gir1.2-gtk-4.0 libglib2.0-dev # Try to install girepository-2.0-dev if available (may not exist in older distros) -sudo apt-get install -y libgirepository-2.0-dev +$SUDO apt-get install -y libgirepository-2.0-dev # dbus-python needs build dependecies -sudo apt-get install -y dbus libdbus-1-dev libdbus-glib-1-dev libdbus-1-3 -sudo apt-get install -y --no-install-recommends dbus-tests +$SUDO apt-get install -y dbus libdbus-1-dev libdbus-glib-1-dev libdbus-1-3 +$SUDO apt-get install -y --no-install-recommends dbus-tests # Pillow needs comprehensive image processing libraries -sudo apt-get install -y \ +$SUDO apt-get install -y \ libjpeg-dev \ libpng-dev \ libtiff5-dev \ @@ -45,37 +51,37 @@ fi #Only ARMv7 if [ "$arch" == "armv7l" ]; then # pip cache permissions to avoid warnings - sudo mkdir -p /github/home/.cache/pip || true - sudo chown -R $USER:$USER /github/home/.cache/pip || true + $SUDO mkdir -p /github/home/.cache/pip || true + $SUDO chown -R $USER:$USER /github/home/.cache/pip || true # ARMv7 specific packages (not already installed globally) - sudo apt-get install -y gobject-introspection + $SUDO apt-get install -y gobject-introspection # Install additional GObject introspection packages if available - sudo apt-get install -y gobject-introspection-dev + $SUDO apt-get install -y gobject-introspection-dev # Install GIR (GObject Introspection Repository) packages that might provide girepository-2.0 - sudo apt-get install -y gir1.2-glib-2.0 gir1.2-gtk-3.0 + $SUDO apt-get install -y gir1.2-glib-2.0 gir1.2-gtk-3.0 # Try alternative package names for girepository-2.0 that might exist in newer repos - sudo apt-get install -y libgirepository-dev - sudo apt-get install -y gobject-introspection-1.0-dev + $SUDO apt-get install -y libgirepository-dev + $SUDO apt-get install -y gobject-introspection-1.0-dev # Additional dbus packages for ARMv7 - sudo apt-get install -y --reinstall dbus-1-dev dbus-1-doc libdbus-1-dev pkg-config + $SUDO apt-get install -y --reinstall dbus-1-dev dbus-1-doc libdbus-1-dev pkg-config # Try to install additional dbus development packages - sudo apt-get install -y libdbus-glib-1-dev + $SUDO apt-get install -y libdbus-glib-1-dev # Force update pkg-config cache - sudo ldconfig + $SUDO ldconfig # cryptography needs Rust # clean the container Rust installation to be sure right interpreter is used - sudo apt remove --auto-remove --purge rust-gdb rustc libstd-rust-dev libstd-rust-1.48 + $SUDO apt remove --auto-remove --purge rust-gdb rustc libstd-rust-dev libstd-rust-1.48 # install Rust dependencies - sudo apt-get install -y libssl-dev libffi-dev gcc musl-dev + $SUDO apt-get install -y libssl-dev libffi-dev gcc musl-dev # install Rust curl --proto '=https' --tlsv1.3 -sSf https://sh.rustup.rs | bash -s -- -y . $HOME/.cargo/env diff --git a/repair_wheels.py b/repair_wheels.py index 9cbd7ed..8afab4f 100644 --- a/repair_wheels.py +++ b/repair_wheels.py @@ -141,8 +141,9 @@ def main() -> None: wheels: list[Path] = list(wheels_dir.rglob("*.whl")) if not wheels: - print_color(f"No wheels found in {wheels_dir}", Fore.RED) - raise SystemExit("No wheels found in downloaded_wheels directory") + print_color(f"No wheels found in {wheels_dir} - nothing to repair", Fore.YELLOW) + print("Exiting successfully (no wheels to process)") + return print_color(f"Found {len(wheels)} wheels\n") diff --git a/test/test_wheels_install.py b/test/test_wheels_install.py index 5a94708..78813ee 100644 --- a/test/test_wheels_install.py +++ b/test/test_wheels_install.py @@ -21,6 +21,7 @@ from colorama import Fore from _helper_functions import EXCLUDE_LIST_PATH +from _helper_functions import get_current_platform from _helper_functions import print_color from _helper_functions import should_exclude_wheel from yaml_list_adapter import YAMLListAdapter @@ -129,8 +130,10 @@ def main() -> int: print_color(f"---------- TEST WHEELS INSTALL (Python {python_version}) ----------") print(f"Platform: {sys.platform}\n") - # Load exclude list using YAMLListAdapter (exclude=True for runtime filtering) - exclude_requirements = YAMLListAdapter(EXCLUDE_LIST_PATH, exclude=True).requirements + # Load exclude list for current platform (exclude=True for runtime filtering) + exclude_requirements = YAMLListAdapter( + EXCLUDE_LIST_PATH, exclude=True, current_platform=get_current_platform() + ).requirements print(f"Loaded {len(exclude_requirements)} exclude requirements from {EXCLUDE_LIST_PATH}\n") # Find compatible wheels diff --git a/yaml_list_adapter.py b/yaml_list_adapter.py index 251a1c7..4f003a6 100644 --- a/yaml_list_adapter.py +++ b/yaml_list_adapter.py @@ -3,16 +3,37 @@ # # SPDX-License-Identifier: Apache-2.0 # +from __future__ import annotations + import re +from typing import List +from typing import Optional +from typing import Set + import yaml from colorama import Fore from packaging.requirements import Requirement +from _helper_functions import exclude_entry_applies_to_platform from _helper_functions import merge_requirements from _helper_functions import print_color +# Map runner platforms to sys.platform (pip markers only know win32/linux/darwin) +SYS_PLATFORM_MAP = { + "linux_armv7": "linux", + "linux_arm64": "linux", + "linux_x86_64": "linux", + "macos_arm64": "darwin", + "macos_x86_64": "darwin", +} + + +def _platform_for_marker(platform): + """Normalize platform to sys_platform value for pip markers.""" + return SYS_PLATFORM_MAP.get(platform, platform) + class YAMLListAdapter: """Class for loading list of requirements defined in exclude or include lists (YAML files) @@ -63,7 +84,7 @@ class YAMLListAdapter: exclude: bool = False requirements: set = set() - def __init__(self, yaml_file: str, exclude: bool = False) -> None: + def __init__(self, yaml_file: str, exclude: bool = False, current_platform: Optional[str] = None) -> None: try: with open(yaml_file, "r") as f: self._yaml_list = yaml.load(f, yaml.Loader) @@ -71,6 +92,10 @@ def __init__(self, yaml_file: str, exclude: bool = False) -> None: print_color(f"File not found, please check the file: {yaml_file}", Fore.RED) self.exclude = exclude + # When building wheels: only exclude entries that apply to this platform + if current_platform and self._yaml_list: + self._yaml_list = [e for e in self._yaml_list if exclude_entry_applies_to_platform(e, current_platform)] + # Assemble duplicates of requirements/packages with the same name and remove them from the YAML list _requirement_duplicates = self._assemble_requirements_duplicates() # Convert YAML list to set of requirements without duplicates @@ -145,7 +170,7 @@ def _yaml_to_requirement(self, yaml: list, exclude: bool = False) -> set: """ yaml_list: list = yaml - requirements_set: set[Requirement] = set() + requirements_set: Set[Requirement] = set() if not yaml_list: return requirements_set @@ -153,7 +178,14 @@ def _yaml_to_requirement(self, yaml: list, exclude: bool = False) -> set: for package in yaml_list: # get attributes of the package if defined to reduce unnecessary complexity package_version = package["version"] if "version" in package else "" - package_platform = package["platform"] if "platform" in package else "" + raw_platform = package["platform"] if "platform" in package else "" + # Normalize to list and map linux_armv7/arm64/x86_64 to "linux" for pip markers + package_platform: List[str] + if raw_platform: + plfs = [raw_platform] if isinstance(raw_platform, str) else raw_platform + package_platform = list(dict.fromkeys(_platform_for_marker(p) for p in plfs)) + else: + package_platform = [] package_python = package["python"] if "python" in package else "" requirement_str_list = [f"{package['package_name']}"]