Merge branch 'v1.4-andium'

Author: evertlammerts

.github/workflows/cleanup_pypi.yml

@@ -6,6 +6,14 @@ on:
         description: CI environment to run in (pypi-test or pypi-prod-nightly)
         type: string
         required: true
+      subcommand:
+        description: List or Delete
+        type: string
+        default: delete
+      verbosity:
+        description: Tool verbosity ("verbose" or "debug")
+        type: string
+        default: verbose
     secrets:
       PYPI_CLEANUP_OTP:
         description: PyPI OTP
@@ -15,17 +23,29 @@ on:
         required: true
   workflow_dispatch:
     inputs:
-      dry-run:
-        description: List packages that would be deleted but don't delete them
-        type: boolean
-        default: false
+      subcommand:
+        description: List or Delete
+        type: choice
+        required: true
+        options:
+          - list
+          - delete
+        default: list
       environment:
         description: CI environment to run in
         type: choice
         required: true
         options:
           - pypi-prod-nightly
           - pypi-test
+      verbosity:
+        description: Tool verbosity
+        type: choice
+        required: true
+        options:
+          - verbose
+          - debug
+        default: verbose
 
 jobs:
   cleanup_pypi:
@@ -50,26 +70,41 @@ jobs:
           exit 1
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.8.16"
+          version: "0.9.0"
+
+      - name: Install dependencies
+        run: uv sync --only-group pypi --no-install-project
+
+      - name: List Stale Packages on PyPI
+        if: inputs.subcommand == 'list'
+        env:
+          PYTHON_UNBUFFERED: 1
+        run: |
+          set -x
+          uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup \
+            --${{ inputs.environment == 'pypi-prod-nightly' && 'prod' || 'test' }} \
+            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} \
+            --${{ inputs.verbosity }} \
+            list  2>&1 | tee cleanup_output
 
-      - name: Run Cleanup
+      - name: Delete Stale Packages from PyPI
+        if: inputs.subcommand == 'delete'
         env:
           PYTHON_UNBUFFERED: 1
         run: |
           set -x
-          uv sync --only-group pypi --no-install-project
-          uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup ${{ inputs.dry-run && '--dry' || '' }} \
-            ${{ inputs.environment == 'pypi-prod-nightly' && '--prod' || '--test' }} \
-            --verbose \
-            --username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
-            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} 2>&1 | tee cleanup_output
+          uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup \
+            --${{ inputs.environment == 'pypi-prod-nightly' && 'prod' || 'test' }} \
+            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} \
+            --${{ inputs.verbosity }} \
+            delete --username "${{ vars.PYPI_CLEANUP_USERNAME }}" 2>&1 | tee cleanup_output
 
       - name: PyPI Cleanup Summary
         run : |
           echo "## PyPI Cleanup Summary" >> $GITHUB_STEP_SUMMARY
-          echo "* Dry run: ${{ inputs.dry-run }}" >> $GITHUB_STEP_SUMMARY
+          echo "* Subcommand: ${{ inputs.subcommand }}" >> $GITHUB_STEP_SUMMARY
           echo "* CI Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
           echo "* Output:" >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY

.github/workflows/code_quality.yml

@@ -29,9 +29,9 @@ jobs:
           persist-credentials: false
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
           python-version: 3.9
 
       - name: pre-commit (cache)

.github/workflows/coverage.yml

@@ -68,9 +68,9 @@ jobs:
           sudo apt-get -y install ccache
 
       - name: Install Astral UV and enable the cache
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.8.16"
+          version: "0.9.0"
           python-version: 3.9
           enable-cache: true
           cache-suffix: -${{ github.workflow }}
@@ -79,7 +79,7 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ inputs.testsuite }}" == "all" ]]; then
-            uv run coverage run -m pytest ./tests --ignore=./tests/stubs
+            uv run coverage run -m pytest ./tests
           elif [[ "${{ inputs.testsuite }}" == "fast" ]]; then
             uv run coverage run -m pytest ./tests/fast
           else

.github/workflows/packaging_sdist.yml

@@ -56,9 +56,9 @@ jobs:
         run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.8.16"
+          version: "0.9.0"
           python-version: 3.11
 
       - name: Build sdist
@@ -80,7 +80,7 @@ jobs:
           # run tests
           tests_root="${{ github.workspace }}/tests"
           tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}"
-          uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs
+          uv run --verbose pytest -c ${{ github.workspace }}/pyproject.toml $tests_dir
 
       - id: versioning
         run: |

.github/workflows/packaging_wheels.yml

@@ -44,16 +44,29 @@ jobs:
           - { minimal: true, python: cp310 }
           - { minimal: true, python: cp311 }
           - { minimal: true, python: cp312 }
+          - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
     runs-on: ${{ matrix.platform.os }}
     env:
+      ### cibuildwheel configuration
+      #
+      # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
+      # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
+      #   built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
+      #   sure that the pytest config in pyproject.toml is available.
+      # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
+      #   `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
+      #   across all platforms we build for. This might be solved in newer uv versions.
+      # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
+      #   traversing the full filesystem, which produces an error on Windows.
+      # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
       CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+        uv export --only-group test --no-emit-project  --quiet --output-file pylock.toml --directory {project} &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
+        uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
 
     steps:
       - name: Checkout DuckDB Python
@@ -78,15 +91,14 @@ jobs:
         run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
 
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
-      - uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@v7
         with:
-          version: "0.8.16"
+          version: "0.9.0"
           enable-cache: false
           cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
-          python-version: ${{ matrix.python }}
 
       - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
-        uses: pypa/cibuildwheel@v3.1
+        uses: pypa/cibuildwheel@v3.2
         env:
           CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}

.pre-commit-config.yaml

@@ -34,7 +34,7 @@ repos:
     rev: v1.18.2
     hooks:
       - id: mypy
-        entry: mypy -v
+        entry: mypy
         files: ^(duckdb/|_duckdb-stubs/)
         exclude: ^duckdb/(experimental|query_graph)/
         additional_dependencies: [ numpy, polars ]

cmake/compiler_launcher.cmake

@@ -8,8 +8,7 @@ include(CMakeParseArguments)
 # Function to look for ccache and sccache to speed up builds, if available
 # ────────────────────────────────────────────
 function(setup_compiler_launcher_if_available)
-  if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED
-                                               ENV{CMAKE_C_COMPILER_LAUNCHER})
+  if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER)
     find_program(COMPILER_LAUNCHER NAMES ccache sccache)
     if(COMPILER_LAUNCHER)
       message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher")
@@ -19,8 +18,7 @@ function(setup_compiler_launcher_if_available)
     endif()
   endif()
 
-  if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER
-     AND NOT DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER})
+  if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER)
     find_program(COMPILER_LAUNCHER NAMES ccache sccache)
     if(COMPILER_LAUNCHER)
       message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher")