diff --git a/.github/workflows/ud-inline-tests.yml b/.github/workflows/ud-inline-tests.yml new file mode 100644 index 0000000000..1f0d0d3834 --- /dev/null +++ b/.github/workflows/ud-inline-tests.yml @@ -0,0 +1,119 @@ +name: Run UD tests inline (manual) + +on: + # TODO: remove pull_request once all tests are passing — keep workflow_dispatch only + pull_request: + workflow_dispatch: + inputs: + ud-branch: + description: 'Branch of universal-driver repo to use' + required: false + default: 'snowpark-compatibility' + type: string + python-version: + description: 'Python version' + required: false + default: '3.13' + type: choice + options: + - '3.10' + - '3.11' + - '3.12' + - '3.13' + cloud-provider: + description: 'Cloud provider' + required: false + default: 'aws' + type: choice + options: + - 'aws' + - 'azure' + - 'gcp' + pytest-addopts: + description: 'Extra pytest args, e.g. "-k test_dataframe" or "--maxfail=5"' + required: false + default: '' + type: string + +permissions: + contents: read + +env: + PYTHON_VERSION: ${{ inputs.python-version || '3.13' }} + CLOUD_PROVIDER: ${{ inputs.cloud-provider || 'aws' }} + UD_BRANCH: ${{ inputs.ud-branch || 'snowpark-compatibility' }} + # TODO: change fallback back to '' once full suite passes against UD + EXTRA_PYTEST_ADDOPTS: ${{ inputs.pytest-addopts || '-v --tb=long' }} + +jobs: + test: + name: UD Test py${{ inputs.python-version || '3.13' }}-${{ inputs.cloud-provider || 'aws' }} (${{ inputs.ud-branch || 'snowpark-compatibility' }}) + runs-on: ubuntu-latest-64-cores + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - run: python -c "import sys; print(sys.version)" + + - uses: astral-sh/setup-uv@v6 + + - name: Decrypt parameters.py + run: .github/scripts/decrypt_parameters.sh + env: + PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }} + CLOUD_PROVIDER: ${{ env.CLOUD_PROVIDER }} + + - name: Install protoc + run: .github/scripts/install_protoc.sh + + - name: Configure git credentials for universal-driver + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + env: + GH_TOKEN: ${{ secrets.SNOWFLAKE_GITHUB_TOKEN }} + + - name: Install tox + run: uv pip install tox --system + + - name: Install MS ODBC Driver + run: | + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list \ + | sudo tee /etc/apt/sources.list.d/mssql-release.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + + - name: Run integration tests + run: python -m tox -e "py${PYTHON_VERSION//.}-notdoctest-ci" + env: + cloud_provider: ${{ env.CLOUD_PROVIDER }} + ud_connector_path: "git+https://github.com/snowflakedb/universal-driver@${{ env.UD_BRANCH }}#subdirectory=python" + GH_TOKEN: ${{ secrets.SNOWFLAKE_GITHUB_TOKEN }} + PYTEST_ADDOPTS: --color=yes --tb=short ${{ env.EXTRA_PYTEST_ADDOPTS }} + TOX_PARALLEL_NO_SPINNER: 1 + + - name: Run doctest + if: ${{ env.EXTRA_PYTEST_ADDOPTS == '' }} + run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci" + env: + cloud_provider: ${{ env.CLOUD_PROVIDER }} + ud_connector_path: "git+https://github.com/snowflakedb/universal-driver@${{ env.UD_BRANCH }}#subdirectory=python" + GH_TOKEN: ${{ secrets.SNOWFLAKE_GITHUB_TOKEN }} + PYTEST_ADDOPTS: --color=yes --tb=short ${{ env.EXTRA_PYTEST_ADDOPTS }} + TOX_PARALLEL_NO_SPINNER: 1 + + - name: Run datasource tests + if: ${{ env.EXTRA_PYTEST_ADDOPTS == '' }} + run: python -m tox -e datasource + env: + cloud_provider: ${{ env.CLOUD_PROVIDER }} + ud_connector_path: "git+https://github.com/snowflakedb/universal-driver@${{ env.UD_BRANCH }}#subdirectory=python" + GH_TOKEN: ${{ secrets.SNOWFLAKE_GITHUB_TOKEN }} + PYTEST_ADDOPTS: --color=yes --tb=short ${{ env.EXTRA_PYTEST_ADDOPTS }} + TOX_PARALLEL_NO_SPINNER: 1 diff --git a/scripts/tox_install_cmd.sh b/scripts/tox_install_cmd.sh index 31ea14f02d..67dce22fe6 100755 --- a/scripts/tox_install_cmd.sh +++ b/scripts/tox_install_cmd.sh @@ -19,17 +19,28 @@ done echo "${uv_options[*]}" -# Default to empty, to ensure snowflake_path variable is defined. +# Default to empty, to ensure variables are defined. snowflake_path=${snowflake_path:-""} +ud_connector_path=${ud_connector_path:-""} python_version=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') -if [[ -z "${snowflake_path}" ]]; then - echo "Using Python Connector from PyPI" +if [[ -n "${ud_connector_path}" ]]; then + echo "Installing Universal Driver connector" + echo "UD connector path: ${ud_connector_path}" + # Install all deps normally (old connector gets pulled in via snowflake-connector-python>=3.17.0) uv pip install ${uv_options[@]} -else + # Remove old connector and install UD in its place. + # --reinstall ensures wheel files are always extracted even if the version + # matches a previous install (uv otherwise skips re-extraction silently). + uv pip uninstall snowflake-connector-python + uv pip install --reinstall "${ud_connector_path}" +elif [[ -n "${snowflake_path}" ]]; then echo "Installing locally built Python Connector" echo "Python Connector path: ${snowflake_path}" ls -al ${snowflake_path} uv pip install ${snowflake_path}/snowflake_connector_python*${python_version}*.whl uv pip install ${uv_options[@]} +else + echo "Using Python Connector from PyPI" + uv pip install ${uv_options[@]} fi diff --git a/tox.ini b/tox.ini index fec74cd6a1..e0a915388a 100644 --- a/tox.ini +++ b/tox.ini @@ -125,6 +125,8 @@ passenv = GITHUB_ENV SNOWPARK_PYTHON_API_TEST_BUCKET_PATH SNOWPARK_PYTHON_API_S3_STORAGE_INTEGRATION + ; Universal Driver connector path (no-op when unset) + ud_connector_path commands = notudf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests udf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} {env:RERUN_FLAGS} src/snowflake/snowpark tests