Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 154 additions & 150 deletions .github/workflows/cpp_extra.yml
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,10 @@ jobs:
export ODBC_INCLUDE_DIR=$LIBIODBC_DIR/include
export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR"
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Register Arrow Flight SQL ODBC Driver
run: |
chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh
sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh $(pwd)/build/cpp/debug/libarrow_flight_sql_odbc.dylib
- name: Test
shell: bash
run: |
Expand All @@ -402,155 +406,155 @@ jobs:
ulimit -c unlimited # must enable within the same shell
ci/scripts/cpp_test.sh $(pwd) $(pwd)/build

odbc-msvc:
needs: check-labels
name: ODBC Windows
runs-on: windows-2022
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') ||
contains(join(github.event.pull_request.changed_files, ' '), 'cpp/src/arrow/flight/sql/odbc/')
timeout-minutes: 240
permissions:
packages: write
env:
ARROW_BUILD_SHARED: ON
ARROW_BUILD_STATIC: OFF
ARROW_BUILD_TESTS: ON
ARROW_BUILD_TYPE: release
ARROW_DEPENDENCY_SOURCE: VCPKG
ARROW_FLIGHT_SQL_ODBC: ON
ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
ARROW_HOME: /usr
CMAKE_GENERATOR: Ninja
CMAKE_INSTALL_PREFIX: /usr
VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite'
VCPKG_DEFAULT_TRIPLET: x64-windows
steps:
- name: Disable Crash Dialogs
run: |
reg add `
"HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
/v DontShowUI `
/t REG_DWORD `
/d 1 `
/f
- name: Checkout Arrow
uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: recursive
- name: Download Timezone Database
shell: bash
run: ci/scripts/download_tz_database.sh
- name: Install ccache
shell: bash
run: |
ci/scripts/install_ccache.sh 4.12.1 /usr
- name: Setup ccache
shell: bash
run: |
ci/scripts/ccache_setup.sh
- name: ccache info
id: ccache-info
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v5
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
restore-keys: cpp-odbc-ccache-windows-x64-
- name: Checkout vcpkg
uses: actions/checkout@v6
with:
fetch-depth: 0
path: vcpkg
repository: microsoft/vcpkg
- name: Bootstrap vcpkg
run: |
vcpkg\bootstrap-vcpkg.bat
$VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString()
Write-Output ${VCPKG_ROOT} | `
Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append
Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | `
Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append
- name: Setup NuGet credentials for vcpkg caching
shell: bash
run: |
$(vcpkg fetch nuget | tail -n 1) \
sources add \
-source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \
-storepasswordincleartext \
-name "GitHub" \
-username "$GITHUB_REPOSITORY_OWNER" \
-password "${{ secrets.GITHUB_TOKEN }}"
$(vcpkg fetch nuget | tail -n 1) \
setapikey "${{ secrets.GITHUB_TOKEN }}" \
-source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
- name: Build
shell: cmd
run: |
set VCPKG_ROOT_KEEP=%VCPKG_ROOT%
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
- name: Register Flight SQL ODBC Driver
shell: cmd
run: |
call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll
# GH-48270 TODO: Resolve segementation fault during Arrow library unload, segementation fault is caught with `ci/scripts/cpp_test.sh`
# GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI and run tests with `ci/scripts/cpp_test.sh` instead
# GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved.
- name: Run ODBC Unit Tests
shell: cmd
run: |
build\cpp\%ARROW_BUILD_TYPE%\arrow-odbc-spi-impl-test.exe
- name: Run ODBC Driver Tests
shell: cmd
run: |
build\cpp\%ARROW_BUILD_TYPE%\arrow-flight-sql-odbc-test.exe
- name: Install WiX Toolset
shell: pwsh
run: |
Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi
Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Build MSI ODBC installer
shell: pwsh
run: |
# Verify WiX version
wix --version
cd build/cpp
cpack
- name: Upload the artifacts to the job
uses: actions/upload-artifact@v6
with:
name: flight-sql-odbc-msi-installer
path: build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi
if-no-files-found: error
# Upload ODBC installer as nightly release in scheduled runs
- name: Prepare ODBC installer for sync
if: github.event_name == 'schedule'
run: |
mkdir odbc-installer
Move-Item "build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi" odbc-installer/
tree odbc-installer /f
- name: Sync to Remote
if: github.event_name == 'schedule'
uses: ./.github/actions/sync-nightlies
with:
upload: true
switches: -avzh --update --delete --progress
local_path: odbc-installer
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
# odbc-msvc:
# needs: check-labels
# name: ODBC Windows
# runs-on: windows-2022
# if: >-
# needs.check-labels.outputs.force == 'true' ||
# contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
# contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') ||
# contains(join(github.event.pull_request.changed_files, ' '), 'cpp/src/arrow/flight/sql/odbc/')
# timeout-minutes: 240
# permissions:
# packages: write
# env:
# ARROW_BUILD_SHARED: ON
# ARROW_BUILD_STATIC: OFF
# ARROW_BUILD_TESTS: ON
# ARROW_BUILD_TYPE: release
# ARROW_DEPENDENCY_SOURCE: VCPKG
# ARROW_FLIGHT_SQL_ODBC: ON
# ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
# ARROW_HOME: /usr
# CMAKE_GENERATOR: Ninja
# CMAKE_INSTALL_PREFIX: /usr
# VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite'
# VCPKG_DEFAULT_TRIPLET: x64-windows
# steps:
# - name: Disable Crash Dialogs
# run: |
# reg add `
# "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
# /v DontShowUI `
# /t REG_DWORD `
# /d 1 `
# /f
# - name: Checkout Arrow
# uses: actions/checkout@v6
# with:
# fetch-depth: 0
# submodules: recursive
# - name: Download Timezone Database
# shell: bash
# run: ci/scripts/download_tz_database.sh
# - name: Install ccache
# shell: bash
# run: |
# ci/scripts/install_ccache.sh 4.12.1 /usr
# - name: Setup ccache
# shell: bash
# run: |
# ci/scripts/ccache_setup.sh
# - name: ccache info
# id: ccache-info
# shell: bash
# run: |
# echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
# - name: Cache ccache
# uses: actions/cache@v5
# with:
# path: ${{ steps.ccache-info.outputs.cache-dir }}
# key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
# restore-keys: cpp-odbc-ccache-windows-x64-
# - name: Checkout vcpkg
# uses: actions/checkout@v6
# with:
# fetch-depth: 0
# path: vcpkg
# repository: microsoft/vcpkg
# - name: Bootstrap vcpkg
# run: |
# vcpkg\bootstrap-vcpkg.bat
# $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString()
# Write-Output ${VCPKG_ROOT} | `
# Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append
# Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | `
# Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append
# - name: Setup NuGet credentials for vcpkg caching
# shell: bash
# run: |
# $(vcpkg fetch nuget | tail -n 1) \
# sources add \
# -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \
# -storepasswordincleartext \
# -name "GitHub" \
# -username "$GITHUB_REPOSITORY_OWNER" \
# -password "${{ secrets.GITHUB_TOKEN }}"
# $(vcpkg fetch nuget | tail -n 1) \
# setapikey "${{ secrets.GITHUB_TOKEN }}" \
# -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
# - name: Build
# shell: cmd
# run: |
# set VCPKG_ROOT_KEEP=%VCPKG_ROOT%
# call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
# set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
# bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
# - name: Register Arrow Flight SQL ODBC Driver
# shell: cmd
# run: |
# call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll
# # GH-48270 TODO: Resolve segementation fault during Arrow library unload, segementation fault is caught with `ci/scripts/cpp_test.sh`
# # GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI and run tests with `ci/scripts/cpp_test.sh` instead
# # GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved.
# - name: Run ODBC Unit Tests
# shell: cmd
# run: |
# build\cpp\%ARROW_BUILD_TYPE%\arrow-odbc-spi-impl-test.exe
# - name: Run ODBC Driver Tests
# shell: cmd
# run: |
# build\cpp\%ARROW_BUILD_TYPE%\arrow-flight-sql-odbc-test.exe
# - name: Install WiX Toolset
# shell: pwsh
# run: |
# Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi
# Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
# echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
# - name: Build MSI ODBC installer
# shell: pwsh
# run: |
# # Verify WiX version
# wix --version
# cd build/cpp
# cpack
# - name: Upload the artifacts to the job
# uses: actions/upload-artifact@v6
# with:
# name: flight-sql-odbc-msi-installer
# path: build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi
# if-no-files-found: error
# # Upload ODBC installer as nightly release in scheduled runs
# - name: Prepare ODBC installer for sync
# if: github.event_name == 'schedule'
# run: |
# mkdir odbc-installer
# Move-Item "build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi" odbc-installer/
# tree odbc-installer /f
# - name: Sync to Remote
# if: github.event_name == 'schedule'
# uses: ./.github/actions/sync-nightlies
# with:
# upload: true
# switches: -avzh --update --delete --progress
# local_path: odbc-installer
# remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc
# remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
# remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
# remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
# remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
# remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}

report-extra-cpp:
if: github.event_name == 'schedule' && always()
Expand All @@ -560,6 +564,6 @@ jobs:
- jni-macos
- msvc-arm64
- odbc-macos
- odbc-msvc
# - odbc-msvc
uses: ./.github/workflows/report_ci.yml
secrets: inherit
1 change: 0 additions & 1 deletion ci/scripts/cpp_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ case "$(uname)" in
;;
Darwin)
n_jobs=$(sysctl -n hw.ncpu)
exclude_tests+=("arrow-flight-sql-odbc-test")
# TODO: https://github.com/apache/arrow/issues/40410
exclude_tests+=("arrow-s3fs-test")
;;
Expand Down
66 changes: 66 additions & 0 deletions ci/scripts/register_odbc_driver_mac.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
echo "SOURCE=$SOURCE"

SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
echo "SCRIPT_DIR=$SCRIPT_DIR"

REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
echo "REPO_DIR=$REPO_DIR"

ODBC_RELEASE_BUILD_DIR="$REPO_DIR/build/cpp/release"
echo "ODBC_RELEASE_BUILD_DIR=$ODBC_RELEASE_BUILD_DIR"

ODBC_DEBUG_BUILD_DIR="$REPO_DIR/build/cpp/debug"
echo "ODBC_DEBUG_BUILD_DIR=$ODBC_DEBUG_BUILD_DIR"

# ODBC_LIB_FILENAME="$ODBC_DEBUG_BUILD_DIR/libarrow_flight_sql_odbc.dylib"
# echo "ODBC_LIB_FILENAME=$ODBC_LIB_FILENAME"

ODBC_LIB_FILENAME="$ODBC_RELEASE_BUILD_DIR/libarrow_flight_sql_odbc.dylib"
echo "ODBC_LIB_FILENAME=$ODBC_LIB_FILENAME"

if [ ! -f "$ODBC_LIB_FILENAME" ]
then
echo "Cannot find ODBC library file: $ODBC_LIB_FILENAME"
exit 1
fi

echo "[ODBC Drivers]" > "$REPO_DIR/arrow-odbc-install.ini"
echo "Apache Arrow Flight SQL ODBC Driver=Installed" >> "$REPO_DIR/arrow-odbc-install.ini"
echo >> "$REPO_DIR/arrow-odbc-install.ini"
echo "[Apache Arrow Flight SQL ODBC Driver]" >> "$REPO_DIR/arrow-odbc-install.ini"
echo "Description=An ODBC Driver for Apache Arrow Flight SQL" >> "$REPO_DIR/arrow-odbc-install.ini"
echo "Driver=$ODBC_LIB_FILENAME" >> "$REPO_DIR/arrow-odbc-install.ini"
echo "Setup=$ODBC_LIB_FILENAME" >> "$REPO_DIR/arrow-odbc-install.ini"

ARM_LIBIODBC_PATH="/opt/homebrew/opt/libiodbc/lib"
INTEL_LIBIODBC_PATH="/usr/local/opt/libiodbc/"

export ODBCINSTINI="$REPO_DIR/arrow-odbc-install.ini"
echo "Exported ODBCINSTINI=$ODBCINSTINI"
export DYLD_LIBRARY_PATH=$ODBC_LIB_PATH:$ARM_LIBIODBC_PATH:$INTEL_LIBIODBC_PATH:$DYLD_LIBRARY_PATH
echo "Exported DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH"
Loading