Skip to content

Commit 702f206

Browse files
pwrliangpwrliang
committed
Add libgpuspatial module
1 parent bfb94e9 commit 702f206

File tree

104 files changed

+19777
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+19777
-0
lines changed

‎.github/workflows/rust-gpu.yml‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# This workflow compiles CUDA code on GitHub-hosted runners (ubuntu-latest).
19+
# CUDA compilation (nvcc) works WITHOUT GPU hardware - only needs CUDA toolkit.
20+
# GPU runtime execution requires actual GPU, so tests are commented out.
21+
#
22+
name: rust-gpu
23+
24+
on:
25+
pull_request:
26+
branches:
27+
- main
28+
paths:
29+
- 'c/sedona-libgpuspatial/**'
30+
- 'rust/sedona-spatial-join-gpu/**'
31+
- '.github/workflows/rust-gpu.yml'
32+
33+
push:
34+
branches:
35+
- main
36+
paths:
37+
- 'c/sedona-libgpuspatial/**'
38+
- 'rust/sedona-spatial-join-gpu/**'
39+
- '.github/workflows/rust-gpu.yml'
40+
41+
concurrency:
42+
group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-rust-gpu
43+
cancel-in-progress: true
44+
45+
permissions:
46+
contents: read
47+
48+
defaults:
49+
run:
50+
shell: bash -l -eo pipefail {0}
51+
52+
# Set workflow timeout to 90 minutes for CUDA compilation
53+
# Expected: ~45-60 minutes first time, ~10-15 minutes cached
54+
env:
55+
WORKFLOW_TIMEOUT_MINUTES: 90
56+
# At GEOS updated to 3.14.0
57+
VCPKG_REF: 5a01de756c28279ddfdd2b061d1c75710a6255fa
58+
59+
jobs:
60+
rust-gpu-build:
61+
# Using GitHub-hosted runner to compile CUDA code
62+
# CUDA compilation works without GPU hardware (only needs CUDA toolkit)
63+
# GPU tests are skipped (no GPU hardware for runtime execution)
64+
# TODO: Once GPU runner is ready, enable GPU tests with:
65+
# runs-on: [self-hosted, gpu, linux, cuda]
66+
strategy:
67+
fail-fast: false
68+
matrix:
69+
name: [ "clippy", "docs", "test", "build" ]
70+
71+
name: "${{ matrix.name }}"
72+
runs-on: ubuntu-latest
73+
timeout-minutes: 60
74+
env:
75+
CARGO_INCREMENTAL: 0
76+
# Disable debug info completely to save disk space
77+
CARGO_PROFILE_DEV_DEBUG: 0
78+
CARGO_PROFILE_TEST_DEBUG: 0
79+
# Limit parallel compilation to reduce memory pressure (GPU compilation is intensive)
80+
CARGO_BUILD_JOBS: 4
81+
82+
83+
steps:
84+
- uses: actions/checkout@v4
85+
with:
86+
submodules: 'recursive'
87+
88+
- name: Clone vcpkg
89+
uses: actions/checkout@v4
90+
with:
91+
repository: microsoft/vcpkg
92+
ref: ${{ env.VCPKG_REF }}
93+
path: vcpkg
94+
95+
# Set up environment variables for vcpkg and CUDA
96+
- name: Set up environment variables and bootstrap vcpkg
97+
env:
98+
VCPKG_ROOT: ${{ github.workspace }}/vcpkg
99+
CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
100+
CUDA_HOME: /usr/local/cuda
101+
run: |
102+
cd vcpkg
103+
./bootstrap-vcpkg.sh
104+
cd ..
105+
106+
echo "VCPKG_ROOT=$VCPKG_ROOT" >> $GITHUB_ENV
107+
echo "PATH=$VCPKG_ROOT:$PATH" >> $GITHUB_ENV
108+
echo "CMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE" >> $GITHUB_ENV
109+
echo "/usr/local/cuda/bin" >> $GITHUB_PATH
110+
111+
- name: Free Disk Space (Ubuntu)
112+
uses: jlumbroso/free-disk-space@main
113+
with:
114+
# Free up space by removing tools we don't need
115+
tool-cache: false # Keep tool cache as we need build tools
116+
android: true # Remove Android SDK (not needed)
117+
dotnet: true # Remove .NET runtime (not needed)
118+
haskell: true # Remove Haskell toolchain (not needed)
119+
large-packages: false # Keep essential packages including build-essential
120+
swap-storage: true # Remove swap file to free space
121+
docker-images: true # Remove docker images (not needed)
122+
123+
# Install system dependencies including CUDA toolkit for compilation
124+
- name: Install system dependencies
125+
run: |
126+
sudo apt-get update
127+
128+
# Install transport tools for Kitware CMake (needed for newer CMake)
129+
sudo apt-get install -y apt-transport-https ca-certificates gnupg software-properties-common wget
130+
131+
# Add Kitware repository for CMake
132+
wget -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add -
133+
sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main'
134+
sudo apt-get update
135+
136+
# Install build tools
137+
sudo apt-get install -y build-essential pkg-config cmake flex bison
138+
139+
# Install libclang for bindgen (Rust FFI binding generator)
140+
sudo apt-get install -y libclang-dev
141+
142+
# Verify compiler and CMake versions
143+
gcc --version
144+
g++ --version
145+
cmake --version
146+
147+
# Install GEOS for spatial operations
148+
sudo apt-get install -y libgeos-dev
149+
150+
# Install CUDA toolkit for compilation (nvcc)
151+
# Note: CUDA compilation works without GPU hardware
152+
# GPU runtime tests still require actual GPU
153+
if ! command -v nvcc &> /dev/null; then
154+
echo "Installing CUDA 12 toolkit for compilation..."
155+
156+
# Add NVIDIA CUDA repository
157+
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
158+
sudo dpkg -i cuda-keyring_1.1-1_all.deb
159+
sudo apt-get update
160+
161+
# Remove any existing CUDA toolkit
162+
sudo apt purge cuda-toolkit* -y || true
163+
164+
# Install CUDA 12
165+
sudo apt-get install -y cuda-toolkit-12
166+
167+
# Set CUDA path
168+
echo "/usr/local/cuda/bin" >> $GITHUB_PATH
169+
170+
nvcc --version
171+
else
172+
echo "CUDA toolkit already installed: $(nvcc --version)"
173+
fi
174+
175+
# Cache vcpkg installed packages (expensive to rebuild)
176+
- name: Cache vcpkg binaries
177+
id: cache-vcpkg
178+
uses: actions/cache@v4
179+
with:
180+
path: vcpkg/packages
181+
# Bump the number at the end of this line to force a new dependency build
182+
key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{ env.VCPKG_REF }}-2
183+
184+
# Install vcpkg dependencies from vcpkg.json manifest
185+
- name: Install vcpkg dependencies
186+
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
187+
run: |
188+
./vcpkg/vcpkg install abseil openssl
189+
# Clean up vcpkg buildtrees and downloads to save space
190+
rm -rf vcpkg/buildtrees
191+
rm -rf vcpkg/downloads
192+
193+
- name: Use stable Rust
194+
id: rust
195+
run: |
196+
rustup toolchain install stable --no-self-update
197+
rustup default stable
198+
199+
- uses: Swatinem/rust-cache@v2
200+
with:
201+
prefix-key: "rust-gpu-v3"
202+
# Cache key includes GPU packages and vcpkg config
203+
key: "${{ runner.os }}-${{ hashFiles('c/sedona-libgpuspatial/**', 'vcpkg.json') }}"
204+
205+
# Build WITH GPU feature to compile CUDA code
206+
# CUDA compilation (nvcc) works without GPU hardware
207+
# Only GPU runtime execution requires actual GPU
208+
- name: Build libgpuspatial (with CUDA compilation)
209+
run: |
210+
echo "=== Building libgpuspatial WITH GPU feature ==="
211+
echo "Compiling CUDA code using nvcc (no GPU hardware needed for compilation)"
212+
echo "Note: First build with CUDA takes 45-60 minutes (CMake + CUDA compilation)"
213+
echo "Subsequent builds: 10-15 minutes (cached)"
214+
echo ""
215+
echo "Build started at: $(date)"
216+
# Build library only (skip tests - they require CUDA driver which isn't available)
217+
# --lib builds only the library, not test binaries
218+
cargo build --locked --package sedona-libgpuspatial --lib --features gpu --verbose

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
members = [
1919
"c/sedona-geoarrow-c",
2020
"c/sedona-geos",
21+
"c/sedona-libgpuspatial",
2122
"c/sedona-proj",
2223
"c/sedona-s2geography",
2324
"c/sedona-tg",

‎c/sedona-libgpuspatial/CMakeLists.txt‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
cmake_minimum_required(VERSION 3.14)
18+
project(sedonadb_libgpuspatial_c)
19+
20+
add_subdirectory(libgpuspatial)

‎c/sedona-libgpuspatial/Cargo.toml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
[package]
18+
name = "sedona-libgpuspatial"
19+
version.workspace = true
20+
authors.workspace = true
21+
license.workspace = true
22+
homepage.workspace = true
23+
repository.workspace = true
24+
description = "GPU spatial operations wrapper for libgpuspatial"
25+
readme.workspace = true
26+
edition.workspace = true
27+
rust-version.workspace = true
28+
29+
[features]
30+
default = []
31+
# Enable GPU acceleration (requires CUDA toolkit)
32+
gpu = []
33+
34+
[build-dependencies]
35+
bindgen = "0.71.0"
36+
cmake = "0.1"
37+
which = "6.0"
38+
39+
[dependencies]
40+
arrow-array = { workspace = true, features = ["ffi"] }
41+
arrow-schema = { workspace = true }
42+
thiserror = { workspace = true }
43+
log = "0.4"
44+
sedona-schema = { path = "../../rust/sedona-schema" }
45+
46+
[dev-dependencies]
47+
sedona-expr = { path = "../../rust/sedona-expr" }
48+
sedona-geos = { path = "../sedona-geos" }
49+
sedona-testing = { path = "../../rust/sedona-testing" }

0 commit comments

Comments
 (0)