diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b2e579..e32b78c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,78 +18,34 @@ jobs: include: - os: ubuntu-latest arch: x86_64 - configure: --with-openssl name: linux - os: LinuxARM64 arch: arm64 - configure: --with-openssl name: linux - os: macos-latest - configure: - --with-secure-transport - CFLAGS="-arch x86_64 -arch arm64" name: macos - os: windows-latest arch: x86_64 - configure: - --with-schannel - CFLAGS="-DCURL_STATICLIB" name: windows - os: ubuntu-latest arch: arm64-v8a - configure: - --host aarch64-linux-android26 - --with-openssl=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - CXX=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ - LD=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip name: android - make: - PLATFORM=android - CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang + make: PLATFORM=android ARCH=arm64-v8a - os: ubuntu-latest arch: x86_64 - configure: - --host x86_64-linux-android26 - --with-openssl=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang - CXX=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang++ - LD=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip name: android - make: - PLATFORM=android - CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang + make: PLATFORM=android ARCH=x86_64 sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip - os: macos-latest - configure: - --host=arm64-apple-darwin - --with-secure-transport - CFLAGS="-arch arm64 -isysroot $(xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0" name: ios make: PLATFORM=ios - os: macos-latest - configure: - --host=arm64-apple-darwin - --with-secure-transport - CFLAGS="-arch x86_64 -arch arm64 -isysroot $(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" name: isim make: PLATFORM=isim defaults: run: - shell: ${{ matrix.os == 'windows-latest' && 'msys2 {0}' || 'bash' }} - env: - MAKEFLAGS: -j 8 + shell: bash steps: @@ -101,123 +57,19 @@ jobs: msystem: mingw64 install: >- mingw-w64-x86_64-cc - mingw-w64-x86_64-autotools make - - - uses: robinraju/release-downloader@v1.12 - with: - repository: curl/curl - tag: 'curl-8_12_1' - extract: true - fileName: curl-*.tar.gz + make - - name: build openssl - if: matrix.name == 'android' - run: | - git clone https://github.com/openssl/openssl.git - cd openssl - export TOOLCHAIN=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64 - export ARCH=${{ matrix.arch }} - export PATH=$TOOLCHAIN/bin:$PATH - - ./Configure android-${{ matrix.arch == 'arm64-v8a' && 'arm64' || matrix.arch }} \ - --prefix=$PWD/build \ - --openssldir=$PWD/build/ssl \ - no-shared \ - no-unit-test \ - -D__ANDROID_API__=26 - make - make install_sw - - cp build/lib/libssl.a $TOOLCHAIN/sysroot/usr/lib/ - cp build/lib/libcrypto.a $TOOLCHAIN/sysroot/usr/lib/ - cp -r build/include/openssl $TOOLCHAIN/sysroot/usr/include/ - - - name: build curl - run: | - - folder=$(ls -d curl-*/ 2>/dev/null | head -n 1) - cd $folder - - ./configure \ - --without-libpsl \ - --disable-alt-svc \ - --disable-ares \ - --disable-cookies \ - --disable-basic-auth \ - --disable-digest-auth \ - --disable-kerberos-auth \ - --disable-negotiate-auth \ - --disable-aws \ - --disable-dateparse \ - --disable-dnsshuffle \ - --disable-doh \ - --disable-form-api \ - --disable-hsts \ - --disable-ipv6 \ - --disable-libcurl-option \ - --disable-manual \ - --disable-mime \ - --disable-netrc \ - --disable-ntlm \ - --disable-ntlm-wb \ - --disable-progress-meter \ - --disable-proxy \ - --disable-pthreads \ - --disable-socketpair \ - --disable-threaded-resolver \ - --disable-tls-srp \ - --disable-verbose \ - --disable-versioned-symbols \ - --enable-symbol-hiding \ - --without-brotli \ - --without-zstd \ - --without-libidn2 \ - --without-librtmp \ - --without-zlib \ - --without-nghttp2 \ - --without-ngtcp2 \ - --disable-shared \ - --disable-ftp \ - --disable-file \ - --disable-ipfs \ - --disable-ldap \ - --disable-ldaps \ - --disable-rtsp \ - --disable-dict \ - --disable-telnet \ - --disable-tftp \ - --disable-pop3 \ - --disable-imap \ - --disable-smb \ - --disable-smtp \ - --disable-gopher \ - --disable-mqtt \ - --disable-docs \ - --enable-static \ - ${{matrix.configure}} - make - - # save avg 1kb more with these options - # --disable-debug \ - # --enable-optimize \ - # --disable-curldebug \ - # --disable-get-easy-options \ - # --without-fish-functions-dir \ - # --without-zsh-functions-dir \ - # --without-libgsasl \ - - cd .. - mkdir -p curl/${{ matrix.name }} - mv $folder/lib/.libs/libcurl.a curl/${{ matrix.name }} + - name: windows build curl + if: matrix.os == 'windows-latest' + run: make curl/windows/libcurl.a -j4 + shell: msys2 {0} - name: build sqlite-sync - run: make ${{ matrix.make && matrix.make || ''}} - shell: bash + run: make extension ${{ matrix.make && matrix.make || ''}} -j4 - name: windows install sqlite3 if: matrix.os == 'windows-latest' run: choco install sqlite -y - shell: bash - name: macos install sqlite3 without SQLITE_OMIT_LOAD_EXTENSION if: matrix.name == 'macos' @@ -237,17 +89,17 @@ jobs: curl -O ${{ matrix.sqlite-amalgamation-zip }} unzip sqlite-amalgamation-*.zip export ${{ matrix.make }} - $CC sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl + $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.arch }}-linux-android26-clang sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl # remove unused folders to save up space - rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* curl-* openssl + rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* openssl echo "::endgroup::" echo "::group::prepare the test script" - make test CC=$CC PLATFORM=$PLATFORM || echo "It should fail. Running remaining commands in the emulator" + make test PLATFORM=$PLATFORM ARCH=$ARCH -j4 || echo "It should fail. Running remaining commands in the emulator" cat > commands.sh << EOF mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp - $(make test CC=$CC PLATFORM=$PLATFORM -n) + $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) EOF echo "::endgroup::" @@ -265,8 +117,7 @@ jobs: - name: test sqlite-sync if: matrix.name == 'linux' || matrix.name == 'windows' - run: make test - shell: bash + run: make test -j4 - name: test sqlite-sync + coverage if: matrix.name == 'macos' diff --git a/.gitignore b/.gitignore index c5781b2..7c74d94 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ *.sqlite *.a unittest -/curl-src +/curl/src diff --git a/Makefile b/Makefile index 3789b6e..a1e3599 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,19 @@ # make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3 SQLITE3 ?= sqlite3 +# set curl version to download and build +CURL_VERSION ?= 8.12.1 + # Set default platform if not specified ifeq ($(OS),Windows_NT) PLATFORM := windows + HOST:= windows else - UNAME_S := $(shell uname -s) - ifeq ($(UNAME_S),Darwin) + HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') + ifeq ($(HOST),darwin) PLATFORM := macos else - PLATFORM := linux + PLATFORM := $(HOST) endif endif @@ -34,6 +38,7 @@ BUILD_RELEASE = build/release BUILD_TEST = build/test BUILD_DIRS = $(BUILD_TEST) $(BUILD_RELEASE) CURL_DIR = curl +CURL_SRC = $(CURL_DIR)/src/curl-$(CURL_VERSION) COV_DIR = coverage CUSTOM_CSS = $(TEST_DIR)/sqliteai.css @@ -48,6 +53,7 @@ TEST_FILES = $(SRC_FILES) $(wildcard $(TEST_DIR)/*.c) $(wildcard $(SQLITE_DIR)/* RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES))) TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES))) COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES)) +CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a # Platform-specific settings ifeq ($(PLATFORM),windows) @@ -57,17 +63,34 @@ ifeq ($(PLATFORM),windows) # Create .def file for Windows DEF_FILE := $(BUILD_RELEASE)/cloudsync.def CFLAGS += -DCURL_STATICLIB + CURL_CONFIG = --with-schannel CFLAGS="-DCURL_STATICLIB" else ifeq ($(PLATFORM),macos) TARGET := $(DIST_DIR)/cloudsync.dylib LDFLAGS += -arch x86_64 -arch arm64 -framework Security -dynamiclib -undefined dynamic_lookup T_LDFLAGS = -framework Security CFLAGS += -arch x86_64 -arch arm64 + CURL_CONFIG = --with-secure-transport CFLAGS="-arch x86_64 -arch arm64" else ifeq ($(PLATFORM),android) - # Use Android NDK's Clang compiler, the user should set the CC - # example CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - ifeq ($(filter %-clang,$(CC)),) - $(error "CC must be set to the Android NDK's Clang compiler") + # Set ARCH to find Android NDK's Clang compiler, the user should set the ARCH + ifeq ($(filter %,$(ARCH)),) + $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") + endif + # Set ANDROID_NDK path to find android build tools + # e.g. on MacOS: export ANDROID_NDK=/Users/username/Library/Android/sdk/ndk/25.2.9519653 + ifeq ($(filter %,$(ANDROID_NDK)),) + $(error "Android NDK must be set") + endif + + BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin + PATH := $(BIN):$(PATH) + + ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) + override ARCH := aarch64 endif + + OPENSSL := $(BIN)/../sysroot/usr/include/openssl + CC = $(BIN)/$(ARCH)-linux-android26-clang + CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$(BIN)/../sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lcrypto -lssl else ifeq ($(PLATFORM),ios) @@ -76,15 +99,18 @@ else ifeq ($(PLATFORM),ios) LDFLAGS += -framework Security -framework CoreFoundation -dynamiclib $(SDK) T_LDFLAGS = -framework Security CFLAGS += -arch arm64 $(SDK) + CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch arm64 -isysroot $$(xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0" else ifeq ($(PLATFORM),isim) TARGET := $(DIST_DIR)/cloudsync.dylib SDK := -isysroot $(shell xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0 LDFLAGS += -arch x86_64 -arch arm64 -framework Security -framework CoreFoundation -dynamiclib $(SDK) T_LDFLAGS = -framework Security CFLAGS += -arch x86_64 -arch arm64 $(SDK) + CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" else # linux TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lssl -lcrypto + CURL_CONFIG = --with-openssl endif ifneq ($(COVERAGE),false) @@ -119,7 +145,7 @@ extension: $(TARGET) all: $(TARGET) # Loadable library -$(TARGET): $(RELEASE_OBJ) $(DEF_FILE) +$(TARGET): $(RELEASE_OBJ) $(DEF_FILE) $(CURL_LIB) $(CC) $(RELEASE_OBJ) $(DEF_FILE) -o $@ $(LDFLAGS) ifeq ($(PLATFORM),windows) # Generate import library for Windows @@ -148,96 +174,118 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif -deps: - brew install autoconf automake libtool pkg-config - -curl-src: - @if [ ! -d curl-src ]; then \ - git clone --depth 1 --branch curl-8_12_1 https://github.com/curl/curl.git curl-src; \ - fi - -libcurl-macos: deps curl-src - @if [ ! -f $(CONFIGURED_MARKER) ]; then \ - cd curl-src && ./buildconf && ./configure \ - --without-libpsl \ - --disable-alt-svc \ - --disable-ares \ - --disable-cookies \ - --disable-basic-auth \ - --disable-digest-auth \ - --disable-kerberos-auth \ - --disable-negotiate-auth \ - --disable-aws \ - --disable-dateparse \ - --disable-dnsshuffle \ - --disable-doh \ - --disable-form-api \ - --disable-hsts \ - --disable-ipv6 \ - --disable-libcurl-option \ - --disable-manual \ - --disable-mime \ - --disable-netrc \ - --disable-ntlm \ - --disable-ntlm-wb \ - --disable-progress-meter \ - --disable-proxy \ - --disable-pthreads \ - --disable-socketpair \ - --disable-threaded-resolver \ - --disable-tls-srp \ - --disable-verbose \ - --disable-versioned-symbols \ - --enable-symbol-hiding \ - --without-brotli \ - --without-zstd \ - --without-libidn2 \ - --without-librtmp \ - --without-zlib \ - --without-nghttp2 \ - --without-ngtcp2 \ - --disable-shared \ - --disable-ftp \ - --disable-file \ - --disable-ipfs \ - --disable-ldap \ - --disable-ldaps \ - --disable-rtsp \ - --disable-dict \ - --disable-telnet \ - --disable-tftp \ - --disable-pop3 \ - --disable-imap \ - --disable-smb \ - --disable-smtp \ - --disable-gopher \ - --disable-mqtt \ - --disable-docs \ - --enable-static \ - --with-secure-transport \ - CFLAGS="-arch x86_64 -arch arm64" \ - else \ - echo "Skipping configure: already configured."; \ - fi - cd curl-src && make - mkdir -p curl/macos - mv curl-src/lib/.libs/libcurl.a curl/macos +$(OPENSSL): + git clone https://github.com/openssl/openssl.git $(CURL_DIR)/src/openssl + + cd $(CURL_DIR)/src/openssl && \ + ./Configure android-$(if $(filter aarch64,$(ARCH)),arm64,$(ARCH)) \ + --prefix=$(BIN)/../sysroot/usr \ + no-shared no-unit-test \ + -D__ANDROID_API__=26 && \ + $(MAKE) && $(MAKE) install_sw + +ifeq ($(PLATFORM),android) +$(CURL_LIB): $(OPENSSL) +else +$(CURL_LIB): +endif + mkdir -p $(CURL_DIR)/src + curl -L -o $(CURL_DIR)/src/curl.zip "https://github.com/curl/curl/releases/download/curl-$(subst .,_,${CURL_VERSION})/curl-$(CURL_VERSION).zip" + + ifeq ($(HOST),windows) + powershell -Command "Expand-Archive -Path '$(CURL_DIR)\src\curl.zip' -DestinationPath '$(CURL_DIR)\src\'" + else + unzip $(CURL_DIR)/src/curl.zip -d $(CURL_DIR)/src/. + endif + + cd $(CURL_SRC) && ./configure \ + --without-libpsl \ + --disable-alt-svc \ + --disable-ares \ + --disable-cookies \ + --disable-basic-auth \ + --disable-digest-auth \ + --disable-kerberos-auth \ + --disable-negotiate-auth \ + --disable-aws \ + --disable-dateparse \ + --disable-dnsshuffle \ + --disable-doh \ + --disable-form-api \ + --disable-hsts \ + --disable-ipv6 \ + --disable-libcurl-option \ + --disable-manual \ + --disable-mime \ + --disable-netrc \ + --disable-ntlm \ + --disable-ntlm-wb \ + --disable-progress-meter \ + --disable-proxy \ + --disable-pthreads \ + --disable-socketpair \ + --disable-threaded-resolver \ + --disable-tls-srp \ + --disable-verbose \ + --disable-versioned-symbols \ + --enable-symbol-hiding \ + --without-brotli \ + --without-zstd \ + --without-libidn2 \ + --without-librtmp \ + --without-zlib \ + --without-nghttp2 \ + --without-ngtcp2 \ + --disable-shared \ + --disable-ftp \ + --disable-file \ + --disable-ipfs \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --disable-mqtt \ + --disable-docs \ + --enable-static \ + $(CURL_CONFIG) + + # save avg 1kb more with these options + # --disable-debug \ + # --enable-optimize \ + # --disable-curldebug \ + # --disable-get-easy-options \ + # --without-fish-functions-dir \ + # --without-zsh-functions-dir \ + # --without-libgsasl \ + + cd $(CURL_SRC) && $(MAKE) + + mkdir -p $(CURL_DIR)/$(PLATFORM) + mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) + rm -rf $(CURL_DIR)/src # Clean up generated files clean: - rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov + rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src # Help message help: @echo "SQLite Sync Extension Makefile" @echo "Usage:" - @echo " make [PLATFORM=platform] [target]" + @echo " make [PLATFORM=platform] [ARCH=arch] [ANDROID_NDK=\$$ANDROID_HOME/ndk/26.1.10909125] [target]" @echo "" @echo "Platforms:" @echo " linux (default on Linux)" @echo " macos (default on macOS)" @echo " windows (default on Windows)" - @echo " android (needs CC to be set to Android NDK's Clang compiler)" + @echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)" @echo " ios (only on macOS)" @echo " isim (only on macOS)" @echo ""