From ebc0539dc00ae16eba6052b8a25738f7b7a3899a Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:07:00 +0200 Subject: [PATCH 01/29] curl build in the makefile --- .github/workflows/main.yml | 127 +---------------------- .gitignore | 2 +- Makefile | 200 ++++++++++++++++++++++--------------- 3 files changed, 126 insertions(+), 203 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b2e579..3391bea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,70 +18,34 @@ jobs: include: - os: ubuntu-latest arch: x86_64 - configure: --with-openssl name: linux - os: LinuxARM64 arch: arm64 - configure: --with-openssl name: linux - os: macos-latest - configure: - --with-secure-transport - CFLAGS="-arch x86_64 -arch arm64" name: macos - os: windows-latest arch: x86_64 - configure: - --with-schannel - CFLAGS="-DCURL_STATICLIB" name: windows - os: ubuntu-latest arch: arm64-v8a - configure: - --host aarch64-linux-android26 - --with-openssl=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - CXX=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ - LD=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip name: android make: PLATFORM=android CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang + ARCH=arm64-v8a - os: ubuntu-latest arch: x86_64 - configure: - --host x86_64-linux-android26 - --with-openssl=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang - CXX=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang++ - LD=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip name: android make: PLATFORM=android CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang + ARCH=x86_64 sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip - os: macos-latest - configure: - --host=arm64-apple-darwin - --with-secure-transport - CFLAGS="-arch arm64 -isysroot $(xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0" name: ios make: PLATFORM=ios - os: macos-latest - configure: - --host=arm64-apple-darwin - --with-secure-transport - CFLAGS="-arch x86_64 -arch arm64 -isysroot $(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" name: isim make: PLATFORM=isim @@ -103,13 +67,6 @@ jobs: mingw-w64-x86_64-cc mingw-w64-x86_64-autotools make - - uses: robinraju/release-downloader@v1.12 - with: - repository: curl/curl - tag: 'curl-8_12_1' - extract: true - fileName: curl-*.tar.gz - - name: build openssl if: matrix.name == 'android' run: | @@ -131,84 +88,6 @@ jobs: cp build/lib/libssl.a $TOOLCHAIN/sysroot/usr/lib/ cp build/lib/libcrypto.a $TOOLCHAIN/sysroot/usr/lib/ cp -r build/include/openssl $TOOLCHAIN/sysroot/usr/include/ - - - name: build curl - run: | - - folder=$(ls -d curl-*/ 2>/dev/null | head -n 1) - cd $folder - - ./configure \ - --without-libpsl \ - --disable-alt-svc \ - --disable-ares \ - --disable-cookies \ - --disable-basic-auth \ - --disable-digest-auth \ - --disable-kerberos-auth \ - --disable-negotiate-auth \ - --disable-aws \ - --disable-dateparse \ - --disable-dnsshuffle \ - --disable-doh \ - --disable-form-api \ - --disable-hsts \ - --disable-ipv6 \ - --disable-libcurl-option \ - --disable-manual \ - --disable-mime \ - --disable-netrc \ - --disable-ntlm \ - --disable-ntlm-wb \ - --disable-progress-meter \ - --disable-proxy \ - --disable-pthreads \ - --disable-socketpair \ - --disable-threaded-resolver \ - --disable-tls-srp \ - --disable-verbose \ - --disable-versioned-symbols \ - --enable-symbol-hiding \ - --without-brotli \ - --without-zstd \ - --without-libidn2 \ - --without-librtmp \ - --without-zlib \ - --without-nghttp2 \ - --without-ngtcp2 \ - --disable-shared \ - --disable-ftp \ - --disable-file \ - --disable-ipfs \ - --disable-ldap \ - --disable-ldaps \ - --disable-rtsp \ - --disable-dict \ - --disable-telnet \ - --disable-tftp \ - --disable-pop3 \ - --disable-imap \ - --disable-smb \ - --disable-smtp \ - --disable-gopher \ - --disable-mqtt \ - --disable-docs \ - --enable-static \ - ${{matrix.configure}} - make - - # save avg 1kb more with these options - # --disable-debug \ - # --enable-optimize \ - # --disable-curldebug \ - # --disable-get-easy-options \ - # --without-fish-functions-dir \ - # --without-zsh-functions-dir \ - # --without-libgsasl \ - - cd .. - mkdir -p curl/${{ matrix.name }} - mv $folder/lib/.libs/libcurl.a curl/${{ matrix.name }} - name: build sqlite-sync run: make ${{ matrix.make && matrix.make || ''}} @@ -239,7 +118,7 @@ jobs: export ${{ matrix.make }} $CC sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl # remove unused folders to save up space - rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* curl-* openssl + rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* openssl echo "::endgroup::" echo "::group::prepare the test script" diff --git a/.gitignore b/.gitignore index c5781b2..7c74d94 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ *.sqlite *.a unittest -/curl-src +/curl/src diff --git a/Makefile b/Makefile index 3789b6e..2042beb 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ # make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3 SQLITE3 ?= sqlite3 +# set curl version to download and build +CURL_VERSION ?= 8.12.1 + # Set default platform if not specified ifeq ($(OS),Windows_NT) PLATFORM := windows @@ -34,6 +37,7 @@ BUILD_RELEASE = build/release BUILD_TEST = build/test BUILD_DIRS = $(BUILD_TEST) $(BUILD_RELEASE) CURL_DIR = curl +CURL_SRC = $(CURL_DIR)/src/curl-$(CURL_VERSION) COV_DIR = coverage CUSTOM_CSS = $(TEST_DIR)/sqliteai.css @@ -48,6 +52,7 @@ TEST_FILES = $(SRC_FILES) $(wildcard $(TEST_DIR)/*.c) $(wildcard $(SQLITE_DIR)/* RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES))) TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES))) COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES)) +CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a # Platform-specific settings ifeq ($(PLATFORM),windows) @@ -57,17 +62,50 @@ ifeq ($(PLATFORM),windows) # Create .def file for Windows DEF_FILE := $(BUILD_RELEASE)/cloudsync.def CFLAGS += -DCURL_STATICLIB + CURL_CONFIG = --with-schannel CFLAGS="-DCURL_STATICLIB" else ifeq ($(PLATFORM),macos) TARGET := $(DIST_DIR)/cloudsync.dylib LDFLAGS += -arch x86_64 -arch arm64 -framework Security -dynamiclib -undefined dynamic_lookup T_LDFLAGS = -framework Security CFLAGS += -arch x86_64 -arch arm64 + CURL_CONFIG = --with-secure-transport CFLAGS="-arch x86_64 -arch arm64" else ifeq ($(PLATFORM),android) - # Use Android NDK's Clang compiler, the user should set the CC + # Set ARCH to find Android NDK's Clang compiler, the user should set the ARCH # example CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - ifeq ($(filter %-clang,$(CC)),) - $(error "CC must be set to the Android NDK's Clang compiler") + ifeq ($(filter %,$(ARCH)),) + $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") + endif + + ifeq ($(ARCH),x86_64) + define CURL_CONFIG + --host x86_64-linux-android26 + --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr + LIBS="-lssl -lcrypto" + AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar + AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as + CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang + CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang++ + LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld + RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib + STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip + endef + else ifeq ($(ARCH),arm64-v8a) + define CURL_CONFIG + --host aarch64-linux-android26 + --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr + LIBS="-lssl -lcrypto" + AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar + AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as + CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang + CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ + LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld + RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib + STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip + endef + else + $(error "Unsupported Android ARCH: $(ARCH)") endif + TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lcrypto -lssl else ifeq ($(PLATFORM),ios) @@ -76,15 +114,18 @@ else ifeq ($(PLATFORM),ios) LDFLAGS += -framework Security -framework CoreFoundation -dynamiclib $(SDK) T_LDFLAGS = -framework Security CFLAGS += -arch arm64 $(SDK) + CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch arm64 -isysroot $$(xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0" else ifeq ($(PLATFORM),isim) TARGET := $(DIST_DIR)/cloudsync.dylib SDK := -isysroot $(shell xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0 LDFLAGS += -arch x86_64 -arch arm64 -framework Security -framework CoreFoundation -dynamiclib $(SDK) T_LDFLAGS = -framework Security CFLAGS += -arch x86_64 -arch arm64 $(SDK) + CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" else # linux TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lssl -lcrypto + CURL_CONFIG = --with-openssl endif ifneq ($(COVERAGE),false) @@ -119,7 +160,7 @@ extension: $(TARGET) all: $(TARGET) # Loadable library -$(TARGET): $(RELEASE_OBJ) $(DEF_FILE) +$(TARGET): $(RELEASE_OBJ) $(DEF_FILE) $(CURL_LIB) $(CC) $(RELEASE_OBJ) $(DEF_FILE) -o $@ $(LDFLAGS) ifeq ($(PLATFORM),windows) # Generate import library for Windows @@ -148,80 +189,83 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif -deps: - brew install autoconf automake libtool pkg-config - -curl-src: - @if [ ! -d curl-src ]; then \ - git clone --depth 1 --branch curl-8_12_1 https://github.com/curl/curl.git curl-src; \ - fi - -libcurl-macos: deps curl-src - @if [ ! -f $(CONFIGURED_MARKER) ]; then \ - cd curl-src && ./buildconf && ./configure \ - --without-libpsl \ - --disable-alt-svc \ - --disable-ares \ - --disable-cookies \ - --disable-basic-auth \ - --disable-digest-auth \ - --disable-kerberos-auth \ - --disable-negotiate-auth \ - --disable-aws \ - --disable-dateparse \ - --disable-dnsshuffle \ - --disable-doh \ - --disable-form-api \ - --disable-hsts \ - --disable-ipv6 \ - --disable-libcurl-option \ - --disable-manual \ - --disable-mime \ - --disable-netrc \ - --disable-ntlm \ - --disable-ntlm-wb \ - --disable-progress-meter \ - --disable-proxy \ - --disable-pthreads \ - --disable-socketpair \ - --disable-threaded-resolver \ - --disable-tls-srp \ - --disable-verbose \ - --disable-versioned-symbols \ - --enable-symbol-hiding \ - --without-brotli \ - --without-zstd \ - --without-libidn2 \ - --without-librtmp \ - --without-zlib \ - --without-nghttp2 \ - --without-ngtcp2 \ - --disable-shared \ - --disable-ftp \ - --disable-file \ - --disable-ipfs \ - --disable-ldap \ - --disable-ldaps \ - --disable-rtsp \ - --disable-dict \ - --disable-telnet \ - --disable-tftp \ - --disable-pop3 \ - --disable-imap \ - --disable-smb \ - --disable-smtp \ - --disable-gopher \ - --disable-mqtt \ - --disable-docs \ - --enable-static \ - --with-secure-transport \ - CFLAGS="-arch x86_64 -arch arm64" \ - else \ - echo "Skipping configure: already configured."; \ - fi - cd curl-src && make - mkdir -p curl/macos - mv curl-src/lib/.libs/libcurl.a curl/macos +$(CURL_LIB): + mkdir -p $(CURL_DIR)/src + curl -L -o $(CURL_DIR)/src/curl.zip "https://github.com/curl/curl/releases/download/curl-$(subst .,_,${CURL_VERSION})/curl-$(CURL_VERSION).zip" + unzip $(CURL_DIR)/src/curl.zip -d $(CURL_DIR)/src/. + + cd $(CURL_SRC) && ./configure \ + --without-libpsl \ + --disable-alt-svc \ + --disable-ares \ + --disable-cookies \ + --disable-basic-auth \ + --disable-digest-auth \ + --disable-kerberos-auth \ + --disable-negotiate-auth \ + --disable-aws \ + --disable-dateparse \ + --disable-dnsshuffle \ + --disable-doh \ + --disable-form-api \ + --disable-hsts \ + --disable-ipv6 \ + --disable-libcurl-option \ + --disable-manual \ + --disable-mime \ + --disable-netrc \ + --disable-ntlm \ + --disable-ntlm-wb \ + --disable-progress-meter \ + --disable-proxy \ + --disable-pthreads \ + --disable-socketpair \ + --disable-threaded-resolver \ + --disable-tls-srp \ + --disable-verbose \ + --disable-versioned-symbols \ + --enable-symbol-hiding \ + --without-brotli \ + --without-zstd \ + --without-libidn2 \ + --without-librtmp \ + --without-zlib \ + --without-nghttp2 \ + --without-ngtcp2 \ + --disable-shared \ + --disable-ftp \ + --disable-file \ + --disable-ipfs \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --disable-dict \ + --disable-telnet \ + --disable-tftp \ + --disable-pop3 \ + --disable-imap \ + --disable-smb \ + --disable-smtp \ + --disable-gopher \ + --disable-mqtt \ + --disable-docs \ + --enable-static \ + $(CURL_CONFIG) + + # save avg 1kb more with these options + # --disable-debug \ + # --enable-optimize \ + # --disable-curldebug \ + # --disable-get-easy-options \ + # --without-fish-functions-dir \ + # --without-zsh-functions-dir \ + # --without-libgsasl \ + + cd $(CURL_SRC) && make + + mkdir -p $(CURL_DIR)/$(PLATFORM) + mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) + rm -rf $(CURL_DIR)/src # Clean up generated files clean: From d2d8c1b518dd5376f14952a7726f83f1494880e6 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:20:32 +0200 Subject: [PATCH 02/29] android fix new line and arch in tests --- .github/workflows/main.yml | 2 +- Makefile | 43 ++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3391bea..064688c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -126,7 +126,7 @@ jobs: cat > commands.sh << EOF mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp - $(make test CC=$CC PLATFORM=$PLATFORM -n) + $(make test CC=$CC PLATFORM=$PLATFORM ARCH=$ARCH -n) EOF echo "::endgroup::" diff --git a/Makefile b/Makefile index 2042beb..ee487ab 100644 --- a/Makefile +++ b/Makefile @@ -75,32 +75,35 @@ else ifeq ($(PLATFORM),android) ifeq ($(filter %,$(ARCH)),) $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") endif + + HOST = $(shell uname -s) + BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifeq ($(ARCH),x86_64) define CURL_CONFIG - --host x86_64-linux-android26 - --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang - CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang++ - LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip + --host x86_64-$(HOST)-android26 \ + --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr \ + LIBS="-lssl -lcrypto" \ + AR=$(BIN)/llvm-ar \ + AS=$(BIN)/llvm-as \ + CC=$(BIN)/x86_64-linux-android26-clang \ + CXX=$(BIN)/x86_64-linux-android26-clang++ \ + LD=$(HOST)/ld \ + RANLIB=$(HOST)/llvm-ranlib \ + STRIP=$(HOST)/llvm-strip \ endef else ifeq ($(ARCH),arm64-v8a) define CURL_CONFIG - --host aarch64-linux-android26 - --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr - LIBS="-lssl -lcrypto" - AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as - CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ - LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld - RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip + --host aarch64-linux-android26 \ + --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr \ + LIBS="-lssl -lcrypto" \ + AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar \ + AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as \ + CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang \ + CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ \ + LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld \ + RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib \ + STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip \ endef else $(error "Unsupported Android ARCH: $(ARCH)") From 7b8467628d86e9772d24d31d25772723ca167e6b Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:26:34 +0200 Subject: [PATCH 03/29] android define curl config inline --- Makefile | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index ee487ab..5698252 100644 --- a/Makefile +++ b/Makefile @@ -80,31 +80,9 @@ else ifeq ($(PLATFORM),android) BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifeq ($(ARCH),x86_64) - define CURL_CONFIG - --host x86_64-$(HOST)-android26 \ - --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr \ - LIBS="-lssl -lcrypto" \ - AR=$(BIN)/llvm-ar \ - AS=$(BIN)/llvm-as \ - CC=$(BIN)/x86_64-linux-android26-clang \ - CXX=$(BIN)/x86_64-linux-android26-clang++ \ - LD=$(HOST)/ld \ - RANLIB=$(HOST)/llvm-ranlib \ - STRIP=$(HOST)/llvm-strip \ - endef + CURL_CONFIG = --host x86_64-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/x86_64-linux-android26-clang CXX=$(BIN)/x86_64-linux-android26-clang++ LD=$(HOST)/ld RANLIB=$(HOST)/llvm-ranlib STRIP=$(HOST)/llvm-strip else ifeq ($(ARCH),arm64-v8a) - define CURL_CONFIG - --host aarch64-linux-android26 \ - --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr \ - LIBS="-lssl -lcrypto" \ - AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar \ - AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as \ - CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang \ - CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ \ - LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld \ - RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib \ - STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip \ - endef + CURL_CONFIG = --host aarch64-linux-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip else $(error "Unsupported Android ARCH: $(ARCH)") endif From a1d6b756daabeebe4057ed8f2995d8143f48f1d4 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:42:44 +0200 Subject: [PATCH 04/29] android fix lowercase uname -s --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5698252..72be5b4 100644 --- a/Makefile +++ b/Makefile @@ -76,12 +76,16 @@ else ifeq ($(PLATFORM),android) $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") endif - HOST = $(shell uname -s) + HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin + + ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) + ARCH = aarch64 + endif ifeq ($(ARCH),x86_64) CURL_CONFIG = --host x86_64-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/x86_64-linux-android26-clang CXX=$(BIN)/x86_64-linux-android26-clang++ LD=$(HOST)/ld RANLIB=$(HOST)/llvm-ranlib STRIP=$(HOST)/llvm-strip - else ifeq ($(ARCH),arm64-v8a) + else ifeq ($(ARCH),aarch64) CURL_CONFIG = --host aarch64-linux-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip else $(error "Unsupported Android ARCH: $(ARCH)") From 5099222a03ac1633253f427435f410e1a67bbe4c Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:49:49 +0200 Subject: [PATCH 05/29] fix windows --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 72be5b4..8957962 100644 --- a/Makefile +++ b/Makefile @@ -246,7 +246,11 @@ $(CURL_LIB): # --without-zsh-functions-dir \ # --without-libgsasl \ + ifeq ($(PLATFORM),windows) + msys2 "cd $(CURL_SRC) && make" + else cd $(CURL_SRC) && make + endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From b1990fdca20fcae3dcbda4e450772c39865a8248 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 16:56:17 +0200 Subject: [PATCH 06/29] android typos --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8957962..992c3cf 100644 --- a/Makefile +++ b/Makefile @@ -80,11 +80,11 @@ else ifeq ($(PLATFORM),android) BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) - ARCH = aarch64 + ARCH := aarch64 endif ifeq ($(ARCH),x86_64) - CURL_CONFIG = --host x86_64-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/x86_64-linux-android26-clang CXX=$(BIN)/x86_64-linux-android26-clang++ LD=$(HOST)/ld RANLIB=$(HOST)/llvm-ranlib STRIP=$(HOST)/llvm-strip + CURL_CONFIG = --host x86_64-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/x86_64-linux-android26-clang CXX=$(BIN)/x86_64-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip else ifeq ($(ARCH),aarch64) CURL_CONFIG = --host aarch64-linux-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip else From b77a02b149f63be36c2235194868459d9b359f2a Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 17:05:17 +0200 Subject: [PATCH 07/29] android arch override --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 992c3cf..667b9d6 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ else ifeq ($(PLATFORM),android) BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) - ARCH := aarch64 + override ARCH := aarch64 endif ifeq ($(ARCH),x86_64) From 056ca1cd96a82b05c5f6536f1e4ecde803a07970 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 17:14:25 +0200 Subject: [PATCH 08/29] android dynamic curl config --- Makefile | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 667b9d6..56977c2 100644 --- a/Makefile +++ b/Makefile @@ -83,14 +83,7 @@ else ifeq ($(PLATFORM),android) override ARCH := aarch64 endif - ifeq ($(ARCH),x86_64) - CURL_CONFIG = --host x86_64-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/x86_64-linux-android26-clang CXX=$(BIN)/x86_64-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip - else ifeq ($(ARCH),aarch64) - CURL_CONFIG = --host aarch64-linux-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar AS=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-as CC=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang CXX=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang++ LD=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/ld RANLIB=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib STRIP=$$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip - else - $(error "Unsupported Android ARCH: $(ARCH)") - endif - + CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lcrypto -lssl else ifeq ($(PLATFORM),ios) From 47a39833731facb2f19b1b3cdad485aac89066a3 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 17:27:33 +0200 Subject: [PATCH 09/29] android set cc in the makefile --- .github/workflows/main.yml | 16 +++++----------- Makefile | 7 ++++--- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 064688c..5ce93fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,17 +30,11 @@ jobs: - os: ubuntu-latest arch: arm64-v8a name: android - make: - PLATFORM=android - CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang - ARCH=arm64-v8a + make: PLATFORM=android ARCH=arm64-v8a - os: ubuntu-latest arch: x86_64 name: android - make: - PLATFORM=android - CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android26-clang - ARCH=x86_64 + make: PLATFORM=android ARCH=x86_64 sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip - os: macos-latest name: ios @@ -116,17 +110,17 @@ jobs: curl -O ${{ matrix.sqlite-amalgamation-zip }} unzip sqlite-amalgamation-*.zip export ${{ matrix.make }} - $CC sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl + $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.arch }}-linux-android26-clang sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl # remove unused folders to save up space rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* openssl echo "::endgroup::" echo "::group::prepare the test script" - make test CC=$CC PLATFORM=$PLATFORM || echo "It should fail. Running remaining commands in the emulator" + make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" cat > commands.sh << EOF mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp - $(make test CC=$CC PLATFORM=$PLATFORM ARCH=$ARCH -n) + $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) EOF echo "::endgroup::" diff --git a/Makefile b/Makefile index 56977c2..a1a4b41 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,8 @@ else ifeq ($(PLATFORM),android) ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) override ARCH := aarch64 endif - + + CC = $(BIN)/$(ARCH)-linux-android26-clang CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lcrypto -lssl @@ -257,13 +258,13 @@ clean: help: @echo "SQLite Sync Extension Makefile" @echo "Usage:" - @echo " make [PLATFORM=platform] [target]" + @echo " make [PLATFORM=platform] [ARCH=arch] [target]" @echo "" @echo "Platforms:" @echo " linux (default on Linux)" @echo " macos (default on macOS)" @echo " windows (default on Windows)" - @echo " android (needs CC to be set to Android NDK's Clang compiler)" + @echo " android (needs ARCH to be set to x86_64 or arm64-v8a)" @echo " ios (only on macOS)" @echo " isim (only on macOS)" @echo "" From 203665abb2557a3e7fc2bb2a3eb23dcc715239ce Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 17:42:32 +0200 Subject: [PATCH 10/29] makefile set android ndk path --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a1a4b41..e06dd7b 100644 --- a/Makefile +++ b/Makefile @@ -71,13 +71,17 @@ else ifeq ($(PLATFORM),macos) CURL_CONFIG = --with-secure-transport CFLAGS="-arch x86_64 -arch arm64" else ifeq ($(PLATFORM),android) # Set ARCH to find Android NDK's Clang compiler, the user should set the ARCH - # example CC=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android26-clang ifeq ($(filter %,$(ARCH)),) $(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a") endif + # Set ANDROID_NDK path to find android build tools + # e.g. on MacOS: export ANDROID_NDK=/Users/username/Library/Android/sdk/ndk/25.2.9519653 + ifeq ($(filter %,$(ANDROID_NDK)),) + $(error "Android NDK must be set") + endif HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') - BIN = $$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin + BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) override ARCH := aarch64 @@ -258,13 +262,13 @@ clean: help: @echo "SQLite Sync Extension Makefile" @echo "Usage:" - @echo " make [PLATFORM=platform] [ARCH=arch] [target]" + @echo " make [PLATFORM=platform] [ARCH=arch] [ANDROID_NDK=\$$ANDROID_HOME/ndk/26.1.10909125] [target]" @echo "" @echo "Platforms:" @echo " linux (default on Linux)" @echo " macos (default on macOS)" @echo " windows (default on Windows)" - @echo " android (needs ARCH to be set to x86_64 or arm64-v8a)" + @echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)" @echo " ios (only on macOS)" @echo " isim (only on macOS)" @echo "" From 0759d7d428dc97b06d1e7d8c31594bdab8338355 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 18:05:31 +0200 Subject: [PATCH 11/29] win build with msys test --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ce93fe..e513b99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,7 +85,6 @@ jobs: - name: build sqlite-sync run: make ${{ matrix.make && matrix.make || ''}} - shell: bash - name: windows install sqlite3 if: matrix.os == 'windows-latest' From 8a057c09c26d5179758384367d661dad6fa86fce Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 18:15:12 +0200 Subject: [PATCH 12/29] remove msys from makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index e06dd7b..d788a07 100644 --- a/Makefile +++ b/Makefile @@ -244,11 +244,7 @@ $(CURL_LIB): # --without-zsh-functions-dir \ # --without-libgsasl \ - ifeq ($(PLATFORM),windows) - msys2 "cd $(CURL_SRC) && make" - else cd $(CURL_SRC) && make - endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From 9c4aa5c81b8e47716a3af4a8b7c4c049eee29940 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 19:34:44 +0200 Subject: [PATCH 13/29] windows use msys bash --- .github/workflows/main.yml | 1 + Makefile | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e513b99..5ce93fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,6 +85,7 @@ jobs: - name: build sqlite-sync run: make ${{ matrix.make && matrix.make || ''}} + shell: bash - name: windows install sqlite3 if: matrix.os == 'windows-latest' diff --git a/Makefile b/Makefile index d788a07..051e9bd 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,13 @@ CURL_VERSION ?= 8.12.1 # Set default platform if not specified ifeq ($(OS),Windows_NT) PLATFORM := windows + HOST:= windows else - UNAME_S := $(shell uname -s) - ifeq ($(UNAME_S),Darwin) + HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') + ifeq ($(HOST),darwin) PLATFORM := macos else - PLATFORM := linux + PLATFORM := $(HOST) endif endif @@ -80,7 +81,6 @@ else ifeq ($(PLATFORM),android) $(error "Android NDK must be set") endif - HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]') BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) @@ -244,7 +244,11 @@ $(CURL_LIB): # --without-zsh-functions-dir \ # --without-libgsasl \ + ifeq ($(PLATFORM),windows) + "C:/msys64/usr/bin/bash.exe" "cd $(CURL_SRC) && make" + else cd $(CURL_SRC) && make + endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From 2b8e2a9addbd530909ee1965ffe65b395b40321e Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 19:59:57 +0200 Subject: [PATCH 14/29] makefile windows use msys make --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 051e9bd..a31f1d0 100644 --- a/Makefile +++ b/Makefile @@ -245,7 +245,7 @@ $(CURL_LIB): # --without-libgsasl \ ifeq ($(PLATFORM),windows) - "C:/msys64/usr/bin/bash.exe" "cd $(CURL_SRC) && make" + cd $(CURL_SRC) && C:\msys64\usr\bin\make.exe else cd $(CURL_SRC) && make endif From ff0eb5b3e3e3b201c6be18309a5565e617e4bd13 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 26 May 2025 20:04:28 +0200 Subject: [PATCH 15/29] fix: quote msys make command for Windows compatibility --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a31f1d0..99ccdac 100644 --- a/Makefile +++ b/Makefile @@ -245,7 +245,7 @@ $(CURL_LIB): # --without-libgsasl \ ifeq ($(PLATFORM),windows) - cd $(CURL_SRC) && C:\msys64\usr\bin\make.exe + cd $(CURL_SRC) && "C:\msys64\usr\bin\make.exe" else cd $(CURL_SRC) && make endif From 47724eea333b1c1aa47a8c71dbd89f2d38c67666 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 18:18:29 +0200 Subject: [PATCH 16/29] android build openssl in the makefile --- .github/workflows/main.yml | 22 ---------------------- Makefile | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ce93fe..311f326 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,28 +61,6 @@ jobs: mingw-w64-x86_64-cc mingw-w64-x86_64-autotools make - - name: build openssl - if: matrix.name == 'android' - run: | - git clone https://github.com/openssl/openssl.git - cd openssl - export TOOLCHAIN=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64 - export ARCH=${{ matrix.arch }} - export PATH=$TOOLCHAIN/bin:$PATH - - ./Configure android-${{ matrix.arch == 'arm64-v8a' && 'arm64' || matrix.arch }} \ - --prefix=$PWD/build \ - --openssldir=$PWD/build/ssl \ - no-shared \ - no-unit-test \ - -D__ANDROID_API__=26 - make - make install_sw - - cp build/lib/libssl.a $TOOLCHAIN/sysroot/usr/lib/ - cp build/lib/libcrypto.a $TOOLCHAIN/sysroot/usr/lib/ - cp -r build/include/openssl $TOOLCHAIN/sysroot/usr/include/ - - name: build sqlite-sync run: make ${{ matrix.make && matrix.make || ''}} shell: bash diff --git a/Makefile b/Makefile index 99ccdac..93474c5 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,7 @@ else ifeq ($(PLATFORM),android) endif BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin + PATH := $(BIN):$(PATH) ifneq (,$(filter $(ARCH),arm64 arm64-v8a)) override ARCH := aarch64 @@ -172,7 +173,21 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif +openssl: + git clone https://github.com/openssl/openssl.git $(CURL_DIR)/src/openssl + + cd $(CURL_DIR)/src/openssl && \ + ./Configure android-$(if $(filter aarch64,$(ARCH)),arm64,$(ARCH)) \ + --prefix=$(ANDROID_NDK)/toolchains/sysroot/usr \ + no-shared no-unit-test \ + -D__ANDROID_API__=26 && \ + make && make install_sw + +ifeq ($(PLATFORM),android) +$(CURL_LIB): openssl +else $(CURL_LIB): +endif mkdir -p $(CURL_DIR)/src curl -L -o $(CURL_DIR)/src/curl.zip "https://github.com/curl/curl/releases/download/curl-$(subst .,_,${CURL_VERSION})/curl-$(CURL_VERSION).zip" unzip $(CURL_DIR)/src/curl.zip -d $(CURL_DIR)/src/. From 165e0a30a68564a271f3c52098ec6bc591f9d82f Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 18:25:55 +0200 Subject: [PATCH 17/29] fix android openssl prefix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 93474c5..12450d2 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ openssl: cd $(CURL_DIR)/src/openssl && \ ./Configure android-$(if $(filter aarch64,$(ARCH)),arm64,$(ARCH)) \ - --prefix=$(ANDROID_NDK)/toolchains/sysroot/usr \ + --prefix=$(BIN)/../sysroot/usr \ no-shared no-unit-test \ -D__ANDROID_API__=26 && \ make && make install_sw From d1a62b47db4e229f2f4bc582e319e76868636175 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 18:55:34 +0200 Subject: [PATCH 18/29] makefile android fix curl dependencies --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 12450d2..14d7632 100644 --- a/Makefile +++ b/Makefile @@ -88,8 +88,12 @@ else ifeq ($(PLATFORM),android) override ARCH := aarch64 endif + SSL := $(BIN)/../sysroot/usr/lib/libssl.a + CRYPTO := $(BIN)/../sysroot/usr/lib/libcrypto.a + OPENSSL := $(BIN)/../sysroot/usr/include/openssl + CC = $(BIN)/$(ARCH)-linux-android26-clang - CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$$ANDROID_NDK/toolchains/llvm/prebuilt/$(HOST)-x86_64/sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip + CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$(BIN)/../sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lcrypto -lssl else ifeq ($(PLATFORM),ios) @@ -173,7 +177,7 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif -openssl: +$(SSL) $(CRYPTO) $(OPENSSL): git clone https://github.com/openssl/openssl.git $(CURL_DIR)/src/openssl cd $(CURL_DIR)/src/openssl && \ @@ -184,7 +188,7 @@ openssl: make && make install_sw ifeq ($(PLATFORM),android) -$(CURL_LIB): openssl +$(CURL_LIB): $(SSL) $(CRYPTO) $(OPENSSL) else $(CURL_LIB): endif From 37b723bec881e45f93a94dcbbea6c539444a667b Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 19:06:52 +0200 Subject: [PATCH 19/29] android fix makefile run recipe only one time --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 14d7632..32b2853 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif -$(SSL) $(CRYPTO) $(OPENSSL): +openssl: git clone https://github.com/openssl/openssl.git $(CURL_DIR)/src/openssl cd $(CURL_DIR)/src/openssl && \ @@ -187,6 +187,8 @@ $(SSL) $(CRYPTO) $(OPENSSL): -D__ANDROID_API__=26 && \ make && make install_sw +$(SSL) $(CRYPTO) $(OPENSSL): openssl + ifeq ($(PLATFORM),android) $(CURL_LIB): $(SSL) $(CRYPTO) $(OPENSSL) else From a809bc6fb2b9c81ea6c76c198a728a6df66e3f30 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 19:19:50 +0200 Subject: [PATCH 20/29] fix android makefile only openssl dependency --- Makefile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 32b2853..504951f 100644 --- a/Makefile +++ b/Makefile @@ -88,10 +88,7 @@ else ifeq ($(PLATFORM),android) override ARCH := aarch64 endif - SSL := $(BIN)/../sysroot/usr/lib/libssl.a - CRYPTO := $(BIN)/../sysroot/usr/lib/libcrypto.a OPENSSL := $(BIN)/../sysroot/usr/include/openssl - CC = $(BIN)/$(ARCH)-linux-android26-clang CURL_CONFIG = --host $(ARCH)-$(HOST)-android26 --with-openssl=$(BIN)/../sysroot/usr LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(BIN)/$(ARCH)-linux-android26-clang CXX=$(BIN)/$(ARCH)-linux-android26-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so @@ -177,7 +174,7 @@ ifneq ($(COVERAGE),false) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif -openssl: +$(OPENSSL): git clone https://github.com/openssl/openssl.git $(CURL_DIR)/src/openssl cd $(CURL_DIR)/src/openssl && \ @@ -187,10 +184,8 @@ openssl: -D__ANDROID_API__=26 && \ make && make install_sw -$(SSL) $(CRYPTO) $(OPENSSL): openssl - ifeq ($(PLATFORM),android) -$(CURL_LIB): $(SSL) $(CRYPTO) $(OPENSSL) +$(CURL_LIB): $(OPENSSL) else $(CURL_LIB): endif From 4de3c7245e624a12d1d2f71311bac6e1a0706294 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 19:50:35 +0200 Subject: [PATCH 21/29] makefile fix windows make path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 504951f..20b5f17 100644 --- a/Makefile +++ b/Makefile @@ -261,7 +261,7 @@ endif # --without-libgsasl \ ifeq ($(PLATFORM),windows) - cd $(CURL_SRC) && "C:\msys64\usr\bin\make.exe" + cd $(CURL_SRC) && "/c/msys64/mingw64/bin/make.exe" else cd $(CURL_SRC) && make endif From 1f5d9bc0f96fa0062b49f439bfc827703a158bdd Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 20:17:30 +0200 Subject: [PATCH 22/29] fix makefile to use msys bash for curl build on Windows --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 20b5f17..729a118 100644 --- a/Makefile +++ b/Makefile @@ -261,7 +261,7 @@ endif # --without-libgsasl \ ifeq ($(PLATFORM),windows) - cd $(CURL_SRC) && "/c/msys64/mingw64/bin/make.exe" + /c/msys64/usr/bin/bash.exe -lc "cd $(CURL_SRC) && make" else cd $(CURL_SRC) && make endif From ece2fb629dd36b6122125f33a343da59d763b933 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 20:21:46 +0200 Subject: [PATCH 23/29] fix makefile spacing --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 729a118..5b504e4 100644 --- a/Makefile +++ b/Makefile @@ -260,11 +260,11 @@ endif # --without-zsh-functions-dir \ # --without-libgsasl \ - ifeq ($(PLATFORM),windows) - /c/msys64/usr/bin/bash.exe -lc "cd $(CURL_SRC) && make" - else + ifeq ($(PLATFORM),windows) + /c/msys64/usr/bin/bash.exe -lc "cd $(CURL_SRC) && make" + else cd $(CURL_SRC) && make - endif + endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From 62c485ba5e67bb1b312b1f04863a635a91d37b35 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 20:28:27 +0200 Subject: [PATCH 24/29] fix makefile tab --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5b504e4..827696a 100644 --- a/Makefile +++ b/Makefile @@ -260,11 +260,11 @@ endif # --without-zsh-functions-dir \ # --without-libgsasl \ - ifeq ($(PLATFORM),windows) + ifeq ($(PLATFORM),windows) /c/msys64/usr/bin/bash.exe -lc "cd $(CURL_SRC) && make" - else + else cd $(CURL_SRC) && make - endif + endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From b21fc4a8bd496f6694a6d99ce9d4b0768b3c1cbd Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Tue, 27 May 2025 21:02:37 +0200 Subject: [PATCH 25/29] fix makefile to use mingw64 make for Windows build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 827696a..881ddc6 100644 --- a/Makefile +++ b/Makefile @@ -261,7 +261,7 @@ endif # --without-libgsasl \ ifeq ($(PLATFORM),windows) - /c/msys64/usr/bin/bash.exe -lc "cd $(CURL_SRC) && make" + cd $(CURL_SRC) && /c/mingw64/bin/make.exe else cd $(CURL_SRC) && make endif From 8fe2d13cedeeb5a873a23d21c2703b1bb9bc3b56 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 28 May 2025 13:40:21 +0200 Subject: [PATCH 26/29] fix: update Windows build process for curl --- .github/workflows/main.yml | 13 ++++++------- Makefile | 11 ++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 311f326..c2ddef9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,9 +43,6 @@ jobs: name: isim make: PLATFORM=isim - defaults: - run: - shell: ${{ matrix.os == 'windows-latest' && 'msys2 {0}' || 'bash' }} env: MAKEFLAGS: -j 8 @@ -59,16 +56,19 @@ jobs: msystem: mingw64 install: >- mingw-w64-x86_64-cc - mingw-w64-x86_64-autotools make + make + + - name: windows build curl + if: matrix.os == 'windows-latest' + run: make curl/windows/libcurl.a + shell: msys2 {0} - name: build sqlite-sync run: make ${{ matrix.make && matrix.make || ''}} - shell: bash - name: windows install sqlite3 if: matrix.os == 'windows-latest' run: choco install sqlite -y - shell: bash - name: macos install sqlite3 without SQLITE_OMIT_LOAD_EXTENSION if: matrix.name == 'macos' @@ -117,7 +117,6 @@ jobs: - name: test sqlite-sync if: matrix.name == 'linux' || matrix.name == 'windows' run: make test - shell: bash - name: test sqlite-sync + coverage if: matrix.name == 'macos' diff --git a/Makefile b/Makefile index 881ddc6..5250f93 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,12 @@ $(CURL_LIB): endif mkdir -p $(CURL_DIR)/src curl -L -o $(CURL_DIR)/src/curl.zip "https://github.com/curl/curl/releases/download/curl-$(subst .,_,${CURL_VERSION})/curl-$(CURL_VERSION).zip" + + ifeq ($(HOST),windows) + powershell -Command "Expand-Archive -Path '$(CURL_DIR)\src\curl.zip' -DestinationPath '$(CURL_DIR)\src\'" + else unzip $(CURL_DIR)/src/curl.zip -d $(CURL_DIR)/src/. + endif cd $(CURL_SRC) && ./configure \ --without-libpsl \ @@ -260,11 +265,7 @@ endif # --without-zsh-functions-dir \ # --without-libgsasl \ - ifeq ($(PLATFORM),windows) - cd $(CURL_SRC) && /c/mingw64/bin/make.exe - else cd $(CURL_SRC) && make - endif mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) @@ -272,7 +273,7 @@ endif # Clean up generated files clean: - rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov + rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src # Help message help: From c165acb417bf7183d6362a45d5c1d741f1a1ddf2 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 28 May 2025 13:56:32 +0200 Subject: [PATCH 27/29] fix: set default shell to bash in GitHub Actions workflow --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c2ddef9..a7afff7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,9 @@ jobs: name: isim make: PLATFORM=isim + defaults: + run: + shell: bash env: MAKEFLAGS: -j 8 From facd13b25ed3c59a229dc0f386d2dc20a218f7fb Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 28 May 2025 14:01:21 +0200 Subject: [PATCH 28/29] fix: optimize make commands for parallel execution in GitHub Actions and Makefile --- .github/workflows/main.yml | 10 ++++------ Makefile | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a7afff7..e32b78c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,8 +46,6 @@ jobs: defaults: run: shell: bash - env: - MAKEFLAGS: -j 8 steps: @@ -63,11 +61,11 @@ jobs: - name: windows build curl if: matrix.os == 'windows-latest' - run: make curl/windows/libcurl.a + run: make curl/windows/libcurl.a -j4 shell: msys2 {0} - name: build sqlite-sync - run: make ${{ matrix.make && matrix.make || ''}} + run: make extension ${{ matrix.make && matrix.make || ''}} -j4 - name: windows install sqlite3 if: matrix.os == 'windows-latest' @@ -97,7 +95,7 @@ jobs: echo "::endgroup::" echo "::group::prepare the test script" - make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" + make test PLATFORM=$PLATFORM ARCH=$ARCH -j4 || echo "It should fail. Running remaining commands in the emulator" cat > commands.sh << EOF mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp @@ -119,7 +117,7 @@ jobs: - name: test sqlite-sync if: matrix.name == 'linux' || matrix.name == 'windows' - run: make test + run: make test -j4 - name: test sqlite-sync + coverage if: matrix.name == 'macos' diff --git a/Makefile b/Makefile index 5250f93..57d6e9f 100644 --- a/Makefile +++ b/Makefile @@ -265,7 +265,7 @@ endif # --without-zsh-functions-dir \ # --without-libgsasl \ - cd $(CURL_SRC) && make + cd $(CURL_SRC) && $(MAKE) mkdir -p $(CURL_DIR)/$(PLATFORM) mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) From 6a92f9ab9b137b336e3e7b5b434961eee927bb49 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 28 May 2025 14:05:36 +0200 Subject: [PATCH 29/29] fix: replace make command with $(MAKE) for consistency and speed up in build openssl --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57d6e9f..a1e3599 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,7 @@ $(OPENSSL): --prefix=$(BIN)/../sysroot/usr \ no-shared no-unit-test \ -D__ANDROID_API__=26 && \ - make && make install_sw + $(MAKE) && $(MAKE) install_sw ifeq ($(PLATFORM),android) $(CURL_LIB): $(OPENSSL)