From d042fe2ce100daf3be5cebd025df88707863122c Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 13:38:02 +0200 Subject: [PATCH 01/39] feat: add WASM support with sqlite3_wasm_extra_init function and related build configurations --- .github/workflows/main.yml | 10 +++++++++- Makefile | 30 +++++++++++++++++++++++++++--- src/wasm.c | 20 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 src/wasm.c diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a86caf1..88d54c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,8 @@ jobs: - os: macos-latest name: isim make: PLATFORM=isim + - os: ubuntu-latest + name: wasm defaults: run: @@ -70,6 +72,10 @@ jobs: run: make curl/windows/libcurl.a shell: msys2 {0} + - name: wasm install wabt + if: matrix.name == 'wasm' + run: apt install wabt + - name: build sqlite-sync run: make extension ${{ matrix.make && matrix.make || ''}} @@ -141,7 +147,9 @@ jobs: if: always() with: name: cloudsync-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} - path: dist/cloudsync.* + path: | + dist/cloudsync.* + dist/sqlite-wasm.zip if-no-files-found: error release: diff --git a/Makefile b/Makefile index 0958776..49eb697 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ SQLITE3 ?= sqlite3 # set curl version to download and build CURL_VERSION ?= 8.12.1 +# set sqlite version for WASM static build +SQLITE_VERSION ?= 3.50.1 + # Set default platform if not specified ifeq ($(OS),Windows_NT) PLATFORM := windows @@ -47,6 +50,7 @@ CURL_DIR = curl CURL_SRC = $(CURL_DIR)/src/curl-$(CURL_VERSION) COV_DIR = coverage CUSTOM_CSS = $(TEST_DIR)/sqliteai.css +BUILD_WASM = build/wasm SRC_FILES = $(wildcard $(SRC_DIR)/*.c) TEST_SRC = $(wildcard $(TEST_DIR)/*.c) @@ -110,6 +114,8 @@ else ifeq ($(PLATFORM),isim) T_LDFLAGS = -framework Security CFLAGS += -arch x86_64 -arch arm64 $(SDK) CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" +else ifeq ($(PLATFORM),wasm) + TARGET := $(DIST_DIR)/sqlite-wasm.zip else # linux TARGET := $(DIST_DIR)/cloudsync.so LDFLAGS += -shared -lssl -lcrypto @@ -127,7 +133,7 @@ endif # Windows .def file generation $(DEF_FILE): ifeq ($(PLATFORM),windows) - @echo "LIBRARY js.dll" > $@ + @echo "LIBRARY cloudsync.dll" > $@ @echo "EXPORTS" >> $@ @echo " sqlite3_cloudsync_init" >> $@ endif @@ -139,12 +145,29 @@ $(shell mkdir -p $(BUILD_DIRS) $(DIST_DIR)) extension: $(TARGET) all: $(TARGET) +ifneq ($(PLATFORM),wasm) # Loadable library $(TARGET): $(RELEASE_OBJ) $(DEF_FILE) $(CURL_LIB) $(CC) $(RELEASE_OBJ) $(DEF_FILE) -o $@ $(LDFLAGS) ifeq ($(PLATFORM),windows) # Generate import library for Windows - dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/js.lib + dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/cloudsync.lib +endif +else +#WASM build +EMSDK := $(BUILD_WASM)/emsdk +$(EMSDK): + git clone https://github.com/emscripten-core/emsdk.git $(EMSDK) + cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest && source ./emsdk_env.sh + +SQLITE_SRC := $(BUILD_WASM)/sqlite +$(SQLITE_SRC): $(EMSDK) + git clone --branch version-$(SQLITE_VERSION) --depth 1 https://github.com/sqlite/sqlite.git $(SQLITE_SRC) + source ./$(EMSDK)/emsdk_env.sh && cd $(SQLITE_SRC) && ./configure --enable-all + +$(TARGET): $(SQLITE_SRC) $(SRC_FILES) + cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c + mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif # Test executable @@ -268,7 +291,7 @@ endif # Clean up generated files clean: - rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite + rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite $(BUILD_WASM) # Help message help: @@ -283,6 +306,7 @@ help: @echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)" @echo " ios (only on macOS)" @echo " isim (only on macOS)" + @echo " wasm (needs wabt[brew install wabt/apt install wabt])" @echo "" @echo "Targets:" @echo " all - Build the extension (default)" diff --git a/src/wasm.c b/src/wasm.c new file mode 100644 index 0000000..5f5d747 --- /dev/null +++ b/src/wasm.c @@ -0,0 +1,20 @@ +#ifdef SQLITE_WASM_EXTRA_INIT + +#include "sqlite3.h" +#include + +#define CLOUDSYNC_OMIT_NETWORK +#include "utils.c" +#include "network.c" +#include "dbutils.c" +#include "cloudsync.c" +#include "vtab.c" +#include "pk.c" +#include "lz4.c" +//ciao +int sqlite3_wasm_extra_init(const char *z) { + fprintf(stderr, "%s: %s()\n", __FILE__, __func__); + return sqlite3_auto_extension((void *) sqlite3_cloudsync_init); +} + +#endif \ No newline at end of file From 2627620a08474be404419fd21172818eefa10885 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 13:42:24 +0200 Subject: [PATCH 02/39] fix: update WASM build instructions and exclude wasm.c from coverage --- .github/workflows/main.yml | 4 ++-- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88d54c9..2f270c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ permissions: jobs: build: runs-on: ${{ matrix.os }} - name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && ' + test' || ''}} + name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && matrix.name != 'wasm' && ' + test' || ''}} timeout-minutes: 20 strategy: fail-fast: false @@ -74,7 +74,7 @@ jobs: - name: wasm install wabt if: matrix.name == 'wasm' - run: apt install wabt + run: sudo apt install wabt - name: build sqlite-sync run: make extension ${{ matrix.make && matrix.make || ''}} diff --git a/Makefile b/Makefile index 49eb697..0412913 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ TEST_SRC = $(wildcard $(TEST_DIR)/*.c) TEST_FILES = $(SRC_FILES) $(TEST_SRC) $(wildcard $(SQLITE_DIR)/*.c) RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES))) TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES))) -COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES)) +COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c $(SRC_DIR)/wasm.c, $(SRC_FILES)) CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC))) @@ -306,7 +306,7 @@ help: @echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)" @echo " ios (only on macOS)" @echo " isim (only on macOS)" - @echo " wasm (needs wabt[brew install wabt/apt install wabt])" + @echo " wasm (needs wabt[brew install wabt/sudo apt install wabt])" @echo "" @echo "Targets:" @echo " all - Build the extension (default)" From 6143d5f409d1b91d4bade9b0bd48d24655818f79 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 13:44:18 +0200 Subject: [PATCH 03/39] feat: add make wasm platform spec to build matrix in GitHub Actions workflow --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f270c7..23e3b8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,6 +45,7 @@ jobs: make: PLATFORM=isim - os: ubuntu-latest name: wasm + make: PLATFORM=wasm defaults: run: From 71ba6363b305dff7e0994a731ac93f1ec151b43b Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 13:50:43 +0200 Subject: [PATCH 04/39] fix: correct sourcing of emsdk_env.sh in Makefile for WASM build with POSIX . --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0412913..0f78577 100644 --- a/Makefile +++ b/Makefile @@ -158,12 +158,12 @@ else EMSDK := $(BUILD_WASM)/emsdk $(EMSDK): git clone https://github.com/emscripten-core/emsdk.git $(EMSDK) - cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest && source ./emsdk_env.sh + cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest && . ./emsdk_env.sh SQLITE_SRC := $(BUILD_WASM)/sqlite $(SQLITE_SRC): $(EMSDK) git clone --branch version-$(SQLITE_VERSION) --depth 1 https://github.com/sqlite/sqlite.git $(SQLITE_SRC) - source ./$(EMSDK)/emsdk_env.sh && cd $(SQLITE_SRC) && ./configure --enable-all + . ./$(EMSDK)/emsdk_env.sh && cd $(SQLITE_SRC) && ./configure --enable-all $(TARGET): $(SQLITE_SRC) $(SRC_FILES) cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c From 3eb08b16d59e6dc47bd1922df9117bdf51351281 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 14:29:10 +0200 Subject: [PATCH 05/39] fix: update emsdk_env.sh sourcing in Makefile for WASM build process --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0f78577..9ec0bb7 100644 --- a/Makefile +++ b/Makefile @@ -158,12 +158,12 @@ else EMSDK := $(BUILD_WASM)/emsdk $(EMSDK): git clone https://github.com/emscripten-core/emsdk.git $(EMSDK) - cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest && . ./emsdk_env.sh + cd $(EMSDK) && ./emsdk install latest && ./emsdk activate latest SQLITE_SRC := $(BUILD_WASM)/sqlite $(SQLITE_SRC): $(EMSDK) git clone --branch version-$(SQLITE_VERSION) --depth 1 https://github.com/sqlite/sqlite.git $(SQLITE_SRC) - . ./$(EMSDK)/emsdk_env.sh && cd $(SQLITE_SRC) && ./configure --enable-all + cd $(EMSDK) && . ./emsdk_env.sh && cd ../sqlite && ./configure --enable-all $(TARGET): $(SQLITE_SRC) $(SRC_FILES) cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c From 7ebd8f0b6420de2c5f2ba0d0c8d2d409effc626c Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 17:52:50 +0200 Subject: [PATCH 06/39] feat: add WASM npm build step and version retrieval in Makefile --- .github/workflows/main.yml | 56 ++++++++++++++++++++++++++++++++++++-- Makefile | 6 ++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23e3b8d..fd7a14b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -144,13 +144,64 @@ jobs: with: path: coverage + - name: wasm npm build + if: matrix.name == 'wasm' + run: | + git clone https://github.com/sqlite/sqlite-wasm.git sqlite-wasm + rm -rf sqlite-wasm/sqlite-wasm/* + unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp + mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm + rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo sqlite-wasm/package-lock.json + + PKG=sqlite-wasm/package.json + TMP=sqlite-wasm/package.tmp.json + + # Get versions from make commands + SQLITE_VERSION=$(make sqlite_version) + CLOUDSYNC_VERSION=$(make version) + + DESC="SQLite Wasm compiled with the automatically initialized cloudsync extension. Conveniently packaged as an ES Module for effortless integration." + + jq \ + --arg name "cloudsync/sqlite-wasm" \ + --arg version "${SQLITE_VERSION}+cloudsync-${CLOUDSYNC_VERSION}" \ + --arg desc "$DESC" \ + --argjson keywords '["offsync","cloudsync","sqliteai"]' \ + --arg repo_url "git+https://github.com/sqliteai/sqlite-sync.git" \ + --arg author "Gioele Cantoni (gioele@sqlitecloud.io)" \ + --arg bugs_url "https://github.com/sqliteai/sqlite-sync/issues" \ + --arg homepage "https://github.com/sqliteai/sqlite-sync#readme" \ + ' + .name = $name + | .version = $version + | .description = $desc + | .keywords += $keywords + | del(.bin) + | .scripts |= with_entries(select( + .key != "build" + and .key != "start" + and .key != "start:node" + and .key != "prepublishOnly" + and .key != "deploy" + )) + | .repository.url = $repo_url + | .author = $author + | .bugs.url = $bugs_url + | .homepage = $homepage + | del(.devDependencies.decompress) + | del(.devDependencies["http-server"]) + | del(.devDependencies.shx) + ' "$PKG" > "$TMP" && mv "$TMP" "$PKG" + + cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && npm pack + - uses: actions/upload-artifact@v4.6.2 if: always() with: name: cloudsync-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} path: | dist/cloudsync.* - dist/sqlite-wasm.zip + sqlite-wasm/*.tgz if-no-files-found: error release: @@ -179,8 +230,7 @@ jobs: - name: release tag version from cloudsync.h id: tag run: | - FILE="src/cloudsync.h" - VERSION=$(grep -oP '#define CLOUDSYNC_VERSION\s+"\K[^"]+' "$FILE") + VERSION=$(make version) if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name') if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then diff --git a/Makefile b/Makefile index 9ec0bb7..fc1eedd 100644 --- a/Makefile +++ b/Makefile @@ -289,6 +289,12 @@ endif mv $(CURL_SRC)/lib/.libs/libcurl.a $(CURL_DIR)/$(PLATFORM) rm -rf $(CURL_DIR)/src +# Tools +sqlite_version: + @echo $(SQLITE_VERSION) +version: + @echo $(shell sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' src/cloudsync.h) + # Clean up generated files clean: rm -rf $(BUILD_DIRS) $(DIST_DIR)/* $(COV_DIR) *.gcda *.gcno *.gcov $(CURL_DIR)/src *.sqlite $(BUILD_WASM) From 6582d6b724752e43c7746799b2684e4f6e52ac10 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 18:00:12 +0200 Subject: [PATCH 07/39] fix: streamline version retrieval in WASM build process and skip npm fix --- .github/workflows/main.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd7a14b..b9570c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -156,15 +156,11 @@ jobs: PKG=sqlite-wasm/package.json TMP=sqlite-wasm/package.tmp.json - # Get versions from make commands - SQLITE_VERSION=$(make sqlite_version) - CLOUDSYNC_VERSION=$(make version) - DESC="SQLite Wasm compiled with the automatically initialized cloudsync extension. Conveniently packaged as an ES Module for effortless integration." jq \ --arg name "cloudsync/sqlite-wasm" \ - --arg version "${SQLITE_VERSION}+cloudsync-${CLOUDSYNC_VERSION}" \ + --arg version "$(make sqlite_version)+cloudsync-$(make version)" \ --arg desc "$DESC" \ --argjson keywords '["offsync","cloudsync","sqliteai"]' \ --arg repo_url "git+https://github.com/sqliteai/sqlite-sync.git" \ @@ -193,7 +189,7 @@ jobs: | del(.devDependencies.shx) ' "$PKG" > "$TMP" && mv "$TMP" "$PKG" - cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && npm pack + cd sqlite-wasm && npm i && npm run publint && npm run check-types && npm pack - uses: actions/upload-artifact@v4.6.2 if: always() From 58c040d445acaaf8951f2484966250d607ae4287 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 18:19:43 +0200 Subject: [PATCH 08/39] fix: rename wasm npm build step to wasm npm pack and update package name format --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9570c4..861376e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -144,7 +144,7 @@ jobs: with: path: coverage - - name: wasm npm build + - name: wasm npm pack if: matrix.name == 'wasm' run: | git clone https://github.com/sqlite/sqlite-wasm.git sqlite-wasm @@ -159,7 +159,7 @@ jobs: DESC="SQLite Wasm compiled with the automatically initialized cloudsync extension. Conveniently packaged as an ES Module for effortless integration." jq \ - --arg name "cloudsync/sqlite-wasm" \ + --arg name "@cloudsync/sqlite-wasm" \ --arg version "$(make sqlite_version)+cloudsync-$(make version)" \ --arg desc "$DESC" \ --argjson keywords '["offsync","cloudsync","sqliteai"]' \ From 8d4fa1397375cd78f375fb925f6903453cf8f48a Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 19:39:07 +0200 Subject: [PATCH 09/39] fix: update README.md for sqlite-wasm with new package name and version info --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 861376e..d561d89 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -189,6 +189,9 @@ jobs: | del(.devDependencies.shx) ' "$PKG" > "$TMP" && mv "$TMP" "$PKG" + sed 's/@sqlite\.org\/sqlite-wasm/@cloudsync\/sqlite-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp + echo -e "# sqlite-sync WASM $(make version)\nThis README and the TypeScript types are from the [official SQLite wasm repository](https://github.com/sqlite/sqlite-wasm)\n\n$(cat sqlite-wasm/README.tmp)" > sqlite-wasm/README.md + rm sqlite-wasm/README.tmp cd sqlite-wasm && npm i && npm run publint && npm run check-types && npm pack - uses: actions/upload-artifact@v4.6.2 From e0c886bf648048ac503c26cdba473609b5abfb73 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 19:43:26 +0200 Subject: [PATCH 10/39] fix: update npm build step in workflow to include 'npm run fix' for sqlite-wasm --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d561d89..cb918e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -192,7 +192,7 @@ jobs: sed 's/@sqlite\.org\/sqlite-wasm/@cloudsync\/sqlite-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp echo -e "# sqlite-sync WASM $(make version)\nThis README and the TypeScript types are from the [official SQLite wasm repository](https://github.com/sqlite/sqlite-wasm)\n\n$(cat sqlite-wasm/README.tmp)" > sqlite-wasm/README.md rm sqlite-wasm/README.tmp - cd sqlite-wasm && npm i && npm run publint && npm run check-types && npm pack + cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && npm pack - uses: actions/upload-artifact@v4.6.2 if: always() From 29d29d2ae5e6fbebdf1344af857d7d41f66be1f8 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 20:19:43 +0200 Subject: [PATCH 11/39] fix: update sqlite version for WASM static build to 3.50.0 and clone the right sqlite-wasm tag --- .github/workflows/main.yml | 16 ++++++++++++++-- Makefile | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb918e1..6667fce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,7 +147,11 @@ jobs: - name: wasm npm pack if: matrix.name == 'wasm' run: | - git clone https://github.com/sqlite/sqlite-wasm.git sqlite-wasm + TAG=$(git ls-remote --tags https://github.com/sqlite/sqlite-wasm.git | \ + awk -v ver=$(make sqlite_version) -F'/' '$NF ~ ver"-build[0-9]+$" {print $NF}' | \ + sort -V | \ + tail -n1) + git clone --branch "$TAG" --depth 1 https://github.com/sqlite/sqlite-wasm.git sqlite-wasm rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm @@ -207,7 +211,7 @@ jobs: runs-on: ubuntu-latest name: release needs: build - if: github.ref == 'refs/heads/main' + #if: github.ref == 'refs/heads/main' env: GH_TOKEN: ${{ github.token }} @@ -221,12 +225,15 @@ jobs: path: artifacts - name: setup GitHub Pages + if: false uses: actions/configure-pages@v5 - name: deploy coverage to GitHub Pages + if: false uses: actions/deploy-pages@v4.0.5 - name: release tag version from cloudsync.h + if: false id: tag run: | VERSION=$(make version) @@ -241,6 +248,11 @@ jobs: fi echo "❌ CLOUDSYNC_VERSION not found in cloudsync.h" exit 1 + + - name: publish sqlite-wasm to npm + run: ls -lah artifacts #npm publish ./artifacts/your-package-1.0.0.tgz + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - name: zip artifacts run: | diff --git a/Makefile b/Makefile index fc1eedd..54c48a2 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SQLITE3 ?= sqlite3 CURL_VERSION ?= 8.12.1 # set sqlite version for WASM static build -SQLITE_VERSION ?= 3.50.1 +SQLITE_VERSION ?= 3.50.0 # Set default platform if not specified ifeq ($(OS),Windows_NT) From ef0d5ea235258212ff2b245207c8fb4ff65cbd89 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 20:36:04 +0200 Subject: [PATCH 12/39] fix: clean up sqlite-wasm build process and fix npm fix command --- .github/workflows/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6667fce..339ea20 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -155,7 +155,9 @@ jobs: rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm - rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo sqlite-wasm/package-lock.json + rm sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo + + cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && cd .. PKG=sqlite-wasm/package.json TMP=sqlite-wasm/package.tmp.json @@ -195,8 +197,8 @@ jobs: sed 's/@sqlite\.org\/sqlite-wasm/@cloudsync\/sqlite-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp echo -e "# sqlite-sync WASM $(make version)\nThis README and the TypeScript types are from the [official SQLite wasm repository](https://github.com/sqlite/sqlite-wasm)\n\n$(cat sqlite-wasm/README.tmp)" > sqlite-wasm/README.md - rm sqlite-wasm/README.tmp - cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && npm pack + rm sqlite-wasm/README.tmp sqlite-wasm/package-lock.json + cd sqlite-wasm && npm pack - uses: actions/upload-artifact@v4.6.2 if: always() From 1ca0b03740db2f26c705374a9db9733db4dd2ec0 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 20:42:33 +0200 Subject: [PATCH 13/39] fix: improve cleanup process for sqlite-wasm --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 339ea20..47814b4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -155,7 +155,7 @@ jobs: rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm - rm sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo + rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && cd .. @@ -252,7 +252,8 @@ jobs: exit 1 - name: publish sqlite-wasm to npm - run: ls -lah artifacts #npm publish ./artifacts/your-package-1.0.0.tgz + #if: steps.tag.outputs.version != '' + run: cd artifacts/cloudsync-wasm && ls -lah #npm publish ./artifacts/your-package-1.0.0.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From eb8e0f69814b6b16148adb58b686765db767160d Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 21:01:55 +0200 Subject: [PATCH 14/39] test with latest main sqlite-wasm --- .github/workflows/main.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47814b4..a60925a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,11 +147,7 @@ jobs: - name: wasm npm pack if: matrix.name == 'wasm' run: | - TAG=$(git ls-remote --tags https://github.com/sqlite/sqlite-wasm.git | \ - awk -v ver=$(make sqlite_version) -F'/' '$NF ~ ver"-build[0-9]+$" {print $NF}' | \ - sort -V | \ - tail -n1) - git clone --branch "$TAG" --depth 1 https://github.com/sqlite/sqlite-wasm.git sqlite-wasm + git clone https://github.com/sqlite/sqlite-wasm.git sqlite-wasm rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm From b8c082afe26f9b13ab733a5c84a834d3edd174e5 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 21:07:39 +0200 Subject: [PATCH 15/39] lowering version should fix the export default issue --- .github/workflows/main.yml | 6 +++++- Makefile | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a60925a..47814b4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,7 +147,11 @@ jobs: - name: wasm npm pack if: matrix.name == 'wasm' run: | - git clone https://github.com/sqlite/sqlite-wasm.git sqlite-wasm + TAG=$(git ls-remote --tags https://github.com/sqlite/sqlite-wasm.git | \ + awk -v ver=$(make sqlite_version) -F'/' '$NF ~ ver"-build[0-9]+$" {print $NF}' | \ + sort -V | \ + tail -n1) + git clone --branch "$TAG" --depth 1 https://github.com/sqlite/sqlite-wasm.git sqlite-wasm rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm diff --git a/Makefile b/Makefile index 54c48a2..a24ce0f 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SQLITE3 ?= sqlite3 CURL_VERSION ?= 8.12.1 # set sqlite version for WASM static build -SQLITE_VERSION ?= 3.50.0 +SQLITE_VERSION ?= 3.49.2 # Set default platform if not specified ifeq ($(OS),Windows_NT) From 9f9a7b9559ea5bc490d8e5979c153f379e705592 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 21:20:15 +0200 Subject: [PATCH 16/39] fix: update package name references for sqlite-wasm to @sqliteai/cloudsync-wasm --- .github/workflows/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47814b4..f82fa75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -155,7 +155,6 @@ jobs: rm -rf sqlite-wasm/sqlite-wasm/* unzip dist/sqlite-wasm.zip -d sqlite-wasm/tmp mv sqlite-wasm/tmp/sqlite-wasm-*/jswasm sqlite-wasm/sqlite-wasm - rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo cd sqlite-wasm && npm i && npm run fix && npm run publint && npm run check-types && cd .. @@ -165,7 +164,7 @@ jobs: DESC="SQLite Wasm compiled with the automatically initialized cloudsync extension. Conveniently packaged as an ES Module for effortless integration." jq \ - --arg name "@cloudsync/sqlite-wasm" \ + --arg name "@sqliteai/cloudsync-wasm" \ --arg version "$(make sqlite_version)+cloudsync-$(make version)" \ --arg desc "$DESC" \ --argjson keywords '["offsync","cloudsync","sqliteai"]' \ @@ -195,9 +194,9 @@ jobs: | del(.devDependencies.shx) ' "$PKG" > "$TMP" && mv "$TMP" "$PKG" - sed 's/@sqlite\.org\/sqlite-wasm/@cloudsync\/sqlite-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp + sed 's/@sqlite\.org\/sqlite-wasm/@sqliteai\/cloudsync-wasm/g' sqlite-wasm/README.md > sqlite-wasm/README.tmp echo -e "# sqlite-sync WASM $(make version)\nThis README and the TypeScript types are from the [official SQLite wasm repository](https://github.com/sqlite/sqlite-wasm)\n\n$(cat sqlite-wasm/README.tmp)" > sqlite-wasm/README.md - rm sqlite-wasm/README.tmp sqlite-wasm/package-lock.json + rm -rf sqlite-wasm/tmp sqlite-wasm/bin sqlite-wasm/demo sqlite-wasm/README.tmp sqlite-wasm/package-lock.json cd sqlite-wasm && npm pack - uses: actions/upload-artifact@v4.6.2 From a87b266e977b5321688ac4b61197f6c5688e597d Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 25 Jun 2025 21:35:47 +0200 Subject: [PATCH 17/39] fix: update path for npm publish command in workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f82fa75..09f9e4d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -252,7 +252,7 @@ jobs: - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - run: cd artifacts/cloudsync-wasm && ls -lah #npm publish ./artifacts/your-package-1.0.0.tgz + run: cd artifacts/cloudsync-wasm/sqlite-wasm && ls -lah #npm publish ./artifacts/your-package-1.0.0.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 856e18048abd9ca8763b51b4ba4f666aa13e9865 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 26 Jun 2025 07:36:53 +0200 Subject: [PATCH 18/39] draft feat: wasm network layer --- .github/workflows/main.yml | 6 +- Makefile | 2 +- src/network.c | 208 ++++++++++++++++++++++++++++++++++++- src/wasm.c | 1 - 4 files changed, 207 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09f9e4d..3c913f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -234,7 +234,6 @@ jobs: uses: actions/deploy-pages@v4.0.5 - name: release tag version from cloudsync.h - if: false id: tag run: | VERSION=$(make version) @@ -252,7 +251,7 @@ jobs: - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - run: cd artifacts/cloudsync-wasm/sqlite-wasm && ls -lah #npm publish ./artifacts/your-package-1.0.0.tgz + run: #npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} @@ -270,7 +269,8 @@ jobs: done - uses: softprops/action-gh-release@v2.2.1 - if: steps.tag.outputs.version != '' + if: false + #if: steps.tag.outputs.version != '' with: generate_release_notes: true tag_name: ${{ steps.tag.outputs.version }} diff --git a/Makefile b/Makefile index a24ce0f..b1c7ee0 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ $(SQLITE_SRC): $(EMSDK) cd $(EMSDK) && . ./emsdk_env.sh && cd ../sqlite && ./configure --enable-all $(TARGET): $(SQLITE_SRC) $(SRC_FILES) - cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c + cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c emcc.jsflags+="-sFETCH -pthread" mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif diff --git a/src/network.c b/src/network.c index 04cb97a..d397064 100644 --- a/src/network.c +++ b/src/network.c @@ -14,9 +14,15 @@ #include "cloudsync_private.h" #include "netword_private.h" +#ifndef SQLITE_WASM_EXTRA_INIT #ifndef CLOUDSYNC_OMIT_CURL #include "curl/curl.h" #endif +#else +#include +#include +#include +#endif #ifdef __ANDROID__ #include "cacert.h" @@ -127,6 +133,98 @@ static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, vo return (size * nmemb); } +#ifdef SQLITE_WASM_EXTRA_INIT +NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { + + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + + // Set method + if (json_payload || is_post_request) { + strcpy(attr.requestMethod, "POST"); + } else { + strcpy(attr.requestMethod, "GET"); + } + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + + // Prepare header array (alternating key, value, NULL-terminated) + const char *headers[11]; + int h = 0; + + // Custom header (must be "Key: Value", split at ':') + char *custom_key = NULL; + if (custom_header) { + const char *colon = strchr(custom_header, ':'); + if (colon) { + size_t klen = colon - custom_header; + custom_key = (char *)malloc(klen + 1); + strncpy(custom_key, custom_header, klen); + custom_key[klen] = 0; + const char *custom_val = colon + 1; + while (*custom_val == ' ') custom_val++; + headers[h++] = custom_key; + headers[h++] = custom_val; + } + } + + // Authorization + char auth_header[256]; + if (authentication) { + snprintf(auth_header, sizeof(auth_header), "Bearer %s", authentication); + headers[h++] = "Authorization"; + headers[h++] = auth_header; + } + + // Content-Type for JSON + if (json_payload) { + headers[h++] = "Content-Type"; + headers[h++] = "application/json"; + } + + // Accept (always) + headers[h++] = "Accept"; + headers[h++] = "application/octet-stream"; + headers[h] = 0; + attr.requestHeaders = headers; + + // Body + if (json_payload) { + attr.requestData = json_payload; + attr.requestDataSize = strlen(json_payload); + } else if (is_post_request) { + attr.requestData = ""; + attr.requestDataSize = 0; + } + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, endpoint); // Blocks here until the operation is complete. + NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; + if (fetch->status == 200) { + result.code = (fetch->numBytes > 0) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; + result.buffer = (char *)malloc(fetch->numBytes + 1); + if (result.buffer && fetch->numBytes > 0) { + memcpy(result.buffer, fetch->data, fetch->numBytes); + result.buffer[fetch->numBytes] = 0; + result.blen = fetch->numBytes; + } else if (result.buffer) { + result.buffer[0] = 0; + result.blen = 0; + } + } else { + result.code = CLOUDSYNC_NETWORK_ERROR; + if (fetch->statusText && fetch->statusText[0]) + result.buffer = strdup(fetch->statusText); + else + result.buffer = strdup("Network error"); + result.blen = 0; + } + + // cleanup + emscripten_fetch_close(fetch); + if (custom_key) free(custom_key); + + return result; +} +#else NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { char *buffer = NULL; size_t blen = 0; @@ -205,6 +303,7 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, return result; } +#endif static size_t network_read_callback(char *buffer, size_t size, size_t nitems, void *userdata) { network_read_data *rd = (network_read_data *)userdata; @@ -220,7 +319,45 @@ static size_t network_read_callback(char *buffer, size_t size, size_t nitems, vo return to_copy; } -bool network_send_buffer (network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { +#ifdef SQLITE_WASM_EXTRA_INIT +bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { + + bool result = false; + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "PUT"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + + // Prepare headers (alternating key, value, NULL-terminated) + // Max 3 headers: Accept, (optional Auth), Content-Type + const char *headers[7]; + int h = 0; + headers[h++] = "Accept"; + headers[h++] = "text/plain"; + char auth_header[256]; + if (authentication) { + snprintf(auth_header, sizeof(auth_header), "Bearer %s", authentication); + headers[h++] = "Authorization"; + headers[h++] = auth_header; + } + headers[h++] = "Content-Type"; + headers[h++] = "application/octet-stream"; + headers[h] = 0; + attr.requestHeaders = headers; + + // Set request body + attr.requestData = (const char *)blob; + attr.requestDataSize = blob_size; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, endpoint); // Blocks here until the operation is complete. + if (fetch->status == 200) result = true; + + emscripten_fetch_close(fetch); + + return result; +} +#else +bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { struct curl_slist *headers = NULL; curl_mime *mime = NULL; bool result = false; @@ -292,6 +429,7 @@ bool network_send_buffer (network_data *data, const char *endpoint, const char * return result; } #endif +#endif int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) { int rc = 0; @@ -360,6 +498,9 @@ int network_extract_query_param(const char *query, const char *key, char *output size_t key_len = strlen(key); const char *p = query; + #ifdef SQLITE_WASM_EXTRA_INIT + if (*p == '?') p++; + #endif while (p && *p) { // Find the start of a key=value pair @@ -394,6 +535,19 @@ int network_extract_query_param(const char *query, const char *key, char *output } #ifndef CLOUDSYNC_OMIT_CURL + +#ifdef SQLITE_WASM_EXTRA_INIT +static char *substr(const char *start, const char *end) { + size_t len = end - start; + char *out = (char *)malloc(len + 1); + if (out) { + memcpy(out, start, len); + out[len] = 0; + } + return out; +} +#endif + bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string) { // compute endpoints bool result = false; @@ -410,12 +564,15 @@ bool network_compute_endpoints (sqlite3_context *context, network_data *data, co char *conn_string_https = NULL; + #ifndef SQLITE_WASM_EXTRA_INIT CURLUcode rc = CURLUE_OUT_OF_MEMORY; CURLU *url = curl_url(); if (!url) goto finalize; + #endif conn_string_https = cloudsync_string_replace_prefix(conn_string, "sqlitecloud://", "https://"); + #ifndef SQLITE_WASM_EXTRA_INIT // set URL: https://UUID.g5.sqlite.cloud:443/chinook.sqlite?apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo rc = curl_url_set(url, CURLUPART_URL, conn_string_https, 0); if (rc != CURLE_OK) goto finalize; @@ -440,6 +597,38 @@ bool network_compute_endpoints (sqlite3_context *context, network_data *data, co // apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo (OPTIONAL) rc = curl_url_get(url, CURLUPART_QUERY, &query, 0); if (rc != CURLE_OK && rc != CURLUE_NO_QUERY) goto finalize; + #else + // Parse: scheme://host[:port]/path?query + const char *p = strstr(conn_string_https, "://"); + if (!p) goto finalize; + scheme = substr(conn_string_https, p); + p += 3; + const char *host_start = p; + const char *host_end = strpbrk(host_start, ":/?"); + if (!host_end) goto finalize; + host = substr(host_start, host_end); + p = host_end; + if (*p == ':') { + ++p; + const char *port_end = strpbrk(p, "/?"); + if (!port_end) goto finalize; + port = substr(p, port_end); + p = port_end; + } + if (*p == '/') { + const char *path_start = p; + const char *path_end = strchr(path_start, '?'); + if (!path_end) path_end = path_start + strlen(path_start); + database = substr(path_start, path_end); + p = path_end; + } + if (*p == '?') { + query = strdup(p); + } + if (!scheme || !host || !database) goto finalize; + if (!port) port = strdup(CLOUDSYNC_DEFAULT_ENDPOINT_PORT); + #define port_or_default port + #endif if (query != NULL) { char value[MAX_QUERY_VALUE_LEN]; if (!authentication && network_extract_query_param(query, "apikey", value, sizeof(value)) == 0) { @@ -463,8 +652,13 @@ bool network_compute_endpoints (sqlite3_context *context, network_data *data, co finalize: if (result == false) { // store proper result code/message + #ifndef SQLITE_WASM_EXTRA_INIT if (rc != CURLE_OK) sqlite3_result_error(context, curl_url_strerror(rc), -1); sqlite3_result_error_code(context, (rc != CURLE_OK) ? SQLITE_ERROR : SQLITE_NOMEM); + #else + sqlite3_result_error(context, "URL parse error", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + #endif // cleanup memory managed by the extension if (authentication) cloudsync_memory_free(authentication); @@ -485,13 +679,17 @@ bool network_compute_endpoints (sqlite3_context *context, network_data *data, co data->upload_endpoint = upload_endpoint; } - // cleanup memory managed by libcurl + // cleanup memory + #ifndef SQLITE_WASM_EXTRA_INIT + if (url) curl_url_cleanup(url); + #else + #define curl_free(x) free(x) + #endif if (scheme) curl_free(scheme); if (host) curl_free(host); if (port) curl_free(port); if (database) curl_free(database); if (query) curl_free(query); - if (url) curl_url_cleanup(url); if (conn_string_https && conn_string_https != conn_string) cloudsync_memory_free(conn_string_https); return result; @@ -518,7 +716,7 @@ network_data *cloudsync_network_data(sqlite3_context *context) { void cloudsync_network_init (sqlite3_context *context, int argc, sqlite3_value **argv) { DEBUG_FUNCTION("cloudsync_network_init"); - #ifndef CLOUDSYNC_OMIT_CURL + #if !defined(CLOUDSYNC_OMIT_CURL) && !defined(SQLITE_WASM_EXTRA_INIT) curl_global_init(CURL_GLOBAL_ALL); #endif @@ -583,7 +781,7 @@ void cloudsync_network_cleanup (sqlite3_context *context, int argc, sqlite3_valu sqlite3_result_int(context, SQLITE_OK); - #ifndef CLOUDSYNC_OMIT_CURL + #if !defined(CLOUDSYNC_OMIT_CURL) && !defined(SQLITE_WASM_EXTRA_INIT) curl_global_cleanup(); #endif } diff --git a/src/wasm.c b/src/wasm.c index 5f5d747..b22fc04 100644 --- a/src/wasm.c +++ b/src/wasm.c @@ -3,7 +3,6 @@ #include "sqlite3.h" #include -#define CLOUDSYNC_OMIT_NETWORK #include "utils.c" #include "network.c" #include "dbutils.c" From 5a65e2bdc45008256282cf1ba329a50a54f403a1 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 26 Jun 2025 07:37:56 +0200 Subject: [PATCH 19/39] fix workflow --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3c913f0..e503227 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -251,7 +251,8 @@ jobs: - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - run: #npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + if: false + run: npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 56c44699c0e8b77e2a4758b9114e6440b5e39349 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 30 Jun 2025 15:16:55 +0200 Subject: [PATCH 20/39] fix: update WASM build process and add flags for fetch support --- Makefile | 7 ++++++- src/wasm.c | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b1c7ee0..71b3154 100644 --- a/Makefile +++ b/Makefile @@ -165,8 +165,13 @@ $(SQLITE_SRC): $(EMSDK) git clone --branch version-$(SQLITE_VERSION) --depth 1 https://github.com/sqlite/sqlite.git $(SQLITE_SRC) cd $(EMSDK) && . ./emsdk_env.sh && cd ../sqlite && ./configure --enable-all +WASM_FLAGS = emcc.jsflags += -sFETCH -pthread +WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUMakefile $(TARGET): $(SQLITE_SRC) $(SRC_FILES) - cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c emcc.jsflags+="-sFETCH -pthread" + if ! grep -Fxq '$(WASM_FLAGS)' '$(WASM_MAKEFILE)'; then \ + echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)'; \ + fi + cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif diff --git a/src/wasm.c b/src/wasm.c index b22fc04..601760a 100644 --- a/src/wasm.c +++ b/src/wasm.c @@ -10,7 +10,7 @@ #include "vtab.c" #include "pk.c" #include "lz4.c" -//ciao + int sqlite3_wasm_extra_init(const char *z) { fprintf(stderr, "%s: %s()\n", __FILE__, __func__); return sqlite3_auto_extension((void *) sqlite3_cloudsync_init); From 2e816cc3fe39b55eed510b77587ab532ca98624b Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 30 Jun 2025 15:33:23 +0200 Subject: [PATCH 21/39] fix: improve WASM_FLAGS handling in Makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 71b3154..acaf868 100644 --- a/Makefile +++ b/Makefile @@ -168,9 +168,7 @@ $(SQLITE_SRC): $(EMSDK) WASM_FLAGS = emcc.jsflags += -sFETCH -pthread WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUMakefile $(TARGET): $(SQLITE_SRC) $(SRC_FILES) - if ! grep -Fxq '$(WASM_FLAGS)' '$(WASM_MAKEFILE)'; then \ - echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)'; \ - fi + @grep '$(WASM_FLAGS)' '$(WASM_MAKEFILE)' >/dev/null 2>&1 || echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)' cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif From 4838dad84c165a350d00c77ae98138de1adde752 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 30 Jun 2025 18:13:05 +0200 Subject: [PATCH 22/39] fix: improve WASM_FLAGS handling in Makefile using awk for uniqueness check --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index acaf868..52b12bf 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ $(SQLITE_SRC): $(EMSDK) WASM_FLAGS = emcc.jsflags += -sFETCH -pthread WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUMakefile $(TARGET): $(SQLITE_SRC) $(SRC_FILES) - @grep '$(WASM_FLAGS)' '$(WASM_MAKEFILE)' >/dev/null 2>&1 || echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)' + awk 'BEGIN{f=0} $$0=="$(WASM_FLAGS)"{f=1} END{exit f}' "$(WASM_MAKEFILE)" || printf '%s\n' "$(WASM_FLAGS)" >> "$(WASM_MAKEFILE)" cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif From 97357f2b51469605080fd6a496afb10e1a0cf9e3 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Mon, 30 Jun 2025 18:19:41 +0200 Subject: [PATCH 23/39] fix: correct WASM makefile name --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 52b12bf..5288a81 100644 --- a/Makefile +++ b/Makefile @@ -166,9 +166,9 @@ $(SQLITE_SRC): $(EMSDK) cd $(EMSDK) && . ./emsdk_env.sh && cd ../sqlite && ./configure --enable-all WASM_FLAGS = emcc.jsflags += -sFETCH -pthread -WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUMakefile +WASM_MAKEFILE = $(SQLITE_SRC)/ext/wasm/GNUmakefile $(TARGET): $(SQLITE_SRC) $(SRC_FILES) - awk 'BEGIN{f=0} $$0=="$(WASM_FLAGS)"{f=1} END{exit f}' "$(WASM_MAKEFILE)" || printf '%s\n' "$(WASM_FLAGS)" >> "$(WASM_MAKEFILE)" + @grep '$(WASM_FLAGS)' '$(WASM_MAKEFILE)' >/dev/null 2>&1 || echo '$(WASM_FLAGS)' >> '$(WASM_MAKEFILE)' cd $(SQLITE_SRC)/ext/wasm && $(MAKE) dist sqlite3_wasm_extra_init.c=../../../../../src/wasm.c mv $(SQLITE_SRC)/ext/wasm/sqlite-wasm-*.zip $(TARGET) endif From c2e11cd9c17d6e2d18c9934fa356ce49c07d8399 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 2 Jul 2025 00:20:51 +0200 Subject: [PATCH 24/39] fix: conditional SQLITE_WASM_EXTRA_INIT escape single quotes instead of double quotes in query strings for wasm support --- src/cloudsync.c | 4 ++++ src/dbutils.c | 4 ++++ src/vtab.c | 15 +++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/src/cloudsync.c b/src/cloudsync.c index 6b1a5d4..ffffd16 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -360,7 +360,11 @@ char *db_version_build_query (sqlite3 *db) { // the good news is that the query can be computed in SQLite without the need to do any extra computation from the host language const char *sql = "WITH table_names AS (" + #ifdef SQLITE_WASM_EXTRA_INIT + "SELECT format('%w', name) as tbl_name " + #else "SELECT format(\"%w\", name) as tbl_name " + #endif "FROM sqlite_master " "WHERE type='table' " "AND name LIKE '%_cloudsync'" diff --git a/src/dbutils.c b/src/dbutils.c index 48e805c..3ab39f2 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -419,7 +419,11 @@ bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const ch // the affinity of a column is determined by the declared type of the column, // according to the following rules in the order shown: // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. + #ifdef SQLITE_WASM_EXTRA_INIT + sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", name); + #else sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE \"%%INT%%\";", name); + #endif sqlite3_int64 count2 = dbutils_int_select(db, sql); if (count == count2) { dbutils_context_result_error(context, "Table %s uses an single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); diff --git a/src/vtab.c b/src/vtab.c index 21ca6dd..33e76d0 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -137,6 +137,20 @@ char *build_changes_sql (sqlite3 *db, const char *idxs) { "changes_query AS ( " " SELECT " " 'SELECT " + #ifdef SQLITE_WASM_EXTRA_INIT + " ''' || table_name || ''' AS tbl, " + " t1.pk AS pk, " + " t1.col_name AS col_name, " + " cloudsync_col_value(''' || table_name || ''', t1.col_name, t1.pk) AS col_value, " + " t1.col_version AS col_version, " + " t1.db_version AS db_version, " + " site_tbl.site_id AS site_id, " + " t1.seq AS seq, " + " COALESCE(t2.col_version, 1) AS cl " + " FROM ''' || table_meta || ''' AS t1 " + " LEFT JOIN cloudsync_site_id AS site_tbl ON t1.site_id = site_tbl.rowid " + " LEFT JOIN ''' || table_meta || ''' AS t2 ON t1.pk = t2.pk AND t2.col_name = ''" CLOUDSYNC_TOMBSTONE_VALUE "'' " + #else " \"' || \"table_name\" || '\" AS tbl, " " t1.pk AS pk, " " t1.col_name AS col_name, " @@ -149,6 +163,7 @@ char *build_changes_sql (sqlite3 *db, const char *idxs) { " FROM \"' || \"table_meta\" || '\" AS t1 " " LEFT JOIN cloudsync_site_id AS site_tbl ON t1.site_id = site_tbl.rowid " " LEFT JOIN \"' || \"table_meta\" || '\" AS t2 ON t1.pk = t2.pk AND t2.col_name = ''" CLOUDSYNC_TOMBSTONE_VALUE "'' " + #endif " WHERE col_value IS NOT ''" CLOUDSYNC_RLS_RESTRICTED_VALUE "''' " " AS query_string FROM table_names " "), " From 6568290967392ab4c2f1e1d45d38e46131fdde39 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 2 Jul 2025 10:55:42 +0200 Subject: [PATCH 25/39] fix: endpoint port handling in network_compute_endpoints function --- src/network.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/network.c b/src/network.c index d397064..47deedf 100644 --- a/src/network.c +++ b/src/network.c @@ -626,8 +626,7 @@ bool network_compute_endpoints (sqlite3_context *context, network_data *data, co query = strdup(p); } if (!scheme || !host || !database) goto finalize; - if (!port) port = strdup(CLOUDSYNC_DEFAULT_ENDPOINT_PORT); - #define port_or_default port + char *port_or_default = port && strcmp(port, "8860") != 0 ? port : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; #endif if (query != NULL) { char value[MAX_QUERY_VALUE_LEN]; From 356fa9bbeb51333a016f635f5d51f5fa30269be9 Mon Sep 17 00:00:00 2001 From: Daniele Briggi <=> Date: Wed, 2 Jul 2025 14:37:07 +0200 Subject: [PATCH 26/39] fix(fetch): sync fetch and single quotes --- src/cloudsync.c | 5 +---- src/dbutils.c | 4 ---- src/network.c | 28 ++++++++++++++++++++++++---- src/vtab.c | 23 ++++------------------- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/cloudsync.c b/src/cloudsync.c index ffffd16..b714997 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -360,11 +360,7 @@ char *db_version_build_query (sqlite3 *db) { // the good news is that the query can be computed in SQLite without the need to do any extra computation from the host language const char *sql = "WITH table_names AS (" - #ifdef SQLITE_WASM_EXTRA_INIT "SELECT format('%w', name) as tbl_name " - #else - "SELECT format(\"%w\", name) as tbl_name " - #endif "FROM sqlite_master " "WHERE type='table' " "AND name LIKE '%_cloudsync'" @@ -3284,6 +3280,7 @@ APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite // register eponymous only changes virtual table rc = cloudsync_vtab_register_changes (db, data); + fprintf(stderr, "clousync.c init - rc: %d\n", rc); if (rc != SQLITE_OK) return rc; // load config, if exists diff --git a/src/dbutils.c b/src/dbutils.c index 3ab39f2..54993fd 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -419,11 +419,7 @@ bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const ch // the affinity of a column is determined by the declared type of the column, // according to the following rules in the order shown: // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. - #ifdef SQLITE_WASM_EXTRA_INIT sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", name); - #else - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%w') WHERE pk=1 AND \"type\" LIKE \"%%INT%%\";", name); - #endif sqlite3_int64 count2 = dbutils_int_select(db, sql); if (count == count2) { dbutils_context_result_error(context, "Table %s uses an single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); diff --git a/src/network.c b/src/network.c index 47deedf..5041d53 100644 --- a/src/network.c +++ b/src/network.c @@ -145,8 +145,11 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, } else { strcpy(attr.requestMethod, "GET"); } - attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; - + attr.onerror = NULL; // No progress callback + attr.onsuccess = NULL; // No success callback + attr.onprogress = NULL; // No progress callback + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_REPLACE; + // Prepare header array (alternating key, value, NULL-terminated) const char *headers[11]; int h = 0; @@ -195,10 +198,27 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, attr.requestData = ""; attr.requestDataSize = 0; } - + //endpoint = "https://echo.free.beeceptor.com"; emscripten_fetch_t *fetch = emscripten_fetch(&attr, endpoint); // Blocks here until the operation is complete. NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; + fprintf(stderr, "network_receive_buffer: %s %s\n", attr.requestMethod, endpoint); + fprintf(stderr, "emscripten_fetch returned, fetch pointer: %p\n", fetch); + fprintf(stderr, "network_receive_buffer: status %u, numBytes %zu\n", fetch->status, fetch->numBytes); + fprintf(stderr, "network_receive_buffer: statusText %s\n", fetch->statusText ? fetch->statusText : "NULL"); + fprintf(stderr, "network_receive_buffer: readyState %u\n", fetch->readyState); + fprintf(stderr, "network_receive_buffer: data pointer %p\n", fetch->data); + + if (fetch->readyState != 4) { + fprintf(stderr, "ERROR: fetch not completed, readyState=%u\n", fetch->readyState); + result.code = CLOUDSYNC_NETWORK_ERROR; + result.buffer = strdup("Network request did not complete"); + emscripten_fetch_close(fetch); + if (custom_key) free(custom_key); + return result; + } + if (fetch->status == 200) { + fprintf(stderr, "status is 200 OK\n"); result.code = (fetch->numBytes > 0) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; result.buffer = (char *)malloc(fetch->numBytes + 1); if (result.buffer && fetch->numBytes > 0) { @@ -822,7 +842,7 @@ void cloudsync_network_set_apikey (sqlite3_context *context, int argc, sqlite3_v void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { sqlite3 *db = sqlite3_context_db_handle(context); - char *sql = "SELECT max(db_version), hex(site_id) FROM cloudsync_changes() WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; + char *sql = "SELECT max(db_version), hex(site_id) FROM cloudsync_changes WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; int last_local_change = (int)dbutils_int_select(db, sql); if (last_local_change == 0) { sqlite3_result_int(context, 0); diff --git a/src/vtab.c b/src/vtab.c index 33e76d0..ca29429 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -137,33 +137,18 @@ char *build_changes_sql (sqlite3 *db, const char *idxs) { "changes_query AS ( " " SELECT " " 'SELECT " - #ifdef SQLITE_WASM_EXTRA_INIT - " ''' || table_name || ''' AS tbl, " + " ''' || \"table_name\" || ''' AS tbl, " " t1.pk AS pk, " " t1.col_name AS col_name, " - " cloudsync_col_value(''' || table_name || ''', t1.col_name, t1.pk) AS col_value, " + " cloudsync_col_value(''' || \"table_name\" || ''', t1.col_name, t1.pk) AS col_value, " " t1.col_version AS col_version, " " t1.db_version AS db_version, " " site_tbl.site_id AS site_id, " " t1.seq AS seq, " " COALESCE(t2.col_version, 1) AS cl " - " FROM ''' || table_meta || ''' AS t1 " + " FROM ''' || \"table_meta\" || ''' AS t1 " " LEFT JOIN cloudsync_site_id AS site_tbl ON t1.site_id = site_tbl.rowid " - " LEFT JOIN ''' || table_meta || ''' AS t2 ON t1.pk = t2.pk AND t2.col_name = ''" CLOUDSYNC_TOMBSTONE_VALUE "'' " - #else - " \"' || \"table_name\" || '\" AS tbl, " - " t1.pk AS pk, " - " t1.col_name AS col_name, " - " cloudsync_col_value(\"' || \"table_name\" || '\", t1.col_name, t1.pk) AS col_value, " - " t1.col_version AS col_version, " - " t1.db_version AS db_version, " - " site_tbl.site_id AS site_id, " - " t1.seq AS seq, " - " COALESCE(t2.col_version, 1) AS cl " - " FROM \"' || \"table_meta\" || '\" AS t1 " - " LEFT JOIN cloudsync_site_id AS site_tbl ON t1.site_id = site_tbl.rowid " - " LEFT JOIN \"' || \"table_meta\" || '\" AS t2 ON t1.pk = t2.pk AND t2.col_name = ''" CLOUDSYNC_TOMBSTONE_VALUE "'' " - #endif + " LEFT JOIN ''' || \"table_meta\" || ''' AS t2 ON t1.pk = t2.pk AND t2.col_name = ''" CLOUDSYNC_TOMBSTONE_VALUE "'' " " WHERE col_value IS NOT ''" CLOUDSYNC_RLS_RESTRICTED_VALUE "''' " " AS query_string FROM table_names " "), " From a4dc460bdf20abcd413bf494e96c33fce29f9a71 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Wed, 2 Jul 2025 16:51:49 +0200 Subject: [PATCH 27/39] fix: remove debug print statement from sqlite3_cloudsync_init function --- src/cloudsync.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cloudsync.c b/src/cloudsync.c index b714997..0f702ce 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -3280,7 +3280,6 @@ APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite // register eponymous only changes virtual table rc = cloudsync_vtab_register_changes (db, data); - fprintf(stderr, "clousync.c init - rc: %d\n", rc); if (rc != SQLITE_OK) return rc; // load config, if exists From 72b57c1cc42e2f49fdf8bb1e4a8d3016f21db6c5 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 3 Jul 2025 20:25:28 +0200 Subject: [PATCH 28/39] refactor: network_receive_buffer and improve buffer management --- src/network.c | 136 ++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 76 deletions(-) diff --git a/src/network.c b/src/network.c index 5041d53..2fa431a 100644 --- a/src/network.c +++ b/src/network.c @@ -102,39 +102,10 @@ bool network_data_set_endpoints (network_data *data, char *auth, char *check, ch // MARK: - Utils - #ifndef CLOUDSYNC_OMIT_CURL - -static bool network_buffer_check (network_buffer *data, size_t needed) { - // alloc/resize buffer - if (data->bused + needed > data->balloc) { - if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; - size_t balloc = (data->balloc * 2) + needed; - - char *buffer = cloudsync_memory_realloc(data->buffer, balloc); - if (!buffer) return false; - - data->buffer = buffer; - data->balloc = balloc; - } - - return true; -} - -static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { - network_buffer *data = (network_buffer *)xdata; - - size_t ptr_size = (size*nmemb); - if (data->zero_term) ptr_size += 1; - - if (network_buffer_check(data, ptr_size) == false) return -1; - memcpy(data->buffer+data->bused, ptr, size*nmemb); - data->bused += size*nmemb; - if (data->zero_term) data->buffer[data->bused] = 0; - - return (size * nmemb); -} - #ifdef SQLITE_WASM_EXTRA_INIT NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { + char *buffer = NULL; + size_t blen = 0; emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); @@ -145,11 +116,8 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, } else { strcpy(attr.requestMethod, "GET"); } - attr.onerror = NULL; // No progress callback - attr.onsuccess = NULL; // No success callback - attr.onprogress = NULL; // No progress callback attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_REPLACE; - + // Prepare header array (alternating key, value, NULL-terminated) const char *headers[11]; int h = 0; @@ -183,10 +151,7 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, headers[h++] = "Content-Type"; headers[h++] = "application/json"; } - - // Accept (always) - headers[h++] = "Accept"; - headers[h++] = "application/octet-stream"; + headers[h] = 0; attr.requestHeaders = headers; @@ -194,48 +159,36 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, if (json_payload) { attr.requestData = json_payload; attr.requestDataSize = strlen(json_payload); - } else if (is_post_request) { - attr.requestData = ""; - attr.requestDataSize = 0; } - //endpoint = "https://echo.free.beeceptor.com"; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, endpoint); // Blocks here until the operation is complete. NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; - fprintf(stderr, "network_receive_buffer: %s %s\n", attr.requestMethod, endpoint); - fprintf(stderr, "emscripten_fetch returned, fetch pointer: %p\n", fetch); - fprintf(stderr, "network_receive_buffer: status %u, numBytes %zu\n", fetch->status, fetch->numBytes); - fprintf(stderr, "network_receive_buffer: statusText %s\n", fetch->statusText ? fetch->statusText : "NULL"); - fprintf(stderr, "network_receive_buffer: readyState %u\n", fetch->readyState); - fprintf(stderr, "network_receive_buffer: data pointer %p\n", fetch->data); - - if (fetch->readyState != 4) { - fprintf(stderr, "ERROR: fetch not completed, readyState=%u\n", fetch->readyState); - result.code = CLOUDSYNC_NETWORK_ERROR; - result.buffer = strdup("Network request did not complete"); - emscripten_fetch_close(fetch); - if (custom_key) free(custom_key); - return result; + + if(fetch->readyState == 4){ + buffer = fetch->data; + blen = fetch->totalBytes; } - if (fetch->status == 200) { - fprintf(stderr, "status is 200 OK\n"); - result.code = (fetch->numBytes > 0) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; - result.buffer = (char *)malloc(fetch->numBytes + 1); - if (result.buffer && fetch->numBytes > 0) { - memcpy(result.buffer, fetch->data, fetch->numBytes); - result.buffer[fetch->numBytes] = 0; - result.blen = fetch->numBytes; - } else if (result.buffer) { - result.buffer[0] = 0; - result.blen = 0; - } + if (fetch->status >= 200 && fetch->status < 300) { + + if (blen > 0 && buffer) { + char *buf = (char*)malloc(blen + 1); + if (buf) { + memcpy(buf, buffer, blen); + buf[blen] = 0; + result.code = CLOUDSYNC_NETWORK_BUFFER; + result.buffer = buf; + result.blen = blen; + result.xfree = free; + } else result.code = CLOUDSYNC_NETWORK_ERROR; + } else result.code = CLOUDSYNC_NETWORK_OK; } else { result.code = CLOUDSYNC_NETWORK_ERROR; - if (fetch->statusText && fetch->statusText[0]) + if (fetch->statusText && fetch->statusText[0]) { result.buffer = strdup(fetch->statusText); - else - result.buffer = strdup("Network error"); - result.blen = 0; + } + result.blen = sizeof(fetch->statusText); + result.xfree = free; } // cleanup @@ -245,6 +198,37 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, return result; } #else + +static bool network_buffer_check (network_buffer *data, size_t needed) { + // alloc/resize buffer + if (data->bused + needed > data->balloc) { + if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; + size_t balloc = data->balloc + needed; + + char *buffer = cloudsync_memory_realloc(data->buffer, balloc); + if (!buffer) return false; + + data->buffer = buffer; + data->balloc = balloc; + } + + return true; +} + +static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { + network_buffer *data = (network_buffer *)xdata; + + size_t ptr_size = (size*nmemb); + if (data->zero_term) ptr_size += 1; + + if (network_buffer_check(data, ptr_size) == false) return -1; + memcpy(data->buffer+data->bused, ptr, size*nmemb); + data->bused += size*nmemb; + if (data->zero_term) data->buffer[data->bused] = 0; + + return (size * nmemb); +} + NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { char *buffer = NULL; size_t blen = 0; @@ -286,7 +270,7 @@ NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, network_buffer netdata = {NULL, 0, 0, (zero_terminated) ? 1 : 0}; curl_easy_setopt(curl, CURLOPT_WRITEDATA, &netdata); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, network_receive_callback); - + // add optional JSON payload (implies setting CURLOPT_POST to 1) // or set the CURLOPT_POST option if (json_payload) { @@ -346,7 +330,7 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "PUT"); - attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_REPLACE; // Prepare headers (alternating key, value, NULL-terminated) // Max 3 headers: Accept, (optional Auth), Content-Type @@ -370,7 +354,7 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a attr.requestDataSize = blob_size; emscripten_fetch_t *fetch = emscripten_fetch(&attr, endpoint); // Blocks here until the operation is complete. - if (fetch->status == 200) result = true; + if (fetch->status >= 200 && fetch->status < 300) result = true; emscripten_fetch_close(fetch); From 0a6216673e739938a69f5ea241d61ec35ce0fe41 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 3 Jul 2025 21:03:21 +0200 Subject: [PATCH 29/39] npm publish --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e503227..c3d982e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -251,7 +251,6 @@ jobs: - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - if: false run: npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 704d9b81a2f6923b0dc8f4cbfd96b6c7309ac4b1 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 3 Jul 2025 21:21:10 +0200 Subject: [PATCH 30/39] fix: missing npm ci and setup --- .github/workflows/main.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3d982e..f732716 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -226,11 +226,9 @@ jobs: path: artifacts - name: setup GitHub Pages - if: false uses: actions/configure-pages@v5 - name: deploy coverage to GitHub Pages - if: false uses: actions/deploy-pages@v4.0.5 - name: release tag version from cloudsync.h @@ -249,9 +247,15 @@ jobs: echo "❌ CLOUDSYNC_VERSION not found in cloudsync.h" exit 1 + - uses: actions/setup-node@v4 + with: + node-version: '20.x' + registry-url: 'https://npm.pkg.github.com' + scope: '@sqliteai' + - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - run: npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + run: npm ci && npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From e7a85eb757a6d3e34fe91d5504ea43546a4fc268 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 3 Jul 2025 21:39:28 +0200 Subject: [PATCH 31/39] fix: rm npm ci --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f732716..deee07a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -255,7 +255,7 @@ jobs: - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' - run: npm ci && npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + run: npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 4bd8a94a360edbeb3e1286b0bf0f9cf437c34e37 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Thu, 3 Jul 2025 21:59:01 +0200 Subject: [PATCH 32/39] fix: update npm registry URL to use npmjs.org --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index deee07a..63f78c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -250,8 +250,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '20.x' - registry-url: 'https://npm.pkg.github.com' - scope: '@sqliteai' + registry-url: 'https://registry.npmjs.org' - name: publish sqlite-wasm to npm #if: steps.tag.outputs.version != '' From f70292126e937204f4b6f6e4a455f1324f39bada Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Fri, 4 Jul 2025 11:43:06 +0200 Subject: [PATCH 33/39] fix: update wasm npm publish step to include provenance and public access --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 63f78c8..23fc84d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -252,9 +252,9 @@ jobs: node-version: '20.x' registry-url: 'https://registry.npmjs.org' - - name: publish sqlite-wasm to npm + - name: wasm publish to npmjs #if: steps.tag.outputs.version != '' - run: npm publish ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 1f44ee44553dfcb02760fc3250adf39f91778026 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Fri, 4 Jul 2025 11:56:44 +0200 Subject: [PATCH 34/39] fix: update wasm npm publish command to remove provenance option --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23fc84d..0ee79a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -254,7 +254,8 @@ jobs: - name: wasm publish to npmjs #if: steps.tag.outputs.version != '' - run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + #use this version when the repo will become public run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + run: npm publish --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 4534c87c1bff5d69f81e329d8cb4856d5eac83dd Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Fri, 4 Jul 2025 12:44:42 +0200 Subject: [PATCH 35/39] test: revert version extraction method in release step to read from cloudsync.h --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ee79a0..127ac09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -234,7 +234,8 @@ jobs: - name: release tag version from cloudsync.h id: tag run: | - VERSION=$(make version) + FILE="src/cloudsync.h" + VERSION=$(grep -oP '#define CLOUDSYNC_VERSION\s+"\K[^"]+' "$FILE") if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name') if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then From 7b1c9c771c269d6ce49417cce6657a065e633347 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Fri, 4 Jul 2025 13:07:14 +0200 Subject: [PATCH 36/39] fix: correct version format in jq command and npm publish step for cloudsync --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 127ac09..580053c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -165,7 +165,7 @@ jobs: jq \ --arg name "@sqliteai/cloudsync-wasm" \ - --arg version "$(make sqlite_version)+cloudsync-$(make version)" \ + --arg version "$(make sqlite_version)-cloudsync-$(make version)" \ --arg desc "$DESC" \ --argjson keywords '["offsync","cloudsync","sqliteai"]' \ --arg repo_url "git+https://github.com/sqliteai/sqlite-sync.git" \ @@ -255,8 +255,8 @@ jobs: - name: wasm publish to npmjs #if: steps.tag.outputs.version != '' - #use this version when the repo will become public run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz - run: npm publish --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*+cloudsync-*.tgz + #use this version when the repo will become public run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz + run: npm publish --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} From 0d76b81cdf2218790da3b630e0a86e3d7d466725 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni Date: Fri, 4 Jul 2025 13:19:38 +0200 Subject: [PATCH 37/39] fix: update version extraction method in release step to use make command --- .github/workflows/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 580053c..e260c3d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -212,7 +212,6 @@ jobs: runs-on: ubuntu-latest name: release needs: build - #if: github.ref == 'refs/heads/main' env: GH_TOKEN: ${{ github.token }} @@ -234,8 +233,7 @@ jobs: - name: release tag version from cloudsync.h id: tag run: | - FILE="src/cloudsync.h" - VERSION=$(grep -oP '#define CLOUDSYNC_VERSION\s+"\K[^"]+' "$FILE") + VERSION=$(make version) if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name') if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then @@ -254,7 +252,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - name: wasm publish to npmjs - #if: steps.tag.outputs.version != '' + if: steps.tag.outputs.version != '' #use this version when the repo will become public run: npm publish --provenance --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz run: npm publish --access public ./artifacts/cloudsync-wasm/sqlite-wasm/sqliteai-cloudsync-wasm-*-cloudsync-*.tgz env: @@ -274,8 +272,7 @@ jobs: done - uses: softprops/action-gh-release@v2.2.1 - if: false - #if: steps.tag.outputs.version != '' + if: steps.tag.outputs.version != '' with: generate_release_notes: true tag_name: ${{ steps.tag.outputs.version }} From 17feec74d017c495a747ce3ceaa2bc7fd4b11b59 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni <48024736+Gioee@users.noreply.github.com> Date: Fri, 4 Jul 2025 14:17:29 +0200 Subject: [PATCH 38/39] fix: release only in main branch --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e260c3d..ba3ace3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -212,6 +212,7 @@ jobs: runs-on: ubuntu-latest name: release needs: build + if: github.ref == 'refs/heads/main' env: GH_TOKEN: ${{ github.token }} From 10c218960faa7f8f98ca8abc710b94f1ce2ff2c2 Mon Sep 17 00:00:00 2001 From: Gioele Cantoni <48024736+Gioee@users.noreply.github.com> Date: Fri, 4 Jul 2025 14:21:37 +0200 Subject: [PATCH 39/39] bump version --- src/cloudsync.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudsync.h b/src/cloudsync.h index 5f4238c..71915d6 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -16,7 +16,7 @@ #include "sqlite3.h" #endif -#define CLOUDSYNC_VERSION "0.8.5" +#define CLOUDSYNC_VERSION "0.8.6" int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);