From c47ddbe17e42a30eeea108a3ca3bf436c6f11971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 12 Jun 2025 13:45:04 +0200 Subject: [PATCH 01/13] feat: multiple versions for the index_advisor extension Build multiple versions of the index_advisor extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. --- nix/ext/index_advisor.nix | 88 +++++++++++++++---- nix/ext/tests/index_advisor.nix | 148 ++++++++++++++++++++++++++++++++ nix/ext/versions.json | 9 ++ 3 files changed, 226 insertions(+), 19 deletions(-) create mode 100644 nix/ext/tests/index_advisor.nix diff --git a/nix/ext/index_advisor.nix b/nix/ext/index_advisor.nix index 19777f39e..4e40214ec 100644 --- a/nix/ext/index_advisor.nix +++ b/nix/ext/index_advisor.nix @@ -1,34 +1,84 @@ { + pkgs, lib, stdenv, fetchFromGitHub, postgresql, }: -stdenv.mkDerivation rec { +let pname = "index_advisor"; - version = "0.2.0"; + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; - buildInputs = [ postgresql ]; + buildInputs = [ postgresql ]; - src = fetchFromGitHub { - owner = "olirice"; - repo = pname; - rev = "v${version}"; - hash = "sha256-G0eQk2bY5CNPMeokN/nb05g03CuiplRf902YXFVQFbs="; - }; + src = fetchFromGitHub { + owner = "olirice"; + repo = pname; + rev = "v${version}"; + inherit hash; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + create_sql_files() { + echo "Creating SQL files for previous versions..." + if [[ "${version}" == "${latestVersion}" ]]; then + cp *.sql $out/share/postgresql/extension + fi + } + + create_control_files() { + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \ + ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + fi + } - installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} + create_sql_files + create_control_files + ''; - cp *.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension - ''; + meta = with lib; { + description = "Recommend indexes to improve query performance in PostgreSQL"; + homepage = "https://github.com/olirice/index_advisor"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); +in +pkgs.buildEnv { + name = pname; + paths = packages; + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; - meta = with lib; { - description = "Recommend indexes to improve query performance in PostgreSQL"; - homepage = "https://github.com/olirice/index_advisor"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/index_advisor.nix b/nix/ext/tests/index_advisor.nix new file mode 100644 index 000000000..e337829c3 --- /dev/null +++ b/nix/ext/tests/index_advisor.nix @@ -0,0 +1,148 @@ +{ self, pkgs }: +let + pname = "index_advisor"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + self.packages.${pkgs.system}."psql_${majorVersion}/exts/hypopg" # dependency + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["15"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index e5c4e90b1..c5f5e24e3 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -10,5 +10,14 @@ "pgrx": "0.14.3", "rust": "1.87.0" } + }, + "index_advisor": { + "0.2.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-G0eQk2bY5CNPMeokN/nb05g03CuiplRf902YXFVQFbs=" + } } } From faa7cc17a827395d2621bff009b76f4bd488ae3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 23 Jun 2025 21:21:27 +0200 Subject: [PATCH 02/13] Change VM name suffix --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d82a06a65..210347185 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.104-orioledb" - postgres17: "17.4.1.054" - postgres15: "15.8.1.111" + postgresorioledb-17: "17.0.1.095-orioledb-ia-1" + postgres17: "17.4.1.046-ia-1" + postgres15: "15.8.1.103-ia-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 815a2cdb67ca83c9865442d47cdceab4a2e666d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Jun 2025 14:06:08 -0400 Subject: [PATCH 03/13] chore: bump testing version --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 210347185..5d71e6188 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.095-orioledb-ia-1" - postgres17: "17.4.1.046-ia-1" - postgres15: "15.8.1.103-ia-1" + postgresorioledb-17: "17.0.1.095-orioledb-ia-3" + postgres17: "17.4.1.046-ia-3" + postgres15: "15.8.1.103-ia-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From be25c7ee6fa4b1793e3f63bd75a580e3ba00a075 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Jul 2025 10:32:50 -0400 Subject: [PATCH 04/13] chore: formatting --- nix/ext/tests/index_advisor.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/ext/tests/index_advisor.nix b/nix/ext/tests/index_advisor.nix index e337829c3..9f4b22024 100644 --- a/nix/ext/tests/index_advisor.nix +++ b/nix/ext/tests/index_advisor.nix @@ -96,7 +96,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { requires = [ "postgresql-migrate.service" ]; }; }; - }; testScript = { nodes, ... }: From a819165e23846fd693cfd3e905c08f49b56f5f7a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Jul 2025 15:47:51 -0400 Subject: [PATCH 05/13] tests: bump version to test --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 5d71e6188..d853f53dc 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.095-orioledb-ia-3" - postgres17: "17.4.1.046-ia-3" - postgres15: "15.8.1.103-ia-3" + postgresorioledb-17: "17.0.1.095-orioledb-ia-4" + postgres17: "17.4.1.046-ia-4" + postgres15: "15.8.1.103-ia-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 7d5f16b8a9bea5a7364e910618e1888194e51e95 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Jul 2025 16:17:16 -0400 Subject: [PATCH 06/13] fix: account for access needed by macos --- nix/checks.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/checks.nix b/nix/checks.nix index 13be10f23..d29d49ef5 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -171,7 +171,9 @@ #First we need to create a generic pg cluster for pgtap tests and run those export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins PGTAP_CLUSTER=$(mktemp -d) - initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" + + # Use --allow-group-access to allow initdb to run as root (needed on macOS) + initdb --locale=C --username=supabase_admin --allow-group-access -D "$PGTAP_CLUSTER" substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf From e0b791ed11b25268b387ec5f84cc987c52c21267 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 07:04:25 -0400 Subject: [PATCH 07/13] fix: handle flake check user in darwin env --- nix/checks.nix | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index d29d49ef5..fbca44c49 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -171,9 +171,29 @@ #First we need to create a generic pg cluster for pgtap tests and run those export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins PGTAP_CLUSTER=$(mktemp -d) + # Set USER environment variable to handle root user restriction on macOS + # This is a common pattern in Nix builds to handle user-related issues + export USER="$(whoami)" + if [[ "$(uname)" == "Darwin" ]]; then + # On macOS, ensure we're not running as root for initdb + if [[ "$USER" == "root" ]]; then + export USER="nobody" + fi + fi - # Use --allow-group-access to allow initdb to run as root (needed on macOS) - initdb --locale=C --username=supabase_admin --allow-group-access -D "$PGTAP_CLUSTER" + # Initialize database with proper flags based on version + if [ "${majorVersion}" = "orioledb-17" ]; then + initdb -D "$PGTAP_CLUSTER" \ + --allow-group-access \ + --username=supabase_admin \ + --locale-provider=icu \ + --encoding=UTF-8 \ + --icu-locale=en_US.UTF-8 + else + initdb -D "$PGTAP_CLUSTER" \ + --allow-group-access \ + --username=supabase_admin + fi substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf From 4d8479ad6f3d09ba3383566c84419dff3cdc7a26 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 07:20:21 -0400 Subject: [PATCH 08/13] fix: pin version of macos --- .github/workflows/nix-build.yml | 2 +- nix/checks.nix | 24 +----------------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 077c525dd..e91816ec1 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -24,7 +24,7 @@ jobs: arch: amd64 - runner: large-linux-arm arch: arm64 - - runner: macos-latest-xlarge + - runner: macos-14-xlarge arch: arm64 runs-on: ${{ matrix.runner }} timeout-minutes: 180 diff --git a/nix/checks.nix b/nix/checks.nix index fbca44c49..13be10f23 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -171,29 +171,7 @@ #First we need to create a generic pg cluster for pgtap tests and run those export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins PGTAP_CLUSTER=$(mktemp -d) - # Set USER environment variable to handle root user restriction on macOS - # This is a common pattern in Nix builds to handle user-related issues - export USER="$(whoami)" - if [[ "$(uname)" == "Darwin" ]]; then - # On macOS, ensure we're not running as root for initdb - if [[ "$USER" == "root" ]]; then - export USER="nobody" - fi - fi - - # Initialize database with proper flags based on version - if [ "${majorVersion}" = "orioledb-17" ]; then - initdb -D "$PGTAP_CLUSTER" \ - --allow-group-access \ - --username=supabase_admin \ - --locale-provider=icu \ - --encoding=UTF-8 \ - --icu-locale=en_US.UTF-8 - else - initdb -D "$PGTAP_CLUSTER" \ - --allow-group-access \ - --username=supabase_admin - fi + initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf From be92344bf98c27649df076456c7c69d1cfd1cafe Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 07:39:35 -0400 Subject: [PATCH 09/13] fix: trying without cleanup --- .github/workflows/nix-build.yml | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index e91816ec1..1683537f4 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -79,30 +79,30 @@ jobs: extra_nix_config: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Aggressive disk cleanup for DuckDB build - if: matrix.runner == 'macos-latest-xlarge' - run: | - echo "=== BEFORE CLEANUP ===" - df -h - # Remove major space consumers - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/local/lib/android || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true - # Clean everything possible - sudo rm -rf /opt/ghc || true - sudo rm -rf /usr/local/share/boost || true - sudo rm -rf /opt/homebrew || true - sudo xcrun simctl delete all 2>/dev/null || true - # Aggressive cache cleanup - sudo rm -rf /System/Library/Caches/* 2>/dev/null || true - sudo rm -rf /Library/Caches/* 2>/dev/null || true - sudo rm -rf ~/Library/Caches/* 2>/dev/null || true - sudo rm -rf /private/var/log/* 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== AFTER CLEANUP ===" - df -h + # - name: Aggressive disk cleanup for DuckDB build + # if: matrix.runner == 'macos-latest-xlarge' + # run: | + # echo "=== BEFORE CLEANUP ===" + # df -h + # # Remove major space consumers + # sudo rm -rf /usr/share/dotnet || true + # sudo rm -rf /usr/local/lib/android || true + # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true + # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true + # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true + # # Clean everything possible + # sudo rm -rf /opt/ghc || true + # sudo rm -rf /usr/local/share/boost || true + # sudo rm -rf /opt/homebrew || true + # sudo xcrun simctl delete all 2>/dev/null || true + # # Aggressive cache cleanup + # sudo rm -rf /System/Library/Caches/* 2>/dev/null || true + # sudo rm -rf /Library/Caches/* 2>/dev/null || true + # sudo rm -rf ~/Library/Caches/* 2>/dev/null || true + # sudo rm -rf /private/var/log/* 2>/dev/null || true + # sudo rm -rf /tmp/* 2>/dev/null || true + # echo "=== AFTER CLEANUP ===" + # df -h - name: Build psql bundle run: > nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" From cda7c668454894858661514be74776593684f36b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 07:51:44 -0400 Subject: [PATCH 10/13] fix: revert workflow --- .github/workflows/nix-build.yml | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 1683537f4..077c525dd 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -24,7 +24,7 @@ jobs: arch: amd64 - runner: large-linux-arm arch: arm64 - - runner: macos-14-xlarge + - runner: macos-latest-xlarge arch: arm64 runs-on: ${{ matrix.runner }} timeout-minutes: 180 @@ -79,30 +79,30 @@ jobs: extra_nix_config: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - # - name: Aggressive disk cleanup for DuckDB build - # if: matrix.runner == 'macos-latest-xlarge' - # run: | - # echo "=== BEFORE CLEANUP ===" - # df -h - # # Remove major space consumers - # sudo rm -rf /usr/share/dotnet || true - # sudo rm -rf /usr/local/lib/android || true - # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true - # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true - # sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true - # # Clean everything possible - # sudo rm -rf /opt/ghc || true - # sudo rm -rf /usr/local/share/boost || true - # sudo rm -rf /opt/homebrew || true - # sudo xcrun simctl delete all 2>/dev/null || true - # # Aggressive cache cleanup - # sudo rm -rf /System/Library/Caches/* 2>/dev/null || true - # sudo rm -rf /Library/Caches/* 2>/dev/null || true - # sudo rm -rf ~/Library/Caches/* 2>/dev/null || true - # sudo rm -rf /private/var/log/* 2>/dev/null || true - # sudo rm -rf /tmp/* 2>/dev/null || true - # echo "=== AFTER CLEANUP ===" - # df -h + - name: Aggressive disk cleanup for DuckDB build + if: matrix.runner == 'macos-latest-xlarge' + run: | + echo "=== BEFORE CLEANUP ===" + df -h + # Remove major space consumers + sudo rm -rf /usr/share/dotnet || true + sudo rm -rf /usr/local/lib/android || true + sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true + sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true + sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true + # Clean everything possible + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/local/share/boost || true + sudo rm -rf /opt/homebrew || true + sudo xcrun simctl delete all 2>/dev/null || true + # Aggressive cache cleanup + sudo rm -rf /System/Library/Caches/* 2>/dev/null || true + sudo rm -rf /Library/Caches/* 2>/dev/null || true + sudo rm -rf ~/Library/Caches/* 2>/dev/null || true + sudo rm -rf /private/var/log/* 2>/dev/null || true + sudo rm -rf /tmp/* 2>/dev/null || true + echo "=== AFTER CLEANUP ===" + df -h - name: Build psql bundle run: > nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" From c151370966d9c8e2cd6c865192cd1c4020592100 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 08:08:42 -0400 Subject: [PATCH 11/13] fix: disable pgtap --- nix/checks.nix | 140 ++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 13be10f23..5db4daa3b 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -168,80 +168,80 @@ '' set -e - #First we need to create a generic pg cluster for pgtap tests and run those - export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins - PGTAP_CLUSTER=$(mktemp -d) - initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" - substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" - echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf - echo "Checking shared_preload_libraries setting:" - grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf - # Remove timescaledb if running orioledb-17 check - echo "I AM ${pgpkg.version}====================================================" - if [[ "${pgpkg.version}" == *"17"* ]]; then - perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" - fi - #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole - echo "PGTAP_CLUSTER directory contents:" - ls -la "$PGTAP_CLUSTER" + # #First we need to create a generic pg cluster for pgtap tests and run those + # export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins + # PGTAP_CLUSTER=$(mktemp -d) + # initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" + # substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ + # --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" + # echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf + # echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf + # echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + # echo "Checking shared_preload_libraries setting:" + # grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf + # # Remove timescaledb if running orioledb-17 check + # echo "I AM ${pgpkg.version}====================================================" + # if [[ "${pgpkg.version}" == *"17"* ]]; then + # perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" + # fi + # #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole + # echo "PGTAP_CLUSTER directory contents:" + # ls -la "$PGTAP_CLUSTER" - # Check if postgresql.conf exists - if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then - echo "postgresql.conf is missing!" - exit 1 - fi + # # Check if postgresql.conf exists + # if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then + # echo "postgresql.conf is missing!" + # exit 1 + # fi - # PostgreSQL startup - if [[ "$(uname)" == "Darwin" ]]; then - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 - else - mkdir -p "$PGTAP_CLUSTER/sockets" - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 - fi || { - echo "pg_ctl failed to start PostgreSQL" - echo "Contents of postgresql.log:" - cat "$PGTAP_CLUSTER"/postgresql.log - exit 1 - } - for i in {1..60}; do - if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort}; then - echo "PostgreSQL is ready" - break - fi - sleep 1 - if [ $i -eq 60 ]; then - echo "PostgreSQL is not ready after 60 seconds" - echo "PostgreSQL status:" - pg_ctl -D "$PGTAP_CLUSTER" status - echo "PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - exit 1 - fi - done - createdb -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin testing - if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then - echo "Error executing SQL file. PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - pg_ctl -D "$PGTAP_CLUSTER" stop - exit 1 - fi - SORTED_DIR=$(mktemp -d) - for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do - psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true - done - rm -rf "$SORTED_DIR" - pg_ctl -D "$PGTAP_CLUSTER" stop - rm -rf $PGTAP_CLUSTER + # # PostgreSQL startup + # if [[ "$(uname)" == "Darwin" ]]; then + # pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 + # else + # mkdir -p "$PGTAP_CLUSTER/sockets" + # pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 + # fi || { + # echo "pg_ctl failed to start PostgreSQL" + # echo "Contents of postgresql.log:" + # cat "$PGTAP_CLUSTER"/postgresql.log + # exit 1 + # } + # for i in {1..60}; do + # if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort}; then + # echo "PostgreSQL is ready" + # break + # fi + # sleep 1 + # if [ $i -eq 60 ]; then + # echo "PostgreSQL is not ready after 60 seconds" + # echo "PostgreSQL status:" + # pg_ctl -D "$PGTAP_CLUSTER" status + # echo "PostgreSQL log content:" + # cat "$PGTAP_CLUSTER"/postgresql.log + # exit 1 + # fi + # done + # createdb -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin testing + # if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then + # echo "Error executing SQL file. PostgreSQL log content:" + # cat "$PGTAP_CLUSTER"/postgresql.log + # pg_ctl -D "$PGTAP_CLUSTER" stop + # exit 1 + # fi + # SORTED_DIR=$(mktemp -d) + # for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do + # psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true + # done + # rm -rf "$SORTED_DIR" + # pg_ctl -D "$PGTAP_CLUSTER" stop + # rm -rf $PGTAP_CLUSTER - # End of pgtap tests - # from here on out we are running pg_regress tests, we use a different cluster for this - # which is start by the start-postgres-server-bin script - # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations + # # End of pgtap tests + # # from here on out we are running pg_regress tests, we use a different cluster for this + # # which is start by the start-postgres-server-bin script + # # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations - unset GRN_PLUGINS_DIR + # unset GRN_PLUGINS_DIR ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize for i in {1..60}; do From 93e9272d61320666acff745726ff6fb9c95e34eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 08:56:31 -0400 Subject: [PATCH 12/13] fix: restore tap tests --- nix/checks.nix | 140 ++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 5db4daa3b..13be10f23 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -168,80 +168,80 @@ '' set -e - # #First we need to create a generic pg cluster for pgtap tests and run those - # export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins - # PGTAP_CLUSTER=$(mktemp -d) - # initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" - # substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ - # --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" - # echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf - # echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf - # echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf - # echo "Checking shared_preload_libraries setting:" - # grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf - # # Remove timescaledb if running orioledb-17 check - # echo "I AM ${pgpkg.version}====================================================" - # if [[ "${pgpkg.version}" == *"17"* ]]; then - # perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" - # fi - # #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole - # echo "PGTAP_CLUSTER directory contents:" - # ls -la "$PGTAP_CLUSTER" + #First we need to create a generic pg cluster for pgtap tests and run those + export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins + PGTAP_CLUSTER=$(mktemp -d) + initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" + substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" + echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + echo "Checking shared_preload_libraries setting:" + grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf + # Remove timescaledb if running orioledb-17 check + echo "I AM ${pgpkg.version}====================================================" + if [[ "${pgpkg.version}" == *"17"* ]]; then + perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" + fi + #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole + echo "PGTAP_CLUSTER directory contents:" + ls -la "$PGTAP_CLUSTER" - # # Check if postgresql.conf exists - # if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then - # echo "postgresql.conf is missing!" - # exit 1 - # fi + # Check if postgresql.conf exists + if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then + echo "postgresql.conf is missing!" + exit 1 + fi - # # PostgreSQL startup - # if [[ "$(uname)" == "Darwin" ]]; then - # pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 - # else - # mkdir -p "$PGTAP_CLUSTER/sockets" - # pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 - # fi || { - # echo "pg_ctl failed to start PostgreSQL" - # echo "Contents of postgresql.log:" - # cat "$PGTAP_CLUSTER"/postgresql.log - # exit 1 - # } - # for i in {1..60}; do - # if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort}; then - # echo "PostgreSQL is ready" - # break - # fi - # sleep 1 - # if [ $i -eq 60 ]; then - # echo "PostgreSQL is not ready after 60 seconds" - # echo "PostgreSQL status:" - # pg_ctl -D "$PGTAP_CLUSTER" status - # echo "PostgreSQL log content:" - # cat "$PGTAP_CLUSTER"/postgresql.log - # exit 1 - # fi - # done - # createdb -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin testing - # if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then - # echo "Error executing SQL file. PostgreSQL log content:" - # cat "$PGTAP_CLUSTER"/postgresql.log - # pg_ctl -D "$PGTAP_CLUSTER" stop - # exit 1 - # fi - # SORTED_DIR=$(mktemp -d) - # for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do - # psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true - # done - # rm -rf "$SORTED_DIR" - # pg_ctl -D "$PGTAP_CLUSTER" stop - # rm -rf $PGTAP_CLUSTER + # PostgreSQL startup + if [[ "$(uname)" == "Darwin" ]]; then + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 + else + mkdir -p "$PGTAP_CLUSTER/sockets" + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 + fi || { + echo "pg_ctl failed to start PostgreSQL" + echo "Contents of postgresql.log:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 + } + for i in {1..60}; do + if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort}; then + echo "PostgreSQL is ready" + break + fi + sleep 1 + if [ $i -eq 60 ]; then + echo "PostgreSQL is not ready after 60 seconds" + echo "PostgreSQL status:" + pg_ctl -D "$PGTAP_CLUSTER" status + echo "PostgreSQL log content:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 + fi + done + createdb -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin testing + if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then + echo "Error executing SQL file. PostgreSQL log content:" + cat "$PGTAP_CLUSTER"/postgresql.log + pg_ctl -D "$PGTAP_CLUSTER" stop + exit 1 + fi + SORTED_DIR=$(mktemp -d) + for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do + psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true + done + rm -rf "$SORTED_DIR" + pg_ctl -D "$PGTAP_CLUSTER" stop + rm -rf $PGTAP_CLUSTER - # # End of pgtap tests - # # from here on out we are running pg_regress tests, we use a different cluster for this - # # which is start by the start-postgres-server-bin script - # # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations + # End of pgtap tests + # from here on out we are running pg_regress tests, we use a different cluster for this + # which is start by the start-postgres-server-bin script + # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations - # unset GRN_PLUGINS_DIR + unset GRN_PLUGINS_DIR ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize for i in {1..60}; do From 01231548d6250f934dcb49818c7f123cbb29215e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 11 Jul 2025 09:25:25 -0400 Subject: [PATCH 13/13] fix: restore and check version --- .github/workflows/nix-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 077c525dd..362d3ff5e 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -82,6 +82,7 @@ jobs: - name: Aggressive disk cleanup for DuckDB build if: matrix.runner == 'macos-latest-xlarge' run: | + nix --version echo "=== BEFORE CLEANUP ===" df -h # Remove major space consumers