diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..6eea24e6a --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - aarch64-darwin + - aarch64-linux + - blacksmith-32vcpu-ubuntu-2404 diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 82eaa32bd..caa9a051d 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -44,4 +44,3 @@ runs: substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }} - max-jobs = 8 diff --git a/.github/actions/nix-install-self-hosted/action.yml b/.github/actions/nix-install-self-hosted/action.yml new file mode 100644 index 000000000..52e49db9d --- /dev/null +++ b/.github/actions/nix-install-self-hosted/action.yml @@ -0,0 +1,30 @@ +name: 'Configure Nix on self hosted runners' +description: 'Sets up AWS credentials to push to the Nix binary cache' +inputs: + aws-role-duration: + description: 'AWS role session duration in seconds' + required: false + default: '18000' + +runs: + using: 'composite' + steps: + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-duration-seconds: ${{ inputs.aws-role-duration }} + + - name: Write creds files + shell: bash + run: | + umask 006 + cat > /etc/nix/aws/nix-aws-credentials <- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-linux) + needs: nix-eval + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false - matrix: - include: - - runner: blacksmith-32vcpu-ubuntu-2404 - arch: amd64 - - runner: blacksmith-32vcpu-ubuntu-2404-arm - arch: arm64 - - runner: macos-latest-xlarge - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: ./.github/actions/nix-install-ephemeral + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix (ephemeral) + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-ephemeral with: - push-to-cache: ${{ github.secret_source == 'Actions' && 'true' || 'false' }} + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Aggressive disk cleanup for DuckDB build - if: matrix.runner == 'macos-latest-xlarge' - run: | - nix --version - echo "=== BEFORE CLEANUP ===" - df -h - # Remove major space consumers - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/local/lib/android || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true - # Clean everything possible - sudo rm -rf /opt/ghc || true - sudo rm -rf /usr/local/share/boost || true - sudo rm -rf /opt/homebrew || true - sudo xcrun simctl delete all 2>/dev/null || true - # Aggressive cache cleanup - sudo rm -rf /System/Library/Caches/* 2>/dev/null || true - sudo rm -rf /Library/Caches/* 2>/dev/null || true - sudo rm -rf ~/Library/Caches/* 2>/dev/null || true - sudo rm -rf /private/var/log/* 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== AFTER CLEANUP ===" - df -h - - - name: Build psql bundle - run: > - nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" - -- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} - --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - name: Install nix (self-hosted) + if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} + + nix-build-aarch64-darwin: + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-darwin) + needs: nix-eval + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} + steps: + - name: Checkout Repo + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix + if: ${{ matrix.attr != '' }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} + + nix-build-x86_64-linux: + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (x86_64-linux) + needs: nix-eval + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} + steps: + - name: Checkout Repo + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix + if: ${{ matrix.attr != '' }} + uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: build-run-image - if: ${{ success() }} + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + if: | + !cancelled() && + needs.nix-eval.result == 'success' && + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml secrets: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: build-run-image - if: ${{ success() }} + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + if: | + !cancelled() && + needs.nix-eval.result == 'success' && + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/test.yml diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml new file mode 100644 index 000000000..cc092b355 --- /dev/null +++ b/.github/workflows/nix-eval.yml @@ -0,0 +1,34 @@ +name: Nix Eval + +on: + workflow_call: + outputs: + matrix: + description: 'Generated build matrix' + value: ${{ jobs.eval.outputs.matrix }} + secrets: + DEV_AWS_ROLE: + required: false + NIX_SIGN_SECRET_KEY: + required: false + +jobs: + eval: + runs-on: blacksmith-32vcpu-ubuntu-2404 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu -o pipefail + nix run --accept-flake-config .\#github-matrix -- checks legacyPackages diff --git a/flake.lock b/flake.lock index 0b1ec8191..78fb1a714 100644 --- a/flake.lock +++ b/flake.lock @@ -34,6 +34,27 @@ "type": "github" } }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1759362264, + "narHash": "sha256-wfG0S7pltlYyZTM+qqlhJ7GMw2fTF4mLKCIVhLii/4M=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "758cf7296bee11f1706a574c77d072b8a7baa881", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -95,6 +116,22 @@ "type": "github" } }, + "nix": { + "flake": false, + "locked": { + "lastModified": 1760472641, + "narHash": "sha256-BuKtM7Vr5EcxBXxUENBQPlOBwmNd5mkTRkSmlJi/iQ4=", + "owner": "NixOS", + "repo": "nix", + "rev": "4041bfdb401ad6d1c31a292fab90392254be506a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, "nix-editor": { "inputs": { "nixpkgs": [ @@ -131,11 +168,11 @@ ] }, "locked": { - "lastModified": 1749427739, - "narHash": "sha256-Nm0oMqFNRnJsiZYeNChmefmjeVCOzngikpSQhgs7iXI=", + "lastModified": 1763868605, + "narHash": "sha256-wO8Lk66GPQeSpzXUzXCBpe2Pj1De17xByrROXxnwIPY=", "owner": "Mic92", "repo": "nix-fast-build", - "rev": "b1dae483ab7d4139a6297e02b6de9e5d30e43d48", + "rev": "0c7fd47fb587250e931a01c6645ab5c0cd737af8", "type": "github" }, "original": { @@ -144,6 +181,27 @@ "type": "github" } }, + "nix-eval-jobs": { + "inputs": { + "flake-parts": "flake-parts_2", + "nix": "nix", + "nixpkgs": "nixpkgs_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1760478325, + "narHash": "sha256-hA+NOH8KDcsuvH7vJqSwk74PyZP3MtvI/l+CggZcnTc=", + "owner": "nix-community", + "repo": "nix-eval-jobs", + "rev": "daa42f9e9c84aeff1e325dd50fda321f53dfd02c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-eval-jobs", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": [ @@ -230,6 +288,66 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 315532800, + "narHash": "sha256-vhAtaRMIQiEghARviANBmSnhGz9Qf2IQJ+nQgsDXnVs=", + "rev": "c12c63cd6c5eb34c7b4c3076c6a99e00fcab86ec", + "type": "tarball", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre877036.c12c63cd6c5e/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1697269602, + "narHash": "sha256-dSzV7Ud+JH4DPVD9od53EgDrxUVQOcSj4KGjggCDVJI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9cb540e9c1910d74a7e10736277f6eb9dff51c81", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1712666087, + "narHash": "sha256-WwjUkWsjlU8iUImbivlYxNyMB1L5YVqE8QotQdL9jWc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a76c4553d7e741e17f289224eda135423de0491d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1744536153, + "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", @@ -239,10 +357,11 @@ "nix-fast-build": "nix-fast-build", "nix2container": "nix2container", "nixpkgs": "nixpkgs", + "nix-eval-jobs": "nix-eval-jobs", "nixpkgs-go124": "nixpkgs-go124", "nixpkgs-pgbackrest": "nixpkgs-pgbackrest", "rust-overlay": "rust-overlay", - "treefmt-nix": "treefmt-nix" + "treefmt-nix": "treefmt-nix_2" } }, "rust-overlay": { @@ -281,6 +400,27 @@ } }, "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1760120816, + "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "761ae7aff00907b607125b2f57338b74177697ed", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, + "treefmt-nix_2": { "inputs": { "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index 8f83a0a25..efd0b7e9e 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; + nix-eval-jobs.url = "github:nix-community/nix-eval-jobs"; }; outputs = diff --git a/nix/checks.nix b/nix/checks.nix index 0a1ab9d8e..553d9b24e 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -397,24 +397,22 @@ pg_regress ; } - // pkgs.lib.optionalAttrs (system == "aarch64-linux") { - inherit (self'.packages) - postgresql_15_debug - postgresql_15_src - postgresql_orioledb-17_debug - postgresql_orioledb-17_src - postgresql_17_debug - postgresql_17_src - ; - } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + // pkgs.lib.optionalAttrs (system == "aarch64-linux") ( { - devShell = self'.devShells.default; + inherit (self'.packages) + postgresql_15_debug + postgresql_15_src + postgresql_orioledb-17_debug + postgresql_orioledb-17_src + postgresql_17_debug + postgresql_17_src + ; } // (import ./ext/tests { inherit self; inherit pkgs; }) - ); + ) + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ({ devShell = self'.devShells.default; }); }; } diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 7d9fbdae9..d5620de09 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -42,7 +42,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; + maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index d944d5ede..76fa2c47d 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -79,7 +79,7 @@ let preCheck = '' export PGRX_HOME="$(mktemp -d)" - export PG_VERSION="${lib.versions.major postgresql.version}" + export PG_VERSION="${pgVersion}" export NIX_PGLIBDIR="$PGRX_HOME/$PG_VERSION/lib" export PATH="$PGRX_HOME/$PG_VERSION/bin:$PATH" ${lib.getExe rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ "$PGRX_HOME/$PG_VERSION/" @@ -119,8 +119,9 @@ let } ); allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_graphql; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -128,10 +129,54 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + buildUnsupported = + # Build SQL-only packages for unsupported versions needed by pg_upgrade. + # When upgrading PostgreSQL, pg_upgrade requires old extension versions to exist + # even if they can't compile against the new PostgreSQL version. + version: hash: _rustVersion: _pgrxVersion: + stdenv.mkDerivation { + inherit pname version; + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v${version}"; + inherit hash; + }; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/share/postgresql/extension + for file in $src/sql/*.sql; do + filename=$(basename "$file") + if [[ "$filename" != "load_sql_config.sql" && "$filename" != "load_sql_context.sql" ]]; then + cat "$file" + echo ";" + fi + done > $out/share/postgresql/extension/${pname}--${version}.sql + ''; + meta = with lib; { + description = "GraphQL support for PostreSQL"; + homepage = "https://github.com/supabase/${pname}"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + unsupportedVersions = lib.filterAttrs ( + _: value: !builtins.elem pgVersion value.postgresql + ) allVersions; + unsupportedPackages = + if pgVersion == 15 then + [ ] + else + # Include SQL-only packages for PG15 extension versions incompatible with current PG + builtins.attrValues ( + lib.mapAttrs ( + name: value: buildUnsupported name value.hash value.rust value.pgrx + ) unsupportedVersions + ); in -buildEnv { +(buildEnv { name = pname; - paths = packages; + paths = packages ++ unsupportedPackages; pathsToLink = [ "/lib" "/share/postgresql/extension" @@ -177,4 +222,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 74742bed5..fb83122b9 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -49,16 +49,6 @@ let # update the following array when the pg_jsonschema version is updated # required to ensure that extensions update scripts from previous versions are generated - previousVersions = [ - "0.3.1" - "0.3.0" - "0.2.0" - "0.1.4" - "0.1.3" - "0.1.2" - "0.1.1" - "0.1.0" - ]; CARGO = "${cargo}/bin/cargo"; #darwin env needs PGPORT to be unique for build to not clash with other pgrx extensions env = lib.optionalAttrs stdenv.isDarwin { @@ -79,19 +69,15 @@ let preCheck = '' export PGRX_HOME=$(mktemp -d) - export NIX_PGLIBDIR=$PGRX_HOME/${lib.versions.major postgresql.version}/lib - ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${lib.versions.major postgresql.version}/ - cargo pgrx init --pg${lib.versions.major postgresql.version} $PGRX_HOME/${lib.versions.major postgresql.version}/bin/pg_config + export NIX_PGLIBDIR=$PGRX_HOME/${pgVersion}/lib + ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${pgVersion}/ + cargo pgrx init --pg${pgVersion} $PGRX_HOME/${pgVersion}/bin/pg_config ''; doCheck = true; - preBuild = '' - echo "Processing git tags..." - echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt - ''; - postInstall = '' + find $out mv $out/lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} create_control_files() { @@ -120,8 +106,9 @@ let }; }; allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_jsonschema; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -129,8 +116,10 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + unsupportedVersionsItems = lib.filterAttrs (_: value: value.postgresql == [ "15" ]) allVersions; + unsupportedVersions = if pgVersion == "17" then lib.attrNames unsupportedVersionsItems else [ ]; in -pkgs.buildEnv { +(pkgs.buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -145,6 +134,10 @@ pkgs.buildEnv { }" ) + for v in ${lib.concatStringsSep " " unsupportedVersions}; do + cp $out/share/postgresql/extension/${pname}--${lib.head versions}.sql $out/share/postgresql/extension/${pname}--$v.sql + done + create_sql_files() { PREVIOUS_VERSION="" while IFS= read -r i; do @@ -170,4 +163,7 @@ pkgs.buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 2a2ce0097..37f0e64c2 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -17,11 +17,16 @@ let ) allVersions; # Derived version information - versions = lib.naturalSort (lib.attrNames supportedVersions); - latestVersion = lib.last versions; + allVersionsList = lib.naturalSort (lib.attrNames supportedVersions); + versions = builtins.filter (v: (allVersions.${v}.pgUpgradeCompatible or true)) allVersionsList; + latestVersion = lib.last allVersionsList; numberOfVersions = builtins.length versions; + # Filter to only build pg_upgrade compatible versions + pgUpgradeCompatibleVersions = lib.filterAttrs ( + name: _: allVersions.${name}.pgUpgradeCompatible or true + ) supportedVersions; packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.revision) supportedVersions + lib.mapAttrs (name: value: build name value.hash value.revision) pgUpgradeCompatibleVersions ); # Build function for individual versions diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index fa6dba77d..b8f8347a3 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -69,7 +69,6 @@ let meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index ed1b738e4..e5b19ccfb 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -179,7 +179,7 @@ let }; }; in -buildEnv { +(buildEnv { name = pname; paths = packages; @@ -209,4 +209,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index facd250f6..c803f32f2 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -136,9 +136,6 @@ let }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -146,7 +143,9 @@ let "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -198,6 +197,33 @@ let with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; }; in @@ -224,6 +250,7 @@ builtins.listToAttrs ( "pg_stat_monitor" "pg_tle" "pgaudit" + "postgis" "vector" "wal2json" "wrappers" diff --git a/nix/ext/tests/lib.py b/nix/ext/tests/lib.py index 42b5eb361..a28361c3b 100644 --- a/nix/ext/tests/lib.py +++ b/nix/ext/tests/lib.py @@ -47,7 +47,7 @@ def run_sql_file(self, file: str) -> str: ).strip() def drop_extension(self): - self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};") + self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name} CASCADE;") def install_extension(self, version: str): self.run_sql( diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index 94438d612..e0fe1d826 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -39,7 +39,7 @@ let psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { - name = "timescaledb"; + name = "pgmq"; hostPkgs = pkgs; nodes.server = { config, ... }: @@ -105,9 +105,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -116,7 +113,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { } extension_name = "${pname}" support_upgrade = True - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -173,5 +172,30 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) ''; } diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix index 83af20ac8..54d5627f9 100644 --- a/nix/ext/tests/pgroonga.nix +++ b/nix/ext/tests/pgroonga.nix @@ -125,9 +125,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -135,7 +132,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -187,5 +186,32 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix index 9ad1aec4b..f97c8b3de 100644 --- a/nix/ext/tests/pgsodium.nix +++ b/nix/ext/tests/pgsodium.nix @@ -41,6 +41,8 @@ let echo 0000000000000000000000000000000000000000000000000000000000000000 '' ); + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { name = pname; @@ -60,7 +62,21 @@ self.inputs.nixpkgs.lib.nixos.runTest { services.postgresql = { enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + package = psql_15; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; settings = { "shared_preload_libraries" = pname; "pgsodium.getkey_script" = pgsodiumGetKey; @@ -69,7 +85,7 @@ self.inputs.nixpkgs.lib.nixos.runTest { specialisation.postgresql17.configuration = { services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + package = lib.mkForce psql_17; }; systemd.services.postgresql-migrate = { @@ -83,8 +99,8 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; script = let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldPostgresql = psql_15; + newPostgresql = psql_17; oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; in @@ -110,49 +126,93 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' + from pathlib import Path versions = { "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } + extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + ${builtins.readFile ./lib.py} start_all() server.wait_for_unit("multi-user.target") server.wait_for_unit("postgresql.service") - check_upgrade_path("15") + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) with subtest("switch to postgresql 17"): server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" + f"{pg17_configuration}/bin/switch-to-configuration test >&2" ) - check_upgrade_path("17") + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix deleted file mode 100644 index ab6a4b3f8..000000000 --- a/nix/ext/tests/postgis.nix +++ /dev/null @@ -1,156 +0,0 @@ -{ self, pkgs }: -let - pname = "postgis"; - inherit (pkgs) lib; - installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; - versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; - postgresqlWithExtension = - postgresql: - let - majorVersion = lib.versions.major postgresql.version; - pkg = pkgs.buildEnv { - name = "postgresql-${majorVersion}-${pname}"; - paths = [ - postgresql - postgresql.lib - (installedExtension majorVersion) - ]; - passthru = { - inherit (postgresql) version psqlSchema; - lib = pkg; - withPackages = _: pkg; - }; - nativeBuildInputs = [ pkgs.makeWrapper ]; - pathsToLink = [ - "/" - "/bin" - "/lib" - ]; - postBuild = '' - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib - ''; - }; - in - pkg; -in -self.inputs.nixpkgs.lib.nixos.runTest { - name = pname; - hostPkgs = pkgs; - nodes.server = - { config, ... }: - { - virtualisation = { - forwardPorts = [ - { - from = "host"; - host.port = 13022; - guest.port = 22; - } - ]; - }; - services.openssh = { - enable = true; - }; - - services.postgresql = { - enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - }; - - specialisation.postgresql17.configuration = { - services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); - }; - - systemd.services.postgresql-migrate = { - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - User = "postgres"; - Group = "postgres"; - StateDirectory = "postgresql"; - WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; - }; - script = - let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; - oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; - newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; - in - '' - if [[ ! -d ${newDataDir} ]]; then - install -d -m 0700 -o postgres -g postgres "${newDataDir}" - ${newPostgresql}/bin/initdb -D "${newDataDir}" - ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ - --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" - else - echo "${newDataDir} already exists" - fi - ''; - }; - - systemd.services.postgresql = { - after = [ "postgresql-migrate.service" ]; - requires = [ "postgresql-migrate.service" ]; - }; - }; - }; - testScript = - { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in - '' - versions = { - "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], - "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], - } - - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" - - start_all() - - server.wait_for_unit("multi-user.target") - server.wait_for_unit("postgresql.service") - - check_upgrade_path("15") - - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension where extname = '${pname}';""") - latestVersion = versions["15"][-1] - majMinVersion = ".".join(latestVersion.split('.')[:1]) - assert f"${pname},{majMinVersion}" in installed_extensions, f"Expected ${pname} version {latestVersion}, but found {installed_extensions}" - - with subtest("switch to postgresql 17"): - server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" - ) - - with subtest("Check ${pname} latest extension version after upgrade"): - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["17"][-1] - majMinVersion = ".".join(latestVersion.split('.')[:1]) - assert f"${pname},{majMinVersion}" in installed_extensions - - check_upgrade_path("17") - ''; -} diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix index 7b16247a5..480ef5ec9 100644 --- a/nix/ext/tests/vault.nix +++ b/nix/ext/tests/vault.nix @@ -133,9 +133,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -143,8 +140,10 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" support_upgrade = True - pg17_configuration = "${pg17-configuration}" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -197,5 +196,33 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/versions.json b/nix/ext/versions.json index d1a78336a..fbafe2371 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -755,7 +755,8 @@ "15" ], "revision": "1.0.1", - "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=" + "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=", + "pgUpgradeCompatible": false }, "2.1": { "postgresql": [ diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 89effcfb3..4a7f03653 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -218,7 +218,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -buildEnv { +(buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -303,4 +303,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/fmt.nix b/nix/fmt.nix index 562c3b3c5..08763e5b8 100644 --- a/nix/fmt.nix +++ b/nix/fmt.nix @@ -4,6 +4,7 @@ perSystem = { pkgs, ... }: { + treefmt.flakeCheck = false; treefmt.programs = { deadnix.enable = true; nixfmt = { diff --git a/nix/hooks.nix b/nix/hooks.nix index 896c262ba..b659635f8 100644 --- a/nix/hooks.nix +++ b/nix/hooks.nix @@ -1,4 +1,11 @@ { inputs, ... }: +let + ghWorkflows = builtins.attrNames (builtins.readDir ../.github/workflows); + lintedWorkflows = [ + "nix-eval.yml" + "nix-build.yml" + ]; +in { imports = [ inputs.git-hooks.flakeModule ]; perSystem = @@ -8,9 +15,17 @@ check.enable = true; settings = { hooks = { + actionlint = { + enable = true; + excludes = builtins.filter (name: !builtins.elem name lintedWorkflows) ghWorkflows; + verbose = true; + }; + treefmt = { enable = true; package = config.treefmt.build.wrapper; + pass_filenames = false; + verbose = true; }; }; }; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index c8eb02ef0..fa6ab10a4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -33,6 +33,9 @@ cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; docs = pkgs.callPackage ./docs.nix { }; + github-matrix = pkgs.callPackage ./github-matrix { + nix-eval-jobs = inputs'.nix-eval-jobs.packages.default; + }; supabase-groonga = pkgs.callPackage ./groonga { }; http-mock-server = pkgs.callPackage ./http-mock-server.nix { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix new file mode 100644 index 000000000..3e1be2df3 --- /dev/null +++ b/nix/packages/github-matrix/default.nix @@ -0,0 +1,51 @@ +{ + lib, + nix-eval-jobs, + python3Packages, +}: +let + pname = "github-matrix"; + + github-action-utils = python3Packages.buildPythonPackage rec { + pname = "github-action-utils"; + version = "1.1.0"; + pyproject = true; + + src = python3Packages.fetchPypi { + inherit pname version; + sha256 = "0q9xrb4jcvbn6954lvpn85gva1yc885ykdqb2q2410cxp280v94a"; + }; + + build-system = with python3Packages; [ setuptools ]; + + meta = with lib; { + description = "Collection of Python functions for GitHub Action Workflow Commands"; + homepage = "https://github.com/saadmk11/github-action-utils"; + license = licenses.mit; + }; + }; +in + +python3Packages.buildPythonApplication { + inherit pname; + version = "0.1.0"; + pyproject = false; + + src = ./.; + + propagatedBuildInputs = [ + github-action-utils + python3Packages.result + ]; + + makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; + + nativeCheckInputs = with python3Packages; [ + pytestCheckHook + pytest-mypy + ]; + + installPhase = '' + install -Dm755 github_matrix.py "$out/bin/${pname}" + ''; +} diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py new file mode 100755 index 000000000..7743753e5 --- /dev/null +++ b/nix/packages/github-matrix/github_matrix.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 + +import argparse +from collections import Counter, defaultdict +import graphlib +import json +import os +import subprocess +import sys +from typing import ( + Any, + Dict, + List, + Literal, + NotRequired, + Optional, + Set, + Tuple, + TypedDict, + get_args, +) + +from github_action_utils import debug, notice, error, set_output, warning +from result import Err, Ok, Result + +System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] +RunnerType = Literal["ephemeral", "self-hosted"] + + +class NixEvalJobsOutput(TypedDict): + """Raw output from nix-eval-jobs command.""" + + attr: str + attrPath: List[str] + cacheStatus: Literal["notBuilt", "cached", "local"] + drvPath: str + name: str + system: System + neededBuilds: NotRequired[List[Any]] + neededSubstitutes: NotRequired[List[Any]] + outputs: NotRequired[Dict[str, str]] + error: NotRequired[str] + requiredSystemFeatures: NotRequired[List[str]] + + +class RunsOnConfig(TypedDict): + """GitHub Actions runs-on configuration.""" + + group: NotRequired[str] + labels: List[str] + + +class GitHubActionPackage(TypedDict): + """Final package output for GitHub Actions matrix.""" + + attr: str + name: str + system: System + runs_on: RunsOnConfig + postgresql_version: NotRequired[str] + + +class NixEvalError(TypedDict): + """Error information from nix evaluation.""" + + attr: str + error: str + + +BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { + "ephemeral": { + "aarch64-linux": { + "labels": ["blacksmith-4vcpu-ubuntu-2404-arm"], + }, + "x86_64-linux": { + "labels": ["blacksmith-8vcpu-ubuntu-2404"], + }, + }, + "self-hosted": { + "aarch64-darwin": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + }, + "aarch64-linux": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + }, + }, +} + + +def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[str]: + """Build the nix-eval-jobs command with appropriate flags.""" + nix_eval_cmd = [ + "nix-eval-jobs", + "--flake", + ".", + "--check-cache-status", + "--force-recurse", + "--quiet", + "--option", + "eval-cache", + "false", + "--option", + "accept-flake-config", + "true", + "--workers", + str(max_workers), + "--select", + f"outputs: {{ inherit (outputs) {' '.join(flake_outputs)}; }}", + ] + return nix_eval_cmd + + +def parse_nix_eval_line( + line: str, drv_paths: Set[str] +) -> Result[Optional[NixEvalJobsOutput], NixEvalError]: + """Parse a single line of nix-eval-jobs output. + + Returns: + Ok(package_data) if successful (None for empty/duplicate lines) + Err(NixEvalError) if a nix evaluation error occurred + """ + if not line.strip(): + return Ok(None) + + try: + data: NixEvalJobsOutput = json.loads(line) + if "error" in data: + error_msg = data["error"] + + # Extract the core error message (last "error:" line and following context) + error_lines = error_msg.split("\n") + core_error_idx = -1 + for i in range(len(error_lines) - 1, -1, -1): + if error_lines[i].strip().startswith("error:"): + core_error_idx = i + break + + if core_error_idx >= 0: + # Take the last error line and up to 3 lines of context after it + error_msg = "\n".join( + error_lines[ + core_error_idx : min(core_error_idx + 4, len(error_lines)) + ] + ).strip() + + return Err({"attr": data["attr"], "error": error_msg}) + if data["drvPath"] in drv_paths: + return Ok(None) + drv_paths.add(data["drvPath"]) + return Ok(data) + except json.JSONDecodeError as e: + warning(f"Skipping invalid JSON line: {line}", title="JSON Parse Warning") + return Ok(None) + + +def run_nix_eval_jobs( + cmd: List[str], +) -> Tuple[List[NixEvalJobsOutput], List[str], List[NixEvalError]]: + """Run nix-eval-jobs and return parsed package data, warnings, and errors. + + Returns: + Tuple of (packages, warnings_list, errors_list) + """ + debug(f"Running command: {' '.join(cmd)}") + + # Disable colors in nix output + env = os.environ.copy() + env["NO_COLOR"] = "1" + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env + ) + stdout_data, stderr_data = process.communicate() + + # Parse stdout for packages + packages: List[NixEvalJobsOutput] = [] + drv_paths: Set[str] = set() + errors_list: List[NixEvalError] = [] + for line in stdout_data.splitlines(): + result = parse_nix_eval_line(line, drv_paths) + if result.is_err(): + errors_list.append(result.err_value) + elif result._value is not None: + packages.append(result._value) + + # Parse stderr for warnings (lines starting with "warning:") + warnings_list: List[str] = [] + for line in stderr_data.splitlines(): + line = line.strip() + if line.startswith("warning:") or line.startswith("evaluation warning:"): + # Remove "warning:" prefix for cleaner messages + warnings_list.append(line[8:].strip()) + + if process.returncode != 0: + error( + "nix-eval-jobs process failed with non-zero exit code", + title="Process Failure", + ) + + return packages, warnings_list, errors_list + + +def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: + """Check if the package is a postgresql extension package.""" + attrs = pkg["attr"].split(".") + return attrs[-2] == "exts" + + +# thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 +def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutput]: + sorted_jobs = [] + + # Prepare job dependencies + job_set = {job["drvPath"] for job in jobs} + job_closures = { + k["drvPath"]: set(k.get("neededSubstitutes", [])) + .union(set(k.get("neededBuilds", []))) + .intersection(job_set) + .difference({k["drvPath"]}) + for k in jobs + } + + sorter = graphlib.TopologicalSorter(job_closures) + + job_by_drv = {job["drvPath"]: job for job in jobs} + for item in sorter.static_order(): + if item in job_by_drv: + sorted_jobs.append(job_by_drv[item]) + + return sorted_jobs + + +def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package is considered large based on its attribute path.""" + return "big-parallel" in pkg.get("requiredSystemFeatures", []) + + +def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package requires KVM""" + return "kvm" in pkg.get("requiredSystemFeatures", []) + + +def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: + """Determine the appropriate GitHub Actions runner for a package. + + Priority order: + 1. KVM packages → self-hosted runners + 2. Large packages on Linux → 32vcpu ephemeral runners + 3. Darwin packages → self-hosted runners + 4. Default → ephemeral runners + """ + system = pkg["system"] + + if is_kvm_pkg(pkg): + runConfig = BUILD_RUNNER_MAP["self-hosted"].get(system) + if runConfig is None: + raise ValueError( + f"No self-hosted with kvm support available for system: {system}" + ) + return runConfig + + if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): + suffix = "-arm" if system == "aarch64-linux" else "" + return {"labels": [f"blacksmith-32vcpu-ubuntu-2404{suffix}"]} + + if system == "aarch64-darwin": + return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"] + + return BUILD_RUNNER_MAP["ephemeral"].get(system) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate GitHub Actions matrix for Nix builds" + ) + parser.add_argument( + "flake_outputs", nargs="+", help="Nix flake outputs to evaluate" + ) + + args = parser.parse_args() + + max_workers: int = os.cpu_count() or 1 + + cmd = build_nix_eval_command(max_workers, args.flake_outputs) + + # Run evaluation and collect packages, warnings, and errors + packages, warnings_list, errors_list = run_nix_eval_jobs(cmd) + gh_action_packages = sort_pkgs_by_closures(packages) + + def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: + """Convert nix-eval-jobs output to GitHub Actions matrix package""" + runner = get_runner_for_package(pkg) + if runner is None: + raise ValueError(f"No runner configuration for system: {pkg['system']}") + returned_pkg: GitHubActionPackage = { + "attr": pkg["attr"], + "name": pkg["name"], + "system": pkg["system"], + "runs_on": runner, + } + if is_extension_pkg(pkg): + # Extract PostgreSQL version from attribute path + attrs = pkg["attr"].split(".") + returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] + return returned_pkg + + # Group packages by system + grouped_by_system = defaultdict(list) + for pkg in gh_action_packages: + if pkg.get("cacheStatus") == "notBuilt": + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + + # Create output with system-specific matrices + # Ensure that we have at least one entry per system + gh_output = {} + for system, packages in grouped_by_system.items(): + gh_output[system.replace("-", "_")] = {"include": packages} + + for system in get_args(System): + s = system.replace("-", "_") + if s not in gh_output: + gh_output[s] = { + "include": [ + { + "attr": "", + "name": "no packages to build", + "system": system, + "runs_on": {"labels": "ubuntu-latest"}, + } + ] + } + + if warnings_list: + warning_counts = Counter(warnings_list) + for warn_msg, count in warning_counts.items(): + if count > 1: + warning( + f"{warn_msg} (occurred {count} times)", + title="Nix Evaluation Warning", + ) + else: + warning(warn_msg, title="Nix Evaluation Warning") + + if errors_list: + # Group errors by error message + errors_by_message: Dict[str, List[str]] = defaultdict(list) + for err in errors_list: + errors_by_message[err["error"]].append(err["attr"]) + + for error_msg, attrs in errors_by_message.items(): + # Format message with attributes on first line, then error details + if len(attrs) > 1: + formatted_msg = f"Affected attributes ({len(attrs)}): {', '.join(attrs)}\n\n{error_msg}" + else: + formatted_msg = f"Attribute: {attrs[0]}\n\n{error_msg}" + formatted_msg = formatted_msg.replace("\n", "%0A") + error(formatted_msg, title="Nix Evaluation Error") + + if errors_list: + sys.exit(1) + else: + formatted_msg = f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}".replace( + "\n", "%0A" + ) + notice(formatted_msg, title="GitHub Actions Matrix") + set_output("matrix", json.dumps(gh_output)) + + +if __name__ == "__main__": + main() diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py new file mode 100644 index 000000000..d5fed5732 --- /dev/null +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 + +import pytest + +from github_matrix import ( + NixEvalJobsOutput, + get_runner_for_package, + is_extension_pkg, + is_kvm_pkg, + is_large_pkg, + sort_pkgs_by_closures, +) + + +class TestIsExtensionPkg: + def test_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is True + + def test_non_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is False + + +class TestIsLargePkg: + @pytest.mark.parametrize( + "attr,expected", + [ + ("packages.x86_64-linux.psql_15.exts.wrappers", True), + ("packages.x86_64-linux.psql_15.exts.pg_jsonschema", True), + ("packages.x86_64-linux.psql_15.exts.pg_graphql", True), + ("packages.x86_64-linux.psql_15.exts.postgis", True), + ("packages.x86_64-linux.psql_15.exts.pg_cron", False), + ("packages.x86_64-linux.psql_15", False), + ], + ) + def test_large_package_detection(self, attr: str, expected: bool): + pkg: NixEvalJobsOutput = { + "attr": attr, + "attrPath": attr.split("."), + "cacheStatus": "notBuilt", + "drvPath": f"/nix/store/{attr}.drv", + "name": attr.split(".")[-1], + "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"] if expected else [], + } + assert is_large_pkg(pkg) is expected + + +class TestIsKvmPkg: + def test_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + assert is_kvm_pkg(pkg) is True + + def test_non_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_kvm_pkg(pkg) is False + + +class TestGetRunnerForPackage: + def test_kvm_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + with pytest.raises( + ValueError, + match=r"No self-hosted with kvm support available for system: x86_64-linux", + ): + get_runner_for_package(pkg) + + def test_kvm_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.vm-test", + "attrPath": ["packages", "aarch64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "aarch64-linux", + "requiredSystemFeatures": ["kvm"], + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + } + + def test_large_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.postgis", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "postgis"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgis", + "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"], + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + + def test_large_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_graphql", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_graphql"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_graphql", + "system": "aarch64-linux", + "requiredSystemFeatures": ["big-parallel"], + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + + def test_darwin_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-darwin.psql_15", + "attrPath": ["packages", "aarch64-darwin", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "aarch64-darwin", + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + } + + def test_default_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404"]} + + def test_default_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "aarch64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-4vcpu-ubuntu-2404-arm"]} + + +class TestSortPkgsByClosures: + def test_empty_list(self): + result = sort_pkgs_by_closures([]) + assert result == [] + + def test_single_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + result = sort_pkgs_by_closures([pkg]) + assert result == [pkg] + + def test_dependency_order(self): + pkg1: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.lib", + "attrPath": ["packages", "x86_64-linux", "lib"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/lib.drv", + "name": "lib", + "system": "x86_64-linux", + "neededBuilds": [], + "neededSubstitutes": [], + } + pkg2: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.app", + "attrPath": ["packages", "x86_64-linux", "app"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/app.drv", + "name": "app", + "system": "x86_64-linux", + "neededBuilds": ["/nix/store/lib.drv"], + "neededSubstitutes": [], + } + + # Regardless of input order, lib should come before app + result = sort_pkgs_by_closures([pkg2, pkg1]) + assert result == [pkg1, pkg2] + + result = sort_pkgs_by_closures([pkg1, pkg2]) + assert result == [pkg1, pkg2] diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 2ce9410f6..ec25d44e4 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -1,7 +1,7 @@ { inputs, ... }: { perSystem = - { pkgs, ... }: + { pkgs, lib, ... }: let # Custom extensions that exist in our repository. These aren't upstream # either because nobody has done the work, maintaining them here is @@ -153,11 +153,12 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - recurseForDerivations = true; - }; + makePostgres = + version: + lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; @@ -166,5 +167,6 @@ in { packages = inputs.flake-utils.lib.flattenTree basePackages; + legacyPackages = basePackages; }; } diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 76904ced7..ea67cbc91 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -21,7 +21,6 @@ let libxml2, tzdata, libkrb5, - substituteAll, darwin, linux-pam, #orioledb specific @@ -193,11 +192,7 @@ let ./patches/paths-for-split-outputs.patch ./patches/specify_pkglibdir_at_runtime.patch ./patches/paths-with-postgresql-suffix.patch - - (substituteAll { - src = ./patches/locale-binary-path.patch; - locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale"; - }) + ./patches/locale-binary-path.patch ] ++ lib.optionals stdenv'.hostPlatform.isMusl ( # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 @@ -213,6 +208,9 @@ let '' # Hardcode the path to pgxs so pg_config returns the path in $out substituteInPlace "src/common/config_info.c" --subst-var out + substituteInPlace "src/backend/commands/collationcmds.c" --replace-fail '@locale@' '${ + if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc + }/bin/locale' '' + lib.optionalString jitSupport '' # Force lookup of jit stuff in $out instead of $lib