Merge branch 'develop' into sam/oriole-extensions-schema

Author: samrose

Dockerfile-15

@@ -24,7 +24,7 @@ ARG rum_release=1.3.13
 ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
 ARG libsodium_release=1.0.18
 ARG pgsodium_release=3.1.6
-ARG pg_graphql_release=1.5.1
+ARG pg_graphql_release=1.5.11
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
 ARG pg_repack_release=1.4.8

Dockerfile-orioledb-17

@@ -24,7 +24,7 @@ ARG rum_release=1.3.13
 ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
 ARG libsodium_release=1.0.18
 ARG pgsodium_release=3.1.6
-ARG pg_graphql_release=1.5.1
+ARG pg_graphql_release=1.5.11
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
 ARG pg_repack_release=1.4.8

README.md

@@ -9,15 +9,32 @@ if pgrep resizefs; then
     exit 1
 fi
 
+# Parses the output of lsblk to get the root partition number
+# Example output:
+# NAME        MOUNTPOINT
+# nvme0n1
+# ├─nvme0n1p1 /boot
+# └─nvme0n1p3 /
+# nvme1n1     /data
+#
+# Resulting in:
+# └─nvme0n1p3 / -> nvme0n1p3 -> 3
+ROOT_PARTITION_NUMBER=$(lsblk -no NAME,MOUNTPOINT | grep ' /$' | awk '{print $1;}' | sed 's/.*nvme[0-9]n[0-9]p//g')
+
+if ! [[ "$ROOT_PARTITION_NUMBER" =~ ^[0-9]+$ ]]; then
+  echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
+  exit 1
+fi
+
 if [ -b /dev/nvme1n1 ] ; then
     if [[ "${VOLUME_TYPE}" == "data" ]]; then
         resize2fs /dev/nvme1n1
 
     elif [[ "${VOLUME_TYPE}" == "root" ]] ; then
         PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
         rm -f "${PLACEHOLDER_FL}" || true
-        growpart /dev/nvme0n1 2
-        resize2fs /dev/nvme0n1p2
+        growpart /dev/nvme0n1 "${ROOT_PARTITION_NUMBER}"
+        resize2fs "/dev/nvme0n1p${ROOT_PARTITION_NUMBER}"
         if [[ ! -f "${PLACEHOLDER_FL}" ]] ; then
             fallocate -l50M "${PLACEHOLDER_FL}"
         fi
@@ -26,7 +43,7 @@ if [ -b /dev/nvme1n1 ] ; then
         exit 1
     fi
 else
-    growpart /dev/nvme0n1 2
-    resize2fs /dev/nvme0n1p2
+    growpart /dev/nvme0n1 "${ROOT_PARTITION_NUMBER}"
+    resize2fs "/dev/nvme0n1p${ROOT_PARTITION_NUMBER}"
 fi
 echo "Done resizing disk"

ansible/files/admin_api_scripts/grow_fs.sh

@@ -22,7 +22,7 @@ function waitfor_boot_finished {
 }
 
 function install_packages {
-	apt-get update && sudo apt-get install software-properties-common e2fsprogs -y
+	apt-get update && sudo apt-get install software-properties-common e2fsprogs nfs-common -y
 	add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y
 	ansible-galaxy collection install community.general
 }
@@ -143,4 +143,6 @@ function clean_system {
 
 install_nix
 execute_stage2_playbook
+# we do not want to ship an initialized DB as this is performed as needed
+rm -rf /data/pgdata
 cloud-init clean --logs

ebssurrogate/scripts/qemu-bootstrap-nix.sh

@@ -68,6 +68,11 @@
               buildPgrxExtension_0_12_6 = prev.buildPgrxExtension.override {
                 cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_6;
               };
+
+              buildPgrxExtension_0_12_9 = prev.buildPgrxExtension.override {
+                cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_9;
+              };
+
             })
             (final: prev: {
               postgresql = final.callPackage ./nix/postgresql/default.nix {
@@ -394,6 +399,7 @@
           supabase-groonga = supabase-groonga;
           cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3;
           cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6;
+          cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9;
           # PostgreSQL versions.
           psql_15 = postgresVersions.psql_15;
           psql_orioledb-17 = postgresVersions.psql_orioledb-17;
@@ -559,7 +565,17 @@
               chmod +x $out/bin/dbmate-tool
               wrapProgram $out/bin/dbmate-tool \
                 --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]}
-            '';       
+            '';
+          update-readme = pkgs.runCommand "update-readme" {
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            buildInputs = [ pkgs.nushell ];
+          } ''
+            mkdir -p $out/bin
+            cp ${./nix/tools/update_readme.nu} $out/bin/update-readme
+            chmod +x $out/bin/update-readme
+            wrapProgram $out/bin/update-readme \
+              --prefix PATH : ${pkgs.nushell}/bin
+          '';
         };
 
 
@@ -818,6 +834,7 @@
             pg-restore = mkApp "pg-restore" "pg-restore";
             local-infra-bootstrap = mkApp "local-infra-bootstrap" "local-infra-bootstrap";
             dbmate-tool = mkApp "dbmate-tool" "dbmate-tool";
+            update-readme = mkApp "update-readme" "update-readme";
           };
 
         # 'devShells.default' lists the set of packages that are included in the
@@ -857,6 +874,7 @@
             basePackages.migrate-tool
             basePackages.sync-exts-versions
             dbmate
+            nushell
           ];
           shellHook = ''
             export HISTFILE=.history

flake.nix

@@ -5,34 +5,44 @@ DECLARE
   pgsodium_exists boolean;
   vault_exists boolean;
 BEGIN
-  pgsodium_exists = (
-    select count(*) = 1 
-    from pg_available_extensions 
-    where name = 'pgsodium'
-    and default_version in ('3.1.6', '3.1.7', '3.1.8', '3.1.9')
-  );
-  
-  vault_exists = (
+  IF EXISTS (SELECT FROM pg_available_extensions WHERE name = 'supabase_vault' AND default_version != '0.2.8') THEN
+    CREATE EXTENSION IF NOT EXISTS supabase_vault;
+
+    -- for some reason extension custom scripts aren't run during AMI build, so
+    -- we manually run it here
+    GRANT USAGE ON SCHEMA vault TO postgres WITH GRANT OPTION;
+    GRANT SELECT, DELETE ON vault.secrets, vault.decrypted_secrets TO postgres WITH GRANT OPTION;
+    GRANT EXECUTE ON FUNCTION vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt TO postgres WITH GRANT OPTION;
+  ELSE
+    pgsodium_exists = (
       select count(*) = 1 
       from pg_available_extensions 
-      where name = 'supabase_vault'
-  );
-
-  IF pgsodium_exists 
-  THEN
-    create extension if not exists pgsodium;
-
-    grant pgsodium_keyiduser to postgres with admin option;
-    grant pgsodium_keyholder to postgres with admin option;
-    grant pgsodium_keymaker  to postgres with admin option;
-
-    grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role;
-    grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role;
-    grant execute on function pgsodium.crypto_aead_det_keygen to service_role;
-
-    IF vault_exists
+      where name = 'pgsodium'
+      and default_version in ('3.1.6', '3.1.7', '3.1.8', '3.1.9')
+    );
+    
+    vault_exists = (
+        select count(*) = 1 
+        from pg_available_extensions 
+        where name = 'supabase_vault'
+    );
+  
+    IF pgsodium_exists 
     THEN
-      create extension if not exists supabase_vault;
+      create extension if not exists pgsodium;
+  
+      grant pgsodium_keyiduser to postgres with admin option;
+      grant pgsodium_keyholder to postgres with admin option;
+      grant pgsodium_keymaker  to postgres with admin option;
+  
+      grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role;
+      grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role;
+      grant execute on function pgsodium.crypto_aead_det_keygen to service_role;
+  
+      IF vault_exists
+      THEN
+        create extension if not exists supabase_vault;
+      END IF;
     END IF;
   END IF;
 END $$;

migrations/db/migrations/20221207154255_create_pgsodium_and_vault.sql

@@ -4,7 +4,12 @@ ALTER ROLE authenticated inherit;
 ALTER ROLE anon inherit;
 ALTER ROLE service_role inherit;
 
-GRANT pgsodium_keyholder to service_role;
+DO $$
+BEGIN
+  IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'pgsodium_keyholder') THEN
+    GRANT pgsodium_keyholder to service_role;
+  END IF;
+END $$;
 
 -- migrate:down
 

migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql

@@ -1,25 +1,31 @@
 -- migrate:up
-CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
-RETURNS void
-LANGUAGE plpgsql
-SECURITY DEFINER
-SET search_path TO ''
-AS $function$
+
+DO $$
 BEGIN
-  EXECUTE format(
-    'GRANT SELECT ON pgsodium.key TO %s',
-    masked_role);
+  IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pgsodium') THEN
+    CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
+    RETURNS void
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path TO ''
+    AS $function$
+    BEGIN
+      EXECUTE format(
+        'GRANT SELECT ON pgsodium.key TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
-    masked_role);
+      EXECUTE format(
+        'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT ALL ON %I TO %s',
-    view_name,
-    masked_role);
-  RETURN;
-END
-$function$;
+      EXECUTE format(
+        'GRANT ALL ON %I TO %s',
+        view_name,
+        masked_role);
+      RETURN;
+    END
+    $function$;
+  END IF;
+END $$;
 
 -- migrate:down

migrations/db/migrations/20250218031949_pgsodium_mask_role.sql

@@ -0,0 +1,64 @@
+-- migrate:up
+CREATE OR REPLACE FUNCTION extensions.grant_pg_net_access()
+RETURNS event_trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  IF EXISTS (
+    SELECT 1
+    FROM pg_event_trigger_ddl_commands() AS ev
+    JOIN pg_extension AS ext
+    ON ev.objid = ext.oid
+    WHERE ext.extname = 'pg_net'
+  )
+  THEN
+    IF NOT EXISTS (
+      SELECT 1
+      FROM pg_roles
+      WHERE rolname = 'supabase_functions_admin'
+    )
+    THEN
+      CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION;
+    END IF;
+
+    GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role;
+
+    IF EXISTS (
+      SELECT FROM pg_extension
+      WHERE extname = 'pg_net'
+      -- all versions in use on existing projects as of 2025-02-20
+      -- version 0.12.0 onwards don't need these applied
+      AND extversion IN ('0.2', '0.6', '0.7', '0.7.1', '0.8', '0.10.0', '0.11.0')
+    ) THEN
+      ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER;
+      ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER;
+
+      ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net;
+      ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net;
+
+      REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC;
+      REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC;
+
+      GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role;
+      GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role;
+    END IF;
+  END IF;
+END;
+$$;
+
+DO $$
+BEGIN
+  IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pg_net')
+  THEN
+    ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY INVOKER;
+    ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY INVOKER;
+
+    REVOKE EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM supabase_functions_admin, postgres, anon, authenticated, service_role;
+    REVOKE EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM supabase_functions_admin, postgres, anon, authenticated, service_role;
+
+    GRANT ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO PUBLIC;
+    GRANT ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO PUBLIC;
+  END IF;
+END $$;
+
+-- migrate:down