feat: move stage 1 to nightly build

Author: samrose

.github/workflows/ami-release-nix-single.yml

@@ -61,14 +61,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}

.github/workflows/ami-release-nix.yml

@@ -89,15 +89,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}

.github/workflows/testinfra-ami-build.yml

@@ -108,15 +108,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          AWS_MAX_ATTEMPTS: 10
-          AWS_RETRY_MODE: adaptive
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           AWS_MAX_ATTEMPTS: 10

amazon-arm64-nix.pkr.hcl

@@ -92,6 +92,18 @@ variable "force-deregister" {
   default = false
 }
 
+variable "base-image-nightly" {
+  type    = bool
+  default = false
+  description = "Build as version-agnostic base image for nightly"
+}
+
+variable "build-timestamp" {
+  type    = string
+  default = ""
+  description = "Timestamp for nightly builds"
+}
+
 packer {
   required_plugins {
     amazon = {
@@ -106,7 +118,7 @@ source "amazon-ebssurrogate" "source" {
   profile = "${var.profile}"
   #access_key    = "${var.aws_access_key}"
   #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}"
-  ami_name = "${var.ami_name}-${var.postgres-version}-stage-1"
+  ami_name = var.base-image-nightly ? "${var.ami_name}-base-stage-1-nightly" : "${var.ami_name}-${var.postgres-version}-stage-1"
   ami_virtualization_type = "hvm"
   ami_architecture = "arm64"
   ami_regions   = "${var.ami_regions}"
@@ -170,8 +182,10 @@ source "amazon-ebssurrogate" "source" {
   tags = {
     creator = "packer"
     appType = "postgres"
-    postgresVersion = "${var.postgres-version}-stage1"
+    postgresVersion = var.base-image-nightly ? "base-nightly" : "${var.postgres-version}-stage1"
     sourceSha = "${var.git-head-version}"
+    buildTimestamp = var.base-image-nightly ? "${var.build-timestamp}" : ""
+    buildType = var.base-image-nightly ? "nightly" : "release"
   }
 
   communicator = "ssh"

ansible/playbook.yml

@@ -46,13 +46,13 @@
         - install-gotrue
         - install-supabase-internal
       when: debpkg_mode or nixpkg_mode
-  
+
     - name: Install PostgREST
       import_tasks: tasks/setup-postgrest.yml
       tags:
         - install-postgrest
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or stage2_nix
 
     - name: Install Envoy
       import_tasks: tasks/setup-envoy.yml

ansible/tasks/setup-postgrest.yml

@@ -11,7 +11,7 @@
 - name: PostgREST - add Postgres PPA main
   ansible.builtin.apt_repository:
     filename: 'postgresql-pgdg'
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main"
     state: 'present'
 
 - name: PostgREST - install system dependencies
@@ -36,7 +36,7 @@
 
 - name: PostgREST - remove Postgres PPA
   ansible.builtin.apt_repository:
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main"
     state: 'absent'
 
 - name: postgis - ensure dependencies do not get autoremoved

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -303,11 +303,10 @@ tee /etc/ansible/ansible.cfg <<EOF
 callbacks_enabled = timer, profile_tasks, profile_roles
 EOF
 	# Run Ansible playbook
-	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp 
+	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \
 	--extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \
-	--extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \
 	$ARGS
 }
 

stage2-nix-psql.pkr.hcl

@@ -66,12 +66,12 @@ source "amazon-ebs" "ubuntu" {
   region        = "${var.region}"
   source_ami_filter {
     filters = {
-      name   = "${var.ami_name}-${var.postgres-version}-stage-1"
+      name   = "${var.ami_name}-base-stage-1-nightly"
       root-device-type    = "ebs"
       virtualization-type = "hvm"
     }
     most_recent = true
-    owners      = ["amazon", "self"]
+    owners      = ["self"]
   }
 
   communicator = "ssh"