diff --git a/.github/workflows/pr-int-test-terraform.yml b/.github/workflows/pr-int-test-terraform.yml index 2c4ea3f..45e771d 100644 --- a/.github/workflows/pr-int-test-terraform.yml +++ b/.github/workflows/pr-int-test-terraform.yml @@ -37,10 +37,6 @@ jobs: environment: dev gcp_workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} gcp_service_account: ${{ vars.SERVICE_ACCOUNT }} - - name: Authenticate with GKE - uses: entur/gha-meta/.github/actions/k8s-auth@v1 - with: - environment: dev - name: Run recursive Go integration tests working-directory: test/integration run: go test -tags=integration ./... -timeout 30m diff --git a/README.md b/README.md index 445e436..ef8b80c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ A PostgreSQL module that uses the [init module](https://github.com/entur/terrafo ```terraform module "postgresql" { source = "github.com/entur/terraform-google-sql-db//modules/postgresql?ref=v1.7.4" + database_version = "POSTGRES_18" # Use the latest postgres version ... } ``` @@ -42,13 +43,12 @@ If a desired machine size and/or availability type is not explicitly set, defaul | non-production | Shared vCPU | <1 | 600 MB | No | | production | Dedicated vCPU | 1 | 3840 MB | Yes | - ### Edition + Changing this will cause a database restart on existing instances. Choosing **Enterprise Plus** (`ENTERPRISE_PLUS`) over **Enterprise** (`ENTERPRISE`) can also increase costs. Carefully evaluate your requirements before choosing this edition. Ensure you select the appropriate tier for your use case. For more details about instance editions, refer to the [official documentation](https://cloud.google.com/sql/docs/postgres/instance-settings). - ### Sizing To specify the size of a database instance, supply the `cpu` and `memory` attributes in `var.machine_size` (recommended): @@ -82,8 +82,6 @@ Run local integration tests in test/integration folder. > Only Team-Plattform has rights to do this locally. > Contributors can create a PR which will run the tests as well. -Make sure you are connected to the dev kubernetes cluster in GKE (kub-ent-dev-001) - ```bash cd test/integration go test -v -tags=integration -timeout 30m ./... diff --git a/examples/minimal/main.tf b/examples/minimal/main.tf index 4a0985b..69b015e 100644 --- a/examples/minimal/main.tf +++ b/examples/minimal/main.tf @@ -11,8 +11,9 @@ module "init" { # ci: x-release-please-start-version module "postgresql" { - source = "github.com/entur/terraform-google-sql-db//modules/postgresql?ref=v1.7.4" - init = module.init - databases = ["my-database"] + source = "github.com/entur/terraform-google-sql-db//modules/postgresql?ref=v1.7.4" + database_version = "POSTGRES_18" + init = module.init + databases = ["my-database"] } # ci: x-release-please-end diff --git a/modules/postgresql/README.md b/modules/postgresql/README.md index cf27f09..046a51f 100644 --- a/modules/postgresql/README.md +++ b/modules/postgresql/README.md @@ -5,9 +5,8 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >=0.13.2 | +| [terraform](#requirement\_terraform) | >=1.3 | | [google](#requirement\_google) | >=5 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | | [random](#requirement\_random) | >=3.6.2 | ## Providers @@ -15,7 +14,6 @@ | Name | Version | |------|---------| | [google](#provider\_google) | >=5 | -| [kubernetes](#provider\_kubernetes) | ~> 2.0 | | [random](#provider\_random) | >=3.6.2 | ## Modules @@ -34,9 +32,6 @@ No modules. | [google_sql_database_instance.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | | [google_sql_user.additional_users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | | [google_sql_user.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | -| [kubernetes_config_map.main_psql_connection](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | -| [kubernetes_secret.additional_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | -| [kubernetes_secret.main_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | | [random_integer.additional_users_password_length](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | | [random_integer.password_length](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | | [random_password.additional_users_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | @@ -46,17 +41,16 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [database\_version](#input\_database\_version) | The PostgreSQL version (see https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version-1). | `string` | n/a | yes | | [databases](#input\_databases) | Names of databases to create. | `list(string)` | n/a | yes | | [init](#input\_init) | Entur init module output. https://github.com/entur/terraform-google-init. Used to determine application name, application project, labels, and resource names. | 
object({
    app = object({
      id         = string
      name       = string
      owner      = string
      project_id = string
    })
    networks = object({
      project_id = string
      vpc_id     = string
    })
    environment   = string
    labels        = map(string)
    is_production = bool
  })
| n/a | yes | | [add\_additional\_secret\_manager\_credentials](#input\_add\_additional\_secret\_manager\_credentials) | Set to false to not store additional database credentials in secret manager | `bool` | `true` | no | | [add\_main\_secret\_manager\_credentials](#input\_add\_main\_secret\_manager\_credentials) | Set to false to not store main database credentials in secret manager | `bool` | `true` | no | -| [additional\_users](#input\_additional\_users) | A list of user-names in addition to the main user that should be created. | 
map(object({
    username                 = string
    create_kubernetes_secret = bool
  }))
| `{}` | no | +| [additional\_users](#input\_additional\_users) | A list of user-names in addition to the main user that should be created. | 
map(object({
    username = string
  }))
| `{}` | no | | [authorized\_networks](#input\_authorized\_networks) | Values for authorized\_networks, list of objects with name and simple strings of IPs or CIDRs. Ex: {name: supermachine, value: 35.90.103.132/30} or {name: rogersmachine, value: 35.90.103.132} | 
list(object({
    value = string
    name  = string
  }))
| `[]` | no | | [availability\_type](#input\_availability\_type) | Whether to enable high availability with automatic failover over multiple zones ('REGIONAL') vs. single zone ('ZONAL'). | `string` | `null` | no | | [backup\_start\_time](#input\_backup\_start\_time) | Start time in UTC for daily backup job in the format HH:MM. This is the start time of a 4 hour time window. | `string` | `"00:00"` | no | -| [create\_kubernetes\_resources](#input\_create\_kubernetes\_resources) | Optionally disables creating k8s resources -psql-connection and -psql-credentials. Can be used to avoid overwriting existing resources on database creation. | `bool` | `true` | no | | [database\_flags](#input\_database\_flags) | Override default CloudSQL configuration by specifying database-flags. Note that some flags require installing extensions. (see https://cloud.google.com/sql/docs/postgres/extensions#installing-an-extension). | 
map(object({
    name  = string
    value = string
  }))
| `{}` | no | -| [database\_version](#input\_database\_version) | The PostgreSQL version (see https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version). | `string` | `"POSTGRES_14"` | no | | [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `bool` | `null` | no | | [disable\_offsite\_backup](#input\_disable\_offsite\_backup) | Disable offsite backup for this instance. Offsite backup is only applied to production environments. | `bool` | `false` | no | | [disk\_autoresize](#input\_disk\_autoresize) | Whether to enable auto-resizing of the storage disk. | `bool` | `true` | no | @@ -66,7 +60,7 @@ No modules. | [enable\_private\_network](#input\_enable\_private\_network) | Whether to enable private network connectivity for the Cloud SQL instance. Immutable after it has been enabled. | `bool` | `false` | no | | [generation](#input\_generation) | The generation (aka serial no.) of the instance. Starts at 1, ends at 999. Will be padded with leading zeros. | `number` | `1` | no | | [instance\_edition](#input\_instance\_edition) | Override the default instance edition (`ENTERPRISE` or `ENTERPRISE_PLUS`). | `string` | `"ENTERPRISE"` | no | -| [machine\_size](#input\_machine\_size) | Map of the database instance CPU count (cpu) and memory sizes in MB (memory). Optionally, set a tier override (tier). See README.md for examples. | `map(any)` | `null` | no | +| [machine\_size](#input\_machine\_size) | Map of the database instance CPU count (cpu) and memory sizes in MB (memory). Optionally, set a tier override (tier). See README.md for examples. | 
object({
    tier   = optional(string)
    cpu    = optional(number)
    memory = optional(number)
  })
| `null` | no | | [maintenance\_window](#input\_maintenance\_window) | The day of the week (1-7), and hour of the day (0-24) in UTC to perform database instance maintenance. This is the start time of the one hour maintinance window. | 
object({
    day  = number
    hour = number
  })
| 
{
  "day": 2,
  "hour": 0
}
| no | | [point\_in\_time\_recovery\_enabled](#input\_point\_in\_time\_recovery\_enabled) | Whether to enable PITR on database instance. Requires enable\_backup to be true. | `bool` | `true` | no | | [query\_insights\_config](#input\_query\_insights\_config) | Advanced config for Query Insights. | 
object({
    query_string_length     = number
    record_application_tags = bool
    record_client_address   = bool
  })
| 
{
  "query_string_length": 1024,
  "record_application_tags": false,
  "record_client_address": false
}
| no | @@ -86,6 +80,5 @@ No modules. | [databases](#output\_databases) | Databases created on this instance. | | [init](#output\_init) | The output of the consumed init module. | | [instance](#output\_instance) | The database instance output, as described in https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance. | -| [kubernetes\_namespace](#output\_kubernetes\_namespace) | Name of the Kubernetes namespace where config maps and secrets are deployed. | | [user](#output\_user) | Map containing the username and password of the default application user. | \ No newline at end of file diff --git a/modules/postgresql/main.tf b/modules/postgresql/main.tf index d3e6d57..8f1f39c 100644 --- a/modules/postgresql/main.tf +++ b/modules/postgresql/main.tf @@ -2,20 +2,20 @@ locals { # If updated, reflect changes in README.md default_tiers = { prod = "db-custom-1-3840" - non-prod = "db-f1-micro" + non-prod = "db-custom-1-3840" } user_name = var.user_name != null ? var.user_name : var.init.app.id retained_backups = var.retained_backups != null ? var.retained_backups : var.init.is_production ? 30 : 7 deletion_protection = var.deletion_protection != null ? var.deletion_protection : var.init.is_production ? true : false availability_type = var.availability_type != null ? var.availability_type : var.init.is_production ? "REGIONAL" : "ZONAL" - machine_size = var.machine_size != null ? try(var.machine_size.tier, "db-custom-${var.machine_size.cpu}-${var.machine_size.memory}") : var.init.is_production ? local.default_tiers.prod : local.default_tiers.non-prod + default_machine_size = var.init.is_production ? local.default_tiers.prod : local.default_tiers.non-prod + machine_size = var.machine_size != null ? var.machine_size.tier != null ? var.machine_size.tier : "db-custom-${var.machine_size.cpu}-${var.machine_size.memory}" : local.default_machine_size offsite_backup_label = var.disable_offsite_backup == true && var.init.is_production ? { offsite_enabled = false } : {} # Add the label for opt-out of offsite backup in prod environments when disable_offsite_backup is true labels = merge(var.init.labels, local.offsite_backup_label) generation = format("%03d", var.generation) disk_autoresize_limit = var.disk_autoresize_limit != null ? var.disk_autoresize_limit : var.init.is_production ? 500 : 50 additional_users = { for key, value in var.additional_users : key => value if value.username != local.user_name } - additional_user_credentials = !var.create_kubernetes_resources ? {} : { for key, value in local.additional_users : key => value if value.create_kubernetes_secret } additional_sm_user_credentials = !var.add_additional_secret_manager_credentials ? {} : { for key, value in local.additional_users : key => value if var.add_additional_secret_manager_credentials } } @@ -108,21 +108,6 @@ resource "google_sql_user" "main" { password = random_password.password.result } -resource "kubernetes_config_map" "main_psql_connection" { - count = var.create_kubernetes_resources ? 1 : 0 - depends_on = [ - google_sql_database_instance.main - ] - metadata { - name = "${var.init.app.name}-psql-connection" - namespace = var.init.app.name - labels = var.init.labels - } - - data = { - INSTANCES = "${google_sql_database_instance.main.connection_name}=tcp:5432" - } -} resource "random_integer" "password_length" { min = 32 max = 64 @@ -133,24 +118,6 @@ resource "random_password" "password" { override_special = "!#$%&*()-_=+[]{}<>:?" } -resource "kubernetes_secret" "main_database_credentials" { - count = var.create_kubernetes_resources ? 1 : 0 - depends_on = [ - google_sql_database_instance.main - ] - metadata { - name = "${var.init.app.name}-psql-credentials" - namespace = var.init.app.name - labels = var.init.labels - } - data = { - PGHOST = "localhost" - PGPORT = 5432 - PGUSER = google_sql_user.main.name - PGPASSWORD = random_password.password.result - } -} - resource "google_sql_user" "additional_users" { for_each = local.additional_users name = each.value.username @@ -172,28 +139,8 @@ resource "random_password" "additional_users_password" { override_special = "!#$%&*()-_=+[]{}<>:?" } -resource "kubernetes_secret" "additional_database_credentials" { - for_each = local.additional_user_credentials - depends_on = [ - google_sql_database_instance.main - ] - metadata { - name = "${var.init.app.name}-${each.value.username}-psql-credentials" - namespace = var.init.app.name - labels = var.init.labels - } - data = { - PGHOST = "localhost" - PGPORT = 5432 - PGUSER = google_sql_user.additional_users[each.key].name - PGPASSWORD = random_password.additional_users_password[each.key].result - } -} - locals { credentials = { - HOST = "localhost", - PORT = 5432, USER = google_sql_user.main.name, PASSWORD = random_password.password.result, INSTANCES = google_sql_database_instance.main.connection_name diff --git a/modules/postgresql/outputs.tf b/modules/postgresql/outputs.tf index 6c5bdc7..4869791 100644 --- a/modules/postgresql/outputs.tf +++ b/modules/postgresql/outputs.tf @@ -32,8 +32,3 @@ output "databases" { description = "Databases created on this instance." value = var.databases } - -output "kubernetes_namespace" { - description = "Name of the Kubernetes namespace where config maps and secrets are deployed." - value = var.create_kubernetes_resources ? kubernetes_secret.main_database_credentials[0].metadata[0].namespace : null -} diff --git a/modules/postgresql/variables.tf b/modules/postgresql/variables.tf index 48ab38a..9399347 100644 --- a/modules/postgresql/variables.tf +++ b/modules/postgresql/variables.tf @@ -25,12 +25,11 @@ variable "region" { variable "machine_size" { description = "Map of the database instance CPU count (cpu) and memory sizes in MB (memory). Optionally, set a tier override (tier). See README.md for examples." - # type = object({ - # tier = optional(string) # Optional attributes not supported until Terraform 1.3 - # cpu = number - # memory = number - # }) - type = map(any) # map(any) forces map to be same type of the any type + type = object({ + tier = optional(string) + cpu = optional(number) + memory = optional(number) + }) #default = { #tier = "db-f1-micro" #cpu = 1 @@ -38,15 +37,15 @@ variable "machine_size" { #} default = null validation { - condition = var.machine_size != null ? try(var.machine_size.cpu, 1) >= 1 && try(var.machine_size.cpu, 1) <= 96 : true + condition = var.machine_size == null || try(var.machine_size.tier, null) != null || try(var.machine_size.cpu, null) == null ? true : var.machine_size.cpu >= 1 && var.machine_size.cpu <= 96 error_message = "CPU must be a whole number between 1 and 96." } validation { - condition = var.machine_size != null ? try(var.machine_size.memory, 256) % 256 == 0 && try(var.machine_size.memory, 3840) >= 3840 && try(var.machine_size.memory, 3840) <= 13312 : true - error_message = "Memory must be divisable by 256, and between 3840 and 13312." + condition = var.machine_size == null || try(var.machine_size.tier, null) != null || try(var.machine_size.memory, null) == null ? true : var.machine_size.memory % 256 == 0 && var.machine_size.memory >= 3840 && var.machine_size.memory <= 13312 + error_message = "Memory must be divisible by 256, larger than 3840, and at most 13312." } validation { - condition = var.machine_size != null ? (try(var.machine_size.tier, null) != null) && (try(var.machine_size.cpu, null) == null || try(var.machine_size.memory, null) == null) || (try(var.machine_size.tier, null) == null) && (try(var.machine_size.cpu, null) != null && try(var.machine_size.memory, null) != null) : true + condition = var.machine_size == null ? true : (try(var.machine_size.tier, null) != null && try(var.machine_size.cpu, null) == null && try(var.machine_size.memory, null) == null) || (try(var.machine_size.tier, null) == null && try(var.machine_size.cpu, null) != null && try(var.machine_size.memory, null) != null) error_message = "Either supply desired resource limits for cpu and memory (recommended), or specify a tier." } } @@ -68,9 +67,8 @@ variable "databases" { } variable "database_version" { - description = "The PostgreSQL version (see https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version)." + description = "The PostgreSQL version (see https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version-1)." type = string - default = "POSTGRES_14" validation { condition = can(regex("^POSTGRES_[1-9][0-9]$", var.database_version)) @@ -87,11 +85,10 @@ variable "user_name" { variable "additional_users" { description = "A list of user-names in addition to the main user that should be created." type = map(object({ - username = string - create_kubernetes_secret = bool + username = string })) default = {} - # validate username since it is used in k8s resource-names + # validate username validation { condition = length([ for user in values(var.additional_users) : true if can(regex("^[0-9a-z-]+$", user.username)) @@ -215,12 +212,6 @@ variable "database_flags" { default = {} } -variable "create_kubernetes_resources" { - description = "Optionally disables creating k8s resources -psql-connection and -psql-credentials. Can be used to avoid overwriting existing resources on database creation." - type = bool - default = true -} - variable "authorized_networks" { description = "Values for authorized_networks, list of objects with name and simple strings of IPs or CIDRs. Ex: {name: supermachine, value: 35.90.103.132/30} or {name: rogersmachine, value: 35.90.103.132}" type = list(object({ diff --git a/modules/postgresql/versions.tf b/modules/postgresql/versions.tf index 185e414..82789ab 100644 --- a/modules/postgresql/versions.tf +++ b/modules/postgresql/versions.tf @@ -1,9 +1,5 @@ terraform { required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.0" - } google = { source = "hashicorp/google" version = ">=5" @@ -13,5 +9,5 @@ terraform { version = ">=3.6.2" } } - required_version = ">=0.13.2" + required_version = ">=1.3" } diff --git a/test/fixtures/postgres-replica/main.tf b/test/fixtures/postgres-replica/main.tf index 0c7f1a9..8d32a50 100644 --- a/test/fixtures/postgres-replica/main.tf +++ b/test/fixtures/postgres-replica/main.tf @@ -22,6 +22,7 @@ module "postgresql" { source = "../../../modules/postgresql" init = module.init generation = random_integer.random_database_generation.result + database_version = "POSTGRES_18" availability_type = "REGIONAL" databases = ["database-1", "database-2"] deletion_protection = false @@ -32,8 +33,8 @@ module "postgresql" { } additional_users = { - user1 = { username = "user1", create_kubernetes_secret = false }, - user2 = { username = "user2", create_kubernetes_secret = false } + user1 = { username = "user1" }, + user2 = { username = "user2" } } } diff --git a/test/integration/postgresql_replica/postgresql_replica_test.go b/test/integration/postgresql_replica/postgresql_replica_test.go index 2bce88e..4fa54f0 100644 --- a/test/integration/postgresql_replica/postgresql_replica_test.go +++ b/test/integration/postgresql_replica/postgresql_replica_test.go @@ -26,16 +26,16 @@ func TestPostgreSqlReplicaModule(t *testing.T) { op := gcloud.Runf(t, "sql instances describe %s --project %s", instanceNames[0], projectId) assert.Equal(1, len(op.Get("replicaNames").Array()), "Expected 1 replicas") - + instanceNames = append(instanceNames, utils.GetResultStrSlice(op.Get("replicaNames").Array())...) for _, instance := range instanceNames { op = gcloud.Runf(t, "sql instances describe %s --project %s", instance, projectId) // assert general database settings - assert.Equal("POSTGRES_14", op.Get("databaseVersion").String(), "Expected POSTGRES_14 databaseVersion") + assert.Equal("POSTGRES_18", op.Get("databaseVersion").String(), "Expected POSTGRES_18 databaseVersion") assert.Equal("RUNNABLE", op.Get("state").String(), "Expected RUNNABLE state") - assert.Equal("europe-west1", op.Get("region").String(), "Expected europe-west1 region") + assert.Equal("europe-west1", op.Get("region").String(), "Expected europe-west1 region") // master specific validation if instance == cloudSqlT.GetStringOutput("instance_name") {