From 775c2ce9d2b669a1d9465d517fc18b86598e3ce9 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 26 Jul 2023 09:27:07 +0200 Subject: [PATCH 01/11] Propagate tags from ECS services. --- modules/metadata-service/ecs.tf | 1 + modules/ui/ecs_ui_backend.tf | 1 + modules/ui/ecs_ui_static.tf | 1 + 3 files changed, 3 insertions(+) diff --git a/modules/metadata-service/ecs.tf b/modules/metadata-service/ecs.tf index 1abd89d..4df73ea 100644 --- a/modules/metadata-service/ecs.tf +++ b/modules/metadata-service/ecs.tf @@ -94,5 +94,6 @@ resource "aws_ecs_service" "this" { ignore_changes = [desired_count] } + propagate_tags = "SERVICE" tags = var.standard_tags } diff --git a/modules/ui/ecs_ui_backend.tf b/modules/ui/ecs_ui_backend.tf index 61b855a..fcf8864 100644 --- a/modules/ui/ecs_ui_backend.tf +++ b/modules/ui/ecs_ui_backend.tf @@ -74,5 +74,6 @@ resource "aws_ecs_service" "ui_backend" { ignore_changes = [desired_count] } + propagate_tags = "SERVICE" tags = var.standard_tags } diff --git a/modules/ui/ecs_ui_static.tf b/modules/ui/ecs_ui_static.tf index dfa99f3..613dcb7 100644 --- a/modules/ui/ecs_ui_static.tf +++ b/modules/ui/ecs_ui_static.tf @@ -66,5 +66,6 @@ resource "aws_ecs_service" "ui_static" { ignore_changes = [desired_count] } + propagate_tags = "SERVICE" tags = var.standard_tags } From 81244876fbcd7d754ab5dae5d597d1f805f7e584 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 28 Aug 2023 16:22:33 +0200 Subject: [PATCH 02/11] Expose DB instance type. --- main.tf | 2 ++ variables.tf | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/main.tf b/main.tf index f789071..61ae727 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,8 @@ module "metaflow-datastore" { source = "./modules/datastore" + db_instance_type = var.db_instance_type + force_destroy_s3_bucket = var.force_destroy_s3_bucket resource_prefix = local.resource_prefix diff --git a/variables.tf b/variables.tf index 9647f5b..406c15e 100644 --- a/variables.tf +++ b/variables.tf @@ -176,3 +176,9 @@ variable "force_destroy_s3_bucket" { description = "Empty S3 bucket before destroying via terraform destroy" default = false } + +variable "db_instance_type" { + type = string + description = "RDS instance type to launch for PostgresQL database." + default = "db.t2.small" +} From 936a93dfac231df1ed8a7cd98fc7578c47dad778 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 9 Feb 2024 13:03:36 +0100 Subject: [PATCH 03/11] Support specifying NLB ARN. --- main.tf | 1 + modules/metadata-service/ec2.tf | 14 ++++++++++++-- modules/metadata-service/variables.tf | 5 +++++ variables.tf | 6 ++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 61ae727..2f21adb 100644 --- a/main.tf +++ b/main.tf @@ -39,6 +39,7 @@ module "metaflow-metadata-service" { subnet2_id = var.subnet2_id vpc_cidr_blocks = var.vpc_cidr_blocks with_public_ip = var.with_public_ip + nlb_arn = var.nlb_arn standard_tags = var.tags } diff --git a/modules/metadata-service/ec2.tf b/modules/metadata-service/ec2.tf index 64ec728..e1ca357 100644 --- a/modules/metadata-service/ec2.tf +++ b/modules/metadata-service/ec2.tf @@ -44,6 +44,16 @@ resource "aws_security_group" "metadata_service_security_group" { ) } +resource "aws_lb" "this" { + count = var.nlb_arn == "" ? 1 : 0 + name = "${var.resource_prefix}nlb${var.resource_suffix}" + internal = true + load_balancer_type = "network" + subnets = [var.subnet1_id, var.subnet2_id] + + tags = var.standard_tags +} + resource "aws_lb" "this" { name = "${var.resource_prefix}nlb${var.resource_suffix}" internal = true @@ -89,7 +99,7 @@ resource "aws_lb_target_group" "db_migrate" { } resource "aws_lb_listener" "this" { - load_balancer_arn = aws_lb.this.arn + load_balancer_arn = var.nlb_arn == "" ? aws_lb.this.arn : var.nlb_arn port = "80" protocol = "TCP" @@ -100,7 +110,7 @@ resource "aws_lb_listener" "this" { } resource "aws_lb_listener" "db_migrate" { - load_balancer_arn = aws_lb.this.arn + load_balancer_arn = var.nlb_arn == "" ? aws_lb.this.arn : var.nlb_arn port = "8082" protocol = "TCP" diff --git a/modules/metadata-service/variables.tf b/modules/metadata-service/variables.tf index a0aae1b..6160a9c 100644 --- a/modules/metadata-service/variables.tf +++ b/modules/metadata-service/variables.tf @@ -120,3 +120,8 @@ variable "with_public_ip" { type = bool description = "Enable public IP assignment for the Metadata Service. Typically you want this to be set to true if using public subnets as subnet1_id and subnet2_id, and false otherwise" } + +variable "nlb_arn" { + type = string + description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled." +} diff --git a/variables.tf b/variables.tf index 406c15e..0fab036 100644 --- a/variables.tf +++ b/variables.tf @@ -182,3 +182,9 @@ variable "db_instance_type" { description = "RDS instance type to launch for PostgresQL database." default = "db.t2.small" } + +variable "nlb_arn" { + type = string + description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled." + default = "" +} From 682ac4ef388b2bdffe81eb43dfc070c5e70a5a84 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 9 Feb 2024 13:57:39 +0100 Subject: [PATCH 04/11] Fixed duplicated resource. --- modules/metadata-service/ec2.tf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/metadata-service/ec2.tf b/modules/metadata-service/ec2.tf index e1ca357..fe63412 100644 --- a/modules/metadata-service/ec2.tf +++ b/modules/metadata-service/ec2.tf @@ -54,15 +54,6 @@ resource "aws_lb" "this" { tags = var.standard_tags } -resource "aws_lb" "this" { - name = "${var.resource_prefix}nlb${var.resource_suffix}" - internal = true - load_balancer_type = "network" - subnets = [var.subnet1_id, var.subnet2_id] - - tags = var.standard_tags -} - resource "aws_lb_target_group" "this" { name = "${var.resource_prefix}mdtg${var.resource_suffix}" port = 8080 From 73929e64d183730844584186d83a97bfd94eb1be Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 9 Feb 2024 14:50:37 +0100 Subject: [PATCH 05/11] Introduced nlb_dns_name variable and fixed references. --- main.tf | 1 + modules/metadata-service/api-gateway.tf | 2 +- modules/metadata-service/ec2.tf | 4 ++-- modules/metadata-service/lambda.tf | 2 +- modules/metadata-service/outputs.tf | 4 ++-- modules/metadata-service/variables.tf | 7 ++++++- variables.tf | 8 +++++++- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 2f21adb..f7eb2ee 100644 --- a/main.tf +++ b/main.tf @@ -40,6 +40,7 @@ module "metaflow-metadata-service" { vpc_cidr_blocks = var.vpc_cidr_blocks with_public_ip = var.with_public_ip nlb_arn = var.nlb_arn + nlb_dns_name = var.nlb_dns_name standard_tags = var.tags } diff --git a/modules/metadata-service/api-gateway.tf b/modules/metadata-service/api-gateway.tf index f01cee9..76e50a6 100644 --- a/modules/metadata-service/api-gateway.tf +++ b/modules/metadata-service/api-gateway.tf @@ -56,7 +56,7 @@ resource "aws_api_gateway_resource" "db" { resource "aws_api_gateway_vpc_link" "this" { count = var.enable_api_gateway ? 1 : 0 name = "${var.resource_prefix}vpclink${var.resource_suffix}" - target_arns = [aws_lb.this.arn] + target_arns = [var.nlb_arn == "" ? aws_lb.this[0].arn : var.nlb_arn] tags = var.standard_tags } diff --git a/modules/metadata-service/ec2.tf b/modules/metadata-service/ec2.tf index fe63412..418a444 100644 --- a/modules/metadata-service/ec2.tf +++ b/modules/metadata-service/ec2.tf @@ -90,7 +90,7 @@ resource "aws_lb_target_group" "db_migrate" { } resource "aws_lb_listener" "this" { - load_balancer_arn = var.nlb_arn == "" ? aws_lb.this.arn : var.nlb_arn + load_balancer_arn = var.nlb_arn == "" ? aws_lb.this[0].arn : var.nlb_arn port = "80" protocol = "TCP" @@ -101,7 +101,7 @@ resource "aws_lb_listener" "this" { } resource "aws_lb_listener" "db_migrate" { - load_balancer_arn = var.nlb_arn == "" ? aws_lb.this.arn : var.nlb_arn + load_balancer_arn = var.nlb_arn == "" ? aws_lb.this[0].arn : var.nlb_arn port = "8082" protocol = "TCP" diff --git a/modules/metadata-service/lambda.tf b/modules/metadata-service/lambda.tf index e4375fc..a214059 100644 --- a/modules/metadata-service/lambda.tf +++ b/modules/metadata-service/lambda.tf @@ -126,7 +126,7 @@ resource "aws_lambda_function" "db_migrate_lambda" { environment { variables = { - MD_LB_ADDRESS = "http://${aws_lb.this.dns_name}:8082" + MD_LB_ADDRESS = "http://${var.nlb_dns_name == "" ? aws_lb.this[0].dns_name : var.nlb_dns_name}:8082" } } diff --git a/modules/metadata-service/outputs.tf b/modules/metadata-service/outputs.tf index 5b93e41..975e1cf 100644 --- a/modules/metadata-service/outputs.tf +++ b/modules/metadata-service/outputs.tf @@ -1,5 +1,5 @@ output "METAFLOW_SERVICE_INTERNAL_URL" { - value = "http://${aws_lb.this.dns_name}/" + value = "http://${var.nlb_dns_name == "" ? aws_lb.this[0].dns_name : var.nlb_dns_name}/" description = "URL for Metadata Service (Accessible in VPC)" } @@ -34,6 +34,6 @@ output "metadata_svc_ecs_task_role_arn" { } output "network_load_balancer_dns_name" { - value = aws_lb.this.dns_name + value = var.nlb_dns_name == "" ? aws_lb.this[0].dns_name : var.nlb_dns_name description = "The DNS addressable name for the Network Load Balancer that accepts requests and forwards them to our Fargate MetaData service instance(s)" } diff --git a/modules/metadata-service/variables.tf b/modules/metadata-service/variables.tf index 6160a9c..ebe7948 100644 --- a/modules/metadata-service/variables.tf +++ b/modules/metadata-service/variables.tf @@ -123,5 +123,10 @@ variable "with_public_ip" { variable "nlb_arn" { type = string - description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled." + description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled. Must be provided together with nlb_dns_name." +} + +variable "nlb_dns_name" { + type = string + description = "The DNS name of the network load balancer to use for Metaflow. Must be provided together with nlb_arn." } diff --git a/variables.tf b/variables.tf index 0fab036..80acbff 100644 --- a/variables.tf +++ b/variables.tf @@ -185,6 +185,12 @@ variable "db_instance_type" { variable "nlb_arn" { type = string - description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled." + description = "The ARN of the network load balancer to use for Metaflow. A new resource will be created if unfilled. Must be provided together with nlb_dns_name." + default = "" +} + +variable "nlb_dns_name" { + type = string + description = "The DNS name of the network load balancer to use for Metaflow. Must be provided together with nlb_arn." default = "" } From 171ee08b138ec073997b04df4d736aa82f0611d9 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 9 Feb 2024 14:56:29 +0100 Subject: [PATCH 06/11] Fixed references to dns_name in api-gateway. --- modules/metadata-service/api-gateway.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/metadata-service/api-gateway.tf b/modules/metadata-service/api-gateway.tf index 76e50a6..71dbf41 100644 --- a/modules/metadata-service/api-gateway.tf +++ b/modules/metadata-service/api-gateway.tf @@ -103,7 +103,7 @@ resource "aws_api_gateway_integration" "this" { } type = "HTTP_PROXY" - uri = "http://${aws_lb.this.dns_name}/{proxy}" + uri = "http://${var.nlb_dns_name == "" ? aws_lb.this[0].dns_name : var.nlb_dns_name}/{proxy}" integration_http_method = "ANY" passthrough_behavior = "WHEN_NO_MATCH" connection_type = "VPC_LINK" @@ -118,7 +118,7 @@ resource "aws_api_gateway_integration" "db" { type = "HTTP_PROXY" - uri = "http://${aws_lb.this.dns_name}:8082/db_schema_status" + uri = "http://${var.nlb_dns_name == "" ? aws_lb.this[0].dns_name : var.nlb_dns_name}:8082/db_schema_status" integration_http_method = "GET" passthrough_behavior = "WHEN_NO_MATCH" connection_type = "VPC_LINK" From a276d0bca4fa1a4b3ae93b1c63765217bdc7866f Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 12 Feb 2024 16:24:24 +0100 Subject: [PATCH 07/11] Upgrade Postgres to version 13. --- modules/datastore/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/datastore/variables.tf b/modules/datastore/variables.tf index 53b6c39..42892c2 100644 --- a/modules/datastore/variables.tf +++ b/modules/datastore/variables.tf @@ -11,7 +11,7 @@ variable "db_engine" { variable "db_engine_version" { type = string - default = "11" + default = "13" } variable "db_name" { From d3289ec974a6cd20e31642daf1bb6c54f83e2751 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 12 Feb 2024 16:30:21 +0100 Subject: [PATCH 08/11] allow_major_version_upgrade = true for RDS instance. --- modules/datastore/rds.tf | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/modules/datastore/rds.tf b/modules/datastore/rds.tf index cddfa76..a41f235 100644 --- a/modules/datastore/rds.tf +++ b/modules/datastore/rds.tf @@ -96,24 +96,25 @@ resource "aws_rds_cluster_instance" "cluster_instances" { Define rds db instance. */ resource "aws_db_instance" "this" { - count = local.use_aurora ? 0 : 1 - publicly_accessible = false - allocated_storage = 20 # Allocate 20GB - storage_type = "gp2" # general purpose SSD - storage_encrypted = true - kms_key_id = aws_kms_key.rds.arn - engine = var.db_engine - engine_version = var.db_engine_version - instance_class = var.db_instance_type # Hardware configuration - identifier = "${var.resource_prefix}${var.db_name}${var.resource_suffix}" # used for dns hostname needs to be customer unique in region - db_name = var.db_name # unique id for CLI commands (name of DB table which is why we're not adding the prefix as no conflicts will occur and the API expects this table name) - username = var.db_username - password = random_password.this.result - db_subnet_group_name = aws_db_subnet_group.this.id - max_allocated_storage = 1000 # Upper limit of automatic scaled storage - multi_az = true # Multiple availability zone? - final_snapshot_identifier = "${var.resource_prefix}${var.db_name}-final-snapshot${var.resource_suffix}-${random_pet.final_snapshot_id.id}" # Snapshot upon delete - vpc_security_group_ids = [aws_security_group.rds_security_group.id] + count = local.use_aurora ? 0 : 1 + publicly_accessible = false + allocated_storage = 20 # Allocate 20GB + storage_type = "gp2" # general purpose SSD + storage_encrypted = true + kms_key_id = aws_kms_key.rds.arn + engine = var.db_engine + engine_version = var.db_engine_version + instance_class = var.db_instance_type # Hardware configuration + identifier = "${var.resource_prefix}${var.db_name}${var.resource_suffix}" # used for dns hostname needs to be customer unique in region + db_name = var.db_name # unique id for CLI commands (name of DB table which is why we're not adding the prefix as no conflicts will occur and the API expects this table name) + username = var.db_username + password = random_password.this.result + db_subnet_group_name = aws_db_subnet_group.this.id + max_allocated_storage = 1000 # Upper limit of automatic scaled storage + multi_az = true # Multiple availability zone? + final_snapshot_identifier = "${var.resource_prefix}${var.db_name}-final-snapshot${var.resource_suffix}-${random_pet.final_snapshot_id.id}" # Snapshot upon delete + vpc_security_group_ids = [aws_security_group.rds_security_group.id] + allow_major_version_upgrade = true tags = merge( var.standard_tags, From 9cd8441b96ff04801a11d3cb14f9205a7557c037 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 12 Feb 2024 16:58:24 +0100 Subject: [PATCH 09/11] Disable multi-AZ deployment. --- modules/datastore/rds.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/datastore/rds.tf b/modules/datastore/rds.tf index a41f235..3bbc9e2 100644 --- a/modules/datastore/rds.tf +++ b/modules/datastore/rds.tf @@ -111,7 +111,7 @@ resource "aws_db_instance" "this" { password = random_password.this.result db_subnet_group_name = aws_db_subnet_group.this.id max_allocated_storage = 1000 # Upper limit of automatic scaled storage - multi_az = true # Multiple availability zone? + multi_az = false # Multiple availability zone? final_snapshot_identifier = "${var.resource_prefix}${var.db_name}-final-snapshot${var.resource_suffix}-${random_pet.final_snapshot_id.id}" # Snapshot upon delete vpc_security_group_ids = [aws_security_group.rds_security_group.id] allow_major_version_upgrade = true From aca8bccbe13d75c8416d356fe578419c92d20ceb Mon Sep 17 00:00:00 2001 From: matebaranyi1995 Date: Thu, 5 Sep 2024 13:31:11 +0200 Subject: [PATCH 10/11] remove duplicate --- main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/main.tf b/main.tf index 6f0ea78..017e6a6 100644 --- a/main.tf +++ b/main.tf @@ -1,8 +1,6 @@ module "metaflow-datastore" { source = "./modules/datastore" - db_instance_type = var.db_instance_type - force_destroy_s3_bucket = var.force_destroy_s3_bucket enable_key_rotation = var.enable_key_rotation From ccf583a77a9a2196ff4f13aec8f112859e3aa58a Mon Sep 17 00:00:00 2001 From: matebaranyi1995 Date: Thu, 5 Sep 2024 13:55:41 +0200 Subject: [PATCH 11/11] change defaults --- variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index 96616bf..2cbb89f 100644 --- a/variables.tf +++ b/variables.tf @@ -70,12 +70,12 @@ variable "compute_environment_egress_cidr_blocks" { variable "db_instance_type" { type = string description = "RDS instance type to launch for PostgresQL database." - default = "db.t2.small" + default = "db.t3.small" } variable "db_engine_version" { type = string - default = "11" + default = "13" } variable "launch_template_http_endpoint" {