From 21b451208e6cc2d9b7f21662746f190df2130cf6 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 14:18:47 +0100 Subject: [PATCH 1/6] NRL-1386 Status lambdas only send logs to splunk --- terraform/infrastructure/lambda.tf | 4 ++-- terraform/infrastructure/locals.tf | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/terraform/infrastructure/lambda.tf b/terraform/infrastructure/lambda.tf index c934452db..8ae406726 100644 --- a/terraform/infrastructure/lambda.tf +++ b/terraform/infrastructure/lambda.tf @@ -322,7 +322,7 @@ module "consumer__status" { local.pointers_kms_read_write_arn, local.auth_store_read_policy_arn ] - firehose_subscriptions = local.firehose_lambda_subscriptions + firehose_subscriptions = local.firehose_lambda_splunk_subscription handler = "status.handler" retention = var.log_retention_period } @@ -351,7 +351,7 @@ module "producer__status" { local.pointers_kms_read_write_arn, local.auth_store_read_policy_arn ] - firehose_subscriptions = local.firehose_lambda_subscriptions + firehose_subscriptions = local.firehose_lambda_splunk_subscription handler = "status.handler" retention = var.log_retention_period } diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 957788553..c7f795543 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -36,6 +36,9 @@ locals { "splunk_subscription" : module.firehose__processor[0].firehose_subscription, "reports_subscription" : module.firehose__processor[0].firehose_reporting_subscription } : {} + firehose_lambda_splunk_subscription = var.use_shared_resources ? { + "splunk_subscription" : module.firehose__processor[0].firehose_subscription + } : {} splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name splunk_index = "aws_recordlocator_${local.splunk_environment}" From 4eb3695a297938aafbf953049607f0fee2e75aea Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 14:50:22 +0100 Subject: [PATCH 2/6] NRL-1386 name update --- terraform/infrastructure/lambda.tf | 4 ++-- terraform/infrastructure/locals.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/infrastructure/lambda.tf b/terraform/infrastructure/lambda.tf index 8ae406726..feb2f8729 100644 --- a/terraform/infrastructure/lambda.tf +++ b/terraform/infrastructure/lambda.tf @@ -322,7 +322,7 @@ module "consumer__status" { local.pointers_kms_read_write_arn, local.auth_store_read_policy_arn ] - firehose_subscriptions = local.firehose_lambda_splunk_subscription + firehose_subscriptions = local.firehost_lambda_splunk_only_subscription handler = "status.handler" retention = var.log_retention_period } @@ -351,7 +351,7 @@ module "producer__status" { local.pointers_kms_read_write_arn, local.auth_store_read_policy_arn ] - firehose_subscriptions = local.firehose_lambda_splunk_subscription + firehose_subscriptions = local.firehost_lambda_splunk_only_subscription handler = "status.handler" retention = var.log_retention_period } diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index c7f795543..4510649f5 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -36,7 +36,7 @@ locals { "splunk_subscription" : module.firehose__processor[0].firehose_subscription, "reports_subscription" : module.firehose__processor[0].firehose_reporting_subscription } : {} - firehose_lambda_splunk_subscription = var.use_shared_resources ? { + firehost_lambda_splunk_only_subscription = var.use_shared_resources ? { "splunk_subscription" : module.firehose__processor[0].firehose_subscription } : {} splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name From 929d82c99b42f99e1cd6b20ca5e9ac1008301836 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 15:10:56 +0100 Subject: [PATCH 3/6] NRL-1386 Logic update for lambda subscriptions --- terraform/infrastructure/locals.tf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 4510649f5..8f96c9137 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -32,10 +32,14 @@ locals { reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn reporting_kms_arn = data.aws_kms_key.glue.arn - firehose_lambda_subscriptions = var.use_shared_resources ? { - "splunk_subscription" : module.firehose__processor[0].firehose_subscription, - "reports_subscription" : module.firehose__processor[0].firehose_reporting_subscription - } : {} + firehose_lambda_subscriptions = var.use_shared_resources ? ( + var.enable_reporting ? { + "splunk_subscription" = module.firehose__processor[0].firehose_subscription, + "reports_subscription" = module.firehose__processor[0].firehose_reporting_subscription + } : { + "splunk_subscription" = module.firehose__processor[0].firehose_subscription + } + ) : {} firehost_lambda_splunk_only_subscription = var.use_shared_resources ? { "splunk_subscription" : module.firehose__processor[0].firehose_subscription } : {} From 0670107f58ca05ddc19bcc0dc98bd8f06f025680 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 16:12:38 +0100 Subject: [PATCH 4/6] NRL-1386 Upgrade to medium ec2 for 4gb memory --- terraform/account-wide-infrastructure/dev/vars.tf | 2 +- terraform/account-wide-infrastructure/prod/vars.tf | 2 +- terraform/account-wide-infrastructure/test/vars.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/vars.tf b/terraform/account-wide-infrastructure/dev/vars.tf index adb15d24a..5ebb7464c 100644 --- a/terraform/account-wide-infrastructure/dev/vars.tf +++ b/terraform/account-wide-infrastructure/dev/vars.tf @@ -59,7 +59,7 @@ variable "enable_powerbi_auto_push" { variable "powerbi_gw_instance_type" { type = string description = "Type for PowerBI GW EC2 Instance" - default = "t2.micro" + default = "t2.medium" } variable "use_powerbi_gw_custom_ami" { diff --git a/terraform/account-wide-infrastructure/prod/vars.tf b/terraform/account-wide-infrastructure/prod/vars.tf index d999261f5..9f3acf808 100644 --- a/terraform/account-wide-infrastructure/prod/vars.tf +++ b/terraform/account-wide-infrastructure/prod/vars.tf @@ -60,7 +60,7 @@ variable "enable_powerbi_auto_push" { variable "powerbi_gw_instance_type" { type = string description = "Type for PowerBI GW EC2 Instance" - default = "t2.micro" + default = "t2.medium" } variable "use_powerbi_gw_custom_ami" { diff --git a/terraform/account-wide-infrastructure/test/vars.tf b/terraform/account-wide-infrastructure/test/vars.tf index 144929512..7598ff13f 100644 --- a/terraform/account-wide-infrastructure/test/vars.tf +++ b/terraform/account-wide-infrastructure/test/vars.tf @@ -74,7 +74,7 @@ variable "enable_powerbi_auto_push" { variable "powerbi_gw_instance_type" { type = string description = "Type for PowerBI GW EC2 Instance" - default = "t2.micro" + default = "t2.medium" } variable "use_powerbi_gw_custom_ami" { From 4d96b3581a19486f4530b16a5906dfa04f3cedd2 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 17:07:06 +0100 Subject: [PATCH 5/6] NRL-1386 temp change for testers --- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 530c9a636..4aca6600e 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -65,7 +65,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { role_arn = aws_iam_role.firehose.arn bucket_arn = var.reporting_bucket_arn buffering_size = 64 - buffering_interval = 600 + buffering_interval = 60 processing_configuration { enabled = true From 041ee5de04806c1a07b4faf2c5cea4cd737e3b35 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 30 Jun 2025 18:13:38 +0100 Subject: [PATCH 6/6] NRL-1386 revert temp change --- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 4aca6600e..530c9a636 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -65,7 +65,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { role_arn = aws_iam_role.firehose.arn bucket_arn = var.reporting_bucket_arn buffering_size = 64 - buffering_interval = 60 + buffering_interval = 600 processing_configuration { enabled = true