From 831eeb13bcf1d13eb5cef8a176f3dd185437fc10 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Thu, 11 Sep 2025 15:57:16 +0100 Subject: [PATCH] [NRL-1591] Add missing backup-infra for prod. Fix all backup infra for prod deployment. Apply glue changes from prod env to TF code --- .../modules/athena/sql/rep_producer.sql | 28 ++++++------- .../modules/glue/glue.tf | 5 ++- .../prod/aws-backup.tf | 2 +- .../account-wide-infrastructure/test/main.tf | 7 ++++ terraform/backup-infrastructure/README.md | 8 ++-- .../backup-infrastructure/prod/aws-backup.tf | 42 +++++++++++++++++++ terraform/backup-infrastructure/prod/data.tf | 1 + .../backup-infrastructure/prod/locals.tf | 8 ++++ terraform/backup-infrastructure/prod/main.tf | 32 ++++++++++++++ terraform/backup-infrastructure/prod/vars.tf | 15 +++++++ terraform/backup-infrastructure/test/main.tf | 2 +- 11 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 terraform/backup-infrastructure/prod/aws-backup.tf create mode 100644 terraform/backup-infrastructure/prod/data.tf create mode 100644 terraform/backup-infrastructure/prod/locals.tf create mode 100644 terraform/backup-infrastructure/prod/main.tf create mode 100644 terraform/backup-infrastructure/prod/vars.tf diff --git a/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql b/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql index 9c294b257..53c968905 100644 --- a/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql +++ b/terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql @@ -33,12 +33,12 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - , event_pointer_types + --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_deletedocumentreference ) -, pr AS ( +/*, pr AS ( SELECT time , event_timestamp @@ -56,8 +56,8 @@ WITH , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_readdocumentreference -) -, ps AS ( +)*/ +/*, ps AS ( SELECT time , event_timestamp @@ -75,7 +75,7 @@ WITH , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_searchdocumentreference -) +)*/ , psp AS ( SELECT time @@ -90,12 +90,12 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - , event_pointer_types + --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_searchpostdocumentreference ) -, pu AS ( +/*, pu AS ( SELECT time , event_timestamp @@ -113,7 +113,7 @@ WITH , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_updatedocumentreference -) +)*/ , pus AS ( SELECT time @@ -128,7 +128,7 @@ WITH , event_function_request_id , event_correlation_id , event_xray_trace_id - , event_pointer_types + --, event_pointer_types , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods FROM producer_upsertdocumentreference @@ -140,18 +140,18 @@ WITH UNION SELECT * FROM pd -UNION SELECT * +/*UNION SELECT * FROM pr UNION SELECT * FROM - ps + ps*/ UNION SELECT * FROM psp -UNION SELECT * +/*UNION SELECT * FROM - pu + pu*/ UNION SELECT * FROM pus @@ -177,7 +177,7 @@ SELECT , event_function_request_id , b.event_correlation_id , b.event_xray_trace_id -, event_pointer_types +--, event_pointer_types , oc.user_ods FROM (base b diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index aced0651d..8ce357b61 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -90,7 +90,8 @@ resource "aws_glue_job" "glue_job" { description = "Transfer logs from source to bucket" glue_version = "5.0" worker_type = "G.1X" - timeout = 2880 + execution_class = "STANDARD" + timeout = 60 # minutes max_retries = 0 number_of_workers = 4 command { @@ -110,5 +111,7 @@ resource "aws_glue_job" "glue_job" { "--enable-continuous-log-filter" = "true" "--enable-metrics" = "true" "--extra-py-files" = "s3://${aws_s3_bucket.code-bucket.id}/src.zip" + "--enable-job-insights" = "true" + "--job-language" = "python" } } diff --git a/terraform/account-wide-infrastructure/prod/aws-backup.tf b/terraform/account-wide-infrastructure/prod/aws-backup.tf index f4aec0652..a7759966b 100644 --- a/terraform/account-wide-infrastructure/prod/aws-backup.tf +++ b/terraform/account-wide-infrastructure/prod/aws-backup.tf @@ -174,7 +174,7 @@ module "source" { "schedule" : "cron(30 0 ? * 4)" # every Thursday from 00:30 to precede releases "copy_action" : [{ "cold_storage_after" : 14 # ensures 2 warm including one from previous release - "delete_after" : 105 + "delete_after" : 15 * 7 }], "lifecycle" : { "delete_after" : 2 diff --git a/terraform/account-wide-infrastructure/test/main.tf b/terraform/account-wide-infrastructure/test/main.tf index e9fdee88a..4aceabd58 100644 --- a/terraform/account-wide-infrastructure/test/main.tf +++ b/terraform/account-wide-infrastructure/test/main.tf @@ -12,6 +12,13 @@ provider "aws" { } } +provider "awscc" { + region = local.region + + assume_role = { + role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + } +} provider "awscc" { region = local.region diff --git a/terraform/backup-infrastructure/README.md b/terraform/backup-infrastructure/README.md index 8af8ae7db..3decbc073 100644 --- a/terraform/backup-infrastructure/README.md +++ b/terraform/backup-infrastructure/README.md @@ -28,8 +28,8 @@ Then, initialise the terraform backup workspace. For the test account: ```shell $ cd test $ terraform init && ( \ - terraform workspace new backup-infra-test || \ - terraform workspace select backup-infra-test ) + terraform workspace new test || \ + terraform workspace select test ) ``` If you want to apply changes to prod, use the `prod` directory and the `backup-infra-prod` terraform workspace. @@ -69,8 +69,8 @@ Then, initialise your terraform workspace. For the test account: ```shell $ cd test $ terraform init && ( \ - terraform workspace new backup-infra-test || \ - terraform workspace select backup-infra-test ) + terraform workspace new test || \ + terraform workspace select test ) ``` If you want to destroy resources in prod, use the `prod` directory and the `backup-infra-prod` terraform workspace. diff --git a/terraform/backup-infrastructure/prod/aws-backup.tf b/terraform/backup-infrastructure/prod/aws-backup.tf new file mode 100644 index 000000000..69e4c03d2 --- /dev/null +++ b/terraform/backup-infrastructure/prod/aws-backup.tf @@ -0,0 +1,42 @@ + +# We need a key for the backup vaults. This key will be used to encrypt the backups themselves. +# We need one per vault (on the assumption that each vault will be in a different account). +resource "aws_kms_key" "destination_backup_key" { + description = "KMS key for AWS Backup vaults" + deletion_window_in_days = 7 + enable_key_rotation = true + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Sid = "Enable IAM User Permissions" + Principal = { + AWS = "arn:aws:iam::${local.destination_account_id}:root" + } + Action = "kms:*" + Resource = "*" + } + ] + }) +} + +module "destination" { + source = "../modules/aws-backup-destination" + + source_account_name = "prod" # please note that the assigned value would be the prefix in aws_backup_vault.vault.name + account_id = local.destination_account_id + source_account_id = local.source_account_id + kms_key = aws_kms_key.destination_backup_key.arn + enable_vault_protection = false +} + +### +# Destination vault ARN output +### + +output "destination_vault_arn" { + # The ARN of the backup vault in the destination account is needed by + # the source account to copy backups into it. + value = module.destination.vault_arn +} diff --git a/terraform/backup-infrastructure/prod/data.tf b/terraform/backup-infrastructure/prod/data.tf new file mode 100644 index 000000000..8fc4b38cc --- /dev/null +++ b/terraform/backup-infrastructure/prod/data.tf @@ -0,0 +1 @@ +data "aws_caller_identity" "current" {} diff --git a/terraform/backup-infrastructure/prod/locals.tf b/terraform/backup-infrastructure/prod/locals.tf new file mode 100644 index 000000000..0564327ac --- /dev/null +++ b/terraform/backup-infrastructure/prod/locals.tf @@ -0,0 +1,8 @@ +locals { + # Adjust these as required + project_name = "nrlf-prod-backup" + environment_name = "prod" + + source_account_id = var.source_account_id + destination_account_id = var.assume_account +} diff --git a/terraform/backup-infrastructure/prod/main.tf b/terraform/backup-infrastructure/prod/main.tf new file mode 100644 index 000000000..ca296ceb4 --- /dev/null +++ b/terraform/backup-infrastructure/prod/main.tf @@ -0,0 +1,32 @@ +provider "aws" { + region = "eu-west-2" + + assume_role { + role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + } + + default_tags { + tags = { + project_name = local.project_name + workspace = terraform.workspace + } + } +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.76.0" + } + } + + backend "s3" { + region = "eu-west-2" + bucket = "nhsd-nrlf--terraform-state" + dynamodb_table = "nhsd-nrlf--terraform-state-lock" + key = "terraform-state-backup-infrastructure" + workspace_key_prefix = "nhsd-nrlf" + encrypt = false + } +} diff --git a/terraform/backup-infrastructure/prod/vars.tf b/terraform/backup-infrastructure/prod/vars.tf new file mode 100644 index 000000000..e091ee9c5 --- /dev/null +++ b/terraform/backup-infrastructure/prod/vars.tf @@ -0,0 +1,15 @@ +variable "assume_account" { + description = "The account id to deploy the infrastructure to" + sensitive = true +} + +variable "assume_role" { + description = "Name of the role to assume to deploy the infrastructure" + type = string +} + +variable "source_account_id" { + description = "The account id of the backup source account" + type = string + sensitive = true +} diff --git a/terraform/backup-infrastructure/test/main.tf b/terraform/backup-infrastructure/test/main.tf index 260e66173..ca296ceb4 100644 --- a/terraform/backup-infrastructure/test/main.tf +++ b/terraform/backup-infrastructure/test/main.tf @@ -25,7 +25,7 @@ terraform { region = "eu-west-2" bucket = "nhsd-nrlf--terraform-state" dynamodb_table = "nhsd-nrlf--terraform-state-lock" - key = "terraform-state-dev-backup-infrastructure" + key = "terraform-state-backup-infrastructure" workspace_key_prefix = "nhsd-nrlf" encrypt = false }