From 001cf2f27c6eb8c268040eaa65aba483998585a5 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Tue, 8 Jul 2025 22:04:41 +0100 Subject: [PATCH 1/4] NRL-853 set up DynamoDB backup plan for weekly and monthly with transitions to cold storage --- .../dev/aws-backup.tf | 31 +++++++++++++++++-- .../modules/backup-source/backup_plan.tf | 6 ++-- .../modules/backup-source/variables.tf | 3 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index d357e6b15..0c8f1f392 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -109,7 +109,7 @@ module "source" { "compliance_resource_types" : [ "S3" ], - "enable" = true, + "enable" : true, "rules" : [ { "copy_action" : { @@ -132,14 +132,39 @@ module "source" { "enable" : true, "rules" : [ { + "name" : "daily", + "schedule" : "cron(0 0 * * ? *)", "copy_action" : { "delete_after" : 4 }, + "lifecycle" : { "delete_after" : 2 + } + }, + { + "name" : "monthly" + "schedule" : "cron(30 0 * * 4#1)" # first Thursday each month from 00:30 + "copy_action" : { + "cold_storage_after" : 3, + "delete_after" : 100 # ensures there will always be min 3 }, - "name" : "daily_kept_for_2_days", - "schedule" : "cron(0 0 * * ? *)" + "lifecycle" : { + "delete_after" : 2 + } + + }, + { + "name" : "weekly" # overlaps with monthly + "schedule" : "cron(30 0 * * 4)" # every Thursday from 00:30 to precede releases + "copy_action" : { + "cold_storage_after" : 14 # ensures 2 warm including one from previous release + "delete_after" : 100 + }, + "lifecycle" : { + "delete_after" : 2 + } + } ], "selection_tag" : "NHSE-Enable-DDB-Backup" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 298d654c1..61cf84d24 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -20,7 +20,8 @@ resource "aws_backup_plan" "default" { for_each = rule.value.copy_action != null ? rule.value.copy_action : {} content { lifecycle { - delete_after = copy_action.value + delete_after = copy_action.value.lifecycle.delete_after + cold_storage_after = copy_action.value.lifecycle.cold_storage_after } destination_vault_arn = var.backup_copy_vault_arn } @@ -51,7 +52,8 @@ resource "aws_backup_plan" "dynamodb" { for_each = rule.value.copy_action != null ? rule.value.copy_action : {} content { lifecycle { - delete_after = copy_action.value + delete_after = copy_action.value.delete_after + cold_storage_after = copy_action.value.cold_storage_after } destination_vault_arn = var.backup_copy_vault_arn } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index e2c5985d3..0cb6d71cb 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -107,7 +107,8 @@ variable "backup_plan_config_dynamodb" { cold_storage_after = optional(number) }) copy_action = optional(object({ - delete_after = optional(number) + delete_after = optional(number) + cold_storage_after = optional(number) })) }))) }) From 4c9cde66d099ac87be632e4ea9f5aa8c464ad057 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 22 Jul 2025 11:30:24 +0100 Subject: [PATCH 2/4] [NRL-853] WIP - Switch copy_action to list, still more to do on that --- .../dev/aws-backup.tf | 16 ++++++++-------- .../modules/backup-source/backup_plan.tf | 6 +++--- .../modules/backup-source/variables.tf | 12 ++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index 0c8f1f392..c54d81544 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -112,9 +112,9 @@ module "source" { "enable" : true, "rules" : [ { - "copy_action" : { + "copy_action" : [{ "delete_after" : 4 - }, + }], "lifecycle" : { "delete_after" : 2 }, @@ -134,9 +134,9 @@ module "source" { { "name" : "daily", "schedule" : "cron(0 0 * * ? *)", - "copy_action" : { + "copy_action" : [{ "delete_after" : 4 - }, + }], "lifecycle" : { "delete_after" : 2 @@ -145,10 +145,10 @@ module "source" { { "name" : "monthly" "schedule" : "cron(30 0 * * 4#1)" # first Thursday each month from 00:30 - "copy_action" : { + "copy_action" : [{ "cold_storage_after" : 3, "delete_after" : 100 # ensures there will always be min 3 - }, + }], "lifecycle" : { "delete_after" : 2 } @@ -157,10 +157,10 @@ module "source" { { "name" : "weekly" # overlaps with monthly "schedule" : "cron(30 0 * * 4)" # every Thursday from 00:30 to precede releases - "copy_action" : { + "copy_action" : [{ "cold_storage_after" : 14 # ensures 2 warm including one from previous release "delete_after" : 100 - }, + }], "lifecycle" : { "delete_after" : 2 } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 61cf84d24..f278374aa 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -17,11 +17,11 @@ resource "aws_backup_plan" "default" { cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null } dynamic "copy_action" { - for_each = rule.value.copy_action != null ? rule.value.copy_action : {} + for_each = rule.value.copy_action content { lifecycle { - delete_after = copy_action.value.lifecycle.delete_after - cold_storage_after = copy_action.value.lifecycle.cold_storage_after + delete_after = copy_action.value.delete_after + cold_storage_after = copy_action.value.cold_storage_after != null ? copy_action.value.cold_storage_after : null } destination_vault_arn = var.backup_copy_vault_arn } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index 0cb6d71cb..61af02831 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -78,7 +78,7 @@ variable "backup_plan_config" { enable = bool selection_tag = string compliance_resource_types = list(string) - rules = list(object({ + rules = optional(list(object({ name = string schedule = string enable_continuous_backup = optional(bool) @@ -86,10 +86,10 @@ variable "backup_plan_config" { delete_after = optional(number) cold_storage_after = optional(number) }) - copy_action = optional(object({ + copy_action = optional(list(object({ delete_after = optional(number) - })) - })) + }))) + }))) }) } variable "backup_plan_config_dynamodb" { @@ -106,10 +106,10 @@ variable "backup_plan_config_dynamodb" { delete_after = number cold_storage_after = optional(number) }) - copy_action = optional(object({ + copy_action = optional(list(object({ delete_after = optional(number) cold_storage_after = optional(number) - })) + }))) }))) }) From 84b3f7dd8ff512bf162a1b98ee6caa8e8b682f7c Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 18 Aug 2025 17:09:24 +0100 Subject: [PATCH 3/4] [NRL-853] Fix backup config params. Fix cron definitions --- .../account-wide-infrastructure/dev/aws-backup.tf | 10 +++++----- .../modules/backup-source/backup_plan.tf | 14 +++++++------- .../modules/backup-source/variables.tf | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index c54d81544..91d4813e6 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -113,7 +113,7 @@ module "source" { "rules" : [ { "copy_action" : [{ - "delete_after" : 4 + "delete_after" : 4, }], "lifecycle" : { "delete_after" : 2 @@ -135,7 +135,7 @@ module "source" { "name" : "daily", "schedule" : "cron(0 0 * * ? *)", "copy_action" : [{ - "delete_after" : 4 + "delete_after" : 4, }], "lifecycle" : { @@ -144,7 +144,7 @@ module "source" { }, { "name" : "monthly" - "schedule" : "cron(30 0 * * 4#1)" # first Thursday each month from 00:30 + "schedule" : "cron(30 0 ? * 4#1)" # first Thursday each month from 00:30 "copy_action" : [{ "cold_storage_after" : 3, "delete_after" : 100 # ensures there will always be min 3 @@ -156,10 +156,10 @@ module "source" { }, { "name" : "weekly" # overlaps with monthly - "schedule" : "cron(30 0 * * 4)" # every Thursday from 00:30 to precede releases + "schedule" : "cron(30 0 ? * 4)" # every Thursday from 00:30 to precede releases "copy_action" : [{ "cold_storage_after" : 14 # ensures 2 warm including one from previous release - "delete_after" : 100 + "delete_after" : 105 }], "lifecycle" : { "delete_after" : 2 diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index f278374aa..7275dd92b 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -11,17 +11,17 @@ resource "aws_backup_plan" "default" { rule_name = rule.value.name target_vault_name = aws_backup_vault.main.name schedule = rule.value.schedule - enable_continuous_backup = rule.value.enable_continuous_backup != null ? rule.value.enable_continuous_backup : null + enable_continuous_backup = rule.value.enable_continuous_backup lifecycle { - delete_after = rule.value.lifecycle.delete_after != null ? rule.value.lifecycle.delete_after : null - cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null + delete_after = rule.value.lifecycle.delete_after + cold_storage_after = rule.value.lifecycle.cold_storage_after } dynamic "copy_action" { for_each = rule.value.copy_action content { lifecycle { delete_after = copy_action.value.delete_after - cold_storage_after = copy_action.value.cold_storage_after != null ? copy_action.value.cold_storage_after : null + cold_storage_after = copy_action.value.cold_storage_after } destination_vault_arn = var.backup_copy_vault_arn } @@ -45,11 +45,11 @@ resource "aws_backup_plan" "dynamodb" { target_vault_name = aws_backup_vault.main.name schedule = rule.value.schedule lifecycle { - delete_after = rule.value.lifecycle.delete_after != null ? rule.value.lifecycle.delete_after : null - cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null + delete_after = rule.value.lifecycle.delete_after + cold_storage_after = rule.value.lifecycle.cold_storage_after } dynamic "copy_action" { - for_each = rule.value.copy_action != null ? rule.value.copy_action : {} + for_each = rule.value.copy_action content { lifecycle { delete_after = copy_action.value.delete_after diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index 61af02831..95eab5216 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -87,7 +87,8 @@ variable "backup_plan_config" { cold_storage_after = optional(number) }) copy_action = optional(list(object({ - delete_after = optional(number) + delete_after = optional(number) + cold_storage_after = optional(number) }))) }))) }) From f99aeff0169715f394310c2fc6333e2db612ed40 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 19 Aug 2025 10:59:19 +0100 Subject: [PATCH 4/4] [NRL-853] Remove incorrect optional. Remove unused cold_storage_after --- .../modules/backup-source/backup_plan.tf | 3 +-- .../modules/backup-source/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 7275dd92b..b352430b7 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -20,8 +20,7 @@ resource "aws_backup_plan" "default" { for_each = rule.value.copy_action content { lifecycle { - delete_after = copy_action.value.delete_after - cold_storage_after = copy_action.value.cold_storage_after + delete_after = copy_action.value.delete_after } destination_vault_arn = var.backup_copy_vault_arn } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index 95eab5216..c05b2178d 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -78,7 +78,7 @@ variable "backup_plan_config" { enable = bool selection_tag = string compliance_resource_types = list(string) - rules = optional(list(object({ + rules = list(object({ name = string schedule = string enable_continuous_backup = optional(bool) @@ -90,7 +90,7 @@ variable "backup_plan_config" { delete_after = optional(number) cold_storage_after = optional(number) }))) - }))) + })) }) } variable "backup_plan_config_dynamodb" {