From 320450930f3e8ed2d6f169df48bf9262527367b6 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Thu, 3 Jul 2025 09:36:41 +0100 Subject: [PATCH 1/2] [NRL-1511] Pre-prod release fixups for dashboards changes --- .../account-wide-infrastructure/README.md | 29 +++++-------------- .../account-wide-infrastructure/dev/athena.tf | 1 + .../account-wide-infrastructure/dev/data.tf | 2 ++ .../account-wide-infrastructure/dev/main.tf | 8 +++++ .../dev/outputs.tf | 24 +++++++++++++++ .../dev/secrets.tf | 10 +++++++ .../mgmt/codebuild.tf | 2 +- .../modules/athena/outputs.tf | 4 +-- .../modules/athena/vars.tf | 5 ++++ .../modules/backup-source/locals.tf | 2 +- .../modules/powerbi-gw-ec2/data.tf | 1 + .../modules/powerbi-gw-ec2/locals.tf | 2 +- .../modules/powerbi-gw-ec2/outputs.tf | 4 +++ .../prod/athena.tf | 1 + .../account-wide-infrastructure/prod/main.tf | 8 +++++ .../prod/outputs.tf | 24 +++++++++++++++ .../prod/secrets.tf | 10 +++++++ .../test/athena.tf | 1 + .../account-wide-infrastructure/test/data.tf | 2 ++ .../account-wide-infrastructure/test/main.tf | 2 +- .../test/outputs.tf | 24 +++++++++++++++ .../test/secrets.tf | 13 +++++++++ .../account-wide-infrastructure/test/vars.tf | 4 +-- 23 files changed, 153 insertions(+), 30 deletions(-) create mode 100644 terraform/account-wide-infrastructure/dev/outputs.tf create mode 100644 terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf create mode 100644 terraform/account-wide-infrastructure/prod/outputs.tf create mode 100644 terraform/account-wide-infrastructure/test/outputs.tf diff --git a/terraform/account-wide-infrastructure/README.md b/terraform/account-wide-infrastructure/README.md index e688f99ac..69fe335f5 100644 --- a/terraform/account-wide-infrastructure/README.md +++ b/terraform/account-wide-infrastructure/README.md @@ -136,35 +136,20 @@ To disable reporting resources for the account, do the following: 1. Set the `enable_reporting` variable to `true` in `./ACCOUNT_NAME/vars.tf` 2. Deploy the account-wide infrastructure to the account -#### Deploying the PowerBI Gateway +#### Deploy the PowerBI Gateway -The first time you deploy the PowerBI Gateway set up to a new account, these steps need to be followed: +The first time you deploy the PowerBI Gateway to an AWS account you need to create, install and configure a gateway image. Instruction on how to do this can be found in [KOP-NRLF-012](https://nhsd-confluence.digital.nhs.uk/x/8BXXQg). -1. Set the `enable_powerbi_auto_push` variable to `true` in `./ACCOUNT_NAME/vars.tf` -2. Set the `use_powerbi_gw_custom_ami` variable to `false` in `./ACCOUNT_NAME/vars.tf` -3. Deploy the account-wide infrastructure for the account -4. Run the below CLI command, and RDP into the newly created EC2 instance (localhost:13389) - -``` -aws ssm start-session --target --document-name AWS-StartPortForwardingSession --parameters "localPortNumber=13389,portNumber=3389" -``` +To enable the PowerBI Gateway in the account: -5. Install Athena ODBC driver and Power BI standard on premises gateway -6. Configure ODBC driver to connect to relevant Athena instance -7. Log in to the gateway using NHS email, name the cluster to nhsd-nrlf-{env}--reporting-gw -8. Log on to power bi, navigate to Manage Connections and Gateways in settings and set up Athena connector with authentication method: Anonymous and privacy level: Private -9. Set dataset to point to this gateway, define schedule as needed -10. In the AWS Console, create an AMI from the instance called `PowerBI_GW` -11. Set the `use_powerbi_gw_custom_ami` variable to `true` -12. Deploy the account-wide infrastructure for the account -13. Run the below CLI command, and RDP into the newly created EC2 instance (localhost:13389) +1. Set the `enable_powerbi_auto_push` variable to `true` in `./ACCOUNT_NAME/vars.tf` +2. Deploy the account-wide infrastructure to the account +3. Access the EC2 Serial Console for the instance and run this command to start the PowerBI Gateway: ``` -aws ssm start-session --target --document-name AWS-StartPortForwardingSession --parameters "localPortNumber=13389,portNumber=3389" +Start-Service -Name "PBIEgwService" ``` -14. Start the PowerBI Gateway service on the instance - To disable the PowerBI Gateway from the account: 1. Set the `enable_powerbi_auto_push` variable to `false` in `./ACCOUNT_NAME/vars.tf` diff --git a/terraform/account-wide-infrastructure/dev/athena.tf b/terraform/account-wide-infrastructure/dev/athena.tf index 77cc82148..f37f9712c 100644 --- a/terraform/account-wide-infrastructure/dev/athena.tf +++ b/terraform/account-wide-infrastructure/dev/athena.tf @@ -3,5 +3,6 @@ module "dev-athena" { source = "../modules/athena" name_prefix = "nhsd-nrlf--dev" target_bucket_name = module.dev-glue.target_bucket_name + bucket_region = data.aws_region.current.region glue_database = module.dev-glue.glue_database } diff --git a/terraform/account-wide-infrastructure/dev/data.tf b/terraform/account-wide-infrastructure/dev/data.tf index bb435ed6b..bbf34c6c2 100644 --- a/terraform/account-wide-infrastructure/dev/data.tf +++ b/terraform/account-wide-infrastructure/dev/data.tf @@ -1,3 +1,5 @@ +data "aws_region" "current" {} + data "aws_secretsmanager_secret_version" "identities_account_id" { secret_id = aws_secretsmanager_secret.identities_account_id.name } diff --git a/terraform/account-wide-infrastructure/dev/main.tf b/terraform/account-wide-infrastructure/dev/main.tf index 6a15ca71b..0c1bbfb58 100644 --- a/terraform/account-wide-infrastructure/dev/main.tf +++ b/terraform/account-wide-infrastructure/dev/main.tf @@ -29,4 +29,12 @@ terraform { key = "terraform-state-account-wide-infrastructure" workspace_key_prefix = "nhsd-nrlf" } + + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } } diff --git a/terraform/account-wide-infrastructure/dev/outputs.tf b/terraform/account-wide-infrastructure/dev/outputs.tf new file mode 100644 index 000000000..9f99d1105 --- /dev/null +++ b/terraform/account-wide-infrastructure/dev/outputs.tf @@ -0,0 +1,24 @@ +output "powerbi_gw_instance_id" { + description = "The ID of the PowerBI Gateway EC2 instance." + value = var.enable_powerbi_auto_push ? module.powerbi_gw_instance[0].instance_id : null +} + +output "reporting_database_name" { + description = "Name of the reporting Athena database" + value = var.enable_reporting ? module.dev-glue.glue_database : null +} + +output "athena_workgroup_name" { + description = "Name of the Athena workgroup" + value = var.enable_reporting ? module.dev-athena[0].workgroup_name : null +} + +output "athena_s3_output_location" { + description = "S3 output location for Athena queries" + value = var.enable_reporting ? "s3://${module.dev-athena[0].bucket.id}/" : null +} + +output "athena_kms_key_arn" { + description = "KMS key ARN for Athena encryption" + value = var.enable_reporting ? module.dev-athena[0].kms_key_arn : null +} diff --git a/terraform/account-wide-infrastructure/dev/secrets.tf b/terraform/account-wide-infrastructure/dev/secrets.tf index dd735c3d7..dfd3333f0 100644 --- a/terraform/account-wide-infrastructure/dev/secrets.tf +++ b/terraform/account-wide-infrastructure/dev/secrets.tf @@ -41,3 +41,13 @@ resource "aws_secretsmanager_secret" "devsandbox_environment_configuration" { name = "${local.project}--dev-sandbox--env-config" description = "The environment configuration for the Dev Sandbox environment" } + +resource "aws_secretsmanager_secret" "powerbi_gw_instance_admin_pwd" { + count = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0 + name = "${local.project}--dev-powerbi-gw-instance-admin-pwd" + description = "Admin password for the PowerBI Gateway EC2 instance" +} +resource "aws_secretsmanager_secret" "powerbi_gw_recovery_key" { + name = "${local.project}--dev-powerbi-gw-recovery-key" + description = "Recovery key for the PowerBI Gateway EC2 instance" +} diff --git a/terraform/account-wide-infrastructure/mgmt/codebuild.tf b/terraform/account-wide-infrastructure/mgmt/codebuild.tf index 7aee98d03..52f7db234 100644 --- a/terraform/account-wide-infrastructure/mgmt/codebuild.tf +++ b/terraform/account-wide-infrastructure/mgmt/codebuild.tf @@ -56,7 +56,7 @@ data "aws_iam_policy_document" "codebuild_policy" { "secretsmanager:ListSecretVersionIds" ] resources = [ - "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.project}--codebuild-github-pat-*", + "arn:aws:secretsmanager:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:secret:${local.project}--codebuild-github-pat-*", ] } diff --git a/terraform/account-wide-infrastructure/modules/athena/outputs.tf b/terraform/account-wide-infrastructure/modules/athena/outputs.tf index 72aaa879c..670ddd9af 100644 --- a/terraform/account-wide-infrastructure/modules/athena/outputs.tf +++ b/terraform/account-wide-infrastructure/modules/athena/outputs.tf @@ -1,5 +1,5 @@ -output "workgroup" { - value = aws_athena_workgroup.athena +output "workgroup_name" { + value = aws_athena_workgroup.athena.name } output "bucket" { diff --git a/terraform/account-wide-infrastructure/modules/athena/vars.tf b/terraform/account-wide-infrastructure/modules/athena/vars.tf index a1d447972..fbacc86d7 100644 --- a/terraform/account-wide-infrastructure/modules/athena/vars.tf +++ b/terraform/account-wide-infrastructure/modules/athena/vars.tf @@ -3,6 +3,11 @@ variable "name_prefix" { description = "The prefix to apply to all resources in the module." } +variable "bucket_region" { + type = string + description = "The AWS region where the S3 bucket will be created." +} + variable "target_bucket_name" { type = string } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/locals.tf b/terraform/account-wide-infrastructure/modules/backup-source/locals.tf index e6929817b..e1b2f2359 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/locals.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/locals.tf @@ -1,3 +1,3 @@ locals { - resource_name_prefix = "${data.aws_region.current.name}-${data.aws_caller_identity.current.account_id}-backup" + resource_name_prefix = "${data.aws_region.current.region}-${data.aws_caller_identity.current.account_id}-backup" } diff --git a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/data.tf b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/data.tf index dfbb5b881..f5c461f0f 100644 --- a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/data.tf +++ b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/data.tf @@ -8,6 +8,7 @@ data "aws_ami" "windows-2019" { } data "aws_ami" "PowerBI_Gateway" { + count = var.use_custom_ami ? 1 : 0 most_recent = true owners = ["self"] filter { diff --git a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/locals.tf b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/locals.tf index ebc33c56a..01ea149af 100644 --- a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/locals.tf +++ b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/locals.tf @@ -1,3 +1,3 @@ locals { - selected_ami_id = var.use_custom_ami ? data.aws_ami.PowerBI_Gateway.id : data.aws_ami.windows-2019.id + selected_ami_id = var.use_custom_ami ? data.aws_ami.PowerBI_Gateway[0].id : data.aws_ami.windows-2019.id } diff --git a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf new file mode 100644 index 000000000..871743c0b --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf @@ -0,0 +1,4 @@ +output "instance_id" { + description = "The ID of the PowerBI Gateway EC2 instance." + value = aws_instance.powerbi_gw.id +} diff --git a/terraform/account-wide-infrastructure/prod/athena.tf b/terraform/account-wide-infrastructure/prod/athena.tf index 9242ddd5e..5a0398c76 100644 --- a/terraform/account-wide-infrastructure/prod/athena.tf +++ b/terraform/account-wide-infrastructure/prod/athena.tf @@ -3,5 +3,6 @@ module "prod-athena" { source = "../modules/athena" name_prefix = "nhsd-nrlf--prod" target_bucket_name = module.prod-glue.target_bucket_name + bucket_region = data.aws_region.current.region glue_database = module.prod-glue.glue_database } diff --git a/terraform/account-wide-infrastructure/prod/main.tf b/terraform/account-wide-infrastructure/prod/main.tf index cfed956f2..f5d3b64a7 100644 --- a/terraform/account-wide-infrastructure/prod/main.tf +++ b/terraform/account-wide-infrastructure/prod/main.tf @@ -22,4 +22,12 @@ terraform { key = "terraform-state-account-wide-infrastructure" workspace_key_prefix = "nhsd-nrlf" } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } + } diff --git a/terraform/account-wide-infrastructure/prod/outputs.tf b/terraform/account-wide-infrastructure/prod/outputs.tf new file mode 100644 index 000000000..270d62ea6 --- /dev/null +++ b/terraform/account-wide-infrastructure/prod/outputs.tf @@ -0,0 +1,24 @@ +output "powerbi_gw_instance_id" { + description = "The ID of the PowerBI Gateway EC2 instance." + value = var.enable_powerbi_auto_push ? module.powerbi_gw_instance[0].instance_id : null +} + +output "reporting_database_name" { + description = "Name of the reporting Athena database" + value = var.enable_reporting ? module.prod-glue.glue_database : null +} + +output "athena_workgroup_name" { + description = "Name of the Athena workgroup" + value = var.enable_reporting ? module.prod-athena[0].workgroup_name : null +} + +output "athena_s3_output_location" { + description = "S3 output location for Athena queries" + value = var.enable_reporting ? "s3://${module.prod-athena[0].bucket.id}/" : null +} + +output "athena_kms_key_arn" { + description = "KMS key ARN for Athena encryption" + value = var.enable_reporting ? module.prod-athena[0].kms_key_arn : null +} diff --git a/terraform/account-wide-infrastructure/prod/secrets.tf b/terraform/account-wide-infrastructure/prod/secrets.tf index 6c7390fee..1cfdfb908 100644 --- a/terraform/account-wide-infrastructure/prod/secrets.tf +++ b/terraform/account-wide-infrastructure/prod/secrets.tf @@ -21,3 +21,13 @@ resource "aws_secretsmanager_secret" "prod_environment_configuration" { name = "${local.project}--prod--env-config" description = "The environment configuration for the Prod environment" } + +resource "aws_secretsmanager_secret" "powerbi_gw_instance_admin_pwd" { + count = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0 + name = "${local.project}--prod-powerbi-gw-instance-admin-pwd" + description = "Admin password for the PowerBI Gateway EC2 instance" +} +resource "aws_secretsmanager_secret" "powerbi_gw_recovery_key" { + name = "${local.project}--prod-powerbi-gw-recovery-key" + description = "Recovery key for the PowerBI Gateway EC2 instance" +} diff --git a/terraform/account-wide-infrastructure/test/athena.tf b/terraform/account-wide-infrastructure/test/athena.tf index c31f4f5af..2c47b81e4 100644 --- a/terraform/account-wide-infrastructure/test/athena.tf +++ b/terraform/account-wide-infrastructure/test/athena.tf @@ -3,5 +3,6 @@ module "test-athena" { source = "../modules/athena" name_prefix = "nhsd-nrlf--test" target_bucket_name = module.test-glue.target_bucket_name + bucket_region = data.aws_region.current.region glue_database = module.test-glue.glue_database } diff --git a/terraform/account-wide-infrastructure/test/data.tf b/terraform/account-wide-infrastructure/test/data.tf index d94f607d5..a8146ceee 100644 --- a/terraform/account-wide-infrastructure/test/data.tf +++ b/terraform/account-wide-infrastructure/test/data.tf @@ -1,3 +1,5 @@ +data "aws_region" "current" {} + data "aws_secretsmanager_secret_version" "identities_account_id" { secret_id = aws_secretsmanager_secret.identities_account_id.name } diff --git a/terraform/account-wide-infrastructure/test/main.tf b/terraform/account-wide-infrastructure/test/main.tf index 61849943f..4bb9ece10 100644 --- a/terraform/account-wide-infrastructure/test/main.tf +++ b/terraform/account-wide-infrastructure/test/main.tf @@ -26,7 +26,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = "~> 6.0" } } } diff --git a/terraform/account-wide-infrastructure/test/outputs.tf b/terraform/account-wide-infrastructure/test/outputs.tf new file mode 100644 index 000000000..096cf026f --- /dev/null +++ b/terraform/account-wide-infrastructure/test/outputs.tf @@ -0,0 +1,24 @@ +output "powerbi_gw_instance_id" { + description = "The ID of the PowerBI Gateway EC2 instance." + value = var.enable_powerbi_auto_push ? module.powerbi_gw_instance[0].instance_id : null +} + +output "reporting_database_name" { + description = "Name of the reporting Athena database" + value = var.enable_reporting ? module.test-glue.glue_database : null +} + +output "athena_workgroup_name" { + description = "Name of the Athena workgroup" + value = var.enable_reporting ? module.test-athena[0].workgroup_name : null +} + +output "athena_s3_output_location" { + description = "S3 output location for Athena queries" + value = var.enable_reporting ? "s3://${module.test-athena[0].bucket.id}/" : null +} + +output "athena_kms_key_arn" { + description = "KMS key ARN for Athena encryption" + value = var.enable_reporting ? module.test-athena[0].kms_key_arn : null +} diff --git a/terraform/account-wide-infrastructure/test/secrets.tf b/terraform/account-wide-infrastructure/test/secrets.tf index 54242a7b7..23cea404c 100644 --- a/terraform/account-wide-infrastructure/test/secrets.tf +++ b/terraform/account-wide-infrastructure/test/secrets.tf @@ -103,3 +103,16 @@ resource "aws_secretsmanager_secret" "ref_environment_configuration" { name = "${local.project}--ref--env-config" description = "The environment configuration for the Ref environment" } + +# +# PowerBI secrets +# +resource "aws_secretsmanager_secret" "powerbi_gw_instance_admin_pwd" { + count = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0 + name = "${local.project}--test-powerbi-gw-instance-admin-pwd" + description = "Admin password for the PowerBI Gateway EC2 instance" +} +resource "aws_secretsmanager_secret" "powerbi_gw_recovery_key" { + name = "${local.project}--test-powerbi-gw-recovery-key" + description = "Recovery key for the PowerBI Gateway EC2 instance" +} diff --git a/terraform/account-wide-infrastructure/test/vars.tf b/terraform/account-wide-infrastructure/test/vars.tf index 7598ff13f..1eb05be54 100644 --- a/terraform/account-wide-infrastructure/test/vars.tf +++ b/terraform/account-wide-infrastructure/test/vars.tf @@ -32,7 +32,7 @@ variable "ref_api_domain_name" { variable "enable_reporting" { type = bool description = "Enable account-wide reporting processes in the test account" - default = false + default = true } variable "aws_azs" { @@ -68,7 +68,7 @@ variable "vpc_private_subnets_cidr_block" { variable "enable_powerbi_auto_push" { type = bool description = "Enable automatic pushing of info into PowerBI" - default = false + default = true } variable "powerbi_gw_instance_type" { From 01b9cbabcb41cbdf37b423cff230b046f5743859 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Thu, 3 Jul 2025 11:30:14 +0100 Subject: [PATCH 2/2] [NRL-1511] Enable reporting for int and prod envs. Disable reporting in dev env --- terraform/infrastructure/etc/dev.tfvars | 2 +- terraform/infrastructure/etc/int.tfvars | 2 +- terraform/infrastructure/etc/prod.tfvars | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/infrastructure/etc/dev.tfvars b/terraform/infrastructure/etc/dev.tfvars index 4ba4b986f..ce5b55be9 100644 --- a/terraform/infrastructure/etc/dev.tfvars +++ b/terraform/infrastructure/etc/dev.tfvars @@ -5,4 +5,4 @@ domain = "api.record-locator.dev.national.nhs.uk" public_domain = "internal-dev.api.service.nhs.uk" public_sandbox_domain = "internal-dev-sandbox.api.service.nhs.uk" log_retention_period = 90 -enable_reporting = true +enable_reporting = false diff --git a/terraform/infrastructure/etc/int.tfvars b/terraform/infrastructure/etc/int.tfvars index 4baa2a936..e63c4680b 100644 --- a/terraform/infrastructure/etc/int.tfvars +++ b/terraform/infrastructure/etc/int.tfvars @@ -7,4 +7,4 @@ deletion_protection = true public_domain = "int.api.service.nhs.uk" public_sandbox_domain = "sandbox.api.service.nhs.uk" log_retention_period = 90 -enable_reporting = false +enable_reporting = true diff --git a/terraform/infrastructure/etc/prod.tfvars b/terraform/infrastructure/etc/prod.tfvars index 4f9ca34e9..7f93103b0 100644 --- a/terraform/infrastructure/etc/prod.tfvars +++ b/terraform/infrastructure/etc/prod.tfvars @@ -5,4 +5,4 @@ domain = "api.record-locator.national.nhs.uk" public_domain = "api.service.nhs.uk" deletion_protection = true log_retention_period = 2192 -enable_reporting = false +enable_reporting = true