NRL 1385 Set up EC2 for automated data refresh #903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

jackleary merged 25 commits into develop from feature/jale13-nrl-1385-set-up-powerbi-gateway

May 30, 2025

terraform/account-wide-infrastructure/README.md

-Original file line number
+Diff line change
@@ Expand Up / @@ -124,6 +124,20 @@ $ terraform apply \ @@
     Replacing AWS_ACCOUNT_ID with the AWS account number of your account.
+    ### Reporting Resources
+    If deploying the EC2 set up to a new environment, these steps need to be followed:
+. Run the below CLI command, and RDP into the newly created EC2 instance (localhost:13389)
+    ```
+    aws ssm start-session --target <AMI> --document-name AWS-StartPortForwardingSession --parameters "localPortNumber=13389,portNumber=3389"
+    ```
+. Install Athena ODBC driver and Power BI personal on premises gateway
+. Configure ODBC driver to connect to relevant Athena instance and log in to the gateway using NHS email
+. Log into power bi and test the refresh on the relevant data sources
     ## Tear down account wide resources
     WARNING - This action will destroy all account-wide resources from the AWS account. This should
@@ Expand Down @@

terraform/account-wide-infrastructure/dev/ec2.tf

-Original file line number
+Diff line change
@@ -0,0 +1,23 @@
+    module "vpc" {
+      source                         = "../modules/vpc"
+      vpc_cidr_block                 = var.vpc_cidr_block
+      enable_dns_hostnames           = var.enable_dns_hostnames
+      vpc_public_subnets_cidr_block  = var.vpc_public_subnets_cidr_block
+      vpc_private_subnets_cidr_block = var.vpc_private_subnets_cidr_block
+      aws_azs                        = var.aws_azs
+      name_prefix                    = "nhsd-nrlf--dev"
+    }
+    module "powerbi_gw_instance_v2" {
+      source             = "../modules/ec2"
+      use_custom_ami     = true
+      instance_type      = var.instance_type
+      name_prefix        = "nhsd-nrlf--dev-powerbi-gw-v2"
+      target_bucket_arn  = module.dev-glue.target_bucket_arn
+      glue_kms_key_arn   = module.dev-glue.aws_kms_key_arn
+      athena_kms_key_arn = module.dev-athena.kms_key_arn
+      athena_bucket_arn  = module.dev-athena.bucket_arn
+      subnet_id       = module.vpc.private_subnet_id
+      security_groups = [module.vpc.powerbi_gw_security_group_id]
+    }

terraform/account-wide-infrastructure/dev/vars.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -13,3 +13,45 @@ variable "devsandbox_api_domain_name" { @@
       description = "The internal DNS name of the API Gateway for the dev sandbox environment"
       default     = "dev-sandbox.api.record-locator.dev.national.nhs.uk"
     }
+    variable "aws_azs" {
+      type        = string
+      description = "AWS Availability Zones"
+      default     = "eu-west-2a"
+    }
+    variable "enable_dns_hostnames" {
+      type        = bool
+      description = "Enable DNS hostnames in VPC"
+      default     = true
+    }
+    variable "vpc_cidr_block" {
+      type        = string
+      description = "Base CIDR Block for VPC"
+      default     = "10.0.0.0/16"
+    }
+    variable "vpc_public_subnets_cidr_block" {
+      type        = string
+      description = "CIDR Block for Public Subnets in VPC"
+      default     = "10.0.0.0/24"
+    }
+    variable "vpc_private_subnets_cidr_block" {
+      type        = string
+      description = "CIDR Block for Private Subnets in VPC"
+      default     = "10.0.1.0/24"
+    }
+    variable "instance_type" {
+      type        = string
+      description = "Type for EC2 Instance"
+      default     = "t2.micro"
+    }
+    variable "use_custom_ami" {
+      type        = bool
+      description = "Use custom image"
+      default     = false
+    }

terraform/account-wide-infrastructure/modules/athena/outputs.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,3 +5,11 @@ output "workgroup" { @@
     output "bucket" {
       value = aws_s3_bucket.athena
     }
+    output "bucket_arn" {
+      value = aws_s3_bucket.athena.arn
+    }
+    output "kms_key_arn" {
+      value = aws_kms_key.athena.arn
+    }

terraform/account-wide-infrastructure/modules/athena/s3.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -26,6 +26,23 @@ resource "aws_s3_bucket_policy" "athena" { @@
               }
             }
           },
+          {
+            Sid : "AllowAthenaAccess",
+            Effect : "Allow",
+            Principal : {
+              Service : "athena.amazonaws.com"
+            },
+            Action : [
+              "s3:PutObject",
+              "s3:GetBucketLocation",
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            Resource : [
+              aws_s3_bucket.athena.arn,
+              "${aws_s3_bucket.athena.arn}/*",
+            ]
+          },
         ]
       })
     }
@@ Expand Down @@

terraform/account-wide-infrastructure/modules/ec2/data.tf

-Original file line number
+Diff line change
@@ -0,0 +1,17 @@
+    data "aws_ami" "windows-2019" {
+      most_recent = true
+      owners      = ["amazon"]
+      filter {
+        name   = "name"
+        values = ["Windows_Server-2019-English-Full-Base*"]
+      }
+    }
+    data "aws_ami" "PowerBI_Gateway" {
+      most_recent = true
+      owners      = ["self"]
+      filter {
+        name   = "name"
+        values = ["PowerBI_GW"]
+      }
+    }

terraform/account-wide-infrastructure/modules/ec2/ec2.tf

-Original file line number
+Diff line change
@@ -0,0 +1,30 @@
+    resource "aws_instance" "web" {
+      associate_public_ip_address = false
+      iam_instance_profile        = aws_iam_instance_profile.powerbi_profile.name
+      ami                         = local.selected_ami_id
+      instance_type               = var.instance_type
+      key_name                    = aws_key_pair.ec2_key_pair.key_name
+      subnet_id                   = var.subnet_id
+      security_groups             = var.security_groups
+      user_data = file("${path.module}/scripts/user_data.tpl")
+      tags = {
+        Name = "${var.name_prefix}-ec2"
+      }
+    }
+    resource "tls_private_key" "instance_key_pair" {
+      algorithm = "RSA"
+    }
+    resource "aws_key_pair" "ec2_key_pair" {
+      key_name   = "${var.name_prefix}_PowerBI-GateWay-Key"
+      public_key = tls_private_key.instance_key_pair.public_key_openssh
+    }
+    resource "local_file" "ssh_key_priv" {
+      filename = "${path.module}/keys/${aws_key_pair.ec2_key_pair.key_name}.pem"
+      content  = tls_private_key.instance_key_pair.private_key_pem
+    }

terraform/account-wide-infrastructure/modules/ec2/iam.tf

-Original file line number
+Diff line change
@@ -0,0 +1,110 @@
+    resource "aws_iam_role" "ec2_service_role" {
+      name = "${var.name_prefix}-ec2_service_role"
+      assume_role_policy = jsonencode({
+        "Version" : "2012-10-17",
+        "Statement" : [
+          {
+            "Effect" : "Allow",
+            "Principal" : {
+              "Service" : "ec2.amazonaws.com"
+            },
+            "Action" : "sts:AssumeRole"
+          }
+        ]
+      })
+    }
+    data "aws_iam_policy_document" "ec2_service" {
+      statement {
+        actions = [
+          "s3:GetBucketLocation",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads",
+          "s3:ListMultipartUploadParts",
+          "s3:AbortMultipartUpload",
+          "s3:CreateBucket",
+          "s3:PutObject",
+          "s3:PutBucketPublicAccessBlock"
+        ]
+        resources = compact([
+          var.target_bucket_arn,
+          "${var.target_bucket_arn}/*",
+          var.athena_bucket_arn,
+          "${var.athena_bucket_arn}/*",
+        ])
+        effect = "Allow"
+      }
+      statement {
+        actions = [
+          "s3:ListBucket",
+          "s3:GetBucketLocation",
+          "s3:ListAllMyBuckets"
+        ]
+        resources = compact([
+          "*"
+        ])
+        effect = "Allow"
+      }
+      statement {
+        actions = [
+          "kms:DescribeKey",
+          "kms:GenerateDataKey*",
+          "kms:Encrypt",
+          "kms:ReEncrypt*",
+          "kms:Decrypt",
+        ]
+        resources = [
+          var.glue_kms_key_arn,
+          var.athena_kms_key_arn,
+        ]
+        effect = "Allow"
+      }
+      statement {
+        actions = [
+          "athena:*",
+        ]
+        effect = "Allow"
+        resources = [
+          "*"
+        ]
+      }
+      statement {
+        actions = [
+          "glue:*",
+        ]
+        effect = "Allow"
+        resources = [
+          "*"
+        ]
+      }
+    }
+    resource "aws_iam_policy" "ec2_service" {
+      name   = "${var.name_prefix}-ec2"
+      policy = data.aws_iam_policy_document.ec2_service.json
+    }
+    resource "aws_iam_role_policy_attachment" "ec2_role_policy" {
+      role       = aws_iam_role.ec2_service_role.name
+      policy_arn = aws_iam_policy.ec2_service.arn
+    }
+    resource "aws_iam_role_policy_attachment" "ec2_role_policy_ssm" {
+      role       = aws_iam_role.ec2_service_role.name
+      policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+    }
+    resource "aws_iam_instance_profile" "powerbi_profile" {
+      name = "${var.name_prefix}-powerbi_instance_profile"
+      role = aws_iam_role.ec2_service_role.name
+    }

terraform/account-wide-infrastructure/modules/ec2/locals.tf

-Original file line number
+Diff line change
@@ -0,0 +1,3 @@
+    locals {
+      selected_ami_id = var.use_custom_ami ? data.aws_ami.PowerBI_Gateway.id : data.aws_ami.windows-2019.id
+    }

terraform/account-wide-infrastructure/modules/ec2/outputs.tf

-Original file line number
+Diff line change
@@ -0,0 +1,7 @@
+    output "instance_id" {
+      value = aws_instance.web.id
+    }
+    output "public_ip" {
+      value = aws_instance.web.public_ip
+    }

terraform/account-wide-infrastructure/modules/ec2/scripts/user_data.tpl

-Original file line number
+Diff line change
@@ -0,0 +1,25 @@
+    <powershell>
+    Install-WindowsFeature -name Web-Server -IncludeManagementTools
+    $instanceId   = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/instance-id -UseBasicParsing).content
+    $instanceAZ   = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/placement/availability-zone -UseBasicParsing).content
+    $pubHostName  = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/public-hostname -UseBasicParsing).content
+    $pubIPv4      = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/public-ipv4 -UseBasicParsing).content
+    $privHostName = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/local-hostname -UseBasicParsing).content
+    $privIPv4     = (Invoke-WebRequest -Uri  http://169.254.169.254/latest/meta-data/local-ipv4 -UseBasicParsing).content
+    New-Item -Path C:\inetpub\wwwroot\index.html -ItemType File -Force
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<font face = "Verdana" size = "5">"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center><h1>AWS Windows VM Deployed with Terraform</h1></center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>EC2 Instance Metadata</b> </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>Instance ID:</b> $instanceId </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>AWS Availablity Zone:</b> $instanceAZ </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>Public Hostname:</b> $pubHostName </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>Public IPv4:</b> $pubIPv4 </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>Private Hostname:</b> $privHostName </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "<center> <b>Private IPv4:</b> $privIPv4 </center>"
+    Add-Content -Path C:\inetpub\wwwroot\index.html "</font>"
+    </powershell>

terraform/account-wide-infrastructure/modules/ec2/vars.tf

-Original file line number
+Diff line change
@@ -0,0 +1,9 @@
+    variable "name_prefix" {}
+    variable "instance_type" {}
+    variable "security_groups" {}
+    variable "subnet_id" {}
+    variable "glue_kms_key_arn" {}
+    variable "athena_kms_key_arn" {}
+    variable "target_bucket_arn" {}
+    variable "athena_bucket_arn" {}
+    variable "use_custom_ami" {}

terraform/account-wide-infrastructure/modules/glue/outputs.tf

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,11 +3,21 @@ output "target_bucket_name" { @@
       value       = aws_s3_bucket.target-data-bucket.id
     }
+    output "target_bucket_arn" {
+      description = "Arn of destination bucket"
+      value       = aws_s3_bucket.target-data-bucket.arn
+    }
     output "source_bucket_name" {
       description = "Name of source bucket"
       value       = aws_s3_bucket.source-data-bucket.id
     }
+    output "aws_kms_key_arn" {
+      description = "Arn of kms key"
+      value       = aws_kms_key.glue.arn
+    }
     output "glue_crawler_name" {
       value = "s3//${aws_s3_bucket.source-data-bucket.id}/"
     }

terraform/account-wide-infrastructure/modules/vpc/outputs.tf

-Original file line number
+Diff line change
@@ -0,0 +1,11 @@
+    output "subnet_id" {
+      value = aws_subnet.public_subnet.id
+    }
+    output "private_subnet_id" {
+      value = aws_subnet.private_subnet.id
+    }
+    output "powerbi_gw_security_group_id" {
+      value = aws_security_group.powerbi_gw_sg.id
+    }

terraform/account-wide-infrastructure/modules/vpc/vars.tf

-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+    variable "aws_azs" {}
+    variable "enable_dns_hostnames" {}
+    variable "vpc_cidr_block" {}
+    variable "vpc_public_subnets_cidr_block" {}
+    variable "vpc_private_subnets_cidr_block" {}
+    variable "name_prefix" {}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

NRL 1385 Set up EC2 for automated data refresh #903

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!