From a53253a5d7e1aef279150276b9318bc0734152f5 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 27 Nov 2024 16:02:19 +0000 Subject: [PATCH 01/60] NRL-1188 S3 reporting module --- .../account-wide-infrastructure/dev/s3.tf | 12 ++++ .../modules/reporting-bucket/output.tf | 9 +++ .../modules/reporting-bucket/s3.tf | 63 +++++++++++++++++++ .../modules/reporting-bucket/vars.tf | 15 +++++ .../modules/truststore-bucket/output.tf | 4 +- 5 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf create mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf create mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf diff --git a/terraform/account-wide-infrastructure/dev/s3.tf b/terraform/account-wide-infrastructure/dev/s3.tf index b90bf677f..073edfa93 100644 --- a/terraform/account-wide-infrastructure/dev/s3.tf +++ b/terraform/account-wide-infrastructure/dev/s3.tf @@ -21,3 +21,15 @@ module "dev-sandbox-truststore-bucket" { name_prefix = "nhsd-nrlf--dev-sandbox" server_certificate_file = "../../../truststore/server/dev.pem" } + +module "dev-reporting-bucket" { + source = "../modules/reporting-bucket" + name_prefix = "nhsd-nrlf-reporting--dev" + server_certificate_file = "../../../truststore/server/dev.pem" +} + +module "dev-sandbox-reporting-bucket" { + source = "../modules/reporting-bucket" + name_prefix = "nhsd-nrlf-reporting--dev-sandbox" + server_certificate_file = "../../../truststore/server/dev.pem" +} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf new file mode 100644 index 000000000..6bd416b8b --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf @@ -0,0 +1,9 @@ +output "bucket_name" { + description = "Name of the truststore S3 bucket" + value = aws_s3_bucket.reporting.bucket +} + +output "certificates_object_key" { + description = "Key of the truststore certificates object" + value = aws_s3_object.reporting.key +} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf new file mode 100644 index 000000000..8ffb497ad --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf @@ -0,0 +1,63 @@ +resource "aws_s3_bucket" "reporting" { + bucket = "${var.name_prefix}-reporting" + force_destroy = var.enable_bucket_force_destroy +} + +resource "aws_s3_bucket_policy" "reporting_bucket_policy" { + bucket = aws_s3_bucket.reporting.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "reporting_bucket_policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [ + aws_s3_bucket.reporting.arn, + "${aws_s3_bucket.reporting.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + +resource "aws_s3_bucket_public_access_block" "reporting-public-access-block" { + bucket = aws_s3_bucket.reporting.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "reporting" { + bucket = aws_s3_bucket.reporting.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_versioning" "reporting" { + bucket = aws_s3_bucket.reporting.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_object" "reporting_certificate" { + bucket = aws_s3_bucket.reporting.bucket + key = "certificates.pem" + source = var.server_certificate_file + etag = filemd5(var.server_certificate_file) +} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf new file mode 100644 index 000000000..3c6fa8790 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf @@ -0,0 +1,15 @@ +variable "name_prefix" { + type = string + description = "The prefix to apply to all resources in the module." +} + +variable "server_certificate_file" { + type = string + description = "The path to the server certificate file." +} + +variable "enable_bucket_force_destroy" { + type = bool + description = "A boolean flag to enable force destroy of the S3 bucket, so that all objects in the bucket are deleted when the bucket is destroyed." + default = false +} diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf index b8cd08057..1c0a2a86e 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf @@ -1,9 +1,9 @@ output "bucket_name" { - description = "Name of the truststore S3 bucket" + description = "Name of the reporting S3 bucket" value = aws_s3_bucket.api_truststore.bucket } output "certificates_object_key" { - description = "Key of the truststore certificates object" + description = "Key of the reporting certificates object" value = aws_s3_object.api_truststore_certificate.key } From 372eeb07062da1684cf7cea98428928b5b53336b Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 29 Nov 2024 16:35:01 +0000 Subject: [PATCH 02/60] NRL-1188 s3 iam --- .../modules/reporting-bucket/iam.tf | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf new file mode 100644 index 000000000..621807bae --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf @@ -0,0 +1,20 @@ +resource "aws_iam_policy" "read-s3-authorization-store" { + name = "${var.name_prefix}-read-s3-authorization-store" + description = "Read the authorization store S3 bucket" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:ListBucket", + ] + Effect = "Allow" + Resource = [ + aws_s3_bucket.authorization-store.arn, + "${aws_s3_bucket.authorization-store.arn}/*", + ] + }, + ] + }) +} From 9f1c2d1c39761c6d2fa6ca2de37189ded13f8030 Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 29 Nov 2024 16:35:17 +0000 Subject: [PATCH 03/60] NRL-1188 reporting kinesis stream --- .../modules/firehose/kinesis.tf | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index de9a65162..077637853 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -56,3 +56,32 @@ resource "aws_kinesis_firehose_delivery_stream" "firehose" { } } } + +resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { + name = "${var.prefix}--cloudwatch-reporting-delivery-stream" + destination = var.destination + + extended_s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.reporting.arn + + processing_configuration { + enabled = "true" + + processors { + type = "CloudWatchLogProcessing" + + parameters { + parameter_name = "DataMessageExtraction" + parameter_value = "true" + } + } + } + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.firehose.name + log_stream_name = aws_cloudwatch_log_stream.firehose.name + } + } +} From 2b2863015b4874228c6e526def48e99ea8c4098a Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 29 Nov 2024 16:35:23 +0000 Subject: [PATCH 04/60] NRL-1188 glue set up --- .../modules/glue/glue.tf | 12 ++++++++++ .../modules/glue/iam.tf | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 terraform/account-wide-infrastructure/modules/glue/glue.tf create mode 100644 terraform/account-wide-infrastructure/modules/glue/iam.tf diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf new file mode 100644 index 000000000..a2c488078 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -0,0 +1,12 @@ +resource "aws_glue_catalog_database" "example" { + name = "example" +} + +resource "aws_glue_job" "example" { + name = "example" + role_arn = aws_iam_role.glue.arn + command { + script_location = "s3://my-script-location/script.py" + python_version = "3" + } +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf new file mode 100644 index 000000000..307b13c79 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -0,0 +1,22 @@ +// Define IAM Role +resource “aws_iam_role” “glue” { + name = “GlueServiceRole” + assume_role_policy = jsonencode({ + “Version”: “2012–10–17”, + “Statement”: [ + { + “Action”: “sts:AssumeRole”, + “Principal”: { + “Service”: “glue.amazonaws.com” + }, + “Effect”: “Allow”, + }, + ] + }) +} + +// Attach Policy +resource “aws_iam_role_policy_attachment” “glue” { + role = aws_iam_role.glue.name + policy_arn = “arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole” +} \ No newline at end of file From 58815a5185a981e2806dce64060797bfd0c2e168 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 4 Dec 2024 15:00:22 +0000 Subject: [PATCH 05/60] NRL-1188 glue set up --- .../account-wide-infrastructure/dev/glue.tf | 5 + .../modules/glue/glue.tf | 63 ++++++++-- .../modules/glue/iam.tf | 109 +++++++++++++++--- .../modules/glue/s3.tf | 22 ++++ .../modules/glue/src/main.py | 45 ++++++++ .../modules/glue/vars.tf | 24 ++++ 6 files changed, 240 insertions(+), 28 deletions(-) create mode 100644 terraform/account-wide-infrastructure/dev/glue.tf create mode 100644 terraform/account-wide-infrastructure/modules/glue/s3.tf create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/main.py create mode 100644 terraform/account-wide-infrastructure/modules/glue/vars.tf diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf new file mode 100644 index 000000000..0cc7c40cc --- /dev/null +++ b/terraform/account-wide-infrastructure/dev/glue.tf @@ -0,0 +1,5 @@ +module "dev-glue" { + source = "../modules/glue" + name_prefix = "nhsd-nrlf--dev" + python_version = "3" +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index a2c488078..aca758989 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -1,12 +1,57 @@ -resource "aws_glue_catalog_database" "example" { - name = "example" +# Create Glue Data Catalog Database +resource "aws_glue_catalog_database" "raw_log_database" { + name = "raw_log" + location_uri = "${aws_s3_bucket.source-data-bucket.id}/" } -resource "aws_glue_job" "example" { - name = "example" - role_arn = aws_iam_role.glue.arn - command { - script_location = "s3://my-script-location/script.py" - python_version = "3" - } +# Create Glue Crawler +resource "aws_glue_crawler" "raw_log_crawler" { + name = "raw-log-crawler" + database_name = aws_glue_catalog_database.raw_log_database.name + role = aws_iam_role.glue_service_role.name + s3_target { + path = "${aws_s3_bucket.source-data-bucket.id}/" + } + schema_change_policy { + delete_behavior = "LOG" + } + configuration = jsonencode({ + "Version":1.0, + "Grouping": { + "TableGroupingPolicy": "CombineCompatibleSchemas" + } + }) +} +resource "aws_glue_trigger""raw_log_trigger" { + name = "org-report-trigger" + type = "ON_DEMAND" + actions { + crawler_name = aws_glue_crawler.raw_log_crawler.name + } +} + +resource "aws_glue_job" "glue_job" { + name = "poc-glue-job" + role_arn = aws_iam_role.glue_service_role.arn + description = "Transfer logs from source to bucket" + glue_version = "4.0" + worker_type = "G.1X" + timeout = 2880 + max_retries = 1 + number_of_workers = 2 + command { + name = "glueetl" + python_version = var.python_version + script_location = "s3://${aws_s3_bucket.code-bucket.id}/script.py" + } + + default_arguments = { + "--enable-auto-scaling" = "true""--enable-continous-cloudwatch-log" = "true""--datalake-formats" = "delta""--source-path" = "s3://${aws_s3_bucket.dc-source-data-bucket.id}/" # Specify the source S3 path + "--destination-path" = "s3://${aws_s3_bucket.target-data-bucket.id}/" # Specify the destination S3 path + "--job-name" = "poc-glue-job""--enable-continuous-log-filter" = "true""--enable-metrics" = "true" + } +} + +output "glue_crawler_name" { + value = "s3//${aws_s3_bucket.source-data-bucket.id}/" } \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 307b13c79..9c4bb0bf1 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -1,22 +1,93 @@ -// Define IAM Role -resource “aws_iam_role” “glue” { - name = “GlueServiceRole” - assume_role_policy = jsonencode({ - “Version”: “2012–10–17”, - “Statement”: [ - { - “Action”: “sts:AssumeRole”, - “Principal”: { - “Service”: “glue.amazonaws.com” - }, - “Effect”: “Allow”, - }, - ] - }) +resource "aws_iam_role""glue_service_role" { + name = "glue_service_role" + + assume_role_policy = jsonencode({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }) } -// Attach Policy -resource “aws_iam_role_policy_attachment” “glue” { - role = aws_iam_role.glue.name - policy_arn = “arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole” +resource "aws_iam_role_policy""glue_service_role_policy" { + name = "glue_service_role_policy" + role = aws_iam_role.glue_service_role.name + policy = jsonencode({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "glue:*", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:GetBucketAcl", + "ec2:DescribeVpcEndpoints", + "ec2:DescribeRouteTables", + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcAttribute", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "cloudwatch:PutMetricData" + ], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": ["s3:CreateBucket"], + "Resource": ["arn:aws:s3:::aws-glue-*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": [ + "arn:aws:s3:::*/*", + "arn:aws:s3:::*/*aws-glue-*/*" + ] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": [ + "arn:aws:s3:::crawler-public*", + "arn:aws:s3:::aws-glue-*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": ["arn:aws:logs:*:*:*:/aws-glue/*"] + }, + { + "Effect": "Allow", + "Action": ["ec2:CreateTags", "ec2:DeleteTags"], + "Condition": { + "ForAllValues:StringEquals": { + "aws:TagKeys": ["aws-glue-service-resource"] + } + }, + "Resource": [ + "arn:aws:ec2:*:*:network-interface/*", + "arn:aws:ec2:*:*:security-group/*", + "arn:aws:ec2:*:*:instance/*" + ] + } + ] +}) } \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf new file mode 100644 index 000000000..1c5701fad --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -0,0 +1,22 @@ +# S3 Bucket for Raw Data +resource "aws_s3_bucket" "source-data-bucket" { + bucket = "source-data-bucket" +} + +# S3 Bucket for Processed Data +resource "aws_s3_bucket" "target-data-bucket" { + bucket = "target-data-bucket" +} + + +# S3 Bucket for Code +resource "aws_s3_bucket" "code-bucket" { + bucket = "code-bucket" +} + +resource "aws_s3_bucket_object""code-data-object" { + bucket = aws_s3_bucket.code-bucket.bucket + key = "main.py" + source = "${path.module}/src/main.py" + etag = "${filemd5("${path.module}/src/main.py")}" +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py new file mode 100644 index 000000000..28d0dcf3b --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -0,0 +1,45 @@ +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job + +# Initialize SparkContext, GlueContext, and SparkSession +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session + +# Get job arguments +args = getResolvedOptions(sys.argv, ['JOB_NAME']) + +# Create Glue job +job = Job(glueContext) +job.init(args['JOB_NAME'], args) + +# Script generated for node AWS Glue Data Catalog +AWSGlueDataCatalog_node1704102689282 = glueContext.create_dynamic_frame.from_catalog( + database="raw-log", + table_name="source_data_bucket", + transformation_ctx="AWSGlueDataCatalog_node1704102689282", +) + +# Script generated for node Change Schema +ChangeSchema_node1704102716061 = ApplyMapping.apply( + frame=AWSGlueDataCatalog_node1704102689282, + mappings=[ + # TBC + ], + transformation_ctx="ChangeSchema_node1704102716061", +) + +# Script generated for node Amazon S3 +AmazonS3_node1704102720699 = glueContext.write_dynamic_frame.from_options( + frame=ChangeSchema_node1704102716061, + connection_type="s3", + format="csv", + connection_options={"path": "s3://target-data-bucket", "partitionKeys": []}, + transformation_ctx="AmazonS3_node1704102720699", +) + +job.commit() \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/glue/vars.tf b/terraform/account-wide-infrastructure/modules/glue/vars.tf new file mode 100644 index 000000000..c1e602513 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/vars.tf @@ -0,0 +1,24 @@ +variable "name_prefix" { + type = string + description = "The prefix to apply to all resources in the module." +} + +variable "python_version" { + type = string + description = "Python version to run in script" +} + +variable "source_bucket" { + description = "S3 bucket for source data" + default = "source-data-bucket" +} + +variable "target_bucket" { + description = "S3 bucket for target data" + default = "target-data-bucket" +} + +variable "code_bucket" { + description = "S3 bucket for Glue job scripts" + default = "code-bucket" +} \ No newline at end of file From 89c8873094bc5868085f920ca15b592afdf33271 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 4 Dec 2024 16:10:46 +0000 Subject: [PATCH 06/60] NRL-1188 athena set up --- .../modules/athena/athena.tf | 32 +++++++++++++++++++ .../modules/athena/outputs.tf | 11 +++++++ .../modules/athena/s3.tf | 15 +++++++++ .../modules/athena/vars.tf | 9 ++++++ 4 files changed, 67 insertions(+) create mode 100644 terraform/account-wide-infrastructure/modules/athena/athena.tf create mode 100644 terraform/account-wide-infrastructure/modules/athena/outputs.tf create mode 100644 terraform/account-wide-infrastructure/modules/athena/s3.tf create mode 100644 terraform/account-wide-infrastructure/modules/athena/vars.tf diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf new file mode 100644 index 000000000..37e6a7a44 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -0,0 +1,32 @@ +resource "aws_athena_database" "reporting-db" { + name = var.database + + bucket = aws_s3_bucket.target-data-bucket.bucket + +# encryption_configuration { +# encryption_option = var.encryption_option +# kms_key = var.kms_key_arn +# } + + force_destroy = true +} + +resource "aws_athena_workgroup" "athena" { + name = var.name_prefix + + configuration { + enforce_workgroup_configuration = true + publish_cloudwatch_metrics_enabled = true + + result_configuration { + output_location = "s3://{aws_s3_bucket.example.bucket}/output/" + + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key_arn = var.kms_key_arn + } + } + } + + tags = var.common_tags +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/athena/outputs.tf b/terraform/account-wide-infrastructure/modules/athena/outputs.tf new file mode 100644 index 000000000..ab68db95b --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/athena/outputs.tf @@ -0,0 +1,11 @@ +output "workgroup" { + value = aws_athena_workgroup.athena +} + +output "bucket" { + value = aws_s3_bucket.athena +} + +output "database" { + value = aws_athena_database.reporting-db +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf new file mode 100644 index 000000000..3f8f77063 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -0,0 +1,15 @@ +resource "aws_s3_bucket" "athena" { + bucket = "athena" +} + + +resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { + bucket = aws_s3_bucket.athena.bucket + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = var.kms_key_arn + } + } + +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/athena/vars.tf b/terraform/account-wide-infrastructure/modules/athena/vars.tf new file mode 100644 index 000000000..7ac48c4f8 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/athena/vars.tf @@ -0,0 +1,9 @@ +variable "database" { + description = "What the db will be called" + default = "NRL-Reporting" +} + +variable "name_prefix" { + type = string + description = "The prefix to apply to all resources in the module." +} \ No newline at end of file From 7b5cdebadc42bc1eae24ac46172af042569c6609 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 4 Dec 2024 16:11:35 +0000 Subject: [PATCH 07/60] NRL-1188 s3 configured in glue module --- .../account-wide-infrastructure/dev/s3.tf | 12 ---- .../modules/reporting-bucket/iam.tf | 20 ------ .../modules/reporting-bucket/output.tf | 9 --- .../modules/reporting-bucket/s3.tf | 63 ------------------- .../modules/reporting-bucket/vars.tf | 15 ----- 5 files changed, 119 deletions(-) delete mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf delete mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf delete mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf delete mode 100644 terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf diff --git a/terraform/account-wide-infrastructure/dev/s3.tf b/terraform/account-wide-infrastructure/dev/s3.tf index 073edfa93..b90bf677f 100644 --- a/terraform/account-wide-infrastructure/dev/s3.tf +++ b/terraform/account-wide-infrastructure/dev/s3.tf @@ -21,15 +21,3 @@ module "dev-sandbox-truststore-bucket" { name_prefix = "nhsd-nrlf--dev-sandbox" server_certificate_file = "../../../truststore/server/dev.pem" } - -module "dev-reporting-bucket" { - source = "../modules/reporting-bucket" - name_prefix = "nhsd-nrlf-reporting--dev" - server_certificate_file = "../../../truststore/server/dev.pem" -} - -module "dev-sandbox-reporting-bucket" { - source = "../modules/reporting-bucket" - name_prefix = "nhsd-nrlf-reporting--dev-sandbox" - server_certificate_file = "../../../truststore/server/dev.pem" -} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf deleted file mode 100644 index 621807bae..000000000 --- a/terraform/account-wide-infrastructure/modules/reporting-bucket/iam.tf +++ /dev/null @@ -1,20 +0,0 @@ -resource "aws_iam_policy" "read-s3-authorization-store" { - name = "${var.name_prefix}-read-s3-authorization-store" - description = "Read the authorization store S3 bucket" - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = [ - "s3:GetObject", - "s3:ListBucket", - ] - Effect = "Allow" - Resource = [ - aws_s3_bucket.authorization-store.arn, - "${aws_s3_bucket.authorization-store.arn}/*", - ] - }, - ] - }) -} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf deleted file mode 100644 index 6bd416b8b..000000000 --- a/terraform/account-wide-infrastructure/modules/reporting-bucket/output.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "bucket_name" { - description = "Name of the truststore S3 bucket" - value = aws_s3_bucket.reporting.bucket -} - -output "certificates_object_key" { - description = "Key of the truststore certificates object" - value = aws_s3_object.reporting.key -} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf deleted file mode 100644 index 8ffb497ad..000000000 --- a/terraform/account-wide-infrastructure/modules/reporting-bucket/s3.tf +++ /dev/null @@ -1,63 +0,0 @@ -resource "aws_s3_bucket" "reporting" { - bucket = "${var.name_prefix}-reporting" - force_destroy = var.enable_bucket_force_destroy -} - -resource "aws_s3_bucket_policy" "reporting_bucket_policy" { - bucket = aws_s3_bucket.reporting.id - - policy = jsonencode({ - Version = "2012-10-17" - Id = "reporting_bucket_policy" - Statement = [ - { - Sid = "HTTPSOnly" - Effect = "Deny" - Principal = "*" - Action = "s3:*" - Resource = [ - aws_s3_bucket.reporting.arn, - "${aws_s3_bucket.reporting.arn}/*", - ] - Condition = { - Bool = { - "aws:SecureTransport" = "false" - } - } - }, - ] - }) -} - -resource "aws_s3_bucket_public_access_block" "reporting-public-access-block" { - bucket = aws_s3_bucket.reporting.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "reporting" { - bucket = aws_s3_bucket.reporting.bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_versioning" "reporting" { - bucket = aws_s3_bucket.reporting.id - versioning_configuration { - status = "Enabled" - } -} - -resource "aws_s3_object" "reporting_certificate" { - bucket = aws_s3_bucket.reporting.bucket - key = "certificates.pem" - source = var.server_certificate_file - etag = filemd5(var.server_certificate_file) -} diff --git a/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf b/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf deleted file mode 100644 index 3c6fa8790..000000000 --- a/terraform/account-wide-infrastructure/modules/reporting-bucket/vars.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "name_prefix" { - type = string - description = "The prefix to apply to all resources in the module." -} - -variable "server_certificate_file" { - type = string - description = "The path to the server certificate file." -} - -variable "enable_bucket_force_destroy" { - type = bool - description = "A boolean flag to enable force destroy of the S3 bucket, so that all objects in the bucket are deleted when the bucket is destroyed." - default = false -} From 84f7ab37aba996ebcf396c9dc1fce6e294dfba81 Mon Sep 17 00:00:00 2001 From: jackleary Date: Thu, 5 Dec 2024 07:26:53 +0000 Subject: [PATCH 08/60] NRL-1188 invoke athena module --- terraform/account-wide-infrastructure/dev/athena.tf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 terraform/account-wide-infrastructure/dev/athena.tf diff --git a/terraform/account-wide-infrastructure/dev/athena.tf b/terraform/account-wide-infrastructure/dev/athena.tf new file mode 100644 index 000000000..b078d462c --- /dev/null +++ b/terraform/account-wide-infrastructure/dev/athena.tf @@ -0,0 +1,4 @@ +module "dev-athena" { + source = "../modules/athena" + name_prefix = "nhsd-nrlf--dev" +} \ No newline at end of file From 9b04618a600cdbaec292ec2e82bca65dd65bb7d1 Mon Sep 17 00:00:00 2001 From: jackleary Date: Thu, 5 Dec 2024 07:36:18 +0000 Subject: [PATCH 09/60] NRL-1188 kinesis set up --- terraform/infrastructure/modules/firehose/iam_subscriptions.tf | 1 + terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf index 05006243b..6dae946fc 100644 --- a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf +++ b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf @@ -22,6 +22,7 @@ data "aws_iam_policy_document" "firehose_subscription" { effect = "Allow" resources = [ aws_kinesis_firehose_delivery_stream.firehose.arn, + aws_kinesis_firehose_delivery_stream.reporting_stream.arn, ] } statement { diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 077637853..62e12ac3f 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.reporting.arn + bucket_arn = aws_s3_bucket.source-data-bucket.arn processing_configuration { enabled = "true" From d99de770490352f1d2158a37a67c4b805829e16f Mon Sep 17 00:00:00 2001 From: jackleary Date: Thu, 5 Dec 2024 08:14:18 +0000 Subject: [PATCH 10/60] NRL-1188 athena kms --- .../account-wide-infrastructure/modules/athena/athena.tf | 9 ++++----- .../account-wide-infrastructure/modules/athena/kms.tf | 7 +++++++ .../account-wide-infrastructure/modules/athena/s3.tf | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/athena/kms.tf diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index 37e6a7a44..bacbd44e3 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -4,8 +4,8 @@ resource "aws_athena_database" "reporting-db" { bucket = aws_s3_bucket.target-data-bucket.bucket # encryption_configuration { -# encryption_option = var.encryption_option -# kms_key = var.kms_key_arn +# encryption_option = "SSE_KMS" +# kms_key = aws_kms_key.athena.arn # } force_destroy = true @@ -19,14 +19,13 @@ resource "aws_athena_workgroup" "athena" { publish_cloudwatch_metrics_enabled = true result_configuration { - output_location = "s3://{aws_s3_bucket.example.bucket}/output/" + output_location = "s3://{aws_s3_bucket.athena.bucket}/output/" encryption_configuration { encryption_option = "SSE_KMS" - kms_key_arn = var.kms_key_arn + kms_key_arn = aws_kms_key.athena.arn } } } - tags = var.common_tags } \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/athena/kms.tf b/terraform/account-wide-infrastructure/modules/athena/kms.tf new file mode 100644 index 000000000..2079ee656 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/athena/kms.tf @@ -0,0 +1,7 @@ +resource "aws_kms_key" "athena" { +} + +resource "aws_kms_alias" "athena" { + name = "alias/${var.prefix}-athena" + target_key_id = aws_kms_key.athena.key_id +} \ No newline at end of file diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 3f8f77063..420cf29c8 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -8,7 +8,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { rule { apply_server_side_encryption_by_default { sse_algorithm = "aws:kms" - kms_master_key_id = var.kms_key_arn + kms_master_key_id = aws_kms_key.athena.arn } } From 1763cfd9866ef419baf20789692075bfef994109 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 10 Dec 2024 10:09:04 +0000 Subject: [PATCH 11/60] NRL-1188 terraform linting and glue python version update --- .../account-wide-infrastructure/dev/athena.tf | 2 +- .../account-wide-infrastructure/dev/glue.tf | 8 +- .../modules/athena/athena.tf | 10 +- .../modules/athena/kms.tf | 2 +- .../modules/athena/outputs.tf | 2 +- .../modules/athena/s3.tf | 2 +- .../modules/athena/vars.tf | 6 +- .../modules/glue/glue.tf | 19 ++- .../modules/glue/iam.tf | 156 +++++++++--------- .../modules/glue/s3.tf | 6 +- .../modules/glue/vars.tf | 6 +- 11 files changed, 112 insertions(+), 107 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/athena.tf b/terraform/account-wide-infrastructure/dev/athena.tf index b078d462c..07dc43554 100644 --- a/terraform/account-wide-infrastructure/dev/athena.tf +++ b/terraform/account-wide-infrastructure/dev/athena.tf @@ -1,4 +1,4 @@ module "dev-athena" { source = "../modules/athena" name_prefix = "nhsd-nrlf--dev" -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf index 0cc7c40cc..aab1623ce 100644 --- a/terraform/account-wide-infrastructure/dev/glue.tf +++ b/terraform/account-wide-infrastructure/dev/glue.tf @@ -1,5 +1,5 @@ module "dev-glue" { - source = "../modules/glue" - name_prefix = "nhsd-nrlf--dev" - python_version = "3" -} \ No newline at end of file + source = "../modules/glue" + name_prefix = "nhsd-nrlf--dev" + python_version = "3.12.2" +} diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index bacbd44e3..b8cf1de1d 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -3,10 +3,10 @@ resource "aws_athena_database" "reporting-db" { bucket = aws_s3_bucket.target-data-bucket.bucket -# encryption_configuration { -# encryption_option = "SSE_KMS" -# kms_key = aws_kms_key.athena.arn -# } + # encryption_configuration { + # encryption_option = "SSE_KMS" + # kms_key = aws_kms_key.athena.arn + # } force_destroy = true } @@ -28,4 +28,4 @@ resource "aws_athena_workgroup" "athena" { } } -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/athena/kms.tf b/terraform/account-wide-infrastructure/modules/athena/kms.tf index 2079ee656..9a9ee0f90 100644 --- a/terraform/account-wide-infrastructure/modules/athena/kms.tf +++ b/terraform/account-wide-infrastructure/modules/athena/kms.tf @@ -4,4 +4,4 @@ resource "aws_kms_key" "athena" { resource "aws_kms_alias" "athena" { name = "alias/${var.prefix}-athena" target_key_id = aws_kms_key.athena.key_id -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/athena/outputs.tf b/terraform/account-wide-infrastructure/modules/athena/outputs.tf index ab68db95b..574aeb3f8 100644 --- a/terraform/account-wide-infrastructure/modules/athena/outputs.tf +++ b/terraform/account-wide-infrastructure/modules/athena/outputs.tf @@ -8,4 +8,4 @@ output "bucket" { output "database" { value = aws_athena_database.reporting-db -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 420cf29c8..71eabdcad 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -12,4 +12,4 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { } } -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/athena/vars.tf b/terraform/account-wide-infrastructure/modules/athena/vars.tf index 7ac48c4f8..285f18823 100644 --- a/terraform/account-wide-infrastructure/modules/athena/vars.tf +++ b/terraform/account-wide-infrastructure/modules/athena/vars.tf @@ -1,9 +1,9 @@ variable "database" { - description = "What the db will be called" - default = "NRL-Reporting" + description = "What the db will be called" + default = "NRL-Reporting" } variable "name_prefix" { type = string description = "The prefix to apply to all resources in the module." -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index aca758989..93ad225e4 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -16,13 +16,13 @@ resource "aws_glue_crawler" "raw_log_crawler" { delete_behavior = "LOG" } configuration = jsonencode({ - "Version":1.0, - "Grouping": { - "TableGroupingPolicy": "CombineCompatibleSchemas" + "Version" : 1.0, + "Grouping" : { + "TableGroupingPolicy" : "CombineCompatibleSchemas" } }) } -resource "aws_glue_trigger""raw_log_trigger" { +resource "aws_glue_trigger" "raw_log_trigger" { name = "org-report-trigger" type = "ON_DEMAND" actions { @@ -46,12 +46,17 @@ resource "aws_glue_job" "glue_job" { } default_arguments = { - "--enable-auto-scaling" = "true""--enable-continous-cloudwatch-log" = "true""--datalake-formats" = "delta""--source-path" = "s3://${aws_s3_bucket.dc-source-data-bucket.id}/" # Specify the source S3 path + "--enable-auto-scaling" = "true" + "--enable-continous-cloudwatch-log" = "true" + "--datalake-formats" = "delta" + "--source-path" = "s3://${aws_s3_bucket.source-data-bucket.id}/" # Specify the source S3 path "--destination-path" = "s3://${aws_s3_bucket.target-data-bucket.id}/" # Specify the destination S3 path - "--job-name" = "poc-glue-job""--enable-continuous-log-filter" = "true""--enable-metrics" = "true" + "--job-name" = "poc-glue-job" + "--enable-continuous-log-filter" = "true" + "--enable-metrics" = "true" } } output "glue_crawler_name" { value = "s3//${aws_s3_bucket.source-data-bucket.id}/" -} \ No newline at end of file +} diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 9c4bb0bf1..d60634adc 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -1,93 +1,93 @@ -resource "aws_iam_role""glue_service_role" { +resource "aws_iam_role" "glue_service_role" { name = "glue_service_role" assume_role_policy = jsonencode({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Service" : "glue.amazonaws.com" }, - "Action": "sts:AssumeRole" - } + "Action" : "sts:AssumeRole" + } ] - }) + }) } -resource "aws_iam_role_policy""glue_service_role_policy" { - name = "glue_service_role_policy" - role = aws_iam_role.glue_service_role.name +resource "aws_iam_role_policy" "glue_service_role_policy" { + name = "glue_service_role_policy" + role = aws_iam_role.glue_service_role.name policy = jsonencode({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "glue:*", - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListAllMyBuckets", - "s3:GetBucketAcl", - "ec2:DescribeVpcEndpoints", - "ec2:DescribeRouteTables", - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeVpcAttribute", - "iam:ListRolePolicies", - "iam:GetRole", - "iam:GetRolePolicy", - "cloudwatch:PutMetricData" + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "glue:*", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:GetBucketAcl", + "ec2:DescribeVpcEndpoints", + "ec2:DescribeRouteTables", + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcAttribute", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "cloudwatch:PutMetricData" ], - "Resource": ["*"] - }, - { - "Effect": "Allow", - "Action": ["s3:CreateBucket"], - "Resource": ["arn:aws:s3:::aws-glue-*"] - }, - { - "Effect": "Allow", - "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": [ - "arn:aws:s3:::*/*", - "arn:aws:s3:::*/*aws-glue-*/*" + "Resource" : ["*"] + }, + { + "Effect" : "Allow", + "Action" : ["s3:CreateBucket"], + "Resource" : ["arn:aws:s3:::aws-glue-*"] + }, + { + "Effect" : "Allow", + "Action" : ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource" : [ + "arn:aws:s3:::*/*", + "arn:aws:s3:::*/*aws-glue-*/*" ] - }, - { - "Effect": "Allow", - "Action": ["s3:GetObject"], - "Resource": [ - "arn:aws:s3:::crawler-public*", - "arn:aws:s3:::aws-glue-*" + }, + { + "Effect" : "Allow", + "Action" : ["s3:GetObject"], + "Resource" : [ + "arn:aws:s3:::crawler-public*", + "arn:aws:s3:::aws-glue-*" ] - }, - { - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" + }, + { + "Effect" : "Allow", + "Action" : [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" ], - "Resource": ["arn:aws:logs:*:*:*:/aws-glue/*"] - }, - { - "Effect": "Allow", - "Action": ["ec2:CreateTags", "ec2:DeleteTags"], - "Condition": { - "ForAllValues:StringEquals": { - "aws:TagKeys": ["aws-glue-service-resource"] - } + "Resource" : ["arn:aws:logs:*:*:*:/aws-glue/*"] + }, + { + "Effect" : "Allow", + "Action" : ["ec2:CreateTags", "ec2:DeleteTags"], + "Condition" : { + "ForAllValues:StringEquals" : { + "aws:TagKeys" : ["aws-glue-service-resource"] + } }, - "Resource": [ - "arn:aws:ec2:*:*:network-interface/*", - "arn:aws:ec2:*:*:security-group/*", - "arn:aws:ec2:*:*:instance/*" + "Resource" : [ + "arn:aws:ec2:*:*:network-interface/*", + "arn:aws:ec2:*:*:security-group/*", + "arn:aws:ec2:*:*:instance/*" ] - } + } ] -}) -} \ No newline at end of file + }) +} diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 1c5701fad..099c5c012 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -14,9 +14,9 @@ resource "aws_s3_bucket" "code-bucket" { bucket = "code-bucket" } -resource "aws_s3_bucket_object""code-data-object" { +resource "aws_s3_bucket_object" "code-data-object" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" source = "${path.module}/src/main.py" - etag = "${filemd5("${path.module}/src/main.py")}" -} \ No newline at end of file + etag = filemd5("${path.module}/src/main.py") +} diff --git a/terraform/account-wide-infrastructure/modules/glue/vars.tf b/terraform/account-wide-infrastructure/modules/glue/vars.tf index c1e602513..63842e2d8 100644 --- a/terraform/account-wide-infrastructure/modules/glue/vars.tf +++ b/terraform/account-wide-infrastructure/modules/glue/vars.tf @@ -4,8 +4,8 @@ variable "name_prefix" { } variable "python_version" { - type = string - description = "Python version to run in script" + type = string + description = "Python version to run in script" } variable "source_bucket" { @@ -21,4 +21,4 @@ variable "target_bucket" { variable "code_bucket" { description = "S3 bucket for Glue job scripts" default = "code-bucket" -} \ No newline at end of file +} From a2502fcd3d28986a1154bbdd7d2d186c19868701 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 10 Dec 2024 10:10:42 +0000 Subject: [PATCH 12/60] NRL-1186 python transformation script update --- .../modules/glue/src/main.py | 154 +++++++++++++----- .../modules/glue/src/transforms.py | 0 2 files changed, 110 insertions(+), 44 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/transforms.py diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 28d0dcf3b..592f01e20 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -1,45 +1,111 @@ -import sys -from awsglue.transforms import * -from awsglue.utils import getResolvedOptions -from pyspark.context import SparkContext from awsglue.context import GlueContext -from awsglue.job import Job - -# Initialize SparkContext, GlueContext, and SparkSession -sc = SparkContext() -glueContext = GlueContext(sc) -spark = glueContext.spark_session - -# Get job arguments -args = getResolvedOptions(sys.argv, ['JOB_NAME']) - -# Create Glue job -job = Job(glueContext) -job.init(args['JOB_NAME'], args) - -# Script generated for node AWS Glue Data Catalog -AWSGlueDataCatalog_node1704102689282 = glueContext.create_dynamic_frame.from_catalog( - database="raw-log", - table_name="source_data_bucket", - transformation_ctx="AWSGlueDataCatalog_node1704102689282", -) - -# Script generated for node Change Schema -ChangeSchema_node1704102716061 = ApplyMapping.apply( - frame=AWSGlueDataCatalog_node1704102689282, - mappings=[ - # TBC - ], - transformation_ctx="ChangeSchema_node1704102716061", -) - -# Script generated for node Amazon S3 -AmazonS3_node1704102720699 = glueContext.write_dynamic_frame.from_options( - frame=ChangeSchema_node1704102716061, - connection_type="s3", - format="csv", - connection_options={"path": "s3://target-data-bucket", "partitionKeys": []}, - transformation_ctx="AmazonS3_node1704102720699", -) - -job.commit() \ No newline at end of file +from awsglue.dynamicframe import DynamicFrame + +# from awsglue.job import Job +from pyspark.context import SparkContext + +# from pyspark.sql import DataFrame + + +def create_glue_context(): + # Initialize the SparkContext and GlueContext + sc = SparkContext() + glueContext = GlueContext(sc) + + return glueContext + + +def load_data_from_s3( + glueContext, s3_path: str, file_type: str = "json", format_options: dict = {} +): + """ + Loads data from S3 into a Glue DynamicFrame. + """ + if file_type == "json": + return glueContext.create_dynamic_frame.from_options( + connection_type="s3", + connection_options={"paths": [s3_path]}, + format=file_type, + ) + else: + raise ValueError(f"Unsupported file_type: {file_type}") + + +def transform_data(dynamic_frame: DynamicFrame) -> DynamicFrame: + """ + Example transformation function. Modify this to suit your transformation logic. + """ + # Convert DynamicFrame to DataFrame to leverage Spark SQL operations if needed + df = dynamic_frame.toDF() + + # Perform any necessary transformations using Spark DataFrame API + df_transformed = df.filter(df["x"] == "placeholder") + + # Convert DataFrame back to DynamicFrame for Glue compatibility + transformed_dynamic_frame = DynamicFrame.fromDF( + df_transformed, dynamic_frame.glue_ctx, "transformed_dynamic_frame" + ) + + return transformed_dynamic_frame + + +def write_data_to_s3( + dynamic_frame: DynamicFrame, + s3_path: str, + file_type: str = "csv", + partition_keys: list = None, +): + """ + Writes a DynamicFrame to S3 with partitioning support for scalability. + """ + if file_type == "csv": + dynamic_frame.toDF().write.option("header", "true").mode( + "overwrite" + ).partitionBy(*partition_keys).csv(s3_path) + elif file_type == "parquet": + dynamic_frame.toDF().write.mode("overwrite").partitionBy( + *partition_keys + ).parquet(s3_path) + elif file_type == "json": + dynamic_frame.toDF().write.mode("overwrite").partitionBy(*partition_keys).json( + s3_path + ) + else: + raise ValueError(f"Unsupported file_type: {file_type}") + + +def handle_error(exception: Exception): + # Custom error handling for logging + raise exception + + +def main(): + try: + # Initialize Glue Context + glueContext = create_glue_context() + + # Example paths and configurations + input_path = "s3://source-data-bucket/input-data/" # probs worth using one bucket and different folders? Cuts costs + output_path = "s3://target-data-bucket/output-data/" + + # Load data from S3 (adjust format if needed) + dynamic_frame = load_data_from_s3(glueContext, input_path, format="json") + + # Transform data + transformed_dynamic_frame = transform_data(dynamic_frame) + + # Write the transformed data back to S3, partitioned by 'date' + write_data_to_s3( + transformed_dynamic_frame, + output_path, + format="csv", + partition_keys=["date"], + ) + + except Exception as e: + handle_error(e) + + +# Entry point for Glue job +if __name__ == "__main__": + main() diff --git a/terraform/account-wide-infrastructure/modules/glue/src/transforms.py b/terraform/account-wide-infrastructure/modules/glue/src/transforms.py new file mode 100644 index 000000000..e69de29bb From 90ab98cc686eeabbfd8260f970a34813e8763857 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 10 Dec 2024 17:53:44 +0000 Subject: [PATCH 13/60] NRL-1188 zip extra python files for use --- .../modules/glue/glue.tf | 3 ++- .../account-wide-infrastructure/modules/glue/s3.tf | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index 93ad225e4..23d80c11a 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -42,7 +42,7 @@ resource "aws_glue_job" "glue_job" { command { name = "glueetl" python_version = var.python_version - script_location = "s3://${aws_s3_bucket.code-bucket.id}/script.py" + script_location = "s3://${aws_s3_bucket.code-bucket.id}/main.py" } default_arguments = { @@ -54,6 +54,7 @@ resource "aws_glue_job" "glue_job" { "--job-name" = "poc-glue-job" "--enable-continuous-log-filter" = "true" "--enable-metrics" = "true" + "--extra-py-files" = "s3://${aws_s3_bucket.code-bucket.id}/src.zip" } } diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 099c5c012..f5506a48e 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -20,3 +20,16 @@ resource "aws_s3_bucket_object" "code-data-object" { source = "${path.module}/src/main.py" etag = filemd5("${path.module}/src/main.py") } + +data "archive_file" "python" { + type = "zip" + output_path = "${path.module}/files/src.zip" + + source_dir = "${path.module}/src" +} + +resource "aws_s3_bucket_object" "code-data-object" { + bucket = aws_s3_bucket.code-bucket.bucket + key = "main.py" + source = data.archive_file.python +} From 479a7a9333cdb87d9d576e656b54478d3054afbc Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 10 Dec 2024 17:54:15 +0000 Subject: [PATCH 14/60] NRL-1186 Lightweight framework for ETL process --- .../glue/src/{transforms.py => __init__.py} | 0 .../modules/glue/src/instances.py | 30 +++++ .../modules/glue/src/main.py | 123 +++--------------- .../modules/glue/src/pipeline.py | 50 +++++++ .../modules/glue/src/transformations.py | 1 + 5 files changed, 100 insertions(+), 104 deletions(-) rename terraform/account-wide-infrastructure/modules/glue/src/{transforms.py => __init__.py} (100%) create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/instances.py create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/pipeline.py create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/transformations.py diff --git a/terraform/account-wide-infrastructure/modules/glue/src/transforms.py b/terraform/account-wide-infrastructure/modules/glue/src/__init__.py similarity index 100% rename from terraform/account-wide-infrastructure/modules/glue/src/transforms.py rename to terraform/account-wide-infrastructure/modules/glue/src/__init__.py diff --git a/terraform/account-wide-infrastructure/modules/glue/src/instances.py b/terraform/account-wide-infrastructure/modules/glue/src/instances.py new file mode 100644 index 000000000..335ba7ecd --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/instances.py @@ -0,0 +1,30 @@ +import logging + +from awsglue.context import GlueContext +from pyspark.sql import SparkSession + + +class GlueContextSingleton: + """Singleton for GlueContext and SparkSession""" + + _instance = None + + def __new__(cls, spark_context): + if not cls._instance: + cls._instance = super().__new__(cls) + cls._instance.spark = SparkSession.builder.getOrCreate() + cls._instance.context = GlueContext(spark_context) + return cls._instance + + +class LoggerSingleton: + """Singleton for logger""" + + _instance = None + + def __new__(cls): + if not cls._instance: + cls._instance = super().__new__(cls) + cls._instance.logger = logging.getLogger("ETLLogger") + cls._instance.logger.setLevel(logging.INFO) + return cls._instance diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 592f01e20..2c321766e 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -1,111 +1,26 @@ -from awsglue.context import GlueContext -from awsglue.dynamicframe import DynamicFrame +import sys -# from awsglue.job import Job +from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext -# from pyspark.sql import DataFrame +# Get arguments from AWS Glue job +args = getResolvedOptions( + sys.argv, ["JOB_NAME", "SOURCE_PATH", "TARGET_PATH", "PARTITION_COLS"] +) +# Start Glue context +sc = SparkContext() -def create_glue_context(): - # Initialize the SparkContext and GlueContext - sc = SparkContext() - glueContext = GlueContext(sc) +partition_cols = args["PARTITION_COLS"].split(",") if "PARTITION_COLS" in args else [] - return glueContext +# Initialize ETL process +etl_job = ETLTemplate( + spark_context=sc, + source_path=args["SOURCE_PATH"], + target_path=args["TARGET_PATH"], + partition_cols=partition_cols, + transformations=[placeholder], +) - -def load_data_from_s3( - glueContext, s3_path: str, file_type: str = "json", format_options: dict = {} -): - """ - Loads data from S3 into a Glue DynamicFrame. - """ - if file_type == "json": - return glueContext.create_dynamic_frame.from_options( - connection_type="s3", - connection_options={"paths": [s3_path]}, - format=file_type, - ) - else: - raise ValueError(f"Unsupported file_type: {file_type}") - - -def transform_data(dynamic_frame: DynamicFrame) -> DynamicFrame: - """ - Example transformation function. Modify this to suit your transformation logic. - """ - # Convert DynamicFrame to DataFrame to leverage Spark SQL operations if needed - df = dynamic_frame.toDF() - - # Perform any necessary transformations using Spark DataFrame API - df_transformed = df.filter(df["x"] == "placeholder") - - # Convert DataFrame back to DynamicFrame for Glue compatibility - transformed_dynamic_frame = DynamicFrame.fromDF( - df_transformed, dynamic_frame.glue_ctx, "transformed_dynamic_frame" - ) - - return transformed_dynamic_frame - - -def write_data_to_s3( - dynamic_frame: DynamicFrame, - s3_path: str, - file_type: str = "csv", - partition_keys: list = None, -): - """ - Writes a DynamicFrame to S3 with partitioning support for scalability. - """ - if file_type == "csv": - dynamic_frame.toDF().write.option("header", "true").mode( - "overwrite" - ).partitionBy(*partition_keys).csv(s3_path) - elif file_type == "parquet": - dynamic_frame.toDF().write.mode("overwrite").partitionBy( - *partition_keys - ).parquet(s3_path) - elif file_type == "json": - dynamic_frame.toDF().write.mode("overwrite").partitionBy(*partition_keys).json( - s3_path - ) - else: - raise ValueError(f"Unsupported file_type: {file_type}") - - -def handle_error(exception: Exception): - # Custom error handling for logging - raise exception - - -def main(): - try: - # Initialize Glue Context - glueContext = create_glue_context() - - # Example paths and configurations - input_path = "s3://source-data-bucket/input-data/" # probs worth using one bucket and different folders? Cuts costs - output_path = "s3://target-data-bucket/output-data/" - - # Load data from S3 (adjust format if needed) - dynamic_frame = load_data_from_s3(glueContext, input_path, format="json") - - # Transform data - transformed_dynamic_frame = transform_data(dynamic_frame) - - # Write the transformed data back to S3, partitioned by 'date' - write_data_to_s3( - transformed_dynamic_frame, - output_path, - format="csv", - partition_keys=["date"], - ) - - except Exception as e: - handle_error(e) - - -# Entry point for Glue job -if __name__ == "__main__": - main() +# Run the job +etl_job.run() diff --git a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py new file mode 100644 index 000000000..9e68181dd --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py @@ -0,0 +1,50 @@ +class ETLTemplate: + def __init__( + self, + spark_context, + source_path, + target_path, + partition_cols=None, + transformations=[], + ): + """Initialize Glue context, Spark session, logger, and paths""" + self.glue_context = GlueContextSingleton(spark_context).context + self.spark = GlueContextSingleton(spark_context).spark + self.logger = LoggerSingleton().logger + self.source_path = source_path + self.target_path = target_path + self.partition_cols = partition_cols + self.transformations = transformations + + def run(self): + """Runs ETL""" + try: + self.logger.info("ETL Process started.") + df = self.extract() + self.logger.info(f"Data extracted from {self.source_path}.") + df = self.transform(df) + self.logger.info("Data transformed successfully.") + self.load(df) + self.logger.info(f"Data loaded into {self.target_path}.") + except Exception as e: + self.logger.error(f"ETL process failed: {e}") + raise e + + def extract(self): + """Extract JSON data from S3""" + self.logger.info(f"Extracting data from {self.source_path} as JSON") + return self.spark.read.json(self.source_path) + + def transform(self, dataframe): + """Apply a list of transformations on the dataframe""" + for transformation in self.transformations: + self.logger.info(f"Applying transformation: {transformation.__name__}") + dataframe = transformation(dataframe) + return dataframe + + def load(self, dataframe): + """Load transformed data into Parquet format""" + self.logger.info(f"Loading data into {self.target_path} as Parquet") + dataframe.write.mode("overwrite").partitionBy(*self.partition_cols).parquet( + self.target_path + ) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/transformations.py b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py new file mode 100644 index 000000000..1d59d52bc --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py @@ -0,0 +1 @@ +def placeholder(): ... From 85e6fa3b6c1a6482a709b51cc5d57bee3992cfc5 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 08:40:19 +0000 Subject: [PATCH 15/60] NRL-1186 imports and name update --- .../account-wide-infrastructure/modules/glue/src/main.py | 3 ++- .../account-wide-infrastructure/modules/glue/src/pipeline.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 2c321766e..f2cf045ff 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -2,6 +2,7 @@ from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext +from src.pipeline import LogPipeline # Get arguments from AWS Glue job args = getResolvedOptions( @@ -14,7 +15,7 @@ partition_cols = args["PARTITION_COLS"].split(",") if "PARTITION_COLS" in args else [] # Initialize ETL process -etl_job = ETLTemplate( +etl_job = LogPipeline( spark_context=sc, source_path=args["SOURCE_PATH"], target_path=args["TARGET_PATH"], diff --git a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py index 9e68181dd..50c34af23 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py @@ -1,4 +1,7 @@ -class ETLTemplate: +from src.instances import GlueContextSingleton, LoggerSingleton + + +class LogPipeline: def __init__( self, spark_context, From b90b97c1260492e5c89c154bb3a4be67b4dd9028 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 08:48:30 +0000 Subject: [PATCH 16/60] NRL-1188 update glue iam policy --- .../modules/glue/iam.tf | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index d60634adc..8e67b5c7b 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -29,17 +29,6 @@ resource "aws_iam_role_policy" "glue_service_role_policy" { "s3:ListBucket", "s3:ListAllMyBuckets", "s3:GetBucketAcl", - "ec2:DescribeVpcEndpoints", - "ec2:DescribeRouteTables", - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeVpcAttribute", - "iam:ListRolePolicies", - "iam:GetRole", - "iam:GetRolePolicy", "cloudwatch:PutMetricData" ], "Resource" : ["*"] @@ -73,20 +62,6 @@ resource "aws_iam_role_policy" "glue_service_role_policy" { "logs:PutLogEvents" ], "Resource" : ["arn:aws:logs:*:*:*:/aws-glue/*"] - }, - { - "Effect" : "Allow", - "Action" : ["ec2:CreateTags", "ec2:DeleteTags"], - "Condition" : { - "ForAllValues:StringEquals" : { - "aws:TagKeys" : ["aws-glue-service-resource"] - } - }, - "Resource" : [ - "arn:aws:ec2:*:*:network-interface/*", - "arn:aws:ec2:*:*:security-group/*", - "arn:aws:ec2:*:*:instance/*" - ] } ] }) From 8535b4df22176c51bcdff8f085b959cba08996b1 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 08:50:29 +0000 Subject: [PATCH 17/60] NRL-1188 Public access block added --- .../modules/glue/s3.tf | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index f5506a48e..891b0c047 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -3,17 +3,44 @@ resource "aws_s3_bucket" "source-data-bucket" { bucket = "source-data-bucket" } +resource "aws_s3_bucket_public_access_block" "source-data-bucket-public-access-block" { + bucket = aws_s3_bucket.source-data-bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + + # S3 Bucket for Processed Data resource "aws_s3_bucket" "target-data-bucket" { bucket = "target-data-bucket" } +resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-block" { + bucket = aws_s3_bucket.target-data-bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} # S3 Bucket for Code resource "aws_s3_bucket" "code-bucket" { bucket = "code-bucket" } +resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" { + bucket = aws_s3_bucket.code-bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + resource "aws_s3_bucket_object" "code-data-object" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" From 3e92fbc97f69283c99f2beec3e15dfe376975594 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 08:53:08 +0000 Subject: [PATCH 18/60] NRL-1188 Public access block added --- .../account-wide-infrastructure/modules/athena/s3.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 71eabdcad..b13348f1b 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -2,6 +2,15 @@ resource "aws_s3_bucket" "athena" { bucket = "athena" } +resource "aws_s3_bucket_public_access_block" "athena-public-access-block" { + bucket = aws_s3_bucket.athena.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { bucket = aws_s3_bucket.athena.bucket From 47844beef44e4a929c3d6515a7250a1d484a4d8f Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 09:07:30 +0000 Subject: [PATCH 19/60] NRL-1188 Update glue iam policy --- .../modules/glue/iam.tf | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 8e67b5c7b..42d0a5d7e 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -21,18 +21,18 @@ resource "aws_iam_role_policy" "glue_service_role_policy" { policy = jsonencode({ "Version" : "2012-10-17", "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "glue:*", - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListAllMyBuckets", - "s3:GetBucketAcl", - "cloudwatch:PutMetricData" - ], - "Resource" : ["*"] - }, + # { + # "Effect" : "Allow", + # "Action" : [ + # "glue:*", + # "s3:GetBucketLocation", + # "s3:ListBucket", + # "s3:ListAllMyBuckets", + # "s3:GetBucketAcl", + # "cloudwatch:PutMetricData" + # ], + # "Resource" : ["*"] + # }, { "Effect" : "Allow", "Action" : ["s3:CreateBucket"], From 68bce407a3fa79af4544e06b6ad72aa46edf1221 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 09:13:21 +0000 Subject: [PATCH 20/60] NRL-1188 Ensure S3 buckets are compliant --- .../modules/athena/s3.tf | 28 +++++++ .../modules/glue/s3.tf | 84 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index b13348f1b..c8283af49 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -2,6 +2,34 @@ resource "aws_s3_bucket" "athena" { bucket = "athena" } +resource "aws_s3_bucket_policy" "athena" { + bucket = "athena" + + policy = jsonencode({ + Version = "2012-10-17" + Id = "athena-policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + "AWS" : "*" + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.athena.arn, + "${aws_s3_bucket.athena.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_public_access_block" "athena-public-access-block" { bucket = aws_s3_bucket.athena.id diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 891b0c047..f9dc53efe 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -3,6 +3,34 @@ resource "aws_s3_bucket" "source-data-bucket" { bucket = "source-data-bucket" } +resource "aws_s3_bucket_policy" "source-data-bucket" { + bucket = "source-data-bucket" + + policy = jsonencode({ + Version = "2012-10-17" + Id = "source-data-bucket-policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + "AWS" : "*" + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.source-data-bucket.arn, + "${aws_s3_bucket.source-data-bucket.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_public_access_block" "source-data-bucket-public-access-block" { bucket = aws_s3_bucket.source-data-bucket.id @@ -18,6 +46,34 @@ resource "aws_s3_bucket" "target-data-bucket" { bucket = "target-data-bucket" } +resource "aws_s3_bucket_policy" "target-data-bucket" { + bucket = "target-data-bucket" + + policy = jsonencode({ + Version = "2012-10-17" + Id = "target-data-bucket-policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + "AWS" : "*" + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.target-data-bucket.arn, + "${aws_s3_bucket.target-data-bucket.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-block" { bucket = aws_s3_bucket.target-data-bucket.id @@ -32,6 +88,34 @@ resource "aws_s3_bucket" "code-bucket" { bucket = "code-bucket" } +resource "aws_s3_bucket_policy" "code-bucket" { + bucket = "code-bucket" + + policy = jsonencode({ + Version = "2012-10-17" + Id = "code-bucket-policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + "AWS" : "*" + } + Action = "s3:*" + Resource = [ + aws_s3_bucket.code-bucket.arn, + "${aws_s3_bucket.code-bucket.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" { bucket = aws_s3_bucket.code-bucket.id From f9f93abcf1d362a503b7b898d54e6c02facf72bb Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 09:29:26 +0000 Subject: [PATCH 21/60] NRL-1188 Reference existing bucket --- terraform/infrastructure/modules/firehose/s3.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 121bfc04b..f6de8058c 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -106,3 +106,7 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { ] }) } + +data "aws_s3_bucket" "selected" { + bucket = "source-data-bucket" +} From c98847ff60ca511643626bed8fef6f9a9a935412 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 09:30:15 +0000 Subject: [PATCH 22/60] NRL-1188 Reference existing bucket --- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 62e12ac3f..812fbdeb3 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.source-data-bucket.arn + bucket_arn = data.aws_s3_bucket.source-data-bucket.arn processing_configuration { enabled = "true" From 58860eebc5b9b6348af90725632d696ee0435391 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 09:46:29 +0000 Subject: [PATCH 23/60] NRL-1188 Bucket name update --- terraform/infrastructure/modules/firehose/s3.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index f6de8058c..d002a2290 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -107,6 +107,6 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { }) } -data "aws_s3_bucket" "selected" { +data "aws_s3_bucket" "source-data-bucket" { bucket = "source-data-bucket" } From c5145510125a5d09a0e673163e81a768583c18a7 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 10:16:01 +0000 Subject: [PATCH 24/60] NRL-1186 import function --- terraform/account-wide-infrastructure/modules/glue/src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index f2cf045ff..a29ef78d8 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -3,6 +3,7 @@ from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext from src.pipeline import LogPipeline +from src.transformations import placeholder # Get arguments from AWS Glue job args = getResolvedOptions( From 38f0d33f7c8c87df4084992544ac1b013f5bf593 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 10:32:51 +0000 Subject: [PATCH 25/60] NRL-1188 specify region in reference --- terraform/infrastructure/modules/firehose/s3.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index d002a2290..8a3b6b7e7 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -109,4 +109,5 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { data "aws_s3_bucket" "source-data-bucket" { bucket = "source-data-bucket" + region = "eu-west-2" } From 8e7798b9a7760aca5a73fb88206dcfb5c206482c Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 11 Dec 2024 11:14:26 +0000 Subject: [PATCH 26/60] NRL-1188 remove region reference --- terraform/infrastructure/modules/firehose/s3.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 8a3b6b7e7..d002a2290 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -109,5 +109,4 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { data "aws_s3_bucket" "source-data-bucket" { bucket = "source-data-bucket" - region = "eu-west-2" } From 55176d75eef59e3a29800bc086e958894831a547 Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 13 Dec 2024 14:12:15 +0000 Subject: [PATCH 27/60] NRL-1188 Resolving comments --- .../account-wide-infrastructure/modules/glue/iam.tf | 12 ------------ .../account-wide-infrastructure/modules/glue/s3.tf | 4 ++-- .../modules/truststore-bucket/output.tf | 4 ++-- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 42d0a5d7e..9b0aecd9d 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -21,18 +21,6 @@ resource "aws_iam_role_policy" "glue_service_role_policy" { policy = jsonencode({ "Version" : "2012-10-17", "Statement" : [ - # { - # "Effect" : "Allow", - # "Action" : [ - # "glue:*", - # "s3:GetBucketLocation", - # "s3:ListBucket", - # "s3:ListAllMyBuckets", - # "s3:GetBucketAcl", - # "cloudwatch:PutMetricData" - # ], - # "Resource" : ["*"] - # }, { "Effect" : "Allow", "Action" : ["s3:CreateBucket"], diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index f9dc53efe..e97e93c26 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -125,7 +125,7 @@ resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" { restrict_public_buckets = true } -resource "aws_s3_bucket_object" "code-data-object" { +resource "aws_s3_bucket_object" "script" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" source = "${path.module}/src/main.py" @@ -139,7 +139,7 @@ data "archive_file" "python" { source_dir = "${path.module}/src" } -resource "aws_s3_bucket_object" "code-data-object" { +resource "aws_s3_bucket_object" "zip" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" source = data.archive_file.python diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf index 1c0a2a86e..b8cd08057 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/output.tf @@ -1,9 +1,9 @@ output "bucket_name" { - description = "Name of the reporting S3 bucket" + description = "Name of the truststore S3 bucket" value = aws_s3_bucket.api_truststore.bucket } output "certificates_object_key" { - description = "Key of the reporting certificates object" + description = "Key of the truststore certificates object" value = aws_s3_object.api_truststore_certificate.key } From 0434eb934802dc928e7d1ee1d2a06ab9443b79e8 Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 13 Dec 2024 14:44:47 +0000 Subject: [PATCH 28/60] NRL-1188 Reference S3 bucket --- .../account-wide-infrastructure/modules/athena/athena.tf | 2 +- terraform/account-wide-infrastructure/modules/athena/s3.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index b8cf1de1d..c547f8c82 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -1,7 +1,7 @@ resource "aws_athena_database" "reporting-db" { name = var.database - bucket = aws_s3_bucket.target-data-bucket.bucket + bucket = data.aws_s3_bucket.target-data-bucket.bucket # encryption_configuration { # encryption_option = "SSE_KMS" diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index c8283af49..6eb8359f7 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -50,3 +50,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { } } + +data "aws_s3_bucket" "target-data-bucket" { + bucket = "target-data-bucket" +} From 601209106596e7e7265eb3cd5b9df974281305ff Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 13 Dec 2024 14:45:51 +0000 Subject: [PATCH 29/60] NRL-1188 Fix var name --- terraform/account-wide-infrastructure/modules/athena/kms.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/athena/kms.tf b/terraform/account-wide-infrastructure/modules/athena/kms.tf index 9a9ee0f90..e53d7c96a 100644 --- a/terraform/account-wide-infrastructure/modules/athena/kms.tf +++ b/terraform/account-wide-infrastructure/modules/athena/kms.tf @@ -2,6 +2,6 @@ resource "aws_kms_key" "athena" { } resource "aws_kms_alias" "athena" { - name = "alias/${var.prefix}-athena" + name = "alias/${var.name_prefix}-athena" target_key_id = aws_kms_key.athena.key_id } From 1e7aa5ca812803c5962fc8885eb6cacf9e2ff994 Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 13 Dec 2024 14:59:56 +0000 Subject: [PATCH 30/60] NRL-1188 name updates --- .../modules/athena/athena.tf | 2 +- .../account-wide-infrastructure/modules/athena/s3.tf | 6 +++--- .../account-wide-infrastructure/modules/glue/glue.tf | 8 ++++---- .../account-wide-infrastructure/modules/glue/iam.tf | 4 ++-- .../account-wide-infrastructure/modules/glue/s3.tf | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index c547f8c82..f48a4861d 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -12,7 +12,7 @@ resource "aws_athena_database" "reporting-db" { } resource "aws_athena_workgroup" "athena" { - name = var.name_prefix + name = "${var.name_prefix}-athena-wg" configuration { enforce_workgroup_configuration = true diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 6eb8359f7..c5fa017a8 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -1,9 +1,9 @@ resource "aws_s3_bucket" "athena" { - bucket = "athena" + bucket = "${var.name_prefix}-athena" } resource "aws_s3_bucket_policy" "athena" { - bucket = "athena" + bucket = "${var.name_prefix}-athena" policy = jsonencode({ Version = "2012-10-17" @@ -52,5 +52,5 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { } data "aws_s3_bucket" "target-data-bucket" { - bucket = "target-data-bucket" + bucket = "${var.name_prefix}-target-data-bucket" } diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index 23d80c11a..431fabc82 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -1,12 +1,12 @@ # Create Glue Data Catalog Database resource "aws_glue_catalog_database" "raw_log_database" { - name = "raw_log" + name = "${var.name_prefix}-raw_log" location_uri = "${aws_s3_bucket.source-data-bucket.id}/" } # Create Glue Crawler resource "aws_glue_crawler" "raw_log_crawler" { - name = "raw-log-crawler" + name = "${var.name_prefix}-raw-log-crawler" database_name = aws_glue_catalog_database.raw_log_database.name role = aws_iam_role.glue_service_role.name s3_target { @@ -23,7 +23,7 @@ resource "aws_glue_crawler" "raw_log_crawler" { }) } resource "aws_glue_trigger" "raw_log_trigger" { - name = "org-report-trigger" + name = "${var.name_prefix}-org-report-trigger" type = "ON_DEMAND" actions { crawler_name = aws_glue_crawler.raw_log_crawler.name @@ -31,7 +31,7 @@ resource "aws_glue_trigger" "raw_log_trigger" { } resource "aws_glue_job" "glue_job" { - name = "poc-glue-job" + name = "${var.name_prefix}-glue-job" role_arn = aws_iam_role.glue_service_role.arn description = "Transfer logs from source to bucket" glue_version = "4.0" diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 9b0aecd9d..568f4589d 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -1,5 +1,5 @@ resource "aws_iam_role" "glue_service_role" { - name = "glue_service_role" + name = "${var.name_prefix}-glue_service_role" assume_role_policy = jsonencode({ "Version" : "2012-10-17", @@ -16,7 +16,7 @@ resource "aws_iam_role" "glue_service_role" { } resource "aws_iam_role_policy" "glue_service_role_policy" { - name = "glue_service_role_policy" + name = "${var.name_prefix}-glue_service_role_policy" role = aws_iam_role.glue_service_role.name policy = jsonencode({ "Version" : "2012-10-17", diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index e97e93c26..a60a63579 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -1,10 +1,10 @@ # S3 Bucket for Raw Data resource "aws_s3_bucket" "source-data-bucket" { - bucket = "source-data-bucket" + bucket = "${var.name_prefix}-source-data-bucket" } resource "aws_s3_bucket_policy" "source-data-bucket" { - bucket = "source-data-bucket" + bucket = "${var.name_prefix}-source-data-bucket" policy = jsonencode({ Version = "2012-10-17" @@ -43,11 +43,11 @@ resource "aws_s3_bucket_public_access_block" "source-data-bucket-public-access-b # S3 Bucket for Processed Data resource "aws_s3_bucket" "target-data-bucket" { - bucket = "target-data-bucket" + bucket = "${var.name_prefix}-target-data-bucket" } resource "aws_s3_bucket_policy" "target-data-bucket" { - bucket = "target-data-bucket" + bucket = "${var.name_prefix}-target-data-bucket" policy = jsonencode({ Version = "2012-10-17" @@ -85,11 +85,11 @@ resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-b # S3 Bucket for Code resource "aws_s3_bucket" "code-bucket" { - bucket = "code-bucket" + bucket = "${var.name_prefix}-code-bucket" } resource "aws_s3_bucket_policy" "code-bucket" { - bucket = "code-bucket" + bucket = "${var.name_prefix}-code-bucket" policy = jsonencode({ Version = "2012-10-17" From 3b35af9e617ebd180021154dae7603e5bded89a4 Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 13 Dec 2024 16:51:50 +0000 Subject: [PATCH 31/60] NRL-1188 use default role policy for glue --- .../modules/glue/iam.tf | 41 ++----------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/iam.tf b/terraform/account-wide-infrastructure/modules/glue/iam.tf index 568f4589d..890b47593 100644 --- a/terraform/account-wide-infrastructure/modules/glue/iam.tf +++ b/terraform/account-wide-infrastructure/modules/glue/iam.tf @@ -15,42 +15,7 @@ resource "aws_iam_role" "glue_service_role" { }) } -resource "aws_iam_role_policy" "glue_service_role_policy" { - name = "${var.name_prefix}-glue_service_role_policy" - role = aws_iam_role.glue_service_role.name - policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : ["s3:CreateBucket"], - "Resource" : ["arn:aws:s3:::aws-glue-*"] - }, - { - "Effect" : "Allow", - "Action" : ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource" : [ - "arn:aws:s3:::*/*", - "arn:aws:s3:::*/*aws-glue-*/*" - ] - }, - { - "Effect" : "Allow", - "Action" : ["s3:GetObject"], - "Resource" : [ - "arn:aws:s3:::crawler-public*", - "arn:aws:s3:::aws-glue-*" - ] - }, - { - "Effect" : "Allow", - "Action" : [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource" : ["arn:aws:logs:*:*:*:/aws-glue/*"] - } - ] - }) +resource "aws_iam_role_policy_attachment" "glue_service" { + role = aws_iam_role.glue_service_role.id + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" } From 8b8865518cb4ae88c37cd11e111a701c36c1d3fc Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 10:30:05 +0000 Subject: [PATCH 32/60] NRL-1188 fix tf plan errors --- .gitignore | 3 +++ terraform/account-wide-infrastructure/dev/athena.tf | 5 +++-- .../account-wide-infrastructure/modules/athena/athena.tf | 2 +- terraform/account-wide-infrastructure/modules/athena/s3.tf | 4 ---- .../account-wide-infrastructure/modules/athena/vars.tf | 6 +++++- .../account-wide-infrastructure/modules/glue/outputs.tf | 4 ++++ terraform/account-wide-infrastructure/modules/glue/s3.tf | 6 +++--- 7 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/glue/outputs.tf diff --git a/.gitignore b/.gitignore index 4bf7237e8..981f56859 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ override.tf.json *_override.tf *_override.tf.json +# Ignore output of data object +terraform/account-wide-infrastructure/modules/glue/files/src.zip + # Include override files you do wish to add to version control using negated pattern # # !example_override.tf diff --git a/terraform/account-wide-infrastructure/dev/athena.tf b/terraform/account-wide-infrastructure/dev/athena.tf index 07dc43554..cb7e78a97 100644 --- a/terraform/account-wide-infrastructure/dev/athena.tf +++ b/terraform/account-wide-infrastructure/dev/athena.tf @@ -1,4 +1,5 @@ module "dev-athena" { - source = "../modules/athena" - name_prefix = "nhsd-nrlf--dev" + source = "../modules/athena" + name_prefix = "nhsd-nrlf--dev" + target_bucket_name = module.dev-glue.target_bucket_name } diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index f48a4861d..fe505226b 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -1,7 +1,7 @@ resource "aws_athena_database" "reporting-db" { name = var.database - bucket = data.aws_s3_bucket.target-data-bucket.bucket + bucket = var.target_bucket_name # encryption_configuration { # encryption_option = "SSE_KMS" diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index c5fa017a8..ea0d144c1 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -50,7 +50,3 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "athena" { } } - -data "aws_s3_bucket" "target-data-bucket" { - bucket = "${var.name_prefix}-target-data-bucket" -} diff --git a/terraform/account-wide-infrastructure/modules/athena/vars.tf b/terraform/account-wide-infrastructure/modules/athena/vars.tf index 285f18823..d09d6f65c 100644 --- a/terraform/account-wide-infrastructure/modules/athena/vars.tf +++ b/terraform/account-wide-infrastructure/modules/athena/vars.tf @@ -1,9 +1,13 @@ variable "database" { description = "What the db will be called" - default = "NRL-Reporting" + default = "nrl_reporting" } variable "name_prefix" { type = string description = "The prefix to apply to all resources in the module." } + +variable "target_bucket_name" { + type = string +} diff --git a/terraform/account-wide-infrastructure/modules/glue/outputs.tf b/terraform/account-wide-infrastructure/modules/glue/outputs.tf new file mode 100644 index 000000000..a5c0e143f --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/outputs.tf @@ -0,0 +1,4 @@ +output "target_bucket_name" { + description = "Name of destination bucket" + value = aws_s3_bucket.target-data-bucket.id +} diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index a60a63579..4777b9e04 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -125,7 +125,7 @@ resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" { restrict_public_buckets = true } -resource "aws_s3_bucket_object" "script" { +resource "aws_s3_object" "script" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" source = "${path.module}/src/main.py" @@ -139,8 +139,8 @@ data "archive_file" "python" { source_dir = "${path.module}/src" } -resource "aws_s3_bucket_object" "zip" { +resource "aws_s3_object" "zip" { bucket = aws_s3_bucket.code-bucket.bucket key = "main.py" - source = data.archive_file.python + source = "${path.module}/files/src.zip" } From 75b84f489869ccf84438bd6bb9f6caa1d30d2859 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 10:33:45 +0000 Subject: [PATCH 33/60] NRL-1188 merge develop --- .github/workflows/daily-build.yml | 85 ++++ .../search_document_reference.py | 7 +- ...test_search_document_reference_consumer.py | 52 +- .../search_post_document_reference.py | 6 +- ...search_post_document_reference_consumer.py | 2 +- api/consumer/swagger.yaml | 98 +++- .../tests/test_create_document_reference.py | 41 ++ .../search_document_reference.py | 6 +- ...test_search_document_reference_producer.py | 2 +- .../search_post_document_reference.py | 6 +- ...search_post_document_reference_producer.py | 2 +- api/producer/swagger.yaml | 103 +++- .../tests/test_upsert_document_reference.py | 41 ++ layer/nrlf/consumer/fhir/r4/model.py | 216 +++++---- layer/nrlf/core/constants.py | 15 +- layer/nrlf/core/errors.py | 35 +- layer/nrlf/core/tests/test_pydantic_errors.py | 303 ++++++++++++ layer/nrlf/core/tests/test_validators.py | 455 +++++++++--------- layer/nrlf/core/validators.py | 127 +++-- layer/nrlf/producer/fhir/r4/model.py | 216 +++++---- layer/nrlf/producer/fhir/r4/strict_model.py | 195 ++++---- reports/find_invalid_pointers.py | 82 ++++ .../fhir/NRLF-ContentStability-ValueSet.json | 37 ++ resources/fhir/NRLF-FormatCode-ValueSet.json | 37 ++ resources/fhir/NRLF-RecordType-ValueSet.json | 2 +- swagger/consumer-static/narrative.yaml | 2 +- swagger/producer-static/components.yaml | 2 +- swagger/producer-static/narrative.yaml | 39 +- .../dev/aws-backup.tf | 11 - .../modules/backup-source/backup_plan.tf | 2 +- .../modules/permissions-store-bucket/s3.tf | 2 +- .../modules/pointers-table/dynamodb.tf | 4 +- .../modules/truststore-bucket/s3.tf | 4 +- .../RQI-736253002-Valid.json | 4 +- ...-Valid-with-date-and-meta-lastupdated.json | 18 +- .../Y05868-736253002-Valid-with-date.json | 18 +- ...736253002-Valid-with-meta-lastupdated.json | 18 +- ...5868-736253002-Valid-with-ssp-content.json | 18 +- .../Y05868-736253002-Valid.json | 4 +- .../readDocumentReference-success.feature | 42 +- .../searchDocumentReference-failure.feature | 30 +- .../searchDocumentReference-success.feature | 38 ++ ...earchPostDocumentReference-failure.feature | 30 +- .../createDocumentReference-failure.feature | 370 ++++++++++++-- .../createDocumentReference-success.feature | 2 +- .../readDocumentReference-success.feature | 21 +- .../updateDocumentReference-failure.feature | 258 ++++++++++ .../upsertDocumentReference-failure.feature | 187 +++++++ tests/features/steps/1_setup.py | 3 +- tests/features/steps/2_request.py | 54 ++- tests/features/steps/3_assert.py | 8 + tests/features/utils/constants.py | 18 +- tests/features/utils/data.py | 52 +- tests/performance/environment.py | 2 +- tests/smoke/setup.py | 36 +- tests/utilities/api_clients.py | 8 + 56 files changed, 2749 insertions(+), 727 deletions(-) create mode 100644 .github/workflows/daily-build.yml create mode 100644 layer/nrlf/core/tests/test_pydantic_errors.py create mode 100644 reports/find_invalid_pointers.py create mode 100644 resources/fhir/NRLF-ContentStability-ValueSet.json create mode 100644 resources/fhir/NRLF-FormatCode-ValueSet.json diff --git a/.github/workflows/daily-build.yml b/.github/workflows/daily-build.yml new file mode 100644 index 000000000..0fc15523c --- /dev/null +++ b/.github/workflows/daily-build.yml @@ -0,0 +1,85 @@ +name: Build NRL Project on Environment +run-name: Build NRL Project on ${{ inputs.environment || 'dev' }} +permissions: + id-token: write + contents: read + actions: write + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + environment: + type: environment + description: "The environment to deploy changes to" + default: "dev" + required: true + +jobs: + build: + name: Build - develop + runs-on: [self-hosted, ci] + + steps: + - name: Git clone - develop + uses: actions/checkout@v4 + with: + ref: develop + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Install zip + run: sudo apt-get install zip + + - name: Setup Python environment + run: | + poetry install --no-root + source $(poetry env info --path)/bin/activate + + - name: Run Linting + run: make lint + + - name: Run Unit Tests + run: make test + + - name: Build Project + run: make build + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ inputs.environment || 'dev' }}-${{ github.run_id }} + + - name: Add S3 Permissions to Lambda + run: | + account=$(echo '${{ inputs.environment || 'dev' }}' | cut -d '-' -f1) + inactive_stack=$(poetry run python ./scripts/get_env_config.py inactive-stack ${{ inputs.environment || 'dev' }}) + make get-s3-perms ENV=${account} TF_WORKSPACE_NAME=${inactive_stack} + + - name: Save Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + dist/*.zip + !dist/nrlf_permissions.zip + + - name: Save NRLF Permissions cache + uses: actions/cache/save@v4 + with: + key: ${{ github.run_id }}-nrlf-permissions + path: dist/nrlf_permissions.zip diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index aee43c7ec..d847e9ef4 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -9,7 +9,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ConsumerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type @request_handler(params=ConsumerRequestParams) @@ -46,15 +46,14 @@ def handler( base_url = f"https://{config.ENVIRONMENT}.api.service.nhs.uk/" self_link = f"{base_url}record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{params.nhs_number}" - # TODO - Add checks for the type code as well as system - if not validate_type_system(params.type, metadata.pointer_types): + if not validate_type(params.type, metadata.pointer_types): logger.log( LogReference.CONSEARCH002, type=params.type, pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + diagnostics="Invalid query parameter (The provided type does not match the allowed types for this organisation)", expression="type", ) diff --git a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py index 1f30aafc5..88d1757c8 100644 --- a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py +++ b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py @@ -59,6 +59,56 @@ def test_search_document_reference_happy_path(repository: DocumentPointerReposit } +@mock_aws +@mock_repository +def test_search_document_reference_accession_number_in_pointer( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.identifier = [ + {"type": {"text": "Accession-Number"}, "value": "Y05868.123456789"} + ] + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + event = create_test_api_gateway_event( + headers=create_headers(), + query_string_parameters={ + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + }, + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "total": 1, + "link": [ + { + "relation": "self", + "url": "https://pytest.api.service.nhs.uk/record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|6700028191", + } + ], + "entry": [{"resource": doc_ref.model_dump(exclude_none=True)}], + } + + created_doc_pointer = repository.get_by_id("Y05868-99999-99999-999999") + + assert created_doc_pointer is not None + assert json.loads(created_doc_pointer.document)["identifier"] == [ + {"type": {"text": "Accession-Number"}, "value": "Y05868.123456789"} + ] + + @mock_aws @mock_repository def test_search_document_reference_happy_path_with_custodian( @@ -451,7 +501,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"], } ], diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index 39964f608..6cb9b7e8c 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -9,7 +9,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ConsumerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type @request_handler(body=ConsumerRequestParams) @@ -50,14 +50,14 @@ def handler( base_url = f"https://{config.ENVIRONMENT}.api.service.nhs.uk/" self_link = f"{base_url}record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{body.nhs_number}" - if not validate_type_system(body.type, metadata.pointer_types): + if not validate_type(body.type, metadata.pointer_types): logger.log( LogReference.CONPOSTSEARCH002, type=body.type, pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid type (The provided type system does not match the allowed types for this organisation)", + diagnostics="The provided type does not match the allowed types for this organisation", expression="type", ) diff --git a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py index 4e3ed4e4d..6616ec51c 100644 --- a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py +++ b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py @@ -424,7 +424,7 @@ def test_search_post_document_reference_invalid_type( } ] }, - "diagnostics": "Invalid type (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"], } ], diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 47c1a072c..5d016c4e7 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -36,7 +36,7 @@ info: * [End of Life Care Coordination Summary](http://snomed.info/sct/861421000000109) * [Emergency health care plan](http://snomed.info/sct/887701000000100) * [Lloyd George record folder](http://snomed.info/sct/16521000000101) - * [Advanced care plan](http://snomed.info/sct/736366004) + * [Advance care plan](http://snomed.info/sct/736366004) * [Treatment escalation plan](http://snomed.info/sct/735324008) * [Summary record]("http://snomed.info/sct|824321000000109") * [Personalised Care and Support Plan]("http://snomed.info/sct|2181441000000107") @@ -788,7 +788,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" @@ -1065,15 +1065,19 @@ components: $ref: "#/components/schemas/Attachment" description: The document or URL of the document along with critical metadata to prove content has integrity. format: - $ref: "#/components/schemas/Coding" + $ref: "#/components/schemas/NRLFormatCode" description: An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType. extension: type: array items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. + $ref: "#/components/schemas/ContentStabilityExtension" + description: Additional extension for content stability. + minItems: 1 + maxItems: 1 required: - attachment + - format + - extension DocumentReferenceRelatesTo: type: object properties: @@ -1130,6 +1134,9 @@ components: type: string pattern: ([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)(-(0[1-9]|1[0-2])(-(0[1-9]|[1-2][0-9]|3[0-1])(T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\.[0-9]+)?(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00)))?)?)? description: The date that the attachment was first created. + required: + - contentType + - url CodeableConcept: type: object properties: @@ -1146,11 +1153,6 @@ components: type: string pattern: "[ \\r\\n\\t\\S]+" description: A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user. - extension: - type: array - items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. Coding: type: object properties: @@ -1187,6 +1189,78 @@ components: type: string pattern: \S* description: The reference details for the link. + ContentStabilityExtension: + allOf: + - $ref: "#/components/schemas/Extension" + - type: object + properties: + url: + type: string + enum: + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + valueCodeableConcept: + $ref: "#/components/schemas/ContentStabilityExtensionValueCodeableConcept" + required: + - url + - valueCodeableConcept + ContentStabilityExtensionValueCodeableConcept: + allOf: + - $ref: "#/components/schemas/CodeableConcept" + - type: object + properties: + coding: + type: array + items: + $ref: "#/components/schemas/ContentStabilityExtensionCoding" + minItems: 1 + maxItems: 1 + required: + - coding + ContentStabilityExtensionCoding: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display + NRLFormatCode: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: @@ -1519,8 +1593,8 @@ components: SNOMED_CODES_LLOYD_GEORGE_RECORD_FOLDER: summary: Lloyd George record folder value: http://snomed.info/sct|16521000000101 - SNOMED_CODES_ADVANCED_CARE_PLAN: - summary: Advanced care plan + SNOMED_CODES_ADVANCE_CARE_PLAN: + summary: Advance care plan value: http://snomed.info/sct|736366004 SNOMED_CODES_TREATMENT_ESCALATION_PLAN: summary: Treatment escalation plan diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index 19e29569b..56daacfc6 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -411,6 +411,47 @@ def test_create_document_reference_with_no_practiceSetting(): } +def test_create_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", + "expression": ["docStatus"], + }, + ], + } + + def test_create_document_reference_invalid_custodian_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index 403d22b8e..e961e7b9f 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -6,7 +6,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ProducerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type from nrlf.producer.fhir.r4.model import Bundle, DocumentReference @@ -48,14 +48,14 @@ def handler( expression="subject:identifier", ) - if not validate_type_system(params.type, metadata.pointer_types): + if not validate_type(params.type, metadata.pointer_types): logger.log( LogReference.PROSEARCH002, type=params.type, pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + diagnostics="Invalid query parameter (The provided type does not match the allowed types for this organisation)", expression="type", ) diff --git a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py index d7a46ece7..b577f7754 100644 --- a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py +++ b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py @@ -201,7 +201,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"], } ], diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index 2c1159653..023dcfe7c 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -6,7 +6,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ProducerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type from nrlf.producer.fhir.r4.model import Bundle, DocumentReference @@ -42,14 +42,14 @@ def handler( expression="subject:identifier", ) - if not validate_type_system(body.type, metadata.pointer_types): + if not validate_type(body.type, metadata.pointer_types): logger.log( LogReference.PROPOSTSEARCH002, type=body.type, pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="The provided type system does not match the allowed types for this organisation", + diagnostics="The provided type does not match the allowed types for this organisation", expression="type", ) diff --git a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py index f17de3fb1..8340ee0d3 100644 --- a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py +++ b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py @@ -206,7 +206,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "The provided type system does not match the allowed types for this organisation", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"], } ], diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bdd0127b8..c927f4871 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -36,7 +36,7 @@ info: * [End of Life Care Coordination Summary](http://snomed.info/sct/861421000000109) * [Emergency health care plan](http://snomed.info/sct/887701000000100) * [Lloyd George record folder](http://snomed.info/sct/16521000000101) - * [Advanced care plan](http://snomed.info/sct/736366004) + * [Advance care plan](http://snomed.info/sct/736366004) * [Treatment escalation plan](http://snomed.info/sct/735324008) * [Summary record]("http://snomed.info/sct|824321000000109") * [Personalised Care and Support Plan]("http://snomed.info/sct|2181441000000107") @@ -334,7 +334,7 @@ paths: { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured" - "display": "Unstructured document" + "display": "Unstructured Document" } ] ``` @@ -1215,7 +1215,7 @@ components: format: system: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode code: "urn:nhs-ic:unstructured" - display: Unstructured document + display: Unstructured Document context: event: - coding: @@ -1351,7 +1351,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" @@ -1630,15 +1630,19 @@ components: $ref: "#/components/schemas/Attachment" description: The document or URL of the document along with critical metadata to prove content has integrity. format: - $ref: "#/components/schemas/Coding" + $ref: "#/components/schemas/NRLFormatCode" description: An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType. extension: type: array items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. + $ref: "#/components/schemas/ContentStabilityExtension" + description: Additional extension for content stability. + minItems: 1 + maxItems: 1 required: - attachment + - format + - extension DocumentReferenceRelatesTo: type: object properties: @@ -1695,6 +1699,9 @@ components: type: string pattern: ([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)(-(0[1-9]|1[0-2])(-(0[1-9]|[1-2][0-9]|3[0-1])(T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\.[0-9]+)?(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00)))?)?)? description: The date that the attachment was first created. + required: + - contentType + - url CodeableConcept: type: object properties: @@ -1711,12 +1718,6 @@ components: type: string pattern: "[ \\r\\n\\t\\S]+" description: A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user. - extension: - type: array - items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. - Coding: type: object properties: @@ -1753,6 +1754,78 @@ components: type: string pattern: \S* description: The reference details for the link. + ContentStabilityExtension: + allOf: + - $ref: "#/components/schemas/Extension" + - type: object + properties: + url: + type: string + enum: + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + valueCodeableConcept: + $ref: "#/components/schemas/ContentStabilityExtensionValueCodeableConcept" + required: + - url + - valueCodeableConcept + ContentStabilityExtensionValueCodeableConcept: + allOf: + - $ref: "#/components/schemas/CodeableConcept" + - type: object + properties: + coding: + type: array + items: + $ref: "#/components/schemas/ContentStabilityExtensionCoding" + minItems: 1 + maxItems: 1 + required: + - coding + ContentStabilityExtensionCoding: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display + NRLFormatCode: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: @@ -2054,8 +2127,8 @@ components: SNOMED_CODES_LLOYD_GEORGE_RECORD_FOLDER: summary: Lloyd George record folder value: http://snomed.info/sct|16521000000101 - SNOMED_CODES_ADVANCED_CARE_PLAN: - summary: Advanced care plan + SNOMED_CODES_ADVANCE_CARE_PLAN: + summary: Advance care plan value: http://snomed.info/sct|736366004 SNOMED_CODES_TREATMENT_ESCALATION_PLAN: summary: Treatment escalation plan diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index 6f3878cfd..090542dae 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -455,6 +455,47 @@ def test_upsert_document_reference_with_no_practiceSetting(): } +def test_upsert_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", + "expression": ["docStatus"], + }, + ], + } + + def test_upsert_document_reference_invalid_producer_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") doc_ref.id = "X26-99999-99999-999999" diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 4665533b5..2fd597b42 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T09:43:58+00:00 +# timestamp: 2024-12-13T11:19:30+00:00 from __future__ import annotations @@ -133,12 +133,12 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[str], + str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", pattern="[^\\s]+(\\s[^\\s]+)*", ), - ] = None + ] language: Annotated[ Optional[str], Field( @@ -154,9 +154,9 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[str], + str, Field(description="A location where the data can be accessed.", pattern="\\S*"), - ] = None + ] size: Annotated[ Optional[int], Field( @@ -230,6 +230,29 @@ class Coding(BaseModel): ] = None +class ContentStabilityExtensionCoding(Coding): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class NRLFormatCode(Coding): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -427,6 +450,43 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[str], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[str], + Field(description="The reference details for the link.", pattern="\\S*"), + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -451,6 +511,52 @@ class CountRequestParams(BaseModel): ] +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + severity: Annotated[ + str, + Field( + description="Indicates whether the issue indicates a variation from successful processing.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + code: Annotated[ + str, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[str], + Field( + description="Additional diagnostic information about the issue.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -489,7 +595,7 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[str], Field( @@ -497,35 +603,21 @@ class OperationOutcomeIssue(BaseModel): pattern="[A-Za-z0-9\\-\\.]{1,64}", ), ] = None - severity: Annotated[ - str, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - str, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[str], - Field( - description="Additional diagnostic information about the issue.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -577,11 +669,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], @@ -818,29 +907,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Optional[Coding], - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] = None - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -861,38 +927,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[str], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[str], - Field(description="The reference details for the link.", pattern="\\S*"), - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[str], @@ -1026,13 +1060,9 @@ class Signature(BaseModel): ] = None -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index 67971859f..d7b94f366 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -60,7 +60,7 @@ class PointerTypes(Enum): CONTINGENCY_PLAN = "http://snomed.info/sct|325691000000100" EOL_CARE_PLAN = "http://snomed.info/sct|736373009" LLOYD_GEORGE_FOLDER = "http://snomed.info/sct|16521000000101" - ADVANCED_CARE_PLAN = "http://snomed.info/sct|736366004" + ADVANCE_CARE_PLAN = "http://snomed.info/sct|736366004" TREATMENT_ESCALATION_PLAN = "http://snomed.info/sct|735324008" SUMMARY_RECORD = "http://snomed.info/sct|824321000000109" PERSONALISED_CARE_AND_SUPPORT_PLAN = "http://snomed.info/sct|2181441000000107" @@ -139,8 +139,8 @@ def coding_value(self): PointerTypes.LLOYD_GEORGE_FOLDER.value: { "display": "Lloyd George record folder", }, - PointerTypes.ADVANCED_CARE_PLAN.value: { - "display": "Advanced care plan", + PointerTypes.ADVANCE_CARE_PLAN.value: { + "display": "Advance care plan", }, PointerTypes.TREATMENT_ESCALATION_PLAN.value: { "display": "Treatment escalation plan", @@ -169,7 +169,7 @@ def coding_value(self): PointerTypes.CONTINGENCY_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.EOL_CARE_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.LLOYD_GEORGE_FOLDER.value: Categories.CARE_PLAN.value, - PointerTypes.ADVANCED_CARE_PLAN.value: Categories.CARE_PLAN.value, + PointerTypes.ADVANCE_CARE_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.TREATMENT_ESCALATION_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.PERSONALISED_CARE_AND_SUPPORT_PLAN.value: Categories.CARE_PLAN.value, # @@ -657,3 +657,10 @@ def coding_value(self): SYSTEM_SHORT_IDS = {"http://snomed.info/sct": "SCT", "https://nicip.nhs.uk": "NICIP"} +CONTENT_STABILITY_EXTENSION_URL = ( + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" +) +CONTENT_STABILITY_SYSTEM_URL = ( + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" +) +CONTENT_FORMAT_CODE_URL = "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index f5bdc3a47..615ef9a2a 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -3,22 +3,45 @@ from pydantic import ValidationError from pydantic_core import ErrorDetails +from nrlf.core.constants import CONTENT_FORMAT_CODE_URL, CONTENT_STABILITY_SYSTEM_URL from nrlf.core.response import Response from nrlf.core.types import CodeableConcept from nrlf.producer.fhir.r4 import model as producer_model from nrlf.producer.fhir.r4.model import OperationOutcome, OperationOutcomeIssue +def format_error_location(loc: List) -> str: + formatted_loc = "" + for each in loc: + if isinstance(each, int): + formatted_loc = f"{formatted_loc}[{each}]" + else: + formatted_loc = f"{formatted_loc}.{each}" if formatted_loc else str(each) + return formatted_loc + + +def append_value_set_url(loc_string: str) -> str: + if loc_string.endswith(("url", "system")): + return "" + + if "content" in loc_string: + if "extension" in loc_string: + return f". See ValueSet: {CONTENT_STABILITY_SYSTEM_URL}" + if "format" in loc_string: + return f". See ValueSet: {CONTENT_FORMAT_CODE_URL}" + + return "" + + def diag_for_error(error: ErrorDetails) -> str: - if error["loc"]: - loc_string = ".".join(each for each in error["loc"]) - return f"{loc_string}: {error['msg']}" - else: - return f"root: {error['msg']}" + loc_string = format_error_location(error["loc"]) + msg = f"{loc_string or 'root'}: {error['msg']}" + msg += append_value_set_url(loc_string) + return msg def expression_for_error(error: ErrorDetails) -> Optional[str]: - return str(".".join(each for each in error["loc"]) if error["loc"] else "root") + return format_error_location(error["loc"]) or "root" class OperationOutcomeError(Exception): diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py new file mode 100644 index 000000000..7ff69e6be --- /dev/null +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -0,0 +1,303 @@ +import pytest + +from nrlf.core.errors import ParseError +from nrlf.core.validators import DocumentReferenceValidator +from nrlf.tests.data import load_document_reference_json + + +def test_validate_content_missing_attachment(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0].pop("attachment") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].attachment: Field required)", + "expression": ["content[0].attachment"], + } + + +def test_validate_content_missing_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"].pop("contentType") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].attachment.contentType: Field required)", + "expression": ["content[0].attachment.contentType"], + } + + +def test_validate_content_missing_format(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0].pop("format") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].format: Field required. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode)", + "expression": ["content[0].format"], + } + + +def test_validate_content_multiple_content_stability_extensions(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Add a second duplicate contentStability extension + document_ref_data["content"][0]["extension"].append( + document_ref_data["content"][0]["extension"][0] + ) + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension: List should have at most 1 item after validation, not 2. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", + "expression": ["content[0].extension"], + } + + +def test_validate_content_invalid_content_stability_code(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid code for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["code"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].code: Input should be 'static' or 'dynamic'. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", + "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].code"], + } + + +def test_validate_content_invalid_content_stability_display(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid display for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["display"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].display: Input should be 'Static' or 'Dynamic'. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", + "expression": [ + "content[0].extension[0].valueCodeableConcept.coding[0].display" + ], + } + + +def test_validate_content_invalid_content_stability_system(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid system for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["system"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].system: Input should be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability')", + "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].system"], + } + + +def test_validate_content_invalid_content_stability_url(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid URL for contentStability extension + document_ref_data["content"][0]["extension"][0]["url"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].url: Input should be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability')", + "expression": ["content[0].extension[0].url"], + } + + +def test_validate_content_empty_content_stability_coding(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an empty coding list for contentStability extension + document_ref_data["content"][0]["extension"][0]["valueCodeableConcept"][ + "coding" + ] = [] + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: List should have at least 1 item after validation, not 0. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", + "expression": ["content[0].extension[0].valueCodeableConcept.coding"], + } + + +def test_validate_content_missing_content_stability_coding(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Remove the coding key from contentStability extension + del document_ref_data["content"][0]["extension"][0]["valueCodeableConcept"][ + "coding" + ] + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: Field required. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", + "expression": ["content[0].extension[0].valueCodeableConcept.coding"], + } diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 169e4af2c..4b13e5d56 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -13,7 +13,7 @@ from nrlf.core.validators import ( DocumentReferenceValidator, ValidationResult, - validate_type_system, + validate_type, ) from nrlf.producer.fhir.r4.model import ( DocumentReference, @@ -23,28 +23,37 @@ from nrlf.tests.data import load_document_reference_json -def test_validate_type_system_valid(): +def test_validate_type_valid(): type_ = RequestQueryType(root=PointerTypes.MENTAL_HEALTH_PLAN.value) pointer_types = [ PointerTypes.MENTAL_HEALTH_PLAN.value, PointerTypes.EOL_CARE_PLAN.value, ] - assert validate_type_system(type_, pointer_types) is True + assert validate_type(type_, pointer_types) is True -def test_validate_type_system_invalid(): +def test_validate_type_invalid_system(): type_ = RequestQueryType(root="http://snomed.info/invalid|736373009") pointer_types = [ PointerTypes.EOL_CARE_PLAN.value, PointerTypes.EOL_CARE_PLAN.value, ] - assert validate_type_system(type_, pointer_types) is False + assert validate_type(type_, pointer_types) is False -def test_validate_type_system_empty(): +def test_validate_type_invalid_code(): + type_ = RequestQueryType(root=PointerTypes.MRA_UPPER_LIMB_ARTERY.value) + pointer_types = [ + PointerTypes.MENTAL_HEALTH_PLAN.value, + PointerTypes.EOL_CARE_PLAN.value, + ] + assert validate_type(type_, pointer_types) is False + + +def test_validate_type_empty(): type_ = None pointer_types: list[str] = [] - assert validate_type_system(type_, pointer_types) is True + assert validate_type(type_, pointer_types) is True def test_validation_result_reset(): @@ -781,47 +790,6 @@ def test_validate_type_coding_display_mismatch(type_str: str, display: str): } -def test_validate_content_extension_too_many_extensions(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"].append( - { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "static", - } - ] - }, - } - ) - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "invalid", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension length: 2 Extension must only contain a single value", - "expression": ["content[0].extension"], - } - - def test_validate_author_too_many_authors(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -956,195 +924,6 @@ def test_validate_author_value_too_long(): } -def test_validate_content_extension_invalid_code(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "invalid", - "display": "invalid", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension code: invalid Extension code must be 'static' or 'dynamic'", - "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].code"], - } - - -def test_validate_content_extension_invalid_display(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "invalid", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension display: invalid Extension display must be the same as code either 'static' or 'dynamic'", - "expression": [ - "content[0].extension[0].valueCodeableConcept.coding[0].display" - ], - } - - -def test_validate_content_extension_invalid_system(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "invalid", - "code": "static", - "display": "static", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension system: invalid Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", - "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].system"], - } - - -def test_validate_content_extension_invalid_url(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "invalid", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "static", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension url: invalid Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", - "expression": ["content[0].extension[0].url"], - } - - -def test_validate_content_extension_missing_coding(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": {"coding": []}, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "required", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Missing content[0].extension[0].valueCodeableConcept.coding, extension must have at least one coding.", - "expression": ["content[0].extension.valueCodeableConcept.coding"], - } - - def test_validate_identifiers_invalid_systems(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -1524,6 +1303,66 @@ def test_validate_ssp_content_with_multiple_asids(): } +def test_validate_content_format_invalid_code_for_unstructured_document(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)", + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "expression": ["content[0].format.code"], + } + + +def test_validate_content_format_invalid_code_for_contact_details(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": ["content[0].format.code"], + } + + def test_validate_practiceSetting_no_coding(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -1724,3 +1563,145 @@ def test_validate_practiceSetting_coding_mismatch_code_and_display(): "diagnostics": "Invalid practice setting coding: display Nephrology service does not match the expected display for 788002001 Practice Setting coding is bound to value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", "expression": ["context.practiceSetting.coding[0]"], } + + +def test_validate_content_extension_invalid_code_and_display_mismatch(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["extension"][0] = { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Dynamic", + } + ] + }, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content extension display: Dynamic Extension display must be the same as code either 'Static' or 'Dynamic'", + "expression": [ + "content[0].extension[0].valueCodeableConcept.coding[0].display" + ], + } + + +def test_validate_content_invalid_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "invalid/type" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid contentType: invalid/type. Must be 'application/pdf' or 'text/html'", + "expression": ["content[0].attachment.contentType"], + } + + +@pytest.mark.parametrize( + "format_code, format_display", + [ + ("urn:nhs-ic:record-contact", "Contact details (HTTP Unsecured)"), + ("urn:nhs-ic:unstructured", "Unstructured Document"), + ], +) +def test_validate_nrl_format_code_valid_match(format_code, format_display): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": format_code, + "display": format_display, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is True + + +@pytest.mark.parametrize( + "format_code, format_display, expected_display", + [ + ( + "urn:nhs-ic:unstructured", + "Contact details (HTTP Unsecured)", + "Unstructured Document", + ), + ( + "urn:nhs-ic:record-contact", + "Unstructured Document", + "Contact details (HTTP Unsecured)", + ), + ], +) +def test_validate_nrl_format_code_display_mismatch( + format_code, format_display, expected_display +): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": format_code, + "display": format_display, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": f"Invalid display for format code '{format_code}'. Expected '{expected_display}'", + "expression": ["content[0].format.display"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index fa1743013..cb8140f1e 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -24,21 +24,14 @@ from nrlf.producer.fhir.r4 import model as producer_model -def validate_type_system( - type_: Optional[RequestQueryType], pointer_types: List[str] -) -> bool: +def validate_type(type_: Optional[RequestQueryType], pointer_types: List[str]) -> bool: """ - Validates if the given type system is present in the list of pointer types. + Validates if the given type is present in the list of pointer types. """ if not type_: return True - type_system = type_.root.split("|", 1)[0] - pointer_type_systems = [ - pointer_type.split("|", 1)[0] for pointer_type in pointer_types - ] - - return type_system in pointer_type_systems + return type_.root in pointer_types # TODO - Validate category is in set permissions once permissioning by category is done. @@ -144,9 +137,10 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_category(resource) self._validate_author(resource) self._validate_type_category_mapping(resource) + self._validate_content(resource) + self._validate_content_format(resource) + self._validate_content_extension(resource) self._validate_practiceSetting(resource) - if resource.content[0].extension: - self._validate_content_extension(resource) except StopValidationError: logger.log(LogReference.VALIDATOR003) @@ -485,77 +479,50 @@ def _validate_type_category_mapping(self, model: DocumentReference): field="category.coding[0].code", ) - def _validate_content_extension(self, model: DocumentReference): + def _validate_content_format(self, model: DocumentReference): """ - Validate the content.extension field contains an appropriate coding. + Validate the content.format field contains an appropriate coding. """ - logger.log(LogReference.VALIDATOR001, step="content_extension") + logger.log(LogReference.VALIDATOR001, step="content_format") - logger.debug("Validating extension") + logger.debug("Validating format") for i, content in enumerate(model.content): - if len(content.extension) > 1: - self.result.add_error( - issue_code="invalid", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension length: {len(content.extension)} Extension must only contain a single value", - field=f"content[{i}].extension", - ) - return - - if len(content.extension[0].valueCodeableConcept.coding) < 1: - self.result.add_error( - issue_code="required", - error_code="INVALID_RESOURCE", - diagnostics=f"Missing content[{i}].extension[0].valueCodeableConcept.coding, extension must have at least one coding.", - field=f"content[{i}].extension.valueCodeableConcept.coding", - ) - return - if ( - content.extension[0].valueCodeableConcept.coding[0].system - != "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + content.attachment.contentType == "text/html" + and content.format.code != "urn:nhs-ic:record-contact" ): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension system: {content.extension[0].valueCodeableConcept.coding[0].system} Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].system", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + field=f"content[{i}].format.code", ) - return - - if content.extension[0].valueCodeableConcept.coding[0].code not in [ - "static", - "dynamic", - ]: - self.result.add_error( - issue_code="value", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension code: {content.extension[0].valueCodeableConcept.coding[0].code} Extension code must be 'static' or 'dynamic'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].code", - ) - return - - if ( - content.extension[0].valueCodeableConcept.coding[0].code - != content.extension[0].valueCodeableConcept.coding[0].display.lower() + elif ( + content.attachment.contentType == "application/pdf" + and content.format.code != "urn:nhs-ic:unstructured" ): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension display: {content.extension[0].valueCodeableConcept.coding[0].display} Extension display must be the same as code either 'static' or 'dynamic'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].display", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + field=f"content[{i}].format.code", ) - return - if ( - content.extension[0].url - != "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ): + def _validate_content_extension(self, model: DocumentReference): + """ + Validate the content.extension field contains an appropriate coding. + """ + logger.log(LogReference.VALIDATOR001, step="content_extension") + + logger.debug("Validating extension") + for i, content in enumerate(model.content): + coding = content.extension[0].valueCodeableConcept.coding[0] + if coding.code != coding.display.lower(): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension url: {content.extension[0].url} Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", - field=f"content[{i}].extension[0].url", + diagnostics=f"Invalid content extension display: {coding.display} Extension display must be the same as code either 'Static' or 'Dynamic'", + field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].display", ) return @@ -667,3 +634,35 @@ def _validate_practiceSetting(self, model: DocumentReference): field="context.practiceSetting.coding[0]", ) return + + def _validate_content(self, model: DocumentReference): + """ + Validate that the contentType is present and is either 'application/pdf' or 'text/html'. + """ + logger.log(LogReference.VALIDATOR001, step="content") + + format_code_display_map = { + "urn:nhs-ic:record-contact": "Contact details (HTTP Unsecured)", + "urn:nhs-ic:unstructured": "Unstructured Document", + } + + for i, content in enumerate(model.content): + if content.attachment.contentType not in ["application/pdf", "text/html"]: + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid contentType: {content.attachment.contentType}. Must be 'application/pdf' or 'text/html'", + field=f"content[{i}].attachment.contentType", + ) + + # Validate NRLFormatCode + format_code = content.format.code + format_display = content.format.display + expected_display = format_code_display_map.get(format_code) + if expected_display and format_display != expected_display: + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid display for format code '{format_code}'. Expected '{expected_display}'", + field=f"content[{i}].format.display", + ) diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index d96b7ce73..945b220ec 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:52+00:00 +# timestamp: 2024-12-13T11:19:26+00:00 from __future__ import annotations @@ -133,12 +133,12 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[str], + str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", pattern="[^\\s]+(\\s[^\\s]+)*", ), - ] = None + ] language: Annotated[ Optional[str], Field( @@ -154,9 +154,9 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[str], + str, Field(description="A location where the data can be accessed.", pattern="\\S*"), - ] = None + ] size: Annotated[ Optional[int], Field( @@ -230,6 +230,29 @@ class Coding(BaseModel): ] = None +class ContentStabilityExtensionCoding(Coding): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class NRLFormatCode(Coding): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -417,6 +440,43 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[str], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[str], + Field(description="The reference details for the link.", pattern="\\S*"), + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -432,6 +492,52 @@ class RequestParams(BaseModel): ] = None +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + severity: Annotated[ + str, + Field( + description="Indicates whether the issue indicates a variation from successful processing.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + code: Annotated[ + str, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[str], + Field( + description="Additional diagnostic information about the issue.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -470,7 +576,7 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[str], Field( @@ -478,35 +584,21 @@ class OperationOutcomeIssue(BaseModel): pattern="[A-Za-z0-9\\-\\.]{1,64}", ), ] = None - severity: Annotated[ - str, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - str, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[str], - Field( - description="Additional diagnostic information about the issue.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -561,11 +653,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], @@ -802,29 +891,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Optional[Coding], - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] = None - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -845,38 +911,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[str], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[str], - Field(description="The reference details for the link.", pattern="\\S*"), - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[str], @@ -1010,13 +1044,9 @@ class Signature(BaseModel): ] = None -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index e4edefc58..0344821fc 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:54+00:00 +# timestamp: 2024-12-13T11:19:28+00:00 from __future__ import annotations @@ -126,11 +126,11 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[StrictStr], + StrictStr, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate." ), - ] = None + ] language: Annotated[ Optional[StrictStr], Field( @@ -144,9 +144,8 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[StrictStr], - Field(description="A location where the data can be accessed."), - ] = None + StrictStr, Field(description="A location where the data can be accessed.") + ] size: Annotated[ Optional[StrictInt], Field( @@ -208,6 +207,29 @@ class Coding(BaseModel): ] = None +class ContentStabilityExtensionCoding(Coding): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class NRLFormatCode(Coding): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[StrictStr], @@ -366,6 +388,40 @@ class RequestHeaderCorrelationId(RootModel[StrictStr]): root: Annotated[StrictStr, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[StrictStr], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user." + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[StrictStr], Field(description="The reference details for the link.") + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -381,6 +437,46 @@ class RequestParams(BaseModel): ] = None +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + severity: Annotated[ + StrictStr, + Field( + description="Indicates whether the issue indicates a variation from successful processing." + ), + ] + code: Annotated[ + StrictStr, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element." + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[StrictStr], + Field(description="Additional diagnostic information about the issue."), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -414,37 +510,28 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[StrictStr], Field( description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." ), ] = None - severity: Annotated[ - StrictStr, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing." + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - StrictStr, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element." + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[StrictStr], - Field(description="Additional diagnostic information about the issue."), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -491,7 +578,8 @@ class DocumentReference(BaseModel): StrictStr, Field(description="The status of this document reference.") ] docStatus: Annotated[ - Optional[StrictStr], Field(description="The status of the underlying document.") + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], @@ -707,28 +795,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Optional[Coding], - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] = None - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[StrictStr], @@ -747,35 +813,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[StrictStr], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user." - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[StrictStr], Field(description="The reference details for the link.") - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[StrictStr], @@ -890,13 +927,9 @@ class Signature(BaseModel): ] = None -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/reports/find_invalid_pointers.py b/reports/find_invalid_pointers.py new file mode 100644 index 000000000..506f27b7a --- /dev/null +++ b/reports/find_invalid_pointers.py @@ -0,0 +1,82 @@ +from datetime import datetime, timedelta, timezone +from typing import Any + +import boto3 +import fire + +from nrlf.consumer.fhir.r4.model import DocumentReference +from nrlf.core.logger import logger +from nrlf.core.validators import DocumentReferenceValidator + +dynamodb = boto3.client("dynamodb") +paginator = dynamodb.get_paginator("scan") + +logger.setLevel("ERROR") + + +def _validate_document(document: str): + docref = DocumentReference.model_validate_json(document) + + validator = DocumentReferenceValidator() + result = validator.validate(data=docref) + + if not result.is_valid: + raise RuntimeError("Failed to validate document: " + str(result.issues)) + + +def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: + """ + Find pointers in the given table that are invalid. + Parameters: + - table_name: The name of the pointers table to use. + """ + + print(f"Finding invalid pointers in table {table_name}....") # noqa + + params: dict[str, Any] = { + "TableName": table_name, + "PaginationConfig": {"PageSize": 50}, + } + + invalid_pointers = [] + total_scanned_count = 0 + + start_time = datetime.now(tz=timezone.utc) + + for page in paginator.paginate(**params): + for item in page["Items"]: + pointer_id = item.get("id", {}).get("S") + document = item.get("document", {}).get("S", "") + try: + _validate_document(document) + except Exception as exc: + invalid_pointers.append((pointer_id, exc)) + + total_scanned_count += page["ScannedCount"] + + if total_scanned_count % 1000 == 0: + print(".", end="", flush=True) # noqa + + if total_scanned_count % 100000 == 0: + print( # noqa + f"scanned={total_scanned_count} invalid={len(invalid_pointers)}" + ) + + end_time = datetime.now(tz=timezone.utc) + + print(" Done") # noqa + + print("Writing invalid_pointers to file ./invalid_pointers.txt ...") # noqa + with open("invalid_pointers.txt", "w") as f: + for _id, err in invalid_pointers: + f.write(f"{_id}: {err}\n") + + return { + "invalid_pointers": len(invalid_pointers), + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + +if __name__ == "__main__": + fire.Fire(_find_invalid_pointers) diff --git a/resources/fhir/NRLF-ContentStability-ValueSet.json b/resources/fhir/NRLF-ContentStability-ValueSet.json new file mode 100644 index 000000000..26a2ca381 --- /dev/null +++ b/resources/fhir/NRLF-ContentStability-ValueSet.json @@ -0,0 +1,37 @@ +{ + "resourceType": "ValueSet", + "id": "NRLF-ContentStability", + "url": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "version": "1.0.0", + "name": "NRLF Content Stability", + "status": "draft", + "date": "2024-12-03T00:00:00+00:00", + "publisher": "NHS Digital", + "contact": { + "name": "NRL Team at NHS Digital", + "telecom": { + "system": "email", + "value": "nrls@nhs.net", + "use": "work" + } + }, + "description": "A code from the NRL Content Stability coding system to represent the stability of the content.", + "copyright": "Copyright 2024 NHS Digital.", + "compose": { + "include": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "concept": [ + { + "code": "static", + "display": "Static" + }, + { + "code": "dynamic", + "display": "Dynamic" + } + ] + } + ] + } +} diff --git a/resources/fhir/NRLF-FormatCode-ValueSet.json b/resources/fhir/NRLF-FormatCode-ValueSet.json new file mode 100644 index 000000000..a0fe1cc77 --- /dev/null +++ b/resources/fhir/NRLF-FormatCode-ValueSet.json @@ -0,0 +1,37 @@ +{ + "resourceType": "ValueSet", + "id": "NRLF-FormatCode", + "url": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "version": "1.0.0", + "name": "NRLF Format Code", + "status": "draft", + "date": "2024-12-03T00:00:00+00:00", + "publisher": "NHS Digital", + "contact": { + "name": "NRL Team at NHS Digital", + "telecom": { + "system": "email", + "value": "nrls@nhs.net", + "use": "work" + } + }, + "description": "A ValueSet that identifies the format of the content of the target document or record of a National Record Locator pointer.", + "copyright": "Copyright © 2024 NHS Digital", + "compose": { + "include": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "concept": [ + { + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)" + }, + { + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + } + ] + } + ] + } +} diff --git a/resources/fhir/NRLF-RecordType-ValueSet.json b/resources/fhir/NRLF-RecordType-ValueSet.json index 4fda682f0..08652fe5f 100644 --- a/resources/fhir/NRLF-RecordType-ValueSet.json +++ b/resources/fhir/NRLF-RecordType-ValueSet.json @@ -56,7 +56,7 @@ }, { "code": "736366004", - "display": "Advanced care plan" + "display": "Advance care plan" }, { "code": "735324008", diff --git a/swagger/consumer-static/narrative.yaml b/swagger/consumer-static/narrative.yaml index a349758ea..6b4522a33 100644 --- a/swagger/consumer-static/narrative.yaml +++ b/swagger/consumer-static/narrative.yaml @@ -190,7 +190,7 @@ info: Right click the icon and save link as... to save the Postman collection to your device - [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/main/postman_collection.json) + [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/develop/postman_collection.json) ### Integration testing diff --git a/swagger/producer-static/components.yaml b/swagger/producer-static/components.yaml index 22a096326..8c984b796 100644 --- a/swagger/producer-static/components.yaml +++ b/swagger/producer-static/components.yaml @@ -122,7 +122,7 @@ components: format: system: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode code: "urn:nhs-ic:unstructured" - display: Unstructured document + display: Unstructured Document context: practiceSetting: coding: diff --git a/swagger/producer-static/narrative.yaml b/swagger/producer-static/narrative.yaml index 4341faf86..8380e0879 100644 --- a/swagger/producer-static/narrative.yaml +++ b/swagger/producer-static/narrative.yaml @@ -179,7 +179,7 @@ info: Right click the icon and save link as... to save the Postman collection to your device - [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/main/postman_collection.json) + [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/develop/postman_collection.json) ### Integration testing @@ -276,13 +276,40 @@ paths: ] ``` * `content` MUST have at least one entry. - * `content[].format[]` SHOULD indicate whether the data is structured or not, e.g. + * `content[]` MUST include an `attachment` entry. + * `content[]` MUST include a `format` entry. (https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode) + * `content[]` MUST include the content stability extension (https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability). + * `content[].attachment` MUST include a `url` to the document. + * `content[].attachment` MUST include a `contentType` and be a valid MIME type, specifically `application/pdf` for documents or `text/html` for contact details. + * `content[].format` MUST indicate whether the data is structured or not + * Example of the content section: ``` - "format": [ + "content": [ { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:unstructured" - "display": "Unstructured document" + "attachment": { + "contentType": "application/pdf", + "url": "https://provider-ods-code.thirdparty.nhs.uk/path/to/document.pdf", + "creation": "2022-12-22T09:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured" + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] ``` diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index fc41d32a8..d357e6b15 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -64,15 +64,6 @@ resource "aws_s3_bucket_acl" "backup_reports" { acl = "private" } -# We need a key for the SNS topic that will be used for notifications from AWS Backup. This key -# will be used to encrypt the messages sent to the topic before they are sent to the subscribers, -# but isn't needed by the recipients of the messages. - -# First we need some contextual data -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - -# Now we can define the key itself resource "aws_kms_key" "backup_notifications" { description = "KMS key for AWS Backup notifications" deletion_window_in_days = 7 @@ -101,8 +92,6 @@ resource "aws_kms_key" "backup_notifications" { }) } -# Now we can deploy the source and destination modules, referencing the resources we've created above. - module "source" { source = "../modules/backup-source" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 0e6cd4ce8..298d654c1 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -82,6 +82,6 @@ resource "aws_backup_selection" "dynamodb" { selection_tag { key = var.backup_plan_config_dynamodb.selection_tag type = "STRINGEQUALS" - value = "true" + value = "True" } } diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 06e61a58e..ab7fe77aa 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -5,7 +5,7 @@ resource "aws_s3_bucket" "authorization-store" { tags = { Name = "authorization store" Environment = "${var.name_prefix}" - NHSE-Enable-S3-Backup = "${var.enable_backups}" + NHSE-Enable-S3-Backup = var.enable_backups ? "True" : "False" } } diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 93e060fdb..06a7428b7 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -52,5 +52,7 @@ resource "aws_dynamodb_table" "pointers" { enabled = var.enable_pitr } - tags = { NHSE-Enable-DDB-Backup = "${var.enable_backups}" } + tags = { + NHSE-Enable-DDB-Backup = var.enable_backups ? "True" : "False" + } } diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index aa32f2f16..1f7bd3e81 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,7 +1,9 @@ resource "aws_s3_bucket" "api_truststore" { bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy - tags = { NHSE-Enable-S3-Backup = "${var.enable_backups}" } + tags = { + NHSE-Enable-S3-Backup = var.enable_backups ? "True" : "False" + } } resource "aws_s3_bucket_policy" "api_truststore_bucket_policy" { diff --git a/tests/data/DocumentReference/RQI-736253002-Valid.json b/tests/data/DocumentReference/RQI-736253002-Valid.json index db7503fbf..873434086 100644 --- a/tests/data/DocumentReference/RQI-736253002-Valid.json +++ b/tests/data/DocumentReference/RQI-736253002-Valid.json @@ -69,7 +69,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { @@ -79,7 +79,7 @@ { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", "code": "static", - "display": "static" + "display": "Static" } ] } diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json index 2bf5f63df..de51a0df6 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json @@ -73,8 +73,22 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" - } + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json index 0aee9f067..49e5484ca 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json @@ -70,8 +70,22 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" - } + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json index ca7dfbcd5..9e6a0e73b 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json @@ -72,8 +72,22 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" - } + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json index c88029c05..a20f81d33 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json @@ -69,8 +69,22 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" - } + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid.json b/tests/data/DocumentReference/Y05868-736253002-Valid.json index b0a18b1d5..6b79042e8 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid.json @@ -69,7 +69,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { @@ -79,7 +79,7 @@ { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", "code": "static", - "display": "static" + "display": "Static" } ] } diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index f0854af16..aa91cd59f 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -69,7 +69,26 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" - } + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { @@ -155,7 +174,26 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" - } + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/features/consumer/searchDocumentReference-failure.feature b/tests/features/consumer/searchDocumentReference-failure.feature index 1de20623a..161e12bcd 100644 --- a/tests/features/consumer/searchDocumentReference-failure.feature +++ b/tests/features/consumer/searchDocumentReference-failure.feature @@ -77,7 +77,35 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", + "expression": ["type"] + } + """ + + Scenario: Search rejects request with type they are not allowed to use + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + | type | http://snomed.info/sct\|887701000000100 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"] } """ diff --git a/tests/features/consumer/searchDocumentReference-success.feature b/tests/features/consumer/searchDocumentReference-success.feature index ead47a40e..1fd296127 100644 --- a/tests/features/consumer/searchDocumentReference-success.feature +++ b/tests/features/consumer/searchDocumentReference-success.feature @@ -36,6 +36,44 @@ Feature: Consumer - searchDocumentReference - Success Scenarios | custodian | 02V | | author | 02V | + Scenario: Search for a DocumentReference and Accession Number is in response + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchDocRefTest | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | 02V | + | author | 02V | + | identifier | 02V.123456789 | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + Then the response status code is 200 + And the response is a searchset Bundle + And the Bundle has a self link matching 'DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|9278693472' + And the Bundle has a total of 1 + And the Bundle has 1 entry + And the Bundle contains an DocumentReference with values + | property | value | + | id | 02V-1111111111-SearchDocRefTest | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | 02V | + | author | 02V | + | identifier | 02V.123456789 | + Scenario: Search for a DocumentReference by NHS Number and Custodian where both search parameters match Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'RX898' is authorised to access pointer types: diff --git a/tests/features/consumer/searchPostDocumentReference-failure.feature b/tests/features/consumer/searchPostDocumentReference-failure.feature index 91380c764..3c76f708a 100644 --- a/tests/features/consumer/searchPostDocumentReference-failure.feature +++ b/tests/features/consumer/searchPostDocumentReference-failure.feature @@ -77,7 +77,35 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid type (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "The provided type does not match the allowed types for this organisation", + "expression": ["type"] + } + """ + + Scenario: Search rejects request with type they are not allowed to use + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences using POST with request body: + | key | value | + | subject | 9278693472 | + | type | http://snomed.info/sct\|887701000000100 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"] } """ diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index d407845b7..af22ae1bc 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -405,22 +405,42 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - Scenario: Invalid practice setting (not in value set) + Scenario: Invalid format code for attachment type contact details Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: + And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | - When producer 'X26' creates a DocumentReference with values: - | property | value | - | subject | 9999999999 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | custodian | X26 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | - | practiceSetting | 12345 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "text/html", + "url": "someContact.co.uk" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: @@ -429,38 +449,60 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid practice setting code: 12345 Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", - "expression": ["context.practiceSetting.coding[0].code"] + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": [ + "content[0].format.code" + ] } """ - Scenario: Invalid practice setting (valid code but wrong display value) + Scenario: Invalid format code for attachment type pdf Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | - When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'context' is: + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: """ - "context": { - "practiceSetting": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "788002001", - "display": "Ophthalmology service" - } + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "language": "en-UK", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } ] - } - } """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue @@ -470,17 +512,17 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid practice setting coding: display Ophthalmology service does not match the expected display for 788002001 Practice Setting coding is bound to value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", "expression": [ - "context.practiceSetting.coding[0]" + "content[0].format.code" ] } """ @@ -614,3 +656,247 @@ Feature: Producer - createDocumentReference - Failure Scenarios | type-system | type-code | category-code | type-display | correct-display | | https://nicip.nhs.uk | MAULR | 721981007 | "Nonsense display" | MRA Upper Limb Rt | | https://nicip.nhs.uk | MAXIB | 103693007 | "Nonsense display" | MRI Axilla Both | + + Scenario: Invalid practice setting (not in value set) + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'X26' creates a DocumentReference with values: + | property | value | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | X26 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 12345 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid practice setting code: 12345 Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0].code"] + } + """ + + Scenario: Invalid practice setting (valid code but wrong display value) + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'context' is: + """ + "context": { + "practiceSetting": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "788002001", + "display": "Ophthalmology service" + } + ] + } + } + """ + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: contentType empty string + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'ANGY1' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'ANGY1' creates a DocumentReference with values: + | property | value | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | contentType | application/invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "text/html", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ diff --git a/tests/features/producer/createDocumentReference-success.feature b/tests/features/producer/createDocumentReference-success.feature index 202595190..8842111fb 100644 --- a/tests/features/producer/createDocumentReference-success.feature +++ b/tests/features/producer/createDocumentReference-success.feature @@ -211,7 +211,7 @@ Feature: Producer - createDocumentReference - Success Scenarios | 325691000000100 | 734163000 | CONTINGENCY_PLAN | | 736373009 | 734163000 | EOL_CARE_PLAN | | 16521000000101 | 734163000 | LLOYD_GEORGE_FOLDER | - | 736366004 | 734163000 | ADVANCED_CARE_PLAN | + | 736366004 | 734163000 | ADVANCE_CARE_PLAN | | 735324008 | 734163000 | TREATMENT_ESCALATION_PLAN | | 2181441000000107 | 734163000 | PERSONALISED_CARE_AND_SUPPORT_PLAN | diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index 2eb19448b..9e4c2f8d7 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -71,7 +71,26 @@ Feature: Producer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" - } + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index b909b46cf..4dd5ea87c 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -48,3 +48,261 @@ Feature: Producer - updateDocumentReference - Failure Scenarios ] } """ + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567890-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567890-updateDocTest' and only changing: + """ + { + "content": [] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: contentType empty string + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567891-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567891-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567892-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567892-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "application/invalid", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567893-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567893-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "text/html", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 3fb5d6ebd..3855b1b34 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -236,3 +236,190 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios | type-system | type-code | category-code | type-display | correct-display | | https://nicip.nhs.uk | MAULR | 721981007 | "Nonsense display" | MRA Upper Limb Rt | | https://nicip.nhs.uk | MAXIB | 103693007 | "Nonsense display" | MRI Axilla Both | + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-sample-id-00001' and default test values except 'content' is: + """ + "content": [] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: contentType empty string + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-sample-id-00002' and default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'ANGY1' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'ANGY1' upserts a DocumentReference with values: + | property | value | + | id | TSTCUS-sample-id-00003 | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | contentType | application/invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-testid-upsert-0001-0001' and default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "text/html", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ diff --git a/tests/features/steps/1_setup.py b/tests/features/steps/1_setup.py index 49269fbff..2a0a51e3b 100644 --- a/tests/features/steps/1_setup.py +++ b/tests/features/steps/1_setup.py @@ -1,5 +1,4 @@ import json -from contextlib import suppress from behave import * # noqa from behave.runner import Context @@ -55,5 +54,5 @@ def create_document_reference_step(context: Context): def clean_up_test_pointer(context: Context, doc_pointer: DocumentPointer): """Remove a pointer during cleanup without failing if it has already been deleted""" - with suppress(Exception): + if context.repository.get_by_id(doc_pointer.id): context.repository.delete(doc_pointer) diff --git a/tests/features/steps/2_request.py b/tests/features/steps/2_request.py index a008aed17..993fae1c7 100644 --- a/tests/features/steps/2_request.py +++ b/tests/features/steps/2_request.py @@ -1,3 +1,5 @@ +import json + from behave import * # noqa from behave.runner import Context @@ -94,17 +96,21 @@ def create_post_document_reference_step(context: Context, ods_code: str): context.add_cleanup(lambda: context.repository.delete_by_id(doc_ref_id)) -@when( - "producer 'TSTCUS' requests creation of a DocumentReference with default test values except '{section}' is" -) -def create_post_body_step(context: Context, section: str): +def _create_or_upsert_body_step( + context: Context, + method: str, + section: str, + pointer_id: str = "TSTCUS-sample-id-00000", +): client = producer_client_from_context(context, "TSTCUS") if not context.text: raise ValueError("No document reference text snippet provided") - doc_ref = create_test_document_reference_with_defaults(section, context.text) - context.response = client.create_text(doc_ref) + doc_ref = create_test_document_reference_with_defaults( + section, context.text, pointer_id + ) + context.response = getattr(client, method)(doc_ref) if context.response.status_code == 201: doc_ref_id = context.response.headers["Location"].split("/")[-1] @@ -114,6 +120,42 @@ def create_post_body_step(context: Context, section: str): context.add_cleanup(lambda: context.repository.delete_by_id(doc_ref_id)) +@when( + "producer 'TSTCUS' requests creation of a DocumentReference with default test values except '{section}' is" +) +def create_post_body_step(context: Context, section: str): + _create_or_upsert_body_step(context, "create_text", section) + + +@when( + "producer 'TSTCUS' requests upsert of a DocumentReference with pointerId '{pointer_id}' and default test values except '{section}' is" +) +def upsert_post_body_step(context: Context, section: str, pointer_id: str): + _create_or_upsert_body_step(context, "upsert_text", section, pointer_id) + + +@when( + "producer 'TSTCUS' requests update of a DocumentReference with pointerId '{pointer_id}' and only changing" +) +def update_post_body_step(context: Context, pointer_id: str): + """ + Updates an existing DocumentReference with new values for a specific section + """ + consumer_client = consumer_client_from_context(context, "TSTCUS") + context.response = consumer_client.read(pointer_id) + + if context.response.status_code != 200: + raise ValueError(f"Failed to read existing pointer: {context.response.text}") + + doc_ref = context.response.json() + custom_data = json.loads(context.text) + for key in custom_data: + doc_ref[key] = custom_data[key] + + producer_client = producer_client_from_context(context, "TSTCUS") + context.response = producer_client.update(doc_ref, pointer_id) + + @when("producer '{ods_code}' upserts a DocumentReference with values") def create_put_document_reference_step(context: Context, ods_code: str): client = producer_client_from_context(context, ods_code) diff --git a/tests/features/steps/3_assert.py b/tests/features/steps/3_assert.py index 1ce9c82f8..445f2268d 100644 --- a/tests/features/steps/3_assert.py +++ b/tests/features/steps/3_assert.py @@ -218,6 +218,14 @@ def assert_document_reference_matches_value( context.response.json(), ) + if identifier := items.get("identifier"): + assert doc_ref.identifier[0].value == identifier, format_error( + "DocumentReference Identifier does not match", + identifier, + doc_ref.identifier[0].value, + context.response.json(), + ) + @then("the Bundle contains an DocumentReference with values") def assert_bundle_contains_documentreference_values_step(context: Context): diff --git a/tests/features/utils/constants.py b/tests/features/utils/constants.py index 7c656824e..2f0e4b2f3 100644 --- a/tests/features/utils/constants.py +++ b/tests/features/utils/constants.py @@ -112,8 +112,22 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" - } + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 5bc300585..35c3eb393 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -1,5 +1,8 @@ from layer.nrlf.core.constants import ( CATEGORY_ATTRIBUTES, + CONTENT_FORMAT_CODE_URL, + CONTENT_STABILITY_EXTENSION_URL, + CONTENT_STABILITY_SYSTEM_URL, SNOMED_PRACTICE_SETTINGS, SNOMED_SYSTEM_URL, TYPE_ATTRIBUTES, @@ -8,11 +11,15 @@ Attachment, CodeableConcept, Coding, + ContentStabilityExtension, + ContentStabilityExtensionCoding, + ContentStabilityExtensionValueCodeableConcept, DocumentReference, DocumentReferenceContent, DocumentReferenceContext, DocumentReferenceRelatesTo, Identifier, + NRLFormatCode, Reference, ) from tests.features.utils.constants import ( @@ -39,14 +46,39 @@ def create_test_document_reference(items: dict) -> DocumentReference: base_doc_ref = DocumentReference.model_construct( resourceType="DocumentReference", status=items.get("status", "current"), - content=[ - DocumentReferenceContent( - attachment=Attachment( - contentType=items.get("contentType", "application/json"), - url=items["url"], + content=items.get( + "content", + [ + DocumentReferenceContent( + attachment=Attachment( + contentType=items.get("contentType", "application/pdf"), + url=items["url"], + ), + format=NRLFormatCode( + system=items.get( + "formatSystem", + CONTENT_FORMAT_CODE_URL, + ), + code=items.get("formatCode", "urn:nhs-ic:unstructured"), + display=items.get("formatDisplay", "Unstructured Document"), + ), + extension=[ + ContentStabilityExtension( + url=CONTENT_STABILITY_EXTENSION_URL, + valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( + coding=[ + ContentStabilityExtensionCoding( + system=CONTENT_STABILITY_SYSTEM_URL, + code="static", + display="Static", + ) + ] + ), + ) + ], ) - ) - ], + ], + ), context=DocumentReferenceContext( practiceSetting=CodeableConcept( coding=[ @@ -128,6 +160,12 @@ def create_test_document_reference(items: dict) -> DocumentReference: ), ) ] + if items.get("identifier"): + base_doc_ref.identifier = [ + Identifier( + type=CodeableConcept(text="Accession-Number"), value=items["identifier"] + ) + ] return base_doc_ref diff --git a/tests/performance/environment.py b/tests/performance/environment.py index 5baa3f7f0..fc7861bd7 100644 --- a/tests/performance/environment.py +++ b/tests/performance/environment.py @@ -29,7 +29,7 @@ class LogReference: "736373009": "End of life care plan", "861421000000109": "End of life care coordination summary", "887701000000100": "Emergency Health Care Plans", - "736366004": "Advanced Care Plan", + "736366004": "Advance Care Plan", "735324008": "Treatment Escalation Plan", "824321000000109": "Summary Record", "2181441000000107": "Personalised Care and Support Plan", diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 69e1f6358..f639e80f8 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -1,13 +1,24 @@ -from nrlf.core.constants import TYPE_ATTRIBUTES, Categories, PointerTypes +from nrlf.core.constants import ( + CONTENT_FORMAT_CODE_URL, + CONTENT_STABILITY_EXTENSION_URL, + CONTENT_STABILITY_SYSTEM_URL, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.producer.fhir.r4.model import ( Attachment, CodeableConcept, Coding, + ContentStabilityExtension, + ContentStabilityExtensionCoding, + ContentStabilityExtensionValueCodeableConcept, DocumentReference, DocumentReferenceContent, DocumentReferenceContext, DocumentReferenceRelatesTo, Identifier, + NRLFormatCode, Reference, ) from tests.utilities.api_clients import ProducerTestClient @@ -20,7 +31,7 @@ def build_document_reference( category: str = Categories.CARE_PLAN.coding_value(), type: str = PointerTypes.MENTAL_HEALTH_PLAN.coding_value(), author: str = "SMOKETEST", - content_type: str = "application/json", + content_type: str = "application/pdf", content_url: str = "https://testing.record-locator.national.nhs.uk/_smoke_test_pointer_content", replaces_id: str | None = None, ) -> DocumentReference: @@ -32,7 +43,26 @@ def build_document_reference( attachment=Attachment( contentType=content_type, url=content_url, - ) + ), + format=NRLFormatCode( + system=CONTENT_FORMAT_CODE_URL, + code="urn:nhs-ic:unstructured", + display="Unstructured Document", + ), + extension=[ + ContentStabilityExtension( + url=CONTENT_STABILITY_EXTENSION_URL, + valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( + coding=[ + ContentStabilityExtensionCoding( + system=CONTENT_STABILITY_SYSTEM_URL, + code="static", + display="Static", + ) + ] + ), + ) + ], ) ], type=CodeableConcept( diff --git a/tests/utilities/api_clients.py b/tests/utilities/api_clients.py index 1d06bf2a8..3b1b78e9f 100644 --- a/tests/utilities/api_clients.py +++ b/tests/utilities/api_clients.py @@ -213,6 +213,14 @@ def upsert(self, doc_ref): cert=self.config.client_cert, ) + def upsert_text(self, doc_ref): + return requests.put( + f"{self.api_url}/DocumentReference", + data=doc_ref, + headers=self.request_headers, + cert=self.config.client_cert, + ) + def update(self, doc_ref, doc_ref_id: str): return requests.put( f"{self.api_url}/DocumentReference/{doc_ref_id}", From 1e56f9b7b4cff0251742931ca3e5706064566734 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 15:32:03 +0000 Subject: [PATCH 34/60] NRL-1188 set up encryption --- .../modules/athena/athena.tf | 8 ++--- .../modules/glue/kms.tf | 7 ++++ .../modules/glue/s3.tf | 33 +++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/glue/kms.tf diff --git a/terraform/account-wide-infrastructure/modules/athena/athena.tf b/terraform/account-wide-infrastructure/modules/athena/athena.tf index fe505226b..b5765e113 100644 --- a/terraform/account-wide-infrastructure/modules/athena/athena.tf +++ b/terraform/account-wide-infrastructure/modules/athena/athena.tf @@ -3,10 +3,10 @@ resource "aws_athena_database" "reporting-db" { bucket = var.target_bucket_name - # encryption_configuration { - # encryption_option = "SSE_KMS" - # kms_key = aws_kms_key.athena.arn - # } + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key = aws_kms_key.athena.arn + } force_destroy = true } diff --git a/terraform/account-wide-infrastructure/modules/glue/kms.tf b/terraform/account-wide-infrastructure/modules/glue/kms.tf new file mode 100644 index 000000000..067c1ad5a --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/kms.tf @@ -0,0 +1,7 @@ +resource "aws_kms_key" "glue" { +} + +resource "aws_kms_alias" "glue" { + name = "alias/${var.name_prefix}-glue" + target_key_id = aws_kms_key.glue.key_id +} diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 4777b9e04..4695f2b5b 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -31,6 +31,17 @@ resource "aws_s3_bucket_policy" "source-data-bucket" { }) } +resource "aws_s3_bucket_server_side_encryption_configuration" "source-data-bucket" { + bucket = aws_s3_bucket.source-data-bucket.bucket + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.glue.arn + sse_algorithm = "aws:kms" + } + } +} + resource "aws_s3_bucket_public_access_block" "source-data-bucket-public-access-block" { bucket = aws_s3_bucket.source-data-bucket.id @@ -74,6 +85,17 @@ resource "aws_s3_bucket_policy" "target-data-bucket" { }) } +resource "aws_s3_bucket_server_side_encryption_configuration" "target-data-bucket" { + bucket = aws_s3_bucket.target-data-bucket.bucket + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.glue.arn + sse_algorithm = "aws:kms" + } + } +} + resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-block" { bucket = aws_s3_bucket.target-data-bucket.id @@ -116,6 +138,17 @@ resource "aws_s3_bucket_policy" "code-bucket" { }) } +resource "aws_s3_bucket_server_side_encryption_configuration" "code-bucket" { + bucket = aws_s3_bucket.code-bucket.bucket + + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.glue.arn + sse_algorithm = "aws:kms" + } + } +} + resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" { bucket = aws_s3_bucket.code-bucket.id From ef476d22cb14b87539880d144470ab57ee37e840 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 15:41:16 +0000 Subject: [PATCH 35/60] NRL-1188 Use latest allowed python version --- terraform/account-wide-infrastructure/dev/glue.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf index aab1623ce..733ee1964 100644 --- a/terraform/account-wide-infrastructure/dev/glue.tf +++ b/terraform/account-wide-infrastructure/dev/glue.tf @@ -1,5 +1,5 @@ module "dev-glue" { source = "../modules/glue" name_prefix = "nhsd-nrlf--dev" - python_version = "3.12.2" + python_version = "3.9" } From 161c6a13bd935db93272fa5c07db0858bc35abf0 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 15:46:57 +0000 Subject: [PATCH 36/60] NRL-1188 Python version var is a number not string --- terraform/account-wide-infrastructure/dev/glue.tf | 2 +- terraform/account-wide-infrastructure/modules/glue/vars.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf index 733ee1964..e665c950c 100644 --- a/terraform/account-wide-infrastructure/dev/glue.tf +++ b/terraform/account-wide-infrastructure/dev/glue.tf @@ -1,5 +1,5 @@ module "dev-glue" { source = "../modules/glue" name_prefix = "nhsd-nrlf--dev" - python_version = "3.9" + python_version = 3.9 } diff --git a/terraform/account-wide-infrastructure/modules/glue/vars.tf b/terraform/account-wide-infrastructure/modules/glue/vars.tf index 63842e2d8..cb03095bf 100644 --- a/terraform/account-wide-infrastructure/modules/glue/vars.tf +++ b/terraform/account-wide-infrastructure/modules/glue/vars.tf @@ -4,7 +4,7 @@ variable "name_prefix" { } variable "python_version" { - type = string + type = number description = "Python version to run in script" } From b974b67fc72012b756008bcbcf8e237ed8a95341 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 15:53:16 +0000 Subject: [PATCH 37/60] NRL-1188 Python version update --- terraform/account-wide-infrastructure/dev/glue.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf index e665c950c..e8fbd713a 100644 --- a/terraform/account-wide-infrastructure/dev/glue.tf +++ b/terraform/account-wide-infrastructure/dev/glue.tf @@ -1,5 +1,5 @@ module "dev-glue" { source = "../modules/glue" name_prefix = "nhsd-nrlf--dev" - python_version = 3.9 + python_version = 3 } From cae47269511b4362092f5da32dbfaa0597cb375a Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 16:19:01 +0000 Subject: [PATCH 38/60] NRL-1188 Fix destination for new stream --- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 812fbdeb3..dd28ee2ce 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -59,7 +59,7 @@ resource "aws_kinesis_firehose_delivery_stream" "firehose" { resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { name = "${var.prefix}--cloudwatch-reporting-delivery-stream" - destination = var.destination + destination = "extended_s3" extended_s3_configuration { role_arn = aws_iam_role.firehose.arn From 0bf39208a5cc62d2487768754e53020ea47184c6 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 16:32:43 +0000 Subject: [PATCH 39/60] NRL-1188 Reference existing s3 bucket in firehose --- .../account-wide-infrastructure/modules/glue/glue.tf | 4 ---- .../account-wide-infrastructure/modules/glue/outputs.tf | 9 +++++++++ terraform/infrastructure/firehose.tf | 1 + terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- terraform/infrastructure/modules/firehose/s3.tf | 4 ---- terraform/infrastructure/modules/firehose/vars.tf | 4 ++++ 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index 431fabc82..64cca24f6 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -57,7 +57,3 @@ resource "aws_glue_job" "glue_job" { "--extra-py-files" = "s3://${aws_s3_bucket.code-bucket.id}/src.zip" } } - -output "glue_crawler_name" { - value = "s3//${aws_s3_bucket.source-data-bucket.id}/" -} diff --git a/terraform/account-wide-infrastructure/modules/glue/outputs.tf b/terraform/account-wide-infrastructure/modules/glue/outputs.tf index a5c0e143f..d17fc4d09 100644 --- a/terraform/account-wide-infrastructure/modules/glue/outputs.tf +++ b/terraform/account-wide-infrastructure/modules/glue/outputs.tf @@ -2,3 +2,12 @@ output "target_bucket_name" { description = "Name of destination bucket" value = aws_s3_bucket.target-data-bucket.id } + +output "source_bucket_name" { + description = "Name of source bucket" + value = aws_s3_bucket.source-data-bucket.id +} + +output "glue_crawler_name" { + value = "s3//${aws_s3_bucket.source-data-bucket.id}/" +} diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 2a0ebd221..55dabb2e7 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -8,4 +8,5 @@ module "firehose__processor" { splunk_environment = local.splunk_environment splunk_index = local.splunk_index destination = "splunk" + reporting_bucket = module.dev-glue.source_bucket_name } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index dd28ee2ce..1498f244c 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = data.aws_s3_bucket.source-data-bucket.arn + bucket_arn = var.reporting_bucket processing_configuration { enabled = "true" diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index d002a2290..121bfc04b 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -106,7 +106,3 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { ] }) } - -data "aws_s3_bucket" "source-data-bucket" { - bucket = "source-data-bucket" -} diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 9d9a70385..83429c171 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -34,3 +34,7 @@ variable "error_prefix" { type = string default = "errors" } + +variable "source_data_bucket" { + type = string +} From fa3d2c5d072bb9da9e142cf84abbdbc7a27c24b8 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 16:39:06 +0000 Subject: [PATCH 40/60] NRL-1188 Update var --- terraform/infrastructure/modules/firehose/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 83429c171..1663f6a71 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -35,6 +35,6 @@ variable "error_prefix" { default = "errors" } -variable "source_data_bucket" { +variable "reporting_bucket" { type = string } From 3c24ddf8c8d9c695243c93ff238c0353a890b460 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 17:24:19 +0000 Subject: [PATCH 41/60] NRL-1188 Reference existing bucket --- terraform/infrastructure/firehose.tf | 1 - terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- terraform/infrastructure/modules/firehose/s3.tf | 4 ++++ terraform/infrastructure/modules/firehose/vars.tf | 4 ---- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 55dabb2e7..2a0ebd221 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -8,5 +8,4 @@ module "firehose__processor" { splunk_environment = local.splunk_environment splunk_index = local.splunk_index destination = "splunk" - reporting_bucket = module.dev-glue.source_bucket_name } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 1498f244c..dd28ee2ce 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket + bucket_arn = data.aws_s3_bucket.source-data-bucket.arn processing_configuration { enabled = "true" diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 121bfc04b..1605706a7 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -106,3 +106,7 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { ] }) } + +data "aws_s3_bucket" "source-data-bucket" { + bucket = "${var.prefix}-source-data-bucket" +} diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 1663f6a71..9d9a70385 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -34,7 +34,3 @@ variable "error_prefix" { type = string default = "errors" } - -variable "reporting_bucket" { - type = string -} From b75bbd06a6a9adf4133f1f3276b612237174bb8a Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 16 Dec 2024 17:33:41 +0000 Subject: [PATCH 42/60] NRL-1188 correct prefix for shared resource --- terraform/infrastructure/modules/firehose/s3.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 1605706a7..5859f8d94 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -108,5 +108,5 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { } data "aws_s3_bucket" "source-data-bucket" { - bucket = "${var.prefix}-source-data-bucket" + bucket = "${local.shared_prefix}-source-data-bucket" } From 7bb824252954035e43823d475c0fecb7abde9e95 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:12:47 +0000 Subject: [PATCH 43/60] NRL-1188 Call existing bucket in correct file --- terraform/infrastructure/data.tf | 5 +++++ terraform/infrastructure/modules/firehose/s3.tf | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/terraform/infrastructure/data.tf b/terraform/infrastructure/data.tf index e66b237ed..85b716377 100644 --- a/terraform/infrastructure/data.tf +++ b/terraform/infrastructure/data.tf @@ -41,3 +41,8 @@ data "external" "current-info" { "../../scripts/get-current-info.sh", ] } + +data "aws_s3_bucket" "source-data-bucket" { + count = var.use_shared_resources ? 1 : 0 + bucket = "${local.shared_prefix}-source-data-bucket" +} diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 5859f8d94..121bfc04b 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -106,7 +106,3 @@ resource "aws_iam_policy" "firehose-alert--s3-read" { ] }) } - -data "aws_s3_bucket" "source-data-bucket" { - bucket = "${local.shared_prefix}-source-data-bucket" -} From fa711eb12563ba70bfb0d0b25977164215bcb224 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:20:35 +0000 Subject: [PATCH 44/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 1 + terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 2a0ebd221..5f550b890 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -8,4 +8,5 @@ module "firehose__processor" { splunk_environment = local.splunk_environment splunk_index = local.splunk_index destination = "splunk" + reporting_bucket = data.aws_s3_bucket.source-data-bucket.arn } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index dd28ee2ce..1498f244c 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = data.aws_s3_bucket.source-data-bucket.arn + bucket_arn = var.reporting_bucket processing_configuration { enabled = "true" From c81bd8a69dabd1b5d5541c95356381eaac39c9dd Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:21:22 +0000 Subject: [PATCH 45/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/modules/firehose/vars.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 9d9a70385..1663f6a71 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -34,3 +34,7 @@ variable "error_prefix" { type = string default = "errors" } + +variable "reporting_bucket" { + type = string +} From 75dddcf771e75719404d8f67ee846cb10fc50585 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:28:02 +0000 Subject: [PATCH 46/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 20 +++++++++---------- .../modules/firehose/kinesis.tf | 2 +- .../infrastructure/modules/firehose/vars.tf | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 5f550b890..035a07463 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -1,12 +1,12 @@ module "firehose__processor" { - source = "./modules/firehose" - assume_account = local.aws_account_id - prefix = local.prefix - region = local.region - environment = local.environment - cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn - splunk_environment = local.splunk_environment - splunk_index = local.splunk_index - destination = "splunk" - reporting_bucket = data.aws_s3_bucket.source-data-bucket.arn + source = "./modules/firehose" + assume_account = local.aws_account_id + prefix = local.prefix + region = local.region + environment = local.environment + cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn + splunk_environment = local.splunk_environment + splunk_index = local.splunk_index + destination = "splunk" + reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 1498f244c..d118223c4 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket + bucket_arn = var.reporting_bucket_arn processing_configuration { enabled = "true" diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 1663f6a71..fa1c06ad1 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -35,6 +35,6 @@ variable "error_prefix" { default = "errors" } -variable "reporting_bucket" { +variable "reporting_bucket_arn" { type = string } From d8de638afd33b81610fe2ed00a560c5875411f8e Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:35:41 +0000 Subject: [PATCH 47/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index d118223c4..f69c26bd6 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket_arn + bucket_arn = var.reporting_bucket_arn[count.index] processing_configuration { enabled = "true" From 9c19ebed3b563df57444eda121ee7b0c849b9582 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:41:28 +0000 Subject: [PATCH 48/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 2 +- terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 035a07463..491d4254b 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -8,5 +8,5 @@ module "firehose__processor" { splunk_environment = local.splunk_environment splunk_index = local.splunk_index destination = "splunk" - reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn + reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn[count.index] } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index f69c26bd6..d118223c4 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket_arn[count.index] + bucket_arn = var.reporting_bucket_arn processing_configuration { enabled = "true" From 9e4e8a662916d3419705bdd12f67009568def1ff Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:48:09 +0000 Subject: [PATCH 49/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 491d4254b..b831791f6 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -8,5 +8,5 @@ module "firehose__processor" { splunk_environment = local.splunk_environment splunk_index = local.splunk_index destination = "splunk" - reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn[count.index] + reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket[count.index].arn } From c994a99f6c995ea62b9b2ce2a43327d12dec3abb Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 08:57:08 +0000 Subject: [PATCH 50/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 20 +++++++++---------- .../modules/firehose/kinesis.tf | 2 +- .../infrastructure/modules/firehose/vars.tf | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index b831791f6..d20f84327 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -1,12 +1,12 @@ module "firehose__processor" { - source = "./modules/firehose" - assume_account = local.aws_account_id - prefix = local.prefix - region = local.region - environment = local.environment - cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn - splunk_environment = local.splunk_environment - splunk_index = local.splunk_index - destination = "splunk" - reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket[count.index].arn + source = "./modules/firehose" + assume_account = local.aws_account_id + prefix = local.prefix + region = local.region + environment = local.environment + cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn + splunk_environment = local.splunk_environment + splunk_index = local.splunk_index + destination = "splunk" + reporting_bucket = data.aws_s3_bucket.source-data-bucket } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index d118223c4..a4ece55b8 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket_arn + bucket_arn = var.reporting_bucket[count.index].arn processing_configuration { enabled = "true" diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index fa1c06ad1..1663f6a71 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -35,6 +35,6 @@ variable "error_prefix" { default = "errors" } -variable "reporting_bucket_arn" { +variable "reporting_bucket" { type = string } From c83ec52dccc63024acf5fe32f12ceb0efdee3ae7 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 09:04:26 +0000 Subject: [PATCH 51/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/data.tf | 1 - terraform/infrastructure/modules/firehose/kinesis.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/infrastructure/data.tf b/terraform/infrastructure/data.tf index 85b716377..2ead5497e 100644 --- a/terraform/infrastructure/data.tf +++ b/terraform/infrastructure/data.tf @@ -43,6 +43,5 @@ data "external" "current-info" { } data "aws_s3_bucket" "source-data-bucket" { - count = var.use_shared_resources ? 1 : 0 bucket = "${local.shared_prefix}-source-data-bucket" } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index a4ece55b8..e4a221706 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket[count.index].arn + bucket_arn = var.reporting_bucket.arn processing_configuration { enabled = "true" From 043ccd3ed399f72661579427755d7e031ad60ce0 Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 09:10:12 +0000 Subject: [PATCH 52/60] NRL-1188 Pass s3 bucket arn as var --- terraform/infrastructure/firehose.tf | 20 +++++++++---------- .../modules/firehose/kinesis.tf | 2 +- .../infrastructure/modules/firehose/vars.tf | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index d20f84327..035a07463 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -1,12 +1,12 @@ module "firehose__processor" { - source = "./modules/firehose" - assume_account = local.aws_account_id - prefix = local.prefix - region = local.region - environment = local.environment - cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn - splunk_environment = local.splunk_environment - splunk_index = local.splunk_index - destination = "splunk" - reporting_bucket = data.aws_s3_bucket.source-data-bucket + source = "./modules/firehose" + assume_account = local.aws_account_id + prefix = local.prefix + region = local.region + environment = local.environment + cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn + splunk_environment = local.splunk_environment + splunk_index = local.splunk_index + destination = "splunk" + reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index e4a221706..d118223c4 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { extended_s3_configuration { role_arn = aws_iam_role.firehose.arn - bucket_arn = var.reporting_bucket.arn + bucket_arn = var.reporting_bucket_arn processing_configuration { enabled = "true" diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index 1663f6a71..fa1c06ad1 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -35,6 +35,6 @@ variable "error_prefix" { default = "errors" } -variable "reporting_bucket" { +variable "reporting_bucket_arn" { type = string } From 3135c3bc4001a5ac0681d9d6e5237f587415c87e Mon Sep 17 00:00:00 2001 From: jackleary Date: Tue, 17 Dec 2024 10:19:17 +0000 Subject: [PATCH 53/60] NRL-1188 Add reporting bucket to firehose policy --- .../infrastructure/modules/firehose/cloudwatch.tf | 10 ++++++++++ .../modules/firehose/iam_firehose.tf | 2 ++ .../infrastructure/modules/firehose/kinesis.tf | 15 +++------------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/terraform/infrastructure/modules/firehose/cloudwatch.tf b/terraform/infrastructure/modules/firehose/cloudwatch.tf index 8e0a1ddac..5539dffa0 100644 --- a/terraform/infrastructure/modules/firehose/cloudwatch.tf +++ b/terraform/infrastructure/modules/firehose/cloudwatch.tf @@ -7,3 +7,13 @@ resource "aws_cloudwatch_log_stream" "firehose" { name = "${var.prefix}-firehose" log_group_name = aws_cloudwatch_log_group.firehose.name } + +resource "aws_cloudwatch_log_group" "firehose_reporting" { + name = "/aws/kinesisfirehose/${var.prefix}-firehose-reporting" + retention_in_days = local.cloudwatch.retention.days +} + +resource "aws_cloudwatch_log_stream" "firehose_reporting" { + name = "${var.prefix}-firehose-reporting" + log_group_name = aws_cloudwatch_log_group.firehose_reporting.name +} diff --git a/terraform/infrastructure/modules/firehose/iam_firehose.tf b/terraform/infrastructure/modules/firehose/iam_firehose.tf index 991d4d84f..56dfad1c2 100644 --- a/terraform/infrastructure/modules/firehose/iam_firehose.tf +++ b/terraform/infrastructure/modules/firehose/iam_firehose.tf @@ -30,6 +30,8 @@ data "aws_iam_policy_document" "firehose" { resources = [ aws_s3_bucket.firehose.arn, "${aws_s3_bucket.firehose.arn}/*", + var.reporting_bucket_arn, + "${var.reporting_bucket_arn}/*", ] effect = "Allow" } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index d118223c4..fc1a4bbcb 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -66,22 +66,13 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { bucket_arn = var.reporting_bucket_arn processing_configuration { - enabled = "true" - - processors { - type = "CloudWatchLogProcessing" - - parameters { - parameter_name = "DataMessageExtraction" - parameter_value = "true" - } - } + enabled = "false" } cloudwatch_logging_options { enabled = true - log_group_name = aws_cloudwatch_log_group.firehose.name - log_stream_name = aws_cloudwatch_log_stream.firehose.name + log_group_name = aws_cloudwatch_log_group.firehose_reporting.name + log_stream_name = aws_cloudwatch_log_stream.firehose_reporting.name } } } From 4f3b00c16a9e30d37d8a32e3a9a94194ac1acdce Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 09:57:20 +0000 Subject: [PATCH 54/60] NRL-1188 env toggle added --- terraform/infrastructure/data.tf | 1 + terraform/infrastructure/firehose.tf | 21 ++++++++++--------- terraform/infrastructure/locals.tf | 4 ++++ .../modules/firehose/cloudwatch.tf | 2 ++ .../modules/firehose/kinesis.tf | 1 + .../infrastructure/modules/firehose/vars.tf | 7 ++++++- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/terraform/infrastructure/data.tf b/terraform/infrastructure/data.tf index 2ead5497e..2c3512fa3 100644 --- a/terraform/infrastructure/data.tf +++ b/terraform/infrastructure/data.tf @@ -43,5 +43,6 @@ data "external" "current-info" { } data "aws_s3_bucket" "source-data-bucket" { + count = local.is_dev_env ? 1 : 0 bucket = "${local.shared_prefix}-source-data-bucket" } diff --git a/terraform/infrastructure/firehose.tf b/terraform/infrastructure/firehose.tf index 035a07463..fea8712ee 100644 --- a/terraform/infrastructure/firehose.tf +++ b/terraform/infrastructure/firehose.tf @@ -1,12 +1,13 @@ module "firehose__processor" { - source = "./modules/firehose" - assume_account = local.aws_account_id - prefix = local.prefix - region = local.region - environment = local.environment - cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn - splunk_environment = local.splunk_environment - splunk_index = local.splunk_index - destination = "splunk" - reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn + source = "./modules/firehose" + assume_account = local.aws_account_id + prefix = local.prefix + region = local.region + environment = local.environment + cloudwatch_kms_arn = module.kms__cloudwatch.kms_arn + splunk_environment = local.splunk_environment + splunk_index = local.splunk_index + destination = "splunk" + reporting_bucket_arn = local.reporting_bucket_arn + reporting_infra_toggle = local.is_dev_env } diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 998bd8ed1..60e755ad5 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -22,11 +22,15 @@ locals { dynamodb_timeout_seconds = "3" is_sandbox_env = length(regexall("-sandbox-", local.stack_name)) > 0 + is_dev_env = length(regexall("dev", local.stack_name)) > 0 environment = local.is_sandbox_env ? "${var.account_name}-sandbox" : var.account_name shared_prefix = "${local.project}--${local.environment}" public_domain = local.is_sandbox_env ? var.public_sandbox_domain : var.public_domain + # Logic / vars for reporting + reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket.arn : null + # Logic / vars for splunk environment splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name splunk_index = "aws_recordlocator_${local.splunk_environment}" diff --git a/terraform/infrastructure/modules/firehose/cloudwatch.tf b/terraform/infrastructure/modules/firehose/cloudwatch.tf index 5539dffa0..5cec0c461 100644 --- a/terraform/infrastructure/modules/firehose/cloudwatch.tf +++ b/terraform/infrastructure/modules/firehose/cloudwatch.tf @@ -9,11 +9,13 @@ resource "aws_cloudwatch_log_stream" "firehose" { } resource "aws_cloudwatch_log_group" "firehose_reporting" { + count = var.reporting_infra_toggle ? 1 : 0 name = "/aws/kinesisfirehose/${var.prefix}-firehose-reporting" retention_in_days = local.cloudwatch.retention.days } resource "aws_cloudwatch_log_stream" "firehose_reporting" { + count = var.reporting_infra_toggle ? 1 : 0 name = "${var.prefix}-firehose-reporting" log_group_name = aws_cloudwatch_log_group.firehose_reporting.name } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index fc1a4bbcb..3427e48c2 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -58,6 +58,7 @@ resource "aws_kinesis_firehose_delivery_stream" "firehose" { } resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { + count = var.reporting_infra_toggle ? 1 : 0 name = "${var.prefix}--cloudwatch-reporting-delivery-stream" destination = "extended_s3" diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf index fa1c06ad1..dec876c12 100644 --- a/terraform/infrastructure/modules/firehose/vars.tf +++ b/terraform/infrastructure/modules/firehose/vars.tf @@ -36,5 +36,10 @@ variable "error_prefix" { } variable "reporting_bucket_arn" { - type = string + type = string + default = null +} + +variable "reporting_infra_toggle" { + type = bool } From c4fa7545df920a9e5909c3a218adbee4d3a48361 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 10:06:46 +0000 Subject: [PATCH 55/60] NRL-1188 add index to instances where count is used --- terraform/infrastructure/locals.tf | 2 +- terraform/infrastructure/modules/firehose/cloudwatch.tf | 2 +- terraform/infrastructure/modules/firehose/kinesis.tf | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 60e755ad5..b3c6d1c7c 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -29,7 +29,7 @@ locals { public_domain = local.is_sandbox_env ? var.public_sandbox_domain : var.public_domain # Logic / vars for reporting - reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket.arn : null + reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket[0].arn : null # Logic / vars for splunk environment splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name diff --git a/terraform/infrastructure/modules/firehose/cloudwatch.tf b/terraform/infrastructure/modules/firehose/cloudwatch.tf index 5cec0c461..86aff3fd2 100644 --- a/terraform/infrastructure/modules/firehose/cloudwatch.tf +++ b/terraform/infrastructure/modules/firehose/cloudwatch.tf @@ -17,5 +17,5 @@ resource "aws_cloudwatch_log_group" "firehose_reporting" { resource "aws_cloudwatch_log_stream" "firehose_reporting" { count = var.reporting_infra_toggle ? 1 : 0 name = "${var.prefix}-firehose-reporting" - log_group_name = aws_cloudwatch_log_group.firehose_reporting.name + log_group_name = aws_cloudwatch_log_group.firehose_reporting[0].name } diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf index 3427e48c2..7c0c4a288 100644 --- a/terraform/infrastructure/modules/firehose/kinesis.tf +++ b/terraform/infrastructure/modules/firehose/kinesis.tf @@ -72,8 +72,8 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" { cloudwatch_logging_options { enabled = true - log_group_name = aws_cloudwatch_log_group.firehose_reporting.name - log_stream_name = aws_cloudwatch_log_stream.firehose_reporting.name + log_group_name = aws_cloudwatch_log_group.firehose_reporting[0].name + log_stream_name = aws_cloudwatch_log_stream.firehose_reporting[0].name } } } From 2f7e3341b280cd59d643ba302f05ab95c899ed0a Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 12:04:31 +0000 Subject: [PATCH 56/60] NRL-1188 make index conditional also --- terraform/infrastructure/locals.tf | 4 ++-- .../infrastructure/modules/firehose/iam_firehose.tf | 4 +++- .../infrastructure/modules/firehose/iam_subscriptions.tf | 2 +- terraform/infrastructure/modules/firehose/locals.tf | 9 +++++++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index b3c6d1c7c..0e24d1817 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -22,14 +22,14 @@ locals { dynamodb_timeout_seconds = "3" is_sandbox_env = length(regexall("-sandbox-", local.stack_name)) > 0 - is_dev_env = length(regexall("dev", local.stack_name)) > 0 + is_dev_env = local.stack_name == "dev" environment = local.is_sandbox_env ? "${var.account_name}-sandbox" : var.account_name shared_prefix = "${local.project}--${local.environment}" public_domain = local.is_sandbox_env ? var.public_sandbox_domain : var.public_domain # Logic / vars for reporting - reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket[0].arn : null + reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket[0].arn : data.aws_s3_bucket.source-data-bucket.arn # Logic / vars for splunk environment splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name diff --git a/terraform/infrastructure/modules/firehose/iam_firehose.tf b/terraform/infrastructure/modules/firehose/iam_firehose.tf index 56dfad1c2..18f239274 100644 --- a/terraform/infrastructure/modules/firehose/iam_firehose.tf +++ b/terraform/infrastructure/modules/firehose/iam_firehose.tf @@ -74,7 +74,9 @@ data "aws_iam_policy_document" "firehose" { ] resources = [ aws_cloudwatch_log_group.firehose.arn, - aws_cloudwatch_log_stream.firehose.arn + aws_cloudwatch_log_stream.firehose.arn, + local.iam_firehose.cloudwatch_reporting_log_group_arn, + local.iam_firehose.cloudwatch_reporting_log_stream_arn, ] effect = "Allow" } diff --git a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf index 6dae946fc..c3ceea10d 100644 --- a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf +++ b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf @@ -22,7 +22,7 @@ data "aws_iam_policy_document" "firehose_subscription" { effect = "Allow" resources = [ aws_kinesis_firehose_delivery_stream.firehose.arn, - aws_kinesis_firehose_delivery_stream.reporting_stream.arn, + local.iam_subscriptions.firehose_reporting_stream_arn, ] } statement { diff --git a/terraform/infrastructure/modules/firehose/locals.tf b/terraform/infrastructure/modules/firehose/locals.tf index 04b405d77..5eeb570c1 100644 --- a/terraform/infrastructure/modules/firehose/locals.tf +++ b/terraform/infrastructure/modules/firehose/locals.tf @@ -31,4 +31,13 @@ locals { compression_format = "GZIP" } + iam_firehose = { + cloudwatch_reporting_log_group_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_group.firehose_reporting[0].arn : aws_cloudwatch_log_group.firehose_reporting.arn + cloudwatch_reporting_log_stream_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_stream.firehose_reporting[0].arn : aws_cloudwatch_log_stream.firehose_reporting.arn + } + + iam_subscriptions = { + firehose_reporting_stream_arn = var.reporting_infra_toggle ? aws_kinesis_firehose_delivery_stream.reporting_stream[0].arn : aws_kinesis_firehose_delivery_stream.reporting_stream.arn + } + } From ab785bd1c09718b46a325e840f394c5ff0f2ba0d Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 12:37:36 +0000 Subject: [PATCH 57/60] NRL-1188 update env condition --- terraform/infrastructure/locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 0e24d1817..37fcdaf6b 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -22,7 +22,7 @@ locals { dynamodb_timeout_seconds = "3" is_sandbox_env = length(regexall("-sandbox-", local.stack_name)) > 0 - is_dev_env = local.stack_name == "dev" + is_dev_env = length(regexall("dev", local.stack_name)) > 0 environment = local.is_sandbox_env ? "${var.account_name}-sandbox" : var.account_name shared_prefix = "${local.project}--${local.environment}" From 467e82515db9b35b890c7cea2f957eadef48879b Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 12:55:28 +0000 Subject: [PATCH 58/60] NRL-1188 use compact to ignore nulls --- terraform/infrastructure/locals.tf | 2 +- terraform/infrastructure/modules/firehose/iam_firehose.tf | 8 ++++---- .../infrastructure/modules/firehose/iam_subscriptions.tf | 4 ++-- terraform/infrastructure/modules/firehose/locals.tf | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index 37fcdaf6b..b3c6d1c7c 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -29,7 +29,7 @@ locals { public_domain = local.is_sandbox_env ? var.public_sandbox_domain : var.public_domain # Logic / vars for reporting - reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket[0].arn : data.aws_s3_bucket.source-data-bucket.arn + reporting_bucket_arn = local.is_dev_env ? data.aws_s3_bucket.source-data-bucket[0].arn : null # Logic / vars for splunk environment splunk_environment = local.is_sandbox_env ? "${var.account_name}sandbox" : var.account_name diff --git a/terraform/infrastructure/modules/firehose/iam_firehose.tf b/terraform/infrastructure/modules/firehose/iam_firehose.tf index 18f239274..2f919d1b6 100644 --- a/terraform/infrastructure/modules/firehose/iam_firehose.tf +++ b/terraform/infrastructure/modules/firehose/iam_firehose.tf @@ -27,12 +27,12 @@ data "aws_iam_policy_document" "firehose" { "s3:PutObject", ] - resources = [ + resources = compact([ aws_s3_bucket.firehose.arn, "${aws_s3_bucket.firehose.arn}/*", var.reporting_bucket_arn, "${var.reporting_bucket_arn}/*", - ] + ]) effect = "Allow" } @@ -72,12 +72,12 @@ data "aws_iam_policy_document" "firehose" { actions = [ "logs:PutLogEvents", ] - resources = [ + resources = compact([ aws_cloudwatch_log_group.firehose.arn, aws_cloudwatch_log_stream.firehose.arn, local.iam_firehose.cloudwatch_reporting_log_group_arn, local.iam_firehose.cloudwatch_reporting_log_stream_arn, - ] + ]) effect = "Allow" } } diff --git a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf index c3ceea10d..3fe217ac3 100644 --- a/terraform/infrastructure/modules/firehose/iam_subscriptions.tf +++ b/terraform/infrastructure/modules/firehose/iam_subscriptions.tf @@ -20,10 +20,10 @@ data "aws_iam_policy_document" "firehose_subscription" { "firehose:*", ] effect = "Allow" - resources = [ + resources = compact([ aws_kinesis_firehose_delivery_stream.firehose.arn, local.iam_subscriptions.firehose_reporting_stream_arn, - ] + ]) } statement { actions = [ diff --git a/terraform/infrastructure/modules/firehose/locals.tf b/terraform/infrastructure/modules/firehose/locals.tf index 5eeb570c1..4658e993a 100644 --- a/terraform/infrastructure/modules/firehose/locals.tf +++ b/terraform/infrastructure/modules/firehose/locals.tf @@ -32,12 +32,12 @@ locals { } iam_firehose = { - cloudwatch_reporting_log_group_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_group.firehose_reporting[0].arn : aws_cloudwatch_log_group.firehose_reporting.arn - cloudwatch_reporting_log_stream_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_stream.firehose_reporting[0].arn : aws_cloudwatch_log_stream.firehose_reporting.arn + cloudwatch_reporting_log_group_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_group.firehose_reporting[0].arn : null + cloudwatch_reporting_log_stream_arn = var.reporting_infra_toggle ? aws_cloudwatch_log_stream.firehose_reporting[0].arn : null } iam_subscriptions = { - firehose_reporting_stream_arn = var.reporting_infra_toggle ? aws_kinesis_firehose_delivery_stream.reporting_stream[0].arn : aws_kinesis_firehose_delivery_stream.reporting_stream.arn + firehose_reporting_stream_arn = var.reporting_infra_toggle ? aws_kinesis_firehose_delivery_stream.reporting_stream[0].arn : null } } From 160fd5bda14a3be2fe839f1adbd9e1ce6b7d691a Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 13:01:28 +0000 Subject: [PATCH 59/60] NRL-1188 use compact to ignore nulls --- terraform/infrastructure/modules/firehose/iam_firehose.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/infrastructure/modules/firehose/iam_firehose.tf b/terraform/infrastructure/modules/firehose/iam_firehose.tf index 2f919d1b6..89e72587d 100644 --- a/terraform/infrastructure/modules/firehose/iam_firehose.tf +++ b/terraform/infrastructure/modules/firehose/iam_firehose.tf @@ -31,7 +31,6 @@ data "aws_iam_policy_document" "firehose" { aws_s3_bucket.firehose.arn, "${aws_s3_bucket.firehose.arn}/*", var.reporting_bucket_arn, - "${var.reporting_bucket_arn}/*", ]) effect = "Allow" } From e331b8f39b1a5d92892eb37cf88158a324d6f760 Mon Sep 17 00:00:00 2001 From: jackleary Date: Wed, 18 Dec 2024 13:11:32 +0000 Subject: [PATCH 60/60] NRL-1188 Update env condition --- terraform/infrastructure/locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf index b3c6d1c7c..dd1cd0f06 100644 --- a/terraform/infrastructure/locals.tf +++ b/terraform/infrastructure/locals.tf @@ -22,7 +22,7 @@ locals { dynamodb_timeout_seconds = "3" is_sandbox_env = length(regexall("-sandbox-", local.stack_name)) > 0 - is_dev_env = length(regexall("dev", local.stack_name)) > 0 + is_dev_env = var.account_name == "dev" environment = local.is_sandbox_env ? "${var.account_name}-sandbox" : var.account_name shared_prefix = "${local.project}--${local.environment}"