From 52bb75d228265f094ce0a8ac337bd9f2079c6fff Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 14 Feb 2025 14:45:05 +0000 Subject: [PATCH 1/8] NRL-1288 framework for multiple schema submission --- .../modules/glue/glue.tf | 4 +- .../modules/glue/src/main.py | 5 +- .../modules/glue/src/pipeline.py | 34 +- .../modules/glue/src/schemas.py | 296 ++++++++++++++++++ .../modules/glue/src/transformations.py | 291 +---------------- 5 files changed, 322 insertions(+), 308 deletions(-) create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/schemas.py diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index e7cd3b810..0d8ab332d 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -49,8 +49,8 @@ resource "aws_glue_job" "glue_job" { "--enable-auto-scaling" = "true" "--enable-continous-cloudwatch-log" = "true" "--datalake-formats" = "delta" - "--source_path" = "s3://${aws_s3_bucket.source-data-bucket.id}/" # Specify the source S3 path - "--target_path" = "s3://${aws_s3_bucket.target-data-bucket.id}/logs" # Specify the destination S3 path + "--source_path" = "s3://${aws_s3_bucket.source-data-bucket.id}/" # Specify the source S3 path + "--target_path" = "s3://${aws_s3_bucket.target-data-bucket.id}/" # Specify the destination S3 path "--job_name" = "${var.name_prefix}-glue-job" "--partition_cols" = "date" "--enable-continuous-log-filter" = "true" diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 5450da0cd..e73166181 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -3,7 +3,8 @@ from awsglue.utils import getResolvedOptions from pipeline import LogPipeline from pyspark.context import SparkContext -from transformations import dtype_conversion, flatten_df, logSchema +from schemas import schemas +from transformations import dtype_conversion, flatten_df # Get arguments from AWS Glue job args = getResolvedOptions( @@ -20,7 +21,7 @@ spark_context=sc, source_path=args["source_path"], target_path=args["target_path"], - schema=logSchema, + schema=schemas, job_name=args["job_name"], partition_cols=partition_cols, transformations=[flatten_df, dtype_conversion], diff --git a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py index 2fb30ff91..eb68be7a6 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py @@ -1,5 +1,6 @@ import boto3 from instances import GlueContextSingleton, LoggerSingleton +from pyspark.sql.functions import col class LogPipeline: @@ -8,7 +9,7 @@ def __init__( spark_context, source_path, target_path, - schema, + schemas, job_name, partition_cols=[], transformations=[], @@ -19,7 +20,7 @@ def __init__( self.logger = LoggerSingleton().logger self.source_path = source_path self.target_path = target_path - self.schema = schema + self.schemas = schemas self.partition_cols = partition_cols self.transformations = transformations self.glue = boto3.client( @@ -33,11 +34,12 @@ def run(self): """Runs ETL""" try: self.logger.info("ETL Process started.") - df = self.extract() + data = self.extract() self.logger.info(f"Data extracted from {self.source_path}.") - df = self.transform(df) + for name, df in data.items(): + data[name] = self.transform(df) self.logger.info("Data transformed successfully.") - self.load(df) + self.load(data) self.logger.info(f"Data loaded into {self.target_path}.") self.logger.info("Trigger glue crawler") self.trigger_crawler() @@ -48,11 +50,14 @@ def run(self): def extract(self): """Extract JSON data from S3""" self.logger.info(f"Extracting data from {self.source_path} as JSON") - return ( - self.spark.read.option("recursiveFileLookup", "true") - .schema(self.schema) - .json(self.source_path) - ) + data = {} + for name, schema in self.schemas.items(): + data[name] = ( + self.spark.read.option("recursiveFileLookup", "true") + .schema(schema) + .json(self.source_path) + ).where(col("host").contains(name)) + return data def transform(self, dataframe): """Apply a list of transformations on the dataframe""" @@ -61,12 +66,13 @@ def transform(self, dataframe): dataframe = transformation(dataframe) return dataframe - def load(self, dataframe): + def load(self, data): """Load transformed data into Parquet format""" self.logger.info(f"Loading data into {self.target_path} as Parquet") - dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet( - self.target_path - ) + for name, dataframe in data.items(): + dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet( + f"{self.target_path}{name}" + ) def trigger_crawler(self): self.glue.start_crawler(Name=f"{self.name_prefix}-log-crawler") diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/schemas.py new file mode 100644 index 000000000..4532168e0 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/schemas.py @@ -0,0 +1,296 @@ +from pyspark.sql.types import ( + BooleanType, + DoubleType, + LongType, + StringType, + StructField, + StructType, +) + +countDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-Id", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Request-Id", StringType(), True), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField( + "params", + StructType( + [StructField("subject:identifier", StringType(), True)] + ), + True, + ), + StructField("model", StringType(), True), + StructField( + "parsed_params", + StructType( + [StructField("subject_identifier", StringType(), True)] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField("Select", StringType(), True), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + ] + ), + True, + ), + StructField("count", LongType(), True), + StructField( + "result", + StructType( + [ + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +schemas = { + "countDocumentReference": countDocumentReferenceSchema, +} diff --git a/terraform/account-wide-infrastructure/modules/glue/src/transformations.py b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py index e4d7b8f1d..eb39d6a4f 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/transformations.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py @@ -5,296 +5,7 @@ to_date, to_timestamp, ) -from pyspark.sql.types import ( - BooleanType, - DoubleType, - LongType, - StringType, - StructField, - StructType, -) - -logSchema = StructType( - [ - StructField("time", DoubleType(), True), - StructField("index", StringType(), True), - StructField("host", StringType(), True), - StructField("source", StringType(), True), - StructField( - "event", - StructType( - [ - StructField("level", StringType(), True), - StructField("location", StringType(), True), - StructField("message", StringType(), True), - StructField("timestamp", StringType(), True), - StructField("service", StringType(), True), - StructField("cold_start", BooleanType(), True), - StructField("function_name", StringType(), True), - StructField("function_memory_size", StringType(), True), - StructField("function_arn", StringType(), True), - StructField("function_request_id", StringType(), True), - StructField("correlation_id", StringType(), True), - StructField("method", StringType(), True), - StructField("path", StringType(), True), - StructField( - "headers", - StructType( - [ - StructField("accept", StringType(), True), - StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), - StructField("Host", StringType(), True), - StructField( - "NHSD-Client-RP-Details", StringType(), True - ), - StructField( - "NHSD-Connection-Metadata", StringType(), True - ), - StructField("NHSD-Correlation-Id", StringType(), True), - StructField("User-Agent", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Request-Id", StringType(), True), - StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True - ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), - ] - ), - True, - ), - StructField("log_reference", StringType(), True), - StructField("xray_trace_id", StringType(), True), - StructField( - "config", - StructType( - [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), - ] - ), - True, - ), - StructField( - "metadata", - StructType( - [ - StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), - StructField( - "client_rp_details", - StructType( - [ - StructField( - "developer_app_name", StringType(), True - ), - StructField( - "developer_app_id", StringType(), True - ), - ] - ), - True, - ), - ] - ), - True, - ), - StructField( - "params", - StructType( - [ - StructField("subject:identifier", StringType(), True), - ] - ), - True, - ), - StructField("model", StringType(), True), - StructField( - "parsed_params", - StructType( - [ - StructField("subject_identifier", StringType(), True), - ] - ), - True, - ), - StructField("table_name", StringType(), True), - StructField("item_type", StringType(), True), - StructField("original_kwargs_keys", StringType(), True), - StructField("filtered_kwargs_keys", StringType(), True), - StructField("nhs_number", StringType(), True), - StructField( - "query", - StructType( - [ - StructField("IndexName", StringType(), True), - StructField( - "KeyConditionExpression", StringType(), True - ), - StructField( - "ExpressionAttributeValues", - StructType( - [ - StructField( - ":patient_key", StringType(), True - ), - StructField( - ":patient_sort", StringType(), True - ), - ] - ), - True, - ), - StructField("Select", StringType(), True), - StructField( - "ReturnConsumedCapacity", StringType(), True - ), - ] - ), - True, - ), - StructField("count", LongType(), True), - StructField( - "result", - StructType( - [ - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), - StructField( - "ConsumedCapacity", - StructType( - [ - StructField( - "TableName", StringType(), True - ), - StructField( - "CapacityUnits", DoubleType(), True - ), - StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ), - ] - ), - True, - ), - StructField( - "GlobalSecondaryIndexes", - StructType( - [ - StructField( - "patient_gsi", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ), - ] - ), - True, - ), - ] - ), - True, - ), - ] - ), - True, - ), - StructField( - "ResponseMetadata", - StructType( - [ - StructField( - "RequestId", StringType(), True - ), - StructField( - "HTTPStatusCode", LongType(), True - ), - StructField( - "HTTPHeaders", - StructType( - [ - StructField( - "server", StringType(), True - ), - StructField( - "date", StringType(), True - ), - StructField( - "content-type", - StringType(), - True, - ), - StructField( - "content-length", - StringType(), - True, - ), - StructField( - "connection", - StringType(), - True, - ), - StructField( - "x-amzn-requestid", - StringType(), - True, - ), - StructField( - "x-amz-crc32", - StringType(), - True, - ), - ] - ), - True, - ), - StructField( - "RetryAttempts", LongType(), True - ), - ] - ), - True, - ), - ] - ), - True, - ), - StructField("status_code", StringType(), True), - StructField( - "response", - StructType( - [ - StructField("statusCode", StringType(), True), - StructField("body", StringType(), True), - StructField("isBase64Encoded", BooleanType(), True), - ] - ), - True, - ), - ] - ), - True, - ), - ] -) +from pyspark.sql.types import StructType def flatten_df(df): From d65768fed634e633c7ca17f702877962d27092cf Mon Sep 17 00:00:00 2001 From: jackleary Date: Fri, 14 Feb 2025 15:26:33 +0000 Subject: [PATCH 2/8] NRL-1288 naming conventions and path updates --- terraform/account-wide-infrastructure/modules/glue/glue.tf | 4 ++-- .../account-wide-infrastructure/modules/glue/src/main.py | 4 ++-- .../account-wide-infrastructure/modules/glue/src/schemas.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index 0d8ab332d..5049bbc1b 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -1,7 +1,7 @@ # Create Glue Data Catalog Database resource "aws_glue_catalog_database" "log_database" { name = "${var.name_prefix}-reporting" - location_uri = "${aws_s3_bucket.target-data-bucket.id}/logs/" + location_uri = "${aws_s3_bucket.target-data-bucket.id}/" } # Create Glue Crawler @@ -10,7 +10,7 @@ resource "aws_glue_crawler" "log_crawler" { database_name = aws_glue_catalog_database.log_database.name role = aws_iam_role.glue_service_role.name s3_target { - path = "${aws_s3_bucket.target-data-bucket.id}/logs/" + path = "${aws_s3_bucket.target-data-bucket.id}/" } schema_change_policy { delete_behavior = "LOG" diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index e73166181..b737d352c 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -3,7 +3,7 @@ from awsglue.utils import getResolvedOptions from pipeline import LogPipeline from pyspark.context import SparkContext -from schemas import schemas +from schemas import schemaList from transformations import dtype_conversion, flatten_df # Get arguments from AWS Glue job @@ -21,7 +21,7 @@ spark_context=sc, source_path=args["source_path"], target_path=args["target_path"], - schema=schemas, + schemas=schemaList, job_name=args["job_name"], partition_cols=partition_cols, transformations=[flatten_df, dtype_conversion], diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/schemas.py index 4532168e0..6ea903a7d 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/schemas.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/schemas.py @@ -291,6 +291,6 @@ ] ) -schemas = { +schemaList = { "countDocumentReference": countDocumentReferenceSchema, } From cc64d6dfc012b0e0a863f6e3759a1d1bbc2fd23f Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 10:08:18 +0000 Subject: [PATCH 3/8] NRL-1288 add schemas in for multiple log groups --- .../modules/glue/src/main.py | 7 +- .../modules/glue/src/pipeline.py | 1 + .../modules/glue/src/schemas.py | 296 -- .../glue/src/schemas/consumer_schemas.py | 1144 ++++++++ .../glue/src/schemas/producer_schemas.py | 2524 +++++++++++++++++ 5 files changed, 3674 insertions(+), 298 deletions(-) delete mode 100644 terraform/account-wide-infrastructure/modules/glue/src/schemas.py create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py create mode 100644 terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index b737d352c..c789342ee 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -3,7 +3,8 @@ from awsglue.utils import getResolvedOptions from pipeline import LogPipeline from pyspark.context import SparkContext -from schemas import schemaList +from schemas.consumer_schemas import consumerSchemaList +from schemas.producer_schemas import producerSchemaList from transformations import dtype_conversion, flatten_df # Get arguments from AWS Glue job @@ -16,12 +17,14 @@ partition_cols = args["partition_cols"].split(",") if "partition_cols" in args else [] +schema_list = consumerSchemaList.extend(producerSchemaList) + # Initialize ETL process etl_job = LogPipeline( spark_context=sc, source_path=args["source_path"], target_path=args["target_path"], - schemas=schemaList, + schemas=schema_list, job_name=args["job_name"], partition_cols=partition_cols, transformations=[flatten_df, dtype_conversion], diff --git a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py index eb68be7a6..2893c6100 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py @@ -70,6 +70,7 @@ def load(self, data): """Load transformed data into Parquet format""" self.logger.info(f"Loading data into {self.target_path} as Parquet") for name, dataframe in data.items(): + name = name.replace("--", "_") dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet( f"{self.target_path}{name}" ) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/schemas.py deleted file mode 100644 index 6ea903a7d..000000000 --- a/terraform/account-wide-infrastructure/modules/glue/src/schemas.py +++ /dev/null @@ -1,296 +0,0 @@ -from pyspark.sql.types import ( - BooleanType, - DoubleType, - LongType, - StringType, - StructField, - StructType, -) - -countDocumentReferenceSchema = StructType( - [ - StructField("raw_message", StringType(), True), - StructField("time", DoubleType(), True), - StructField("index", StringType(), True), - StructField("host", StringType(), True), - StructField("source", StringType(), True), - StructField( - "event", - StructType( - [ - StructField("level", StringType(), True), - StructField("location", StringType(), True), - StructField("message", StringType(), True), - StructField("timestamp", StringType(), True), - StructField("service", StringType(), True), - StructField("cold_start", BooleanType(), True), - StructField("function_name", StringType(), True), - StructField("function_memory_size", StringType(), True), - StructField("function_arn", StringType(), True), - StructField("function_request_id", StringType(), True), - StructField("correlation_id", StringType(), True), - StructField("method", StringType(), True), - StructField("path", StringType(), True), - StructField( - "headers", - StructType( - [ - StructField("accept", StringType(), True), - StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), - StructField("Host", StringType(), True), - StructField( - "NHSD-Client-RP-Details", StringType(), True - ), - StructField( - "NHSD-Connection-Metadata", StringType(), True - ), - StructField("NHSD-Correlation-Id", StringType(), True), - StructField("User-Agent", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Request-Id", StringType(), True), - StructField("NHSD-Correlation-ID", StringType(), True), - StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True - ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), - ] - ), - True, - ), - StructField("log_reference", StringType(), True), - StructField("xray_trace_id", StringType(), True), - StructField( - "config", - StructType( - [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), - ] - ), - True, - ), - StructField( - "metadata", - StructType( - [ - StructField("pointer_types", StringType(), True), - StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), - StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), - StructField( - "client_rp_details", - StructType( - [ - StructField( - "developer_app_name", StringType(), True - ), - StructField( - "developer_app_id", StringType(), True - ), - ] - ), - True, - ), - ] - ), - True, - ), - StructField("pointer_types", StringType(), True), - StructField( - "params", - StructType( - [StructField("subject:identifier", StringType(), True)] - ), - True, - ), - StructField("model", StringType(), True), - StructField( - "parsed_params", - StructType( - [StructField("subject_identifier", StringType(), True)] - ), - True, - ), - StructField("table_name", StringType(), True), - StructField("item_type", StringType(), True), - StructField("original_kwargs_keys", StringType(), True), - StructField("filtered_kwargs_keys", StringType(), True), - StructField("nhs_number", StringType(), True), - StructField( - "query", - StructType( - [ - StructField("IndexName", StringType(), True), - StructField( - "KeyConditionExpression", StringType(), True - ), - StructField( - "ExpressionAttributeValues", - StructType( - [ - StructField( - ":patient_key", StringType(), True - ), - StructField( - ":patient_sort", StringType(), True - ), - ] - ), - True, - ), - StructField("Select", StringType(), True), - StructField( - "ReturnConsumedCapacity", StringType(), True - ), - ] - ), - True, - ), - StructField("count", LongType(), True), - StructField( - "result", - StructType( - [ - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), - StructField( - "ConsumedCapacity", - StructType( - [ - StructField( - "TableName", StringType(), True - ), - StructField( - "CapacityUnits", DoubleType(), True - ), - StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, - ), - StructField( - "GlobalSecondaryIndexes", - StructType( - [ - StructField( - "patient_gsi", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, - ) - ] - ), - True, - ), - ] - ), - True, - ), - StructField( - "ResponseMetadata", - StructType( - [ - StructField( - "RequestId", StringType(), True - ), - StructField( - "HTTPStatusCode", LongType(), True - ), - StructField( - "HTTPHeaders", - StructType( - [ - StructField( - "server", StringType(), True - ), - StructField( - "date", StringType(), True - ), - StructField( - "content-type", - StringType(), - True, - ), - StructField( - "content-length", - StringType(), - True, - ), - StructField( - "connection", - StringType(), - True, - ), - StructField( - "x-amzn-requestid", - StringType(), - True, - ), - StructField( - "x-amz-crc32", - StringType(), - True, - ), - ] - ), - True, - ), - StructField( - "RetryAttempts", LongType(), True - ), - ] - ), - True, - ), - ] - ), - True, - ), - StructField("status_code", StringType(), True), - StructField( - "response", - StructType( - [ - StructField("statusCode", StringType(), True), - StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), - ] - ), - True, - ), - ] - ), - True, - ), - ] -) - -schemaList = { - "countDocumentReference": countDocumentReferenceSchema, -} diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py new file mode 100644 index 000000000..9ddd8bab5 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py @@ -0,0 +1,1144 @@ +from pyspark.sql.types import ( + BooleanType, + DoubleType, + LongType, + StringType, + StructField, + StructType, +) + +searchPostDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("content-type", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("error", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField("frames", StructType([]), True), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), + StructField("pointer_types", StringType(), True), + StructField("body", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_body", + StructType( + [ + StructField("subject_identifier", StringType(), True), + StructField("custodian_identifier", StringType(), True), + StructField("type", StringType(), True), + StructField("category", StringType(), True), + StructField("next_page_token", StringType(), True), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField(":type_0", StringType(), True), + StructField(":type_1", StringType(), True), + StructField(":type_2", StringType(), True), + StructField(":type_3", StringType(), True), + StructField(":type_4", StringType(), True), + StructField(":type_5", StringType(), True), + StructField(":type_6", StringType(), True), + StructField(":type_7", StringType(), True), + StructField(":type_8", StringType(), True), + StructField(":type_9", StringType(), True), + StructField(":type_10", StringType(), True), + StructField(":type_11", StringType(), True), + StructField(":type_12", StringType(), True), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + StructField("FilterExpression", StringType(), True), + StructField( + "ExpressionAttributeNames", + StructType( + [ + StructField( + "#pointer_type", StringType(), True + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table", StringType(), True), + StructField( + "stats", + StructType( + [ + StructField("count", LongType(), True), + StructField("scanned_count", LongType(), True), + StructField("last_evaluated_key", StringType(), True), + ] + ), + True, + ), + StructField( + "result", + StructType( + [ + StructField("Items", StructType([]), True), + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("id", StringType(), True), + StructField("count", LongType(), True), + StructField("subject_identifier", StringType(), True), + StructField("type", StringType(), True), + ] + ), + True, + ), + ] +) + +searchDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField( + "params", + StructType( + [ + StructField("subject:identifier", StringType(), True), + StructField("type", StringType(), True), + ] + ), + True, + ), + StructField("model", StringType(), True), + StructField( + "parsed_params", + StructType( + [ + StructField("subject_identifier", StringType(), True), + StructField("custodian_identifier", StringType(), True), + StructField("type", StringType(), True), + StructField("category", StringType(), True), + StructField("next_page_token", StringType(), True), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("type", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("nhs_number", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField(":type_0", StringType(), True), + StructField(":type_1", StringType(), True), + StructField(":type_2", StringType(), True), + StructField(":type_3", StringType(), True), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + StructField("FilterExpression", StringType(), True), + StructField( + "ExpressionAttributeNames", + StructType( + [ + StructField( + "#pointer_type", StringType(), True + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table", StringType(), True), + StructField( + "stats", + StructType( + [ + StructField("count", LongType(), True), + StructField("scanned_count", LongType(), True), + StructField("last_evaluated_key", StringType(), True), + ] + ), + True, + ), + StructField( + "result", + StructType( + [ + StructField("Items", StructType([]), True), + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("id", StringType(), True), + StructField("count", LongType(), True), + ] + ), + True, + ), + ] +) + +readDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField( + "path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField( + "result", + StructType( + [ + StructField("id", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("custodian", StringType(), True), + StructField("custodian_suffix", StringType(), True), + StructField("producer_id", StringType(), True), + StructField("category_id", StringType(), True), + StructField("category", StringType(), True), + StructField("type_id", StringType(), True), + StructField("type", StringType(), True), + StructField("master_identifier", StringType(), True), + StructField("author", StringType(), True), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField("document", StringType(), True), + StructField("created_on", StringType(), True), + StructField("updated_on", StringType(), True), + StructField("schemas", StringType(), True), + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +countDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-Id", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Request-Id", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField( + "params", + StructType( + [StructField("subject:identifier", StringType(), True)] + ), + True, + ), + StructField("model", StringType(), True), + StructField( + "parsed_params", + StructType( + [StructField("subject_identifier", StringType(), True)] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField("Select", StringType(), True), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + ] + ), + True, + ), + StructField("count", LongType(), True), + StructField( + "result", + StructType( + [ + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +consumerSchemaList = { + "consumer--countDocumentReference": countDocumentReferenceSchema, + "consumer--searchPostDocumentReference": searchPostDocumentReferenceSchema, + "consumer--searchDocumentReference": searchDocumentReferenceSchema, + "consumer--readDocumentReference": readDocumentReferenceSchema, +} diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py new file mode 100644 index 000000000..4f525154c --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py @@ -0,0 +1,2524 @@ +from pyspark.sql.types import ( + BooleanType, + DoubleType, + LongType, + StringType, + StructField, + StructType, +) + +upsertDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("content-type", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("body", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_body", + StructType( + [ + StructField("resourceType", StringType(), True), + StructField("id", StringType(), True), + StructField("meta", StringType(), True), + StructField("implicitRules", StringType(), True), + StructField("language", StringType(), True), + StructField("text", StringType(), True), + StructField( + "masterIdentifier", + StructType( + [ + StructField("id", StringType(), True), + StructField("use", StringType(), True), + StructField("type", StringType(), True), + StructField("system", StringType(), True), + StructField("value", StringType(), True), + StructField("period", StringType(), True), + StructField("assigner", StringType(), True), + ] + ), + True, + ), + StructField("identifier", StringType(), True), + StructField("status", StringType(), True), + StructField("docStatus", StringType(), True), + StructField( + "type", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "coding", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + StructField("text", StringType(), True), + ] + ), + True, + ), + StructField( + "category", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField( + "subject", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("date", StringType(), True), + StructField( + "author", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField("authenticator", StringType(), True), + StructField( + "custodian", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("relatesTo", StringType(), True), + StructField("description", StringType(), True), + StructField("securityLabel", StringType(), True), + StructField( + "content", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField( + "context", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "encounter", StringType(), True + ), + StructField("event", StringType(), True), + StructField( + "period", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "start", StringType(), True + ), + StructField( + "end", StringType(), True + ), + ] + ), + True, + ), + StructField( + "facilityType", StringType(), True + ), + StructField( + "practiceSetting", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "coding", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + StructField( + "text", StringType(), True + ), + ] + ), + True, + ), + StructField( + "sourcePatientInfo", StringType(), True + ), + StructField( + "related", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("resource", StringType(), True), + StructField("resource_type", StringType(), True), + StructField("step", StringType(), True), + StructField("required_fields", StringType(), True), + StructField("reason", StringType(), True), + StructField("is_valid", BooleanType(), True), + StructField("issue_count", LongType(), True), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("type", StringType(), True), + StructField("pointer_id", StringType(), True), + StructField( + "indexes", + StructType( + [ + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), + StructField("masterid_key", StringType(), True), + ] + ), + True, + ), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField("error", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField( + "frames", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), + StructField("ods_code", StringType(), True), + StructField( + "result", + StructType( + [ + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "masterid_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +updateDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField( + "path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("content-type", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("body", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_body", + StructType( + [ + StructField("resourceType", StringType(), True), + StructField("id", StringType(), True), + StructField("meta", StringType(), True), + StructField("implicitRules", StringType(), True), + StructField("language", StringType(), True), + StructField("text", StringType(), True), + StructField("masterIdentifier", StringType(), True), + StructField("identifier", StringType(), True), + StructField("status", StringType(), True), + StructField("docStatus", StringType(), True), + StructField( + "type", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "coding", + StructType( + [ + StructField( + "id", StringType(), True + ) + ] + ), + True, + ), + StructField("text", StringType(), True), + ] + ), + True, + ), + StructField( + "category", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "subject", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("date", StringType(), True), + StructField( + "author", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("authenticator", StringType(), True), + StructField( + "custodian", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("relatesTo", StringType(), True), + StructField("description", StringType(), True), + StructField("securityLabel", StringType(), True), + StructField( + "content", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "context", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "encounter", StringType(), True + ), + StructField("event", StringType(), True), + StructField( + "period", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "start", StringType(), True + ), + StructField( + "end", StringType(), True + ), + ] + ), + True, + ), + StructField( + "facilityType", StringType(), True + ), + StructField( + "practiceSetting", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "coding", + StructType( + [ + StructField( + "id", + StringType(), + True, + ) + ] + ), + True, + ), + StructField( + "text", StringType(), True + ), + ] + ), + True, + ), + StructField( + "sourcePatientInfo", StringType(), True + ), + StructField( + "related", + StructType( + [ + StructField( + "id", StringType(), True + ) + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "parsed_path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("resource_type", StringType(), True), + StructField("step", StringType(), True), + StructField("required_fields", StringType(), True), + StructField("reason", StringType(), True), + StructField("is_valid", BooleanType(), True), + StructField("issue_count", LongType(), True), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("type", StringType(), True), + StructField( + "result", + StructType( + [ + StructField("id", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("custodian", StringType(), True), + StructField("custodian_suffix", StringType(), True), + StructField("producer_id", StringType(), True), + StructField("category_id", StringType(), True), + StructField("category", StringType(), True), + StructField("type_id", StringType(), True), + StructField("type", StringType(), True), + StructField("master_identifier", StringType(), True), + StructField("author", StringType(), True), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField("document", StringType(), True), + StructField("created_on", StringType(), True), + StructField("updated_on", StringType(), True), + StructField("schemas", StringType(), True), + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), + ] + ), + True, + ), + StructField("field", StringType(), True), + StructField("provided", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +searchPostDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("content-type", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("body", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_body", + StructType( + [ + StructField("subject_identifier", StringType(), True), + StructField("type", StringType(), True), + StructField("category", StringType(), True), + StructField("next_page_token", StringType(), True), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("custodian", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("categories", StringType(), True), + StructField("expression", StringType(), True), + StructField("values", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField(":type_0", StringType(), True), + StructField(":type_1", StringType(), True), + StructField(":type_2", StringType(), True), + StructField(":type_3", StringType(), True), + StructField(":type_4", StringType(), True), + StructField(":type_5", StringType(), True), + StructField(":type_6", StringType(), True), + StructField(":type_7", StringType(), True), + StructField(":type_8", StringType(), True), + StructField(":type_9", StringType(), True), + StructField(":type_10", StringType(), True), + StructField(":type_11", StringType(), True), + StructField(":type_12", StringType(), True), + StructField( + ":custodian", StringType(), True + ), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + StructField("FilterExpression", StringType(), True), + StructField( + "ExpressionAttributeNames", + StructType( + [ + StructField( + "#pointer_type", StringType(), True + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table", StringType(), True), + StructField( + "stats", + StructType( + [ + StructField("count", LongType(), True), + StructField("scanned_count", LongType(), True), + StructField("last_evaluated_key", StringType(), True), + ] + ), + True, + ), + StructField( + "result", + StructType( + [ + StructField("Items", StructType([]), True), + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("id", StringType(), True), + StructField("count", LongType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +searchDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField( + "params", + StructType( + [ + StructField("subject:identifier", StringType(), True), + StructField("type", StringType(), True), + ] + ), + True, + ), + StructField("model", StringType(), True), + StructField( + "parsed_params", + StructType( + [ + StructField("subject_identifier", StringType(), True), + StructField("type", StringType(), True), + StructField("category", StringType(), True), + StructField("next_page_token", StringType(), True), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("custodian", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("categories", StringType(), True), + StructField("expression", StringType(), True), + StructField("values", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField(":type_0", StringType(), True), + StructField(":type_1", StringType(), True), + StructField(":type_2", StringType(), True), + StructField(":type_3", StringType(), True), + StructField( + ":custodian", StringType(), True + ), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + StructField("FilterExpression", StringType(), True), + StructField( + "ExpressionAttributeNames", + StructType( + [ + StructField( + "#pointer_type", StringType(), True + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table", StringType(), True), + StructField( + "stats", + StructType( + [ + StructField("count", LongType(), True), + StructField("scanned_count", LongType(), True), + StructField("last_evaluated_key", StringType(), True), + ] + ), + True, + ), + StructField( + "result", + StructType( + [ + StructField("Items", StructType([]), True), + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("id", StringType(), True), + StructField("count", LongType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("headers", StructType([]), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("type", StringType(), True), + ] + ), + True, + ), + ] +) + +readDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField( + "path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField( + "result", + StructType( + [ + StructField("id", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("custodian", StringType(), True), + StructField("custodian_suffix", StringType(), True), + StructField("producer_id", StringType(), True), + StructField("category_id", StringType(), True), + StructField("category", StringType(), True), + StructField("type_id", StringType(), True), + StructField("type", StringType(), True), + StructField("master_identifier", StringType(), True), + StructField("author", StringType(), True), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField("document", StringType(), True), + StructField("created_on", StringType(), True), + StructField("updated_on", StringType(), True), + StructField("schemas", StringType(), True), + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +deleteDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField( + "path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("model", StringType(), True), + StructField( + "parsed_path", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("pointer_id", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType([StructField("id", StringType(), True)]), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + ] + ), + True, + ), + ] +) + +createDocumentReferenceSchema = StructType( + [ + StructField("raw_message", StringType(), True), + StructField("time", DoubleType(), True), + StructField("index", StringType(), True), + StructField("host", StringType(), True), + StructField("source", StringType(), True), + StructField( + "event", + StructType( + [ + StructField("level", StringType(), True), + StructField("location", StringType(), True), + StructField("message", StringType(), True), + StructField("timestamp", StringType(), True), + StructField("service", StringType(), True), + StructField("cold_start", BooleanType(), True), + StructField("function_name", StringType(), True), + StructField("function_memory_size", StringType(), True), + StructField("function_arn", StringType(), True), + StructField("function_request_id", StringType(), True), + StructField("correlation_id", StringType(), True), + StructField("method", StringType(), True), + StructField("path", StringType(), True), + StructField( + "headers", + StructType( + [ + StructField("accept", StringType(), True), + StructField("accept-encoding", StringType(), True), + StructField("Authorization", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("content-type", StringType(), True), + StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), + StructField( + "NHSD-Connection-Metadata", StringType(), True + ), + StructField("NHSD-Correlation-ID", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("Postman-Token", StringType(), True), + StructField("User-Agent", StringType(), True), + StructField("x-correlation-id", StringType(), True), + StructField("X-Forwarded-For", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-request-id", StringType(), True), + ] + ), + True, + ), + StructField("log_reference", StringType(), True), + StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("pointer_types", StringType(), True), + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_permissions", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("body", StringType(), True), + StructField("model", StringType(), True), + StructField("error", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField( + "parsed_body", + StructType( + [ + StructField("resourceType", StringType(), True), + StructField("id", StringType(), True), + StructField( + "meta", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "versionId", StringType(), True + ), + StructField( + "lastUpdated", StringType(), True + ), + StructField("source", StringType(), True), + StructField("profile", StringType(), True), + StructField("security", StringType(), True), + StructField("tag", StringType(), True), + ] + ), + True, + ), + StructField("implicitRules", StringType(), True), + StructField("language", StringType(), True), + StructField("text", StringType(), True), + StructField( + "masterIdentifier", + StructType( + [ + StructField("id", StringType(), True), + StructField("use", StringType(), True), + StructField("type", StringType(), True), + StructField("system", StringType(), True), + StructField("value", StringType(), True), + StructField("period", StringType(), True), + StructField("assigner", StringType(), True), + ] + ), + True, + ), + StructField("identifier", StringType(), True), + StructField("status", StringType(), True), + StructField("docStatus", StringType(), True), + StructField( + "type", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "coding", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + StructField("text", StringType(), True), + ] + ), + True, + ), + StructField( + "category", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField( + "subject", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("date", StringType(), True), + StructField( + "author", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField("authenticator", StringType(), True), + StructField( + "custodian", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "reference", StringType(), True + ), + StructField("type", StringType(), True), + StructField( + "identifier", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "use", StringType(), True + ), + StructField( + "type", StringType(), True + ), + StructField( + "system", StringType(), True + ), + StructField( + "value", StringType(), True + ), + StructField( + "period", StringType(), True + ), + StructField( + "assigner", + StringType(), + True, + ), + ] + ), + True, + ), + StructField("display", StringType(), True), + ] + ), + True, + ), + StructField("relatesTo", StringType(), True), + StructField("description", StringType(), True), + StructField("securityLabel", StringType(), True), + StructField( + "content", + StructType( + [StructField("Location", StringType(), True)] + ), + True, + ), + StructField( + "context", + StructType( + [ + StructField("id", StringType(), True), + StructField( + "encounter", StringType(), True + ), + StructField( + "event", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + StructField( + "period", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "start", StringType(), True + ), + StructField( + "end", StringType(), True + ), + ] + ), + True, + ), + StructField( + "facilityType", StringType(), True + ), + StructField( + "practiceSetting", + StructType( + [ + StructField( + "id", StringType(), True + ), + StructField( + "coding", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + StructField( + "text", StringType(), True + ), + ] + ), + True, + ), + StructField( + "sourcePatientInfo", StringType(), True + ), + StructField( + "related", + StructType( + [ + StructField( + "Location", + StringType(), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("resource", StringType(), True), + StructField("resource_type", StringType(), True), + StructField("step", StringType(), True), + StructField("required_fields", StringType(), True), + StructField("reason", StringType(), True), + StructField("is_valid", BooleanType(), True), + StructField("issue_count", LongType(), True), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("type", StringType(), True), + StructField("pointer_id", StringType(), True), + StructField( + "indexes", + StructType( + [ + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), + StructField("masterid_key", StringType(), True), + ] + ), + True, + ), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField( + "result", + StructType( + [ + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "masterid_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("ods_code", StringType(), True), + StructField("relatesTo", StringType(), True), + StructField("related_identifier", StringType(), True), + ] + ), + True, + ), + ] +) + +producerSchemaList = { + "producer--searchPostDocumentReference": searchPostDocumentReferenceSchema, + "producer--searchDocumentReference": searchDocumentReferenceSchema, + "producer--readDocumentReference": readDocumentReferenceSchema, + "producer--upsertDocumentReference": upsertDocumentReferenceSchema, + "producer--updateDocumentReference": updateDocumentReferenceSchema, + "producer--deleteDocumentReference": deleteDocumentReferenceSchema, + "producer--createDocumentReference": createDocumentReferenceSchema, +} From 341187078db414e1d5da8af6b5230551c4aac238 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 11:14:56 +0000 Subject: [PATCH 4/8] NRL-1288 import pathing --- .../modules/glue/src/{schemas => }/consumer_schemas.py | 0 .../account-wide-infrastructure/modules/glue/src/main.py | 4 ++-- .../modules/glue/src/{schemas => }/producer_schemas.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename terraform/account-wide-infrastructure/modules/glue/src/{schemas => }/consumer_schemas.py (100%) rename terraform/account-wide-infrastructure/modules/glue/src/{schemas => }/producer_schemas.py (100%) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py similarity index 100% rename from terraform/account-wide-infrastructure/modules/glue/src/schemas/consumer_schemas.py rename to terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index c789342ee..1d0f8ba52 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -1,10 +1,10 @@ import sys from awsglue.utils import getResolvedOptions +from consumer_schemas import consumerSchemaList from pipeline import LogPipeline +from producer_schemas import producerSchemaList from pyspark.context import SparkContext -from schemas.consumer_schemas import consumerSchemaList -from schemas.producer_schemas import producerSchemaList from transformations import dtype_conversion, flatten_df # Get arguments from AWS Glue job diff --git a/terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py similarity index 100% rename from terraform/account-wide-infrastructure/modules/glue/src/schemas/producer_schemas.py rename to terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py From 530aef819041c723a3be11ffc0b2186d249d7ee4 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 11:18:28 +0000 Subject: [PATCH 5/8] NRL-1288 syntax six --- terraform/account-wide-infrastructure/modules/glue/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 1d0f8ba52..5fc8375a4 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -17,7 +17,7 @@ partition_cols = args["partition_cols"].split(",") if "partition_cols" in args else [] -schema_list = consumerSchemaList.extend(producerSchemaList) +schema_list = consumerSchemaList.update(producerSchemaList) # Initialize ETL process etl_job = LogPipeline( From 53839cc9e80c4c8931c581405f5e829facd7865f Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 11:21:42 +0000 Subject: [PATCH 6/8] NRL-1288 syntax six --- .../account-wide-infrastructure/modules/glue/src/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/main.py b/terraform/account-wide-infrastructure/modules/glue/src/main.py index 5fc8375a4..e461ecbd4 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/main.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/main.py @@ -17,14 +17,14 @@ partition_cols = args["partition_cols"].split(",") if "partition_cols" in args else [] -schema_list = consumerSchemaList.update(producerSchemaList) +consumerSchemaList.update(producerSchemaList) # Initialize ETL process etl_job = LogPipeline( spark_context=sc, source_path=args["source_path"], target_path=args["target_path"], - schemas=schema_list, + schemas=consumerSchemaList, job_name=args["job_name"], partition_cols=partition_cols, transformations=[flatten_df, dtype_conversion], From 7c669947d79eb97a659ba8bbf1f289aef47c81d0 Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 13:26:28 +0000 Subject: [PATCH 7/8] NRL-1288 update athena data sources --- .../modules/glue/glue.tf | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf index 5049bbc1b..240ea7677 100644 --- a/terraform/account-wide-infrastructure/modules/glue/glue.tf +++ b/terraform/account-wide-infrastructure/modules/glue/glue.tf @@ -10,7 +10,37 @@ resource "aws_glue_crawler" "log_crawler" { database_name = aws_glue_catalog_database.log_database.name role = aws_iam_role.glue_service_role.name s3_target { - path = "${aws_s3_bucket.target-data-bucket.id}/" + path = "${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/" + } + s3_target { + path = "${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference//" } schema_change_policy { delete_behavior = "LOG" From 589a5ae3bba648d22faed90356a7c5f661773b4b Mon Sep 17 00:00:00 2001 From: jackleary Date: Mon, 17 Feb 2025 14:35:20 +0000 Subject: [PATCH 8/8] NRL-1288 update schemas --- .../modules/glue/src/consumer_schemas.py | 410 ++--- .../modules/glue/src/producer_schemas.py | 1474 ++++++++--------- 2 files changed, 830 insertions(+), 1054 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py index 9ddd8bab5..41e45edd4 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/consumer_schemas.py @@ -37,28 +37,28 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("content-type", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), StructField("cache-control", StringType(), True), - StructField("NHSD-Correlation-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), + StructField("postman-token", StringType(), True), ] ), True, @@ -69,13 +69,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -84,12 +84,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -108,23 +104,6 @@ ), True, ), - StructField("error", StringType(), True), - StructField("exception", StringType(), True), - StructField("exception_name", StringType(), True), - StructField( - "stack_trace", - StructType( - [ - StructField("type", StringType(), True), - StructField("value", StringType(), True), - StructField("module", StringType(), True), - StructField("frames", StructType([]), True), - ] - ), - True, - ), - StructField("bucket", StringType(), True), - StructField("key", StringType(), True), StructField("pointer_types", StringType(), True), StructField("body", StringType(), True), StructField("model", StringType(), True), @@ -133,10 +112,8 @@ StructType( [ StructField("subject_identifier", StringType(), True), - StructField("custodian_identifier", StringType(), True), StructField("type", StringType(), True), - StructField("category", StringType(), True), - StructField("next_page_token", StringType(), True), + StructField("custodian_identifier", StringType(), True), ] ), True, @@ -150,12 +127,12 @@ "query", StructType( [ - StructField("IndexName", StringType(), True), + StructField("indexname", StringType(), True), StructField( - "KeyConditionExpression", StringType(), True + "keyconditionexpression", StringType(), True ), StructField( - "ExpressionAttributeValues", + "expressionattributevalues", StructType( [ StructField( @@ -182,11 +159,11 @@ True, ), StructField( - "ReturnConsumedCapacity", StringType(), True + "returnconsumedcapacity", StringType(), True ), - StructField("FilterExpression", StringType(), True), + StructField("filterexpression", StringType(), True), StructField( - "ExpressionAttributeNames", + "expressionattributenames", StructType( [ StructField( @@ -207,7 +184,6 @@ [ StructField("count", LongType(), True), StructField("scanned_count", LongType(), True), - StructField("last_evaluated_key", StringType(), True), ] ), True, @@ -216,34 +192,22 @@ "result", StructType( [ - StructField("Items", StructType([]), True), - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), + StructField("items", StructType([]), True), + StructField("count", LongType(), True), + StructField("scannedcount", LongType(), True), StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True + "tablename", StringType(), True ), StructField( - "CapacityUnits", DoubleType(), True + "capacityunits", DoubleType(), True ), + StructField("table", StructType([]), True), StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, - ), - StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( @@ -251,7 +215,7 @@ StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -268,17 +232,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -316,9 +280,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -327,24 +288,40 @@ ), True, ), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), + StructField("id", StringType(), True), + StructField("count", LongType(), True), StructField("status_code", StringType(), True), StructField( "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, ), - StructField("producer_id", StringType(), True), - StructField("document_id", StringType(), True), - StructField("custodian", StringType(), True), - StructField("id", StringType(), True), - StructField("count", LongType(), True), + StructField("error", StringType(), True), + StructField("document", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField("frames", StructType([]), True), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), StructField("subject_identifier", StringType(), True), StructField("type", StringType(), True), ] @@ -384,26 +361,26 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -415,13 +392,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -430,12 +407,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -471,10 +444,7 @@ StructType( [ StructField("subject_identifier", StringType(), True), - StructField("custodian_identifier", StringType(), True), StructField("type", StringType(), True), - StructField("category", StringType(), True), - StructField("next_page_token", StringType(), True), ] ), True, @@ -489,10 +459,8 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, @@ -502,12 +470,12 @@ "query", StructType( [ - StructField("IndexName", StringType(), True), + StructField("indexname", StringType(), True), StructField( - "KeyConditionExpression", StringType(), True + "keyconditionexpression", StringType(), True ), StructField( - "ExpressionAttributeValues", + "expressionattributevalues", StructType( [ StructField( @@ -520,16 +488,25 @@ StructField( ":patient_sort", StringType(), True ), + StructField(":type_4", StringType(), True), + StructField(":type_5", StringType(), True), + StructField(":type_6", StringType(), True), + StructField(":type_7", StringType(), True), + StructField(":type_8", StringType(), True), + StructField(":type_9", StringType(), True), + StructField(":type_10", StringType(), True), + StructField(":type_11", StringType(), True), + StructField(":type_12", StringType(), True), ] ), True, ), StructField( - "ReturnConsumedCapacity", StringType(), True + "returnconsumedcapacity", StringType(), True ), - StructField("FilterExpression", StringType(), True), + StructField("filterexpression", StringType(), True), StructField( - "ExpressionAttributeNames", + "expressionattributenames", StructType( [ StructField( @@ -550,7 +527,6 @@ [ StructField("count", LongType(), True), StructField("scanned_count", LongType(), True), - StructField("last_evaluated_key", StringType(), True), ] ), True, @@ -559,34 +535,22 @@ "result", StructType( [ - StructField("Items", StructType([]), True), - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), + StructField("items", StructType([]), True), + StructField("count", LongType(), True), + StructField("scannedcount", LongType(), True), StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True - ), - StructField( - "CapacityUnits", DoubleType(), True + "tablename", StringType(), True ), StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, + "capacityunits", DoubleType(), True ), + StructField("table", StructType([]), True), StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( @@ -594,7 +558,7 @@ StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -611,17 +575,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -659,9 +623,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -716,26 +677,26 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -747,13 +708,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -762,12 +723,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -802,14 +759,8 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField( - "headers", - StructType([StructField("id", StringType(), True)]), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, @@ -824,20 +775,16 @@ StructField("id", StringType(), True), StructField("nhs_number", StringType(), True), StructField("custodian", StringType(), True), - StructField("custodian_suffix", StringType(), True), StructField("producer_id", StringType(), True), StructField("category_id", StringType(), True), StructField("category", StringType(), True), StructField("type_id", StringType(), True), StructField("type", StringType(), True), - StructField("master_identifier", StringType(), True), StructField("author", StringType(), True), StructField("source", StringType(), True), StructField("version", LongType(), True), StructField("document", StringType(), True), StructField("created_on", StringType(), True), - StructField("updated_on", StringType(), True), - StructField("schemas", StringType(), True), StructField("pk", StringType(), True), StructField("sk", StringType(), True), StructField("patient_key", StringType(), True), @@ -883,24 +830,24 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), - StructField("Host", StringType(), True), + StructField("authorization", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-Id", StringType(), True), - StructField("User-Agent", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Request-Id", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), + StructField("user-agent", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-request-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), ] ), True, @@ -911,13 +858,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -926,12 +873,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -975,12 +918,12 @@ "query", StructType( [ - StructField("IndexName", StringType(), True), + StructField("indexname", StringType(), True), StructField( - "KeyConditionExpression", StringType(), True + "keyconditionexpression", StringType(), True ), StructField( - "ExpressionAttributeValues", + "expressionattributevalues", StructType( [ StructField( @@ -993,9 +936,9 @@ ), True, ), - StructField("Select", StringType(), True), + StructField("select", StringType(), True), StructField( - "ReturnConsumedCapacity", StringType(), True + "returnconsumedcapacity", StringType(), True ), ] ), @@ -1006,33 +949,21 @@ "result", StructType( [ - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), + StructField("count", LongType(), True), + StructField("scannedcount", LongType(), True), StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True + "tablename", StringType(), True ), StructField( - "CapacityUnits", DoubleType(), True + "capacityunits", DoubleType(), True ), + StructField("table", StructType([]), True), StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, - ), - StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( @@ -1040,7 +971,7 @@ StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -1057,17 +988,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -1105,9 +1036,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -1121,10 +1049,8 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, diff --git a/terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py b/terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py index 4f525154c..410917925 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/producer_schemas.py @@ -37,28 +37,28 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), - StructField("cache-control", StringType(), True), - StructField("content-type", StringType(), True), - StructField("Host", StringType(), True), + StructField("authorization", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("content-type", StringType(), True), + StructField("postman-token", StringType(), True), ] ), True, @@ -69,13 +69,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -84,12 +84,9 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -108,192 +105,78 @@ ), True, ), - StructField("pointer_types", StringType(), True), StructField("body", StringType(), True), StructField("model", StringType(), True), StructField( "parsed_body", StructType( [ - StructField("resourceType", StringType(), True), + StructField("resourcetype", StringType(), True), StructField("id", StringType(), True), - StructField("meta", StringType(), True), - StructField("implicitRules", StringType(), True), - StructField("language", StringType(), True), - StructField("text", StringType(), True), - StructField( - "masterIdentifier", - StructType( - [ - StructField("id", StringType(), True), - StructField("use", StringType(), True), - StructField("type", StringType(), True), - StructField("system", StringType(), True), - StructField("value", StringType(), True), - StructField("period", StringType(), True), - StructField("assigner", StringType(), True), - ] - ), - True, - ), - StructField("identifier", StringType(), True), StructField("status", StringType(), True), - StructField("docStatus", StringType(), True), StructField( "type", StructType( - [ - StructField("id", StringType(), True), - StructField( - "coding", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), - True, - ), - StructField("text", StringType(), True), - ] - ), - True, - ), - StructField( - "category", - StructType( - [StructField("Location", StringType(), True)] + [StructField("coding", StructType([]), True)] ), True, ), + StructField("category", StructType([]), True), StructField( "subject", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, ), StructField("date", StringType(), True), - StructField( - "author", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), - StructField("authenticator", StringType(), True), + StructField("author", StructType([]), True), StructField( "custodian", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, ), - StructField("relatesTo", StringType(), True), - StructField("description", StringType(), True), - StructField("securityLabel", StringType(), True), - StructField( - "content", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), + StructField("content", StructType([]), True), StructField( "context", StructType( [ - StructField("id", StringType(), True), - StructField( - "encounter", StringType(), True - ), - StructField("event", StringType(), True), StructField( "period", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "start", StringType(), True ), @@ -305,55 +188,35 @@ True, ), StructField( - "facilityType", StringType(), True - ), - StructField( - "practiceSetting", + "practicesetting", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "coding", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), + StructType([]), True, - ), - StructField( - "text", StringType(), True - ), + ) ] ), True, ), StructField( - "sourcePatientInfo", StringType(), True - ), - StructField( - "related", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), - True, + "related", StructType([]), True ), ] ), True, ), + StructField( + "masteridentifier", + StructType( + [ + StructField("system", StringType(), True), + StructField("value", StringType(), True), + ] + ), + True, + ), ] ), True, @@ -368,7 +231,8 @@ StructField("required_fields", StringType(), True), StructField("reason", StringType(), True), StructField("is_valid", BooleanType(), True), - StructField("issue_count", LongType(), True), + StructField("id", StringType(), True), + StructField("date", StringType(), True), StructField("producer_id", StringType(), True), StructField("document_id", StringType(), True), StructField("custodian", StringType(), True), @@ -389,69 +253,26 @@ ), StructField("source", StringType(), True), StructField("version", LongType(), True), - StructField("error", StringType(), True), - StructField( - "response", - StructType( - [ - StructField("statusCode", StringType(), True), - StructField("body", StringType(), True), - StructField( - "headers", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), - ] - ), - True, - ), - StructField("status_code", StringType(), True), - StructField("exception", StringType(), True), - StructField("exception_name", StringType(), True), - StructField( - "stack_trace", - StructType( - [ - StructField("type", StringType(), True), - StructField("value", StringType(), True), - StructField("module", StringType(), True), - StructField( - "frames", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), - ] - ), - True, - ), - StructField("bucket", StringType(), True), - StructField("key", StringType(), True), - StructField("ods_code", StringType(), True), StructField( "result", StructType( [ StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True + "tablename", StringType(), True ), StructField( - "CapacityUnits", DoubleType(), True + "capacityunits", DoubleType(), True ), StructField( - "Table", + "table", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -460,15 +281,15 @@ True, ), StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( - "masterid_gsi", + "patient_gsi", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -477,11 +298,11 @@ True, ), StructField( - "patient_gsi", + "masterid_gsi", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -498,17 +319,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -546,9 +367,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -557,6 +375,43 @@ ), True, ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statuscode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType( + [StructField("location", StringType(), True)] + ), + True, + ), + ] + ), + True, + ), + StructField("pointer_types", StringType(), True), + StructField("error", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField("frames", StructType([]), True), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), + StructField("ods_code", StringType(), True), ] ), True, @@ -598,27 +453,27 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), StructField("content-type", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -630,13 +485,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -645,12 +500,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -676,21 +527,23 @@ "parsed_body", StructType( [ - StructField("resourceType", StringType(), True), + StructField("resourcetype", StringType(), True), StructField("id", StringType(), True), - StructField("meta", StringType(), True), - StructField("implicitRules", StringType(), True), - StructField("language", StringType(), True), - StructField("text", StringType(), True), - StructField("masterIdentifier", StringType(), True), - StructField("identifier", StringType(), True), + StructField( + "masteridentifier", + StructType( + [ + StructField("system", StringType(), True), + StructField("value", StringType(), True), + ] + ), + True, + ), StructField("status", StringType(), True), - StructField("docStatus", StringType(), True), StructField( "type", StructType( [ - StructField("id", StringType(), True), StructField( "coding", StructType( @@ -701,8 +554,7 @@ ] ), True, - ), - StructField("text", StringType(), True), + ) ] ), True, @@ -716,43 +568,20 @@ "subject", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, @@ -763,55 +592,28 @@ StructType([StructField("id", StringType(), True)]), True, ), - StructField("authenticator", StringType(), True), StructField( "custodian", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, ), - StructField("relatesTo", StringType(), True), - StructField("description", StringType(), True), - StructField("securityLabel", StringType(), True), StructField( "content", StructType([StructField("id", StringType(), True)]), @@ -821,38 +623,21 @@ "context", StructType( [ - StructField("id", StringType(), True), - StructField( - "encounter", StringType(), True - ), - StructField("event", StringType(), True), StructField( "period", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "start", StringType(), True - ), - StructField( - "end", StringType(), True - ), + ) ] ), True, ), StructField( - "facilityType", StringType(), True - ), - StructField( - "practiceSetting", + "practicesetting", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "coding", StructType( @@ -865,17 +650,11 @@ ] ), True, - ), - StructField( - "text", StringType(), True - ), + ) ] ), True, ), - StructField( - "sourcePatientInfo", StringType(), True - ), StructField( "related", StructType( @@ -909,7 +688,6 @@ StructField("required_fields", StringType(), True), StructField("reason", StringType(), True), StructField("is_valid", BooleanType(), True), - StructField("issue_count", LongType(), True), StructField("producer_id", StringType(), True), StructField("document_id", StringType(), True), StructField("custodian", StringType(), True), @@ -921,24 +699,122 @@ StructField("id", StringType(), True), StructField("nhs_number", StringType(), True), StructField("custodian", StringType(), True), - StructField("custodian_suffix", StringType(), True), StructField("producer_id", StringType(), True), StructField("category_id", StringType(), True), StructField("category", StringType(), True), StructField("type_id", StringType(), True), StructField("type", StringType(), True), - StructField("master_identifier", StringType(), True), StructField("author", StringType(), True), StructField("source", StringType(), True), StructField("version", LongType(), True), StructField("document", StringType(), True), StructField("created_on", StringType(), True), StructField("updated_on", StringType(), True), - StructField("schemas", StringType(), True), StructField("pk", StringType(), True), StructField("sk", StringType(), True), StructField("patient_key", StringType(), True), StructField("patient_sort", StringType(), True), + StructField( + "consumedcapacity", + StructType( + [ + StructField( + "tablename", StringType(), True + ), + StructField( + "capacityunits", DoubleType(), True + ), + StructField( + "table", + StructType( + [ + StructField( + "capacityunits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "globalsecondaryindexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "capacityunits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "responsemetadata", + StructType( + [ + StructField( + "requestid", StringType(), True + ), + StructField( + "httpstatuscode", LongType(), True + ), + StructField( + "httpheaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), ] ), True, @@ -950,14 +826,8 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField( - "headers", - StructType([StructField("id", StringType(), True)]), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, @@ -999,27 +869,27 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), StructField("content-type", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -1031,13 +901,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -1046,12 +916,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -1079,8 +945,6 @@ [ StructField("subject_identifier", StringType(), True), StructField("type", StringType(), True), - StructField("category", StringType(), True), - StructField("next_page_token", StringType(), True), ] ), True, @@ -1091,19 +955,18 @@ StructField("filtered_kwargs_keys", StringType(), True), StructField("custodian", StringType(), True), StructField("nhs_number", StringType(), True), - StructField("categories", StringType(), True), StructField("expression", StringType(), True), StructField("values", StringType(), True), StructField( "query", StructType( [ - StructField("IndexName", StringType(), True), + StructField("indexname", StringType(), True), StructField( - "KeyConditionExpression", StringType(), True + "keyconditionexpression", StringType(), True ), StructField( - "ExpressionAttributeValues", + "expressionattributevalues", StructType( [ StructField( @@ -1133,11 +996,11 @@ True, ), StructField( - "ReturnConsumedCapacity", StringType(), True + "returnconsumedcapacity", StringType(), True ), - StructField("FilterExpression", StringType(), True), + StructField("filterexpression", StringType(), True), StructField( - "ExpressionAttributeNames", + "expressionattributenames", StructType( [ StructField( @@ -1158,7 +1021,6 @@ [ StructField("count", LongType(), True), StructField("scanned_count", LongType(), True), - StructField("last_evaluated_key", StringType(), True), ] ), True, @@ -1167,34 +1029,22 @@ "result", StructType( [ - StructField("Items", StructType([]), True), - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), + StructField("items", StructType([]), True), + StructField("count", LongType(), True), + StructField("scannedcount", LongType(), True), StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True - ), - StructField( - "CapacityUnits", DoubleType(), True + "tablename", StringType(), True ), StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, + "capacityunits", DoubleType(), True ), + StructField("table", StructType([]), True), StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( @@ -1202,7 +1052,7 @@ StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -1219,17 +1069,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -1267,9 +1117,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -1287,14 +1134,30 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, ), + StructField("error", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField("frames", StructType([]), True), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), + StructField("ods_code", StringType(), True), ] ), True, @@ -1332,26 +1195,26 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -1363,13 +1226,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -1378,12 +1241,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -1420,8 +1279,6 @@ [ StructField("subject_identifier", StringType(), True), StructField("type", StringType(), True), - StructField("category", StringType(), True), - StructField("next_page_token", StringType(), True), ] ), True, @@ -1432,19 +1289,18 @@ StructField("filtered_kwargs_keys", StringType(), True), StructField("custodian", StringType(), True), StructField("nhs_number", StringType(), True), - StructField("categories", StringType(), True), StructField("expression", StringType(), True), StructField("values", StringType(), True), StructField( "query", StructType( [ - StructField("IndexName", StringType(), True), + StructField("indexname", StringType(), True), StructField( - "KeyConditionExpression", StringType(), True + "keyconditionexpression", StringType(), True ), StructField( - "ExpressionAttributeValues", + "expressionattributevalues", StructType( [ StructField( @@ -1460,16 +1316,25 @@ StructField( ":patient_sort", StringType(), True ), + StructField(":type_4", StringType(), True), + StructField(":type_5", StringType(), True), + StructField(":type_6", StringType(), True), + StructField(":type_7", StringType(), True), + StructField(":type_8", StringType(), True), + StructField(":type_9", StringType(), True), + StructField(":type_10", StringType(), True), + StructField(":type_11", StringType(), True), + StructField(":type_12", StringType(), True), ] ), True, ), StructField( - "ReturnConsumedCapacity", StringType(), True + "returnconsumedcapacity", StringType(), True ), - StructField("FilterExpression", StringType(), True), + StructField("filterexpression", StringType(), True), StructField( - "ExpressionAttributeNames", + "expressionattributenames", StructType( [ StructField( @@ -1490,7 +1355,6 @@ [ StructField("count", LongType(), True), StructField("scanned_count", LongType(), True), - StructField("last_evaluated_key", StringType(), True), ] ), True, @@ -1499,34 +1363,22 @@ "result", StructType( [ - StructField("Items", StructType([]), True), - StructField("Count", LongType(), True), - StructField("ScannedCount", LongType(), True), + StructField("items", StructType([]), True), + StructField("count", LongType(), True), + StructField("scannedcount", LongType(), True), StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True + "tablename", StringType(), True ), StructField( - "CapacityUnits", DoubleType(), True + "capacityunits", DoubleType(), True ), + StructField("table", StructType([]), True), StructField( - "Table", - StructType( - [ - StructField( - "CapacityUnits", - DoubleType(), - True, - ) - ] - ), - True, - ), - StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( @@ -1534,7 +1386,7 @@ StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -1551,17 +1403,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -1599,9 +1451,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -1619,15 +1468,31 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField("headers", StructType([]), True), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, ), StructField("type", StringType(), True), + StructField("error", StringType(), True), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField("frames", StructType([]), True), + ] + ), + True, + ), + StructField("bucket", StringType(), True), + StructField("key", StringType(), True), + StructField("ods_code", StringType(), True), ] ), True, @@ -1669,26 +1534,26 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -1700,13 +1565,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -1715,12 +1580,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -1755,14 +1616,8 @@ "response", StructType( [ - StructField("statusCode", StringType(), True), + StructField("statuscode", StringType(), True), StructField("body", StringType(), True), - StructField( - "headers", - StructType([StructField("id", StringType(), True)]), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), ] ), True, @@ -1777,20 +1632,16 @@ StructField("id", StringType(), True), StructField("nhs_number", StringType(), True), StructField("custodian", StringType(), True), - StructField("custodian_suffix", StringType(), True), StructField("producer_id", StringType(), True), StructField("category_id", StringType(), True), StructField("category", StringType(), True), StructField("type_id", StringType(), True), StructField("type", StringType(), True), - StructField("master_identifier", StringType(), True), StructField("author", StringType(), True), StructField("source", StringType(), True), StructField("version", LongType(), True), StructField("document", StringType(), True), StructField("created_on", StringType(), True), - StructField("updated_on", StringType(), True), - StructField("schemas", StringType(), True), StructField("pk", StringType(), True), StructField("sk", StringType(), True), StructField("patient_key", StringType(), True), @@ -1799,6 +1650,25 @@ ), True, ), + StructField("exception", StringType(), True), + StructField("exception_name", StringType(), True), + StructField( + "stack_trace", + StructType( + [ + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("module", StringType(), True), + StructField( + "frames", + StructType([StructField("id", StringType(), True)]), + True, + ), + ] + ), + True, + ), + StructField("ods_code_parts", StringType(), True), ] ), True, @@ -1840,27 +1710,27 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), - StructField("cache-control", StringType(), True), - StructField("Host", StringType(), True), + StructField("authorization", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), + StructField("user-agent", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-request-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), + StructField("cache-control", StringType(), True), + StructField("postman-token", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), - StructField("x-request-id", StringType(), True), ] ), True, @@ -1871,13 +1741,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -1886,12 +1756,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -1921,25 +1787,151 @@ StructField("item_type", StringType(), True), StructField("original_kwargs_keys", StringType(), True), StructField("filtered_kwargs_keys", StringType(), True), - StructField("pointer_id", StringType(), True), - StructField("status_code", StringType(), True), + StructField("producer_id", StringType(), True), + StructField("document_id", StringType(), True), + StructField("custodian", StringType(), True), StructField( - "response", + "result", StructType( [ - StructField("statusCode", StringType(), True), - StructField("body", StringType(), True), + StructField("id", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField("custodian", StringType(), True), + StructField("producer_id", StringType(), True), + StructField("category_id", StringType(), True), + StructField("category", StringType(), True), + StructField("type_id", StringType(), True), + StructField("type", StringType(), True), + StructField("author", StringType(), True), + StructField("source", StringType(), True), + StructField("version", LongType(), True), + StructField("document", StringType(), True), + StructField("created_on", StringType(), True), + StructField("pk", StringType(), True), + StructField("sk", StringType(), True), + StructField("patient_key", StringType(), True), + StructField("patient_sort", StringType(), True), StructField( - "headers", - StructType([StructField("id", StringType(), True)]), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), - ] - ), - True, - ), - ] + "consumedcapacity", + StructType( + [ + StructField( + "tablename", StringType(), True + ), + StructField( + "capacityunits", DoubleType(), True + ), + StructField( + "table", + StructType( + [ + StructField( + "capacityunits", + DoubleType(), + True, + ) + ] + ), + True, + ), + StructField( + "globalsecondaryindexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "capacityunits", + DoubleType(), + True, + ) + ] + ), + True, + ) + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "responsemetadata", + StructType( + [ + StructField( + "requestid", StringType(), True + ), + StructField( + "httpstatuscode", LongType(), True + ), + StructField( + "httpheaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("updated_on", StringType(), True), + ] + ), + True, + ), + StructField("partition_key", StringType(), True), + StructField("sort_key", StringType(), True), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statuscode", StringType(), True), + StructField("body", StringType(), True), + ] + ), + True, + ), + StructField("pointer_id", StringType(), True), + ] ), True, ), @@ -1976,27 +1968,27 @@ [ StructField("accept", StringType(), True), StructField("accept-encoding", StringType(), True), - StructField("Authorization", StringType(), True), + StructField("authorization", StringType(), True), StructField("cache-control", StringType(), True), StructField("content-type", StringType(), True), - StructField("Host", StringType(), True), + StructField("host", StringType(), True), StructField( - "NHSD-Client-RP-Details", StringType(), True + "nhsd-client-rp-details", StringType(), True ), StructField( - "NHSD-Connection-Metadata", StringType(), True + "nhsd-connection-metadata", StringType(), True ), - StructField("NHSD-Correlation-ID", StringType(), True), + StructField("nhsd-correlation-id", StringType(), True), StructField( - "NHSD-End-User-Organisation-ODS", StringType(), True + "nhsd-end-user-organisation-ods", StringType(), True ), - StructField("NHSD-Request-ID", StringType(), True), - StructField("Postman-Token", StringType(), True), - StructField("User-Agent", StringType(), True), + StructField("nhsd-request-id", StringType(), True), + StructField("postman-token", StringType(), True), + StructField("user-agent", StringType(), True), StructField("x-correlation-id", StringType(), True), - StructField("X-Forwarded-For", StringType(), True), - StructField("X-Forwarded-Port", StringType(), True), - StructField("X-Forwarded-Proto", StringType(), True), + StructField("x-forwarded-for", StringType(), True), + StructField("x-forwarded-port", StringType(), True), + StructField("x-forwarded-proto", StringType(), True), StructField("x-request-id", StringType(), True), ] ), @@ -2008,13 +2000,13 @@ "config", StructType( [ - StructField("AWS_REGION", StringType(), True), - StructField("PREFIX", StringType(), True), - StructField("ENVIRONMENT", StringType(), True), - StructField("SPLUNK_INDEX", StringType(), True), - StructField("SOURCE", StringType(), True), - StructField("AUTH_STORE", StringType(), True), - StructField("TABLE_NAME", StringType(), True), + StructField("aws_region", StringType(), True), + StructField("prefix", StringType(), True), + StructField("environment", StringType(), True), + StructField("splunk_index", StringType(), True), + StructField("source", StringType(), True), + StructField("auth_store", StringType(), True), + StructField("table_name", StringType(), True), ] ), True, @@ -2023,12 +2015,8 @@ "metadata", StructType( [ - StructField("pointer_types", StringType(), True), StructField("ods_code", StringType(), True), - StructField("ods_code_extension", StringType(), True), - StructField("nrl_permissions", StringType(), True), StructField("nrl_app_id", StringType(), True), - StructField("is_test_event", BooleanType(), True), StructField( "client_rp_details", StructType( @@ -2050,239 +2038,75 @@ StructField("pointer_types", StringType(), True), StructField("body", StringType(), True), StructField("model", StringType(), True), - StructField("error", StringType(), True), - StructField( - "response", - StructType( - [ - StructField("statusCode", StringType(), True), - StructField("body", StringType(), True), - StructField( - "headers", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), - StructField("isBase64Encoded", BooleanType(), True), - ] - ), - True, - ), - StructField("status_code", StringType(), True), StructField( "parsed_body", StructType( [ - StructField("resourceType", StringType(), True), - StructField("id", StringType(), True), - StructField( - "meta", - StructType( - [ - StructField("id", StringType(), True), - StructField( - "versionId", StringType(), True - ), - StructField( - "lastUpdated", StringType(), True - ), - StructField("source", StringType(), True), - StructField("profile", StringType(), True), - StructField("security", StringType(), True), - StructField("tag", StringType(), True), - ] - ), - True, - ), - StructField("implicitRules", StringType(), True), - StructField("language", StringType(), True), - StructField("text", StringType(), True), - StructField( - "masterIdentifier", - StructType( - [ - StructField("id", StringType(), True), - StructField("use", StringType(), True), - StructField("type", StringType(), True), - StructField("system", StringType(), True), - StructField("value", StringType(), True), - StructField("period", StringType(), True), - StructField("assigner", StringType(), True), - ] - ), - True, - ), - StructField("identifier", StringType(), True), + StructField("resourcetype", StringType(), True), StructField("status", StringType(), True), - StructField("docStatus", StringType(), True), StructField( "type", StructType( - [ - StructField("id", StringType(), True), - StructField( - "coding", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), - True, - ), - StructField("text", StringType(), True), - ] - ), - True, - ), - StructField( - "category", - StructType( - [StructField("Location", StringType(), True)] + [StructField("coding", StructType([]), True)] ), True, ), + StructField("category", StructType([]), True), StructField( "subject", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, ), StructField("date", StringType(), True), - StructField( - "author", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), - StructField("authenticator", StringType(), True), + StructField("author", StructType([]), True), StructField( "custodian", StructType( [ - StructField("id", StringType(), True), - StructField( - "reference", StringType(), True - ), - StructField("type", StringType(), True), StructField( "identifier", StructType( [ - StructField( - "id", StringType(), True - ), - StructField( - "use", StringType(), True - ), - StructField( - "type", StringType(), True - ), StructField( "system", StringType(), True ), StructField( "value", StringType(), True ), - StructField( - "period", StringType(), True - ), - StructField( - "assigner", - StringType(), - True, - ), ] ), True, - ), - StructField("display", StringType(), True), + ) ] ), True, ), - StructField("relatesTo", StringType(), True), - StructField("description", StringType(), True), - StructField("securityLabel", StringType(), True), - StructField( - "content", - StructType( - [StructField("Location", StringType(), True)] - ), - True, - ), + StructField("content", StructType([]), True), StructField( "context", StructType( [ - StructField("id", StringType(), True), - StructField( - "encounter", StringType(), True - ), - StructField( - "event", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), - True, - ), StructField( "period", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "start", StringType(), True ), @@ -2294,55 +2118,46 @@ True, ), StructField( - "facilityType", StringType(), True - ), - StructField( - "practiceSetting", + "practicesetting", StructType( [ - StructField( - "id", StringType(), True - ), StructField( "coding", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), + StructType([]), True, - ), - StructField( - "text", StringType(), True - ), + ) ] ), True, ), StructField( - "sourcePatientInfo", StringType(), True - ), - StructField( - "related", - StructType( - [ - StructField( - "Location", - StringType(), - True, - ) - ] - ), - True, + "related", StructType([]), True ), + StructField("event", StructType([]), True), ] ), True, ), + StructField("relatesto", StructType([]), True), + StructField("id", StringType(), True), + StructField( + "meta", + StructType( + [StructField("lastupdated", StringType(), True)] + ), + True, + ), + StructField( + "masteridentifier", + StructType( + [ + StructField("system", StringType(), True), + StructField("value", StringType(), True), + ] + ), + True, + ), + StructField("identifier", StructType([]), True), ] ), True, @@ -2357,7 +2172,6 @@ StructField("required_fields", StringType(), True), StructField("reason", StringType(), True), StructField("is_valid", BooleanType(), True), - StructField("issue_count", LongType(), True), StructField("producer_id", StringType(), True), StructField("document_id", StringType(), True), StructField("custodian", StringType(), True), @@ -2383,21 +2197,21 @@ StructType( [ StructField( - "ConsumedCapacity", + "consumedcapacity", StructType( [ StructField( - "TableName", StringType(), True + "tablename", StringType(), True ), StructField( - "CapacityUnits", DoubleType(), True + "capacityunits", DoubleType(), True ), StructField( - "Table", + "table", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -2406,15 +2220,15 @@ True, ), StructField( - "GlobalSecondaryIndexes", + "globalsecondaryindexes", StructType( [ StructField( - "masterid_gsi", + "patient_gsi", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -2423,11 +2237,11 @@ True, ), StructField( - "patient_gsi", + "masterid_gsi", StructType( [ StructField( - "CapacityUnits", + "capacityunits", DoubleType(), True, ) @@ -2444,17 +2258,17 @@ True, ), StructField( - "ResponseMetadata", + "responsemetadata", StructType( [ StructField( - "RequestId", StringType(), True + "requestid", StringType(), True ), StructField( - "HTTPStatusCode", LongType(), True + "httpstatuscode", LongType(), True ), StructField( - "HTTPHeaders", + "httpheaders", StructType( [ StructField( @@ -2492,9 +2306,6 @@ ), True, ), - StructField( - "RetryAttempts", LongType(), True - ), ] ), True, @@ -2503,8 +2314,47 @@ ), True, ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statuscode", StringType(), True), + StructField("body", StringType(), True), + StructField( + "headers", + StructType( + [StructField("location", StringType(), True)] + ), + True, + ), + ] + ), + True, + ), + StructField("error", StringType(), True), + StructField( + "issue", + StructType( + [ + StructField("severity", StringType(), True), + StructField("code", StringType(), True), + StructField( + "details", + StructType( + [StructField("coding", StructType([]), True)] + ), + True, + ), + StructField("diagnostics", StringType(), True), + StructField("expression", StringType(), True), + ] + ), + True, + ), + StructField("issue_count", LongType(), True), StructField("ods_code", StringType(), True), - StructField("relatesTo", StringType(), True), + StructField("relatesto", StringType(), True), StructField("related_identifier", StringType(), True), ] ),