diff --git a/terraform/account-wide-infrastructure/modules/glue/src/transformations.py b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py index 35e2a6b45..e4d7b8f1d 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/transformations.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/transformations.py @@ -8,6 +8,7 @@ from pyspark.sql.types import ( BooleanType, DoubleType, + LongType, StringType, StructField, StructType, @@ -44,6 +45,9 @@ StructField("accept-encoding", StringType(), True), StructField("Authorization", StringType(), True), StructField("Host", StringType(), True), + StructField( + "NHSD-Client-RP-Details", StringType(), True + ), StructField( "NHSD-Connection-Metadata", StringType(), True ), @@ -51,12 +55,240 @@ StructField("User-Agent", StringType(), True), StructField("X-Forwarded-For", StringType(), True), StructField("X-Request-Id", StringType(), True), + StructField( + "NHSD-End-User-Organisation-ODS", StringType(), True + ), + StructField("NHSD-Request-ID", StringType(), True), + StructField("X-Forwarded-Port", StringType(), True), + StructField("X-Forwarded-Proto", StringType(), True), ] ), True, ), StructField("log_reference", StringType(), True), StructField("xray_trace_id", StringType(), True), + StructField( + "config", + StructType( + [ + StructField("AWS_REGION", StringType(), True), + StructField("PREFIX", StringType(), True), + StructField("ENVIRONMENT", StringType(), True), + StructField("SPLUNK_INDEX", StringType(), True), + StructField("SOURCE", StringType(), True), + StructField("AUTH_STORE", StringType(), True), + StructField("TABLE_NAME", StringType(), True), + ] + ), + True, + ), + StructField( + "metadata", + StructType( + [ + StructField("ods_code", StringType(), True), + StructField("ods_code_extension", StringType(), True), + StructField("nrl_app_id", StringType(), True), + StructField("is_test_event", BooleanType(), True), + StructField( + "client_rp_details", + StructType( + [ + StructField( + "developer_app_name", StringType(), True + ), + StructField( + "developer_app_id", StringType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "params", + StructType( + [ + StructField("subject:identifier", StringType(), True), + ] + ), + True, + ), + StructField("model", StringType(), True), + StructField( + "parsed_params", + StructType( + [ + StructField("subject_identifier", StringType(), True), + ] + ), + True, + ), + StructField("table_name", StringType(), True), + StructField("item_type", StringType(), True), + StructField("original_kwargs_keys", StringType(), True), + StructField("filtered_kwargs_keys", StringType(), True), + StructField("nhs_number", StringType(), True), + StructField( + "query", + StructType( + [ + StructField("IndexName", StringType(), True), + StructField( + "KeyConditionExpression", StringType(), True + ), + StructField( + "ExpressionAttributeValues", + StructType( + [ + StructField( + ":patient_key", StringType(), True + ), + StructField( + ":patient_sort", StringType(), True + ), + ] + ), + True, + ), + StructField("Select", StringType(), True), + StructField( + "ReturnConsumedCapacity", StringType(), True + ), + ] + ), + True, + ), + StructField("count", LongType(), True), + StructField( + "result", + StructType( + [ + StructField("Count", LongType(), True), + StructField("ScannedCount", LongType(), True), + StructField( + "ConsumedCapacity", + StructType( + [ + StructField( + "TableName", StringType(), True + ), + StructField( + "CapacityUnits", DoubleType(), True + ), + StructField( + "Table", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ), + ] + ), + True, + ), + StructField( + "GlobalSecondaryIndexes", + StructType( + [ + StructField( + "patient_gsi", + StructType( + [ + StructField( + "CapacityUnits", + DoubleType(), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField( + "ResponseMetadata", + StructType( + [ + StructField( + "RequestId", StringType(), True + ), + StructField( + "HTTPStatusCode", LongType(), True + ), + StructField( + "HTTPHeaders", + StructType( + [ + StructField( + "server", StringType(), True + ), + StructField( + "date", StringType(), True + ), + StructField( + "content-type", + StringType(), + True, + ), + StructField( + "content-length", + StringType(), + True, + ), + StructField( + "connection", + StringType(), + True, + ), + StructField( + "x-amzn-requestid", + StringType(), + True, + ), + StructField( + "x-amz-crc32", + StringType(), + True, + ), + ] + ), + True, + ), + StructField( + "RetryAttempts", LongType(), True + ), + ] + ), + True, + ), + ] + ), + True, + ), + StructField("status_code", StringType(), True), + StructField( + "response", + StructType( + [ + StructField("statusCode", StringType(), True), + StructField("body", StringType(), True), + StructField("isBase64Encoded", BooleanType(), True), + ] + ), + True, + ), ] ), True,