Up to date for msk-cluster module (#35)

Author: posquit0

modules/msk-cluster/cluster.tf

@@ -44,7 +44,6 @@ resource "aws_msk_configuration" "this" {
 # MSK Cluster
 ###################################################
 
-# TODO: public access cidrs
 resource "aws_msk_cluster" "this" {
   cluster_name           = var.name
   kafka_version          = var.kafka_version
@@ -55,10 +54,12 @@ resource "aws_msk_cluster" "this" {
     az_distribution = "DEFAULT"
     client_subnets  = var.broker_subnets
     security_groups = concat(
-      module.security_group[*].id,
+      [module.security_group.id],
       var.broker_additional_security_groups
     )
 
+    # TODO: `vpc_connectivity`
+    # TODO: public access cidrs
     connectivity_info {
       public_access {
         type = var.broker_public_access_enabled ? "SERVICE_PROVIDED_EIPS" : "DISABLED"
@@ -67,52 +68,53 @@ resource "aws_msk_cluster" "this" {
 
     storage_info {
       ebs_storage_info {
-        volume_size = var.broker_volume_size
+        volume_size = var.broker_storage.volume_size
 
         dynamic "provisioned_throughput" {
-          for_each = var.broker_volume_provisioned_throughput_enabled ? ["go"] : []
+          for_each = var.broker_storage.provisioned_throughput.enabled ? [var.broker_storage.provisioned_throughput] : []
 
           content {
-            enabled           = true
-            volume_throughput = var.broker_volume_provisioned_throughput
+            enabled           = provisioned_throughput.value.enabled
+            volume_throughput = provisioned_throughput.value.throughput
           }
         }
       }
     }
   }
+  storage_mode = var.cluster_storage_mode
 
   configuration_info {
     arn      = aws_msk_configuration.this.arn
     revision = aws_msk_configuration.this.latest_revision
   }
 
 
-  ## Auth
+  ## Authentiation
   client_authentication {
-    unauthenticated = var.auth_unauthenticated_access_enabled
+    unauthenticated = var.authentication.unauthenticated_access.enabled
 
     sasl {
-      iam   = var.auth_sasl_iam_enabled
-      scram = var.auth_sasl_scram_enabled
+      iam   = var.authentication.sasl_iam.enabled
+      scram = var.authentication.sasl_scram.enabled
     }
 
     dynamic "tls" {
-      for_each = var.auth_tls_enabled ? ["go"] : []
+      for_each = var.authentication.tls.enabled ? [var.authentication.tls] : []
 
       content {
-        certificate_authority_arns = var.auth_tls_acm_ca_arns
+        certificate_authority_arns = tls.value.acm_private_certificate_authorities
       }
     }
   }
 
 
   ## Encryption
   encryption_info {
-    encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key
+    encryption_at_rest_kms_key_arn = var.encryption_at_rest.kms_key
 
     encryption_in_transit {
-      in_cluster    = var.encryption_in_transit_in_cluster_enabled
-      client_broker = var.encryption_in_transit_client_mode
+      in_cluster    = var.encryption_in_transit.in_cluster_enabled
+      client_broker = var.encryption_in_transit.client_mode
     }
   }
 
@@ -121,33 +123,33 @@ resource "aws_msk_cluster" "this" {
   logging_info {
     broker_logs {
       cloudwatch_logs {
-        enabled   = var.logging_cloudwatch_enabled
-        log_group = var.logging_cloudwatch_log_group
+        enabled   = var.logging.cloudwatch_logs.enabled
+        log_group = var.logging.cloudwatch_logs.log_group
       }
       firehose {
-        enabled         = var.logging_firehose_enabled
-        delivery_stream = var.logging_firehose_delivery_stream
+        enabled         = var.logging.firehose.enabled
+        delivery_stream = var.logging.firehose.delivery_stream
       }
       s3 {
-        enabled = var.logging_s3_enabled
-        bucket  = var.logging_s3_bucket
-        prefix  = var.logging_s3_prefix
+        enabled = var.logging.s3.enabled
+        bucket  = var.logging.s3.bucket
+        prefix  = var.logging.s3.key_prefix
       }
     }
   }
 
 
   ## Monitoring
-  enhanced_monitoring = var.monitoring_cloudwatch_level
+  enhanced_monitoring = var.cloudwatch_metrics.monitoring_level
 
   open_monitoring {
     prometheus {
       jmx_exporter {
-        enabled_in_broker = var.monitoring_prometheus_jmx_exporter_enabled
+        enabled_in_broker = var.prometheus.jmx_exporter_enabled
       }
 
       node_exporter {
-        enabled_in_broker = var.monitoring_prometheus_node_exporter_enabled
+        enabled_in_broker = var.prometheus.node_exporter_enabled
       }
     }
   }

modules/msk-cluster/outputs.tf

@@ -46,8 +46,6 @@ output "broker" {
 
     `public_access_enabled` - Whether public access to MSK brokers is enabled.
     `security_groups` - A list of the security groups associated with the MSK cluster.
-
-    `volume` - A EBS volume information for MSK brokers.
   EOF
   value = {
     size          = aws_msk_cluster.this.number_of_broker_nodes
@@ -56,19 +54,27 @@ output "broker" {
     subnets                   = aws_msk_cluster.this.broker_node_group_info[0].client_subnets
     public_access_enabled     = var.broker_public_access_enabled
     security_groups           = aws_msk_cluster.this.broker_node_group_info[0].security_groups
-    default_security_group_id = try(module.security_group[*].id[0], null)
+    default_security_group_id = module.security_group.id
+  }
+}
 
-    volume = {
-      size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size
-      provisioned_throughput = {
-        enabled    = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false)
-        throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null)
-      }
+output "broker_storage" {
+  description = "The configuration for broker storage of the MSK cluster."
+  value = {
+    volume_size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size
+    provisioned_throughput = {
+      enabled    = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false)
+      throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null)
     }
   }
 }
 
-output "auth" {
+output "cluster_storage_mode" {
+  description = "The storage mode of the MSK cluster."
+  value       = aws_msk_cluster.this.storage_mode
+}
+
+output "authentication" {
   description = "A configuration for authentication of the Kafka cluster."
   value = {
     unauthenticated_access = {
@@ -80,31 +86,33 @@ output "auth" {
       }
       scram = {
         enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].scram
-        kms_key = var.auth_sasl_scram_kms_key
-        users   = var.auth_sasl_scram_users
+        kms_key = var.authentication.sasl_scram.kms_key
+        users   = var.authentication.sasl_scram.users
       }
     }
     tls = {
-      enabled     = var.auth_tls_enabled
-      acm_ca_arns = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, [])
+      enabled                             = var.authentication.tls.enabled
+      acm_private_certificate_authorities = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, [])
     }
   }
 }
 
-output "encryption" {
+output "encryption_at_rest" {
   description = <<EOF
-  A configuration for encryption of the Kafka cluster.
-    `at_rest` - The configuration for encryption at rest.
-    `in_transit` - The configuration for encryption in transit.
+  The configuration for encryption at rest of the Kafka cluster.
   EOF
   value = {
-    at_rest = {
-      kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn
-    }
-    in_transit = {
-      in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster
-      client_mode        = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker
-    }
+    kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn
+  }
+}
+
+output "encryption_in_transit" {
+  description = <<EOF
+  The configuration for encryption in transit of the Kafka cluster.
+  EOF
+  value = {
+    in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster
+    client_mode        = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker
   }
 }
 
@@ -125,22 +133,22 @@ output "logging" {
       delivery_stream = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].delivery_stream
     }
     s3 = {
-      enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled
-      bucket  = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket
-      prefix  = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix
+      enabled    = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled
+      bucket     = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket
+      key_prefix = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix
     }
   }
 }
 
 output "monitoring" {
   description = <<EOF
   A configuration for monitoring of the Kafka cluster.
-    `cloudwatch` - The configuration for MSK CloudWatch Metrics.
+    `cloudwatch_metrics` - The configuration for MSK CloudWatch Metrics.
     `prometheus` - The configuration for Prometheus open monitoring.
   EOF
   value = {
-    cloudwatch = {
-      level = aws_msk_cluster.this.enhanced_monitoring
+    cloudwatch_metrics = {
+      monitoring_level = aws_msk_cluster.this.enhanced_monitoring
     }
     prometheus = {
       jmx_exporter_enabled  = aws_msk_cluster.this.open_monitoring[0].prometheus[0].jmx_exporter[0].enabled_in_broker

modules/msk-cluster/scram-secrets.tf

@@ -1,5 +1,5 @@
 resource "random_password" "this" {
-  for_each = var.auth_sasl_scram_users
+  for_each = var.authentication.sasl_scram.users
 
   length = 16
 
@@ -19,9 +19,9 @@ resource "random_password" "this" {
 # TODO: Create an independant module for msk-scram-users
 module "secret" {
   source  = "tedilabs/secret/aws//modules/secrets-manager-secret"
-  version = "~> 0.2.0"
+  version = "~> 0.5.0"
 
-  for_each = var.auth_sasl_scram_users
+  for_each = var.authentication.sasl_scram.users
 
   name        = "AmazonMSK_SCRAM/${var.name}/${each.key}"
   description = "The SASL/SCRAM secret to provide username and password for MSK cluster authenticaiton."
@@ -32,7 +32,7 @@ module "secret" {
     password = random_password.this[each.key].result
   }
 
-  kms_key             = var.auth_sasl_scram_kms_key
+  kms_key             = var.authentication.sasl_scram.kms_key
   policy              = null
   block_public_policy = true