Skip to content

Commit 57ff77b

Browse files
posquit0posquit0
authored
Migrate msk-cluster module from terraform-aws-misc (#4)
1 parent f7b9384 commit 57ff77b

File tree

11 files changed

+974
-1
lines changed

11 files changed

+974
-1
lines changed

‎.github/labeler.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
# Modules
22
":floppy_disk: eventbridge-event-bus":
33
- modules/eventbridge-event-bus/**/*
4+
":floppy_disk: msk-cluster":
5+
- modules/msk-cluster/**/*

‎.github/labels.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,6 @@
4343
- color: "fbca04"
4444
description: "This issue or pull request is related to eventbridge-event-bus module."
4545
name: ":floppy_disk: eventbridge-event-bus"
46+
- color: "fbca04"
47+
description: "This issue or pull request is related to msk-cluster module."
48+
name: ":floppy_disk: msk-cluster"

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
Terraform module which creates messaging related resources on AWS.
88

99
- [eventbridge-event-bus](./modules/eventbridge-event-bus)
10+
- [msk-cluster](./modules/msk-cluster)
1011

1112

1213
## Target AWS Services
@@ -16,7 +17,7 @@ Terraform Modules from [this package](https://github.com/tedilabs/terraform-aws-
1617
- **AWS EventBridge (Formerly known as CloudWatch Events)**
1718
- Event Bus
1819
- **AWS MSK (Managed Streaming for Apache Kafka)**
19-
- Comming Soon!
20+
- Cluster
2021
- **AWS SNS (Simple Notification Service)**
2122
- Comming Soon!
2223
- **AWS SQS (Simple Queue Service)**

‎modules/msk-cluster/README.md‎

Lines changed: 108 additions & 0 deletions
Large diffs are not rendered by default.

‎modules/msk-cluster/cluster.tf‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
locals {
2+
metadata = {
3+
package = "terraform-aws-messaging"
4+
version = trimspace(file("${path.module}/../../VERSION"))
5+
module = basename(path.module)
6+
name = var.name
7+
}
8+
module_tags = var.module_tags_enabled ? {
9+
"module.terraform.io/package" = local.metadata.package
10+
"module.terraform.io/version" = local.metadata.version
11+
"module.terraform.io/name" = local.metadata.module
12+
"module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}"
13+
"module.terraform.io/instance" = local.metadata.name
14+
} : {}
15+
}
16+
17+
18+
###################################################
19+
# Configuration for MSK Cluster
20+
###################################################
21+
22+
locals {
23+
server_properties = <<EOT
24+
%{for k, v in var.kafka_server_properties~}
25+
${k} = ${v}
26+
%{endfor~}
27+
EOT
28+
}
29+
30+
resource "aws_msk_configuration" "this" {
31+
name = var.name
32+
description = "Configuration for ${var.name} Kafka Cluster."
33+
kafka_versions = [var.kafka_version]
34+
35+
server_properties = local.server_properties
36+
37+
lifecycle {
38+
create_before_destroy = true
39+
}
40+
}
41+
42+
43+
###################################################
44+
# MSK Cluster
45+
###################################################
46+
47+
# TODO: public access cidrs
48+
resource "aws_msk_cluster" "this" {
49+
cluster_name = var.name
50+
kafka_version = var.kafka_version
51+
number_of_broker_nodes = var.broker_size
52+
53+
broker_node_group_info {
54+
instance_type = var.broker_instance_type
55+
az_distribution = "DEFAULT"
56+
client_subnets = var.broker_subnets
57+
security_groups = concat(
58+
module.security_group[*].id,
59+
var.broker_additional_security_groups
60+
)
61+
62+
connectivity_info {
63+
public_access {
64+
type = var.broker_public_access_enabled ? "SERVICE_PROVIDED_EIPS" : "DISABLED"
65+
}
66+
}
67+
68+
storage_info {
69+
ebs_storage_info {
70+
volume_size = var.broker_volume_size
71+
72+
dynamic "provisioned_throughput" {
73+
for_each = var.broker_volume_provisioned_throughput_enabled ? ["go"] : []
74+
75+
content {
76+
enabled = true
77+
volume_throughput = var.broker_volume_provisioned_throughput
78+
}
79+
}
80+
}
81+
}
82+
}
83+
84+
configuration_info {
85+
arn = aws_msk_configuration.this.arn
86+
revision = aws_msk_configuration.this.latest_revision
87+
}
88+
89+
90+
## Auth
91+
client_authentication {
92+
unauthenticated = var.auth_unauthenticated_access_enabled
93+
94+
sasl {
95+
iam = var.auth_sasl_iam_enabled
96+
scram = var.auth_sasl_scram_enabled
97+
}
98+
99+
dynamic "tls" {
100+
for_each = var.auth_tls_enabled ? ["go"] : []
101+
102+
content {
103+
certificate_authority_arns = var.auth_tls_acm_ca_arns
104+
}
105+
}
106+
}
107+
108+
109+
## Encryption
110+
encryption_info {
111+
encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key
112+
113+
encryption_in_transit {
114+
in_cluster = var.encryption_in_transit_in_cluster_enabled
115+
client_broker = var.encryption_in_transit_client_mode
116+
}
117+
}
118+
119+
120+
## Logging
121+
logging_info {
122+
broker_logs {
123+
cloudwatch_logs {
124+
enabled = var.logging_cloudwatch_enabled
125+
log_group = var.logging_cloudwatch_log_group
126+
}
127+
firehose {
128+
enabled = var.logging_firehose_enabled
129+
delivery_stream = var.logging_firehose_delivery_stream
130+
}
131+
s3 {
132+
enabled = var.logging_s3_enabled
133+
bucket = var.logging_s3_bucket
134+
prefix = var.logging_s3_prefix
135+
}
136+
}
137+
}
138+
139+
140+
## Monitoring
141+
enhanced_monitoring = var.monitoring_cloudwatch_level
142+
143+
open_monitoring {
144+
prometheus {
145+
jmx_exporter {
146+
enabled_in_broker = var.monitoring_prometheus_jmx_exporter_enabled
147+
}
148+
149+
node_exporter {
150+
enabled_in_broker = var.monitoring_prometheus_node_exporter_enabled
151+
}
152+
}
153+
}
154+
155+
timeouts {
156+
create = var.timeouts.create
157+
update = var.timeouts.update
158+
delete = var.timeouts.delete
159+
}
160+
161+
tags = merge(
162+
{
163+
"Name" = local.metadata.name
164+
},
165+
local.module_tags,
166+
var.tags,
167+
)
168+
}
169+
170+
data "aws_msk_broker_nodes" "this" {
171+
cluster_arn = aws_msk_cluster.this.arn
172+
}

‎modules/msk-cluster/outputs.tf‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
output "arn" {
2+
description = "The ARN of the MSK cluster."
3+
value = aws_msk_cluster.this.arn
4+
}
5+
6+
output "name" {
7+
description = "The MSK cluster name."
8+
value = var.name
9+
}
10+
11+
output "version" {
12+
description = "Current version of the MSK Cluster used for updates."
13+
value = aws_msk_cluster.this.current_version
14+
}
15+
16+
output "kafka_version" {
17+
description = "The MSK cluster version."
18+
value = var.kafka_version
19+
}
20+
21+
output "kafka_config" {
22+
description = "The MSK configuration."
23+
value = {
24+
arn = aws_msk_configuration.this.arn
25+
name = aws_msk_configuration.this.name
26+
latest_revision = aws_msk_configuration.this.latest_revision
27+
properties = aws_msk_configuration.this.server_properties
28+
}
29+
}
30+
31+
output "broker_security_group_id" {
32+
description = "The id of security group that were created for the MSK cluster."
33+
value = try(module.security_group[*].id[0], null)
34+
}
35+
36+
output "broker_nodes" {
37+
description = "The information of broker nodes in the kafka cluster."
38+
value = data.aws_msk_broker_nodes.this.node_info_list
39+
}
40+
41+
output "broker" {
42+
description = <<EOF
43+
A configuration for brokers of the Kafka cluster.
44+
`size` - The number of broker nodes in the kafka cluster.
45+
`instance_type` - The instance type used by the kafka brokers.
46+
47+
`public_access_enabled` - Whether public access to MSK brokers is enabled.
48+
`security_groups` - A list of the security groups associated with the MSK cluster.
49+
50+
`volume` - A EBS volume information for MSK brokers.
51+
EOF
52+
value = {
53+
size = aws_msk_cluster.this.number_of_broker_nodes
54+
instance_type = aws_msk_cluster.this.broker_node_group_info[0].instance_type
55+
56+
subnets = aws_msk_cluster.this.broker_node_group_info[0].client_subnets
57+
public_access_enabled = var.broker_public_access_enabled
58+
security_groups = aws_msk_cluster.this.broker_node_group_info[0].security_groups
59+
default_security_group_id = try(module.security_group[*].id[0], null)
60+
61+
volume = {
62+
size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size
63+
provisioned_throughput = {
64+
enabled = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false)
65+
throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null)
66+
}
67+
}
68+
}
69+
}
70+
71+
output "auth" {
72+
description = "A configuration for authentication of the Kafka cluster."
73+
value = {
74+
unauthenticated_access = {
75+
enabled = aws_msk_cluster.this.client_authentication[0].unauthenticated
76+
}
77+
sasl = {
78+
iam = {
79+
enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].iam
80+
}
81+
scram = {
82+
enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].scram
83+
kms_key = var.auth_sasl_scram_kms_key
84+
users = var.auth_sasl_scram_users
85+
}
86+
}
87+
tls = {
88+
enabled = var.auth_tls_enabled
89+
acm_ca_arns = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, [])
90+
}
91+
}
92+
}
93+
94+
output "encryption" {
95+
description = <<EOF
96+
A configuration for encryption of the Kafka cluster.
97+
`at_rest` - The configuration for encryption at rest.
98+
`in_transit` - The configuration for encryption in transit.
99+
EOF
100+
value = {
101+
at_rest = {
102+
kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn
103+
}
104+
in_transit = {
105+
in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster
106+
client_mode = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker
107+
}
108+
}
109+
}
110+
111+
output "logging" {
112+
description = <<EOF
113+
A configuration for logging of the Kafka cluster.
114+
`cloudwatch` - The configuration for MSK broker logs to CloudWatch Logs.
115+
`firehose` - The configuration for MSK broker logs to Kinesis Firehose.
116+
`s3` - The configuration for MSK broker logs to S3 Bucket.
117+
EOF
118+
value = {
119+
cloudwatch = {
120+
enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].cloudwatch_logs[0].enabled
121+
log_group = aws_msk_cluster.this.logging_info[0].broker_logs[0].cloudwatch_logs[0].log_group
122+
}
123+
firehose = {
124+
enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].enabled
125+
delivery_stream = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].delivery_stream
126+
}
127+
s3 = {
128+
enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled
129+
bucket = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket
130+
prefix = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix
131+
}
132+
}
133+
}
134+
135+
output "monitoring" {
136+
description = <<EOF
137+
A configuration for monitoring of the Kafka cluster.
138+
`cloudwatch` - The configuration for MSK CloudWatch Metrics.
139+
`prometheus` - The configuration for Prometheus open monitoring.
140+
EOF
141+
value = {
142+
cloudwatch = {
143+
level = aws_msk_cluster.this.enhanced_monitoring
144+
}
145+
prometheus = {
146+
jmx_exporter_enabled = aws_msk_cluster.this.open_monitoring[0].prometheus[0].jmx_exporter[0].enabled_in_broker
147+
node_exporter_enabled = aws_msk_cluster.this.open_monitoring[0].prometheus[0].node_exporter[0].enabled_in_broker
148+
}
149+
}
150+
}
151+
152+
output "bootstrap_brokers" {
153+
description = <<EOF
154+
A configuration for connecting to the Kafka cluster.
155+
`plaintext` - A comma separated list of one or more hostname:port pairs of kafka brokers suitable to boostrap connectivity to the kafka cluster. Only contains value if `client_encryption_in_transit_mode` is set to PLAINTEXT or TLS_PLAINTEXT. AWS may not always return all endpoints so the values may not be stable across applies.
156+
`sasl_iam` - A comma separated list of one or more DNS names (or IPs) and SASL IAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
157+
`sasl_scram` - A comma separated list of one or more DNS names (or IPs) and SASL SCRAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
158+
`tls` - A comma separated list of one or more DNS names (or IPs) and TLS port pairs kafka brokers suitable to boostrap connectivity to the kafka cluster. Only contains value if `client_encryption_in_transit_mode is set to TLS_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
159+
`public_sasl_iam` - A comma separated list of one or more DNS names (or IPs) and SASL IAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS_PLAINTEXT or TLS and `auth_sasl_iam_enabled` is `true` and `broker_public_access_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
160+
`public_sasl_scram` - A comma separated list of one or more DNS names (or IPs) and SASL SCRAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS_PLAINTEXT or TLS and `auth_sasl_scram_enabled` is `true` and `broker_public_access_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
161+
`public_tls` - A comma separated list of one or more DNS names (or IPs) and TLS port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS_PLAINTEXT or TLS and `broker_public_access_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
162+
EOF
163+
value = {
164+
plaintext = aws_msk_cluster.this.bootstrap_brokers
165+
sasl_iam = aws_msk_cluster.this.bootstrap_brokers_sasl_iam
166+
sasl_scram = aws_msk_cluster.this.bootstrap_brokers_sasl_scram
167+
tls = aws_msk_cluster.this.bootstrap_brokers_tls
168+
169+
public_sasl_iam = aws_msk_cluster.this.bootstrap_brokers_public_sasl_iam
170+
public_sasl_scram = aws_msk_cluster.this.bootstrap_brokers_public_sasl_scram
171+
public_tls = aws_msk_cluster.this.bootstrap_brokers_public_tls
172+
}
173+
}
174+
175+
output "zookeeper_connections" {
176+
description = <<EOF
177+
A configuration for connecting to the Apache Zookeeper cluster.
178+
`tcp` - A comma separated list of one or more IP:port pairs to use to connect to the Apache Zookeeper cluster.
179+
`tls` - A comma separated list of one or more IP:port pairs to use to connect to the Apache Zookeeper cluster via TLS.
180+
EOF
181+
value = {
182+
tcp = aws_msk_cluster.this.zookeeper_connect_string
183+
tls = aws_msk_cluster.this.zookeeper_connect_string_tls
184+
}
185+
}

0 commit comments

Comments
 (0)