From e10785b982d064238b3733750083d5fd4cc4e9ed Mon Sep 17 00:00:00 2001 From: Mark Tinderholt Date: Thu, 11 Dec 2025 12:24:27 -0500 Subject: [PATCH 1/2] two scenarios --- src/terraform/aks-baseline/main.tf | 18 +--- src/terraform/aks-baseline/outputs.tf | 2 +- src/terraform/aks-baseline/variables.tf | 3 + src/terraform/aks-managed-id/acr.tf | 65 +++++++++++++ src/terraform/aks-managed-id/aks.tf | 93 +++++++++++++++++++ src/terraform/aks-managed-id/identity.tf | 42 +++++++++ src/terraform/aks-managed-id/main.tf | 8 ++ src/terraform/aks-managed-id/monitoring.tf | 29 ++++++ src/terraform/aks-managed-id/variables.tf | 51 ++++++++++ .../terraform/aks-managed-id}/versions.tf | 0 testing/prereq-name/main.tf | 5 + testing/prereq-name/outputs.tf | 3 + testing/prereq-name/versions.tf | 8 ++ testing/prereq-network/dns.tf | 68 ++++++++++++++ testing/prereq-network/main.tf | 7 ++ testing/prereq-network/outputs.tf | 12 +++ testing/prereq-network/subnet-acr.tf | 6 ++ testing/prereq-network/subnet-aks.tf | 33 +++++++ testing/prereq-network/subnet-api-server.tf | 16 ++++ testing/prereq-network/subnet-cloudshell.tf | 60 ++++++++++++ testing/prereq-network/variables.tf | 19 ++++ testing/prereq-network/versions.tf | 12 +++ testing/prereq-rg/main.tf | 5 + testing/prereq-rg/outputs.tf | 6 ++ testing/prereq-rg/versions.tf | 10 ++ .../{setup => prereq-vm-size}/compute-skus.tf | 0 testing/{setup => prereq-vm-size}/outputs.tf | 0 .../{setup => prereq-vm-size}/variables.tf | 0 testing/prereq-vm-size/versions.tf | 16 ++++ tests/README.md | 6 ++ tests/aks-baseline.tftest.hcl | 31 +++++-- tests/aks-managed-id.tftest.hcl | 87 +++++++++++++++++ 32 files changed, 695 insertions(+), 26 deletions(-) create mode 100644 src/terraform/aks-managed-id/acr.tf create mode 100644 src/terraform/aks-managed-id/aks.tf create mode 100644 src/terraform/aks-managed-id/identity.tf create mode 100644 src/terraform/aks-managed-id/main.tf create mode 100644 src/terraform/aks-managed-id/monitoring.tf create mode 100644 src/terraform/aks-managed-id/variables.tf rename {testing/setup => src/terraform/aks-managed-id}/versions.tf (100%) create mode 100644 testing/prereq-name/main.tf create mode 100644 testing/prereq-name/outputs.tf create mode 100644 testing/prereq-name/versions.tf create mode 100644 testing/prereq-network/dns.tf create mode 100644 testing/prereq-network/main.tf create mode 100644 testing/prereq-network/outputs.tf create mode 100644 testing/prereq-network/subnet-acr.tf create mode 100644 testing/prereq-network/subnet-aks.tf create mode 100644 testing/prereq-network/subnet-api-server.tf create mode 100644 testing/prereq-network/subnet-cloudshell.tf create mode 100644 testing/prereq-network/variables.tf create mode 100644 testing/prereq-network/versions.tf create mode 100644 testing/prereq-rg/main.tf create mode 100644 testing/prereq-rg/outputs.tf create mode 100644 testing/prereq-rg/versions.tf rename testing/{setup => prereq-vm-size}/compute-skus.tf (100%) rename testing/{setup => prereq-vm-size}/outputs.tf (100%) rename testing/{setup => prereq-vm-size}/variables.tf (100%) create mode 100644 testing/prereq-vm-size/versions.tf create mode 100644 tests/README.md create mode 100644 tests/aks-managed-id.tftest.hcl diff --git a/src/terraform/aks-baseline/main.tf b/src/terraform/aks-baseline/main.tf index c457c9d..6199115 100644 --- a/src/terraform/aks-baseline/main.tf +++ b/src/terraform/aks-baseline/main.tf @@ -1,20 +1,8 @@ -data "azurerm_client_config" "current" {} - -resource "random_string" "suffix" { - length = 8 - upper = false - special = false -} - -data "azurerm_resource_group" "main" { - name = "rg-pvt-aks-cluster-tf-test" -} - resource "azurerm_kubernetes_cluster" "main" { - name = "aks-${var.application_name}-${var.environment_name}-${random_string.suffix.result}" - resource_group_name = data.azurerm_resource_group.main.name + name = "aks-${var.application_name}-${var.environment_name}" + resource_group_name = var.resource_group_name location = var.location - dns_prefix = "aks${var.application_name}${random_string.suffix.result}" + dns_prefix = "aks${var.application_name}" default_node_pool { name = "default" diff --git a/src/terraform/aks-baseline/outputs.tf b/src/terraform/aks-baseline/outputs.tf index d73c042..70c3f60 100644 --- a/src/terraform/aks-baseline/outputs.tf +++ b/src/terraform/aks-baseline/outputs.tf @@ -1,3 +1,3 @@ output "resource_group_name" { - value = data.azurerm_resource_group.main.name + value = var.resource_group_name } diff --git a/src/terraform/aks-baseline/variables.tf b/src/terraform/aks-baseline/variables.tf index d9b395f..331d441 100644 --- a/src/terraform/aks-baseline/variables.tf +++ b/src/terraform/aks-baseline/variables.tf @@ -4,6 +4,9 @@ variable "application_name" { variable "environment_name" { type = string } +variable "resource_group_name" { + type = string +} variable "location" { type = string } diff --git a/src/terraform/aks-managed-id/acr.tf b/src/terraform/aks-managed-id/acr.tf new file mode 100644 index 0000000..876f13c --- /dev/null +++ b/src/terraform/aks-managed-id/acr.tf @@ -0,0 +1,65 @@ +locals { + acr_name = "cr${random_string.suffix.result}" +} + +resource "azurerm_container_registry" "acr" { + name = local.acr_name + resource_group_name = var.resource_group_name + location = var.location + sku = "Premium" + public_network_access_enabled = false + tags = var.tags +} + +######################################## +# ACR Cache Rules +######################################## + +# CRITICAL for network isolated clusters - caches ALL MCR images for AKS bootstrap +# BYO ACR requires EXACT settings per MS docs: +# - name: aks-managed-mcr +# - source_repo: mcr.microsoft.com/* +# - target_repo: aks-managed-repository/* +# DO NOT modify this cache rule - it is required for cluster creation/functioning/upgrading +resource "azurerm_container_registry_cache_rule" "aks_managed" { + name = "aks-managed-mcr" + container_registry_id = azurerm_container_registry.acr.id + source_repo = "mcr.microsoft.com/*" + target_repo = "aks-managed-repository/*" + credential_set_id = null +} + +######################################## +# ACR Private Endpoint +######################################## + +resource "azurerm_private_endpoint" "acr" { + name = "pe-acr-${local.acr_name}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + subnet_id = azurerm_subnet.acr_subnet.id + tags = var.tags + + private_service_connection { + name = "acr-connection" + private_connection_resource_id = azurerm_container_registry.acr.id + is_manual_connection = false + subresource_names = ["registry"] + } + + private_dns_zone_group { + name = "acr-dns-zone-group" + private_dns_zone_ids = [azurerm_private_dns_zone.acr.id] + } +} + +######################################## +# ACR Role Assignments +######################################## + +resource "azurerm_role_assignment" "kubelet_acr_pull" { + scope = azurerm_container_registry.acr.id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.kubelet_identity.principal_id + skip_service_principal_aad_check = true +} diff --git a/src/terraform/aks-managed-id/aks.tf b/src/terraform/aks-managed-id/aks.tf new file mode 100644 index 0000000..1007d71 --- /dev/null +++ b/src/terraform/aks-managed-id/aks.tf @@ -0,0 +1,93 @@ +locals { + cluster_name = "aks-${var.application_name}-${var.environment_name}" +} + +resource "azurerm_kubernetes_cluster" "main" { + name = local.cluster_name + location = var.location + resource_group_name = var.resource_group_name + dns_prefix = local.cluster_name + tags = var.tags + + sku_tier = var.aks_sku_tier # SKU Tier - Standard/Premium includes Uptime SLA + kubernetes_version = var.kubernetes_version + automatic_upgrade_channel = "patch" # Automatic upgrade channels + node_os_upgrade_channel = "NodeImage" + azure_policy_enabled = true # Azure Policy for governance + private_cluster_enabled = true # Private cluster configuration + private_cluster_public_fqdn_enabled = false + private_dns_zone_id = "System" + local_account_disabled = true # Disable local accounts for enhanced security + oidc_issuer_enabled = true # Enable OIDC issuer and workload identity + workload_identity_enabled = true + + default_node_pool { + name = "system" + node_count = var.default_node_count + vm_size = var.vm_size + vnet_subnet_id = var.aks_subnet_id + + upgrade_settings { + max_surge = "10%" + } + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.aks_identity.id] + } + + kubelet_identity { + client_id = azurerm_user_assigned_identity.kubelet_identity.client_id + object_id = azurerm_user_assigned_identity.kubelet_identity.principal_id + user_assigned_identity_id = azurerm_user_assigned_identity.kubelet_identity.id + } + + api_server_access_profile { + virtual_network_integration_enabled = true + subnet_id = azurerm_subnet.api_server_subnet.id + } + + azure_active_directory_role_based_access_control { + azure_rbac_enabled = true + admin_group_object_ids = var.aks_admin_group_object_ids + } + + network_profile { + network_plugin = "azure" + network_plugin_mode = "overlay" + outbound_type = "none" + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + dns_service_ip = var.dns_service_ip + } + + oms_agent { + log_analytics_workspace_id = azurerm_log_analytics_workspace.aks.id + msi_auth_for_monitoring_enabled = true + } + + bootstrap_profile { + artifact_source = "Cache" + container_registry_id = azurerm_container_registry.acr.id + } + + maintenance_window_auto_upgrade { + frequency = "Weekly" + interval = 1 + duration = 4 + day_of_week = "Sunday" + start_time = "02:00" + utc_offset = "+00:00" + } + + maintenance_window_node_os { + frequency = "Weekly" + interval = 1 + duration = 4 + day_of_week = "Sunday" + start_time = "06:00" + utc_offset = "+00:00" + } + +} diff --git a/src/terraform/aks-managed-id/identity.tf b/src/terraform/aks-managed-id/identity.tf new file mode 100644 index 0000000..a4cde82 --- /dev/null +++ b/src/terraform/aks-managed-id/identity.tf @@ -0,0 +1,42 @@ +resource "azurerm_user_assigned_identity" "aks_identity" { + name = "id-${var.application_name}-${var.environment_name}-cluster" + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags +} + +resource "azurerm_user_assigned_identity" "kubelet_identity" { + name = "id-${var.application_name}-${var.environment_name}-kubelet" + resource_group_name = var.resource_group_name + location = var.location + tags = var.tags +} + + +# AKS identity needs Network Contributor on VNet +resource "azurerm_role_assignment" "aks_network_contributor" { + scope = azurerm_virtual_network.vnet.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks_identity.principal_id +} + +# AKS identity needs Managed Identity Operator on kubelet identity +resource "azurerm_role_assignment" "aks_identity_operator" { + scope = azurerm_user_assigned_identity.kubelet_identity.id + role_definition_name = "Managed Identity Operator" + principal_id = azurerm_user_assigned_identity.aks_identity.principal_id +} + +# Current user - Cluster Admin Role (control plane access) +resource "azurerm_role_assignment" "aks_cluster_admin_current_user" { + scope = azurerm_kubernetes_cluster.aks.id + role_definition_name = "Azure Kubernetes Service Cluster Admin Role" + principal_id = data.azurerm_client_config.current.object_id +} + +# Current user - RBAC Cluster Admin (data plane access) +resource "azurerm_role_assignment" "aks_rbac_cluster_admin_current_user" { + scope = azurerm_kubernetes_cluster.aks.id + role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin" + principal_id = data.azurerm_client_config.current.object_id +} diff --git a/src/terraform/aks-managed-id/main.tf b/src/terraform/aks-managed-id/main.tf new file mode 100644 index 0000000..b0dc39c --- /dev/null +++ b/src/terraform/aks-managed-id/main.tf @@ -0,0 +1,8 @@ +data "azurerm_client_config" "current" {} + +resource "random_string" "suffix" { + length = 8 + upper = false + special = false +} + diff --git a/src/terraform/aks-managed-id/monitoring.tf b/src/terraform/aks-managed-id/monitoring.tf new file mode 100644 index 0000000..f8783ea --- /dev/null +++ b/src/terraform/aks-managed-id/monitoring.tf @@ -0,0 +1,29 @@ +######################################## +# Log Analytics Workspace +######################################## + +resource "azurerm_log_analytics_workspace" "aks" { + name = "log-${var.application_name}-${var.environment_name}" + resource_group_name = var.resource_group_name + location = var.location + retention_in_days = var.log_analytics_retention_days + tags = var.tags +} + +######################################## +# Container Insights Solution +######################################## + +resource "azurerm_log_analytics_solution" "container_insights" { + solution_name = "ContainerInsights" + resource_group_name = var.resource_group_name + location = var.location + workspace_resource_id = azurerm_log_analytics_workspace.aks.id + workspace_name = azurerm_log_analytics_workspace.aks.name + tags = var.tags + + plan { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } +} diff --git a/src/terraform/aks-managed-id/variables.tf b/src/terraform/aks-managed-id/variables.tf new file mode 100644 index 0000000..0a9390b --- /dev/null +++ b/src/terraform/aks-managed-id/variables.tf @@ -0,0 +1,51 @@ +variable "application_name" { + type = string +} +variable "environment_name" { + type = string +} +variable "resource_group_name" { + type = string +} +variable "location" { + type = string +} +variable "vm_size" { + type = string +} +variable "tags" { + type = map(string) + default = {} +} +variable "aks_sku_tier" { + type = string + default = "Standard" +} +variable "aks_admin_group_object_ids" { + type = list(string) + default = [] +} +variable "kubernetes_version" { + type = string + default = "1.32.0" +} +variable "log_analytics_retention_days" { + type = number + default = 30 +} +variable "default_node_count" { + type = number + default = 3 +} +variable "aks_subnet_id" { + type = string +} +variable "pod_cidr" { + type = string +} +variable "service_cidr" { + type = string +} +variable "dns_service_ip" { + type = string +} diff --git a/testing/setup/versions.tf b/src/terraform/aks-managed-id/versions.tf similarity index 100% rename from testing/setup/versions.tf rename to src/terraform/aks-managed-id/versions.tf diff --git a/testing/prereq-name/main.tf b/testing/prereq-name/main.tf new file mode 100644 index 0000000..b4bdcd4 --- /dev/null +++ b/testing/prereq-name/main.tf @@ -0,0 +1,5 @@ +resource "random_string" "suffix" { + length = 8 + upper = false + special = false +} diff --git a/testing/prereq-name/outputs.tf b/testing/prereq-name/outputs.tf new file mode 100644 index 0000000..1733f46 --- /dev/null +++ b/testing/prereq-name/outputs.tf @@ -0,0 +1,3 @@ +output "suffix" { + value = random_string.suffix.result +} diff --git a/testing/prereq-name/versions.tf b/testing/prereq-name/versions.tf new file mode 100644 index 0000000..3f4ca54 --- /dev/null +++ b/testing/prereq-name/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.7.2" + } + } +} diff --git a/testing/prereq-network/dns.tf b/testing/prereq-network/dns.tf new file mode 100644 index 0000000..12f234e --- /dev/null +++ b/testing/prereq-network/dns.tf @@ -0,0 +1,68 @@ + +######################################## +# Private DNS Zones - ACR +######################################## + +resource "azurerm_private_dns_zone" "acr" { + name = "privatelink.azurecr.io" + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "acr" { + name = "acr-vnetlink" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.acr.name + virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags +} + +######################################## +# Private DNS Zones - Azure Relay +######################################## + +resource "azurerm_private_dns_zone" "relay" { + name = "privatelink.servicebus.windows.net" + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "relay" { + name = "relay-vnetlink" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.relay.name + virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags +} + +######################################## +# Private DNS Zones - Storage +######################################## + +resource "azurerm_private_dns_zone" "storage_blob" { + name = "privatelink.blob.core.windows.net" + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_private_dns_zone" "storage_file" { + name = "privatelink.file.core.windows.net" + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "storage_blob" { + name = "storage-blob-vnetlink" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.storage_blob.name + virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "storage_file" { + name = "storage-file-vnetlink" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.storage_file.name + virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags +} diff --git a/testing/prereq-network/main.tf b/testing/prereq-network/main.tf new file mode 100644 index 0000000..c7efcb2 --- /dev/null +++ b/testing/prereq-network/main.tf @@ -0,0 +1,7 @@ +resource "azurerm_virtual_network" "main" { + name = "vnet-${var.application_name}-${var.environment_name}" + resource_group_name = var.resource_group_name + location = var.location + address_space = [var.vnet_address_space] + tags = var.tags +} diff --git a/testing/prereq-network/outputs.tf b/testing/prereq-network/outputs.tf new file mode 100644 index 0000000..66e5036 --- /dev/null +++ b/testing/prereq-network/outputs.tf @@ -0,0 +1,12 @@ +output "aks_subnet_id" { + value = azurerm_subnet.aks.id +} +output "api_server_subnet_id" { + value = azurerm_subnet.api_server.id +} +output "acr_subnet_id" { + value = azurerm_subnet.acr.id +} +output "vnet_id" { + value = azurerm_virtual_network.main.id +} diff --git a/testing/prereq-network/subnet-acr.tf b/testing/prereq-network/subnet-acr.tf new file mode 100644 index 0000000..011c0d2 --- /dev/null +++ b/testing/prereq-network/subnet-acr.tf @@ -0,0 +1,6 @@ +resource "azurerm_subnet" "acr" { + name = "acr-subnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 8, 3)] # x.x.3.0/24 +} diff --git a/testing/prereq-network/subnet-aks.tf b/testing/prereq-network/subnet-aks.tf new file mode 100644 index 0000000..349cb7e --- /dev/null +++ b/testing/prereq-network/subnet-aks.tf @@ -0,0 +1,33 @@ +resource "azurerm_subnet" "aks" { + name = "aks-subnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 4, 1)] # x.x.1.0/24 +} + +# NSG for AKS subnet - blocks outbound internet traffic +resource "azurerm_network_security_group" "aks" { + name = "nsg-aks-subnet" + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_network_security_rule" "aks_deny_internet_outbound" { + name = "DenyInternetOutbound" + priority = 4000 + direction = "Outbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Internet" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.aks.name +} + +resource "azurerm_subnet_network_security_group_association" "aks" { + subnet_id = azurerm_subnet.aks.id + network_security_group_id = azurerm_network_security_group.aks.id +} diff --git a/testing/prereq-network/subnet-api-server.tf b/testing/prereq-network/subnet-api-server.tf new file mode 100644 index 0000000..8be1b94 --- /dev/null +++ b/testing/prereq-network/subnet-api-server.tf @@ -0,0 +1,16 @@ +resource "azurerm_subnet" "api_server" { + name = "api-server-subnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 8, 2)] # x.x.2.0/24 + + delegation { + name = "aks-delegation" + service_delegation { + name = "Microsoft.ContainerService/managedClusters" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + ] + } + } +} diff --git a/testing/prereq-network/subnet-cloudshell.tf b/testing/prereq-network/subnet-cloudshell.tf new file mode 100644 index 0000000..72de23b --- /dev/null +++ b/testing/prereq-network/subnet-cloudshell.tf @@ -0,0 +1,60 @@ +resource "azurerm_subnet" "cloudshell_container" { + name = "cloudshellsubnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 8, 4)] # x.x.4.0/24 + + delegation { + name = "cloudshell-delegation" + service_delegation { + name = "Microsoft.ContainerInstance/containerGroups" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/action", + ] + } + } +} + +resource "azurerm_subnet" "cloudshell_relay" { + name = "relaysubnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 8, 5)] # x.x.5.0/24 +} + +resource "azurerm_subnet" "cloudshell_storage_pe" { + name = "storage-pe-subnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = [cidrsubnet(var.vnet_address_space, 8, 6)] # x.x.6.0/24 + + # Enable network policies for private endpoints per MS Cloud Shell best practices + private_endpoint_network_policies = "Enabled" +} + +# NSG for Cloud Shell - requires outbound internet +resource "azurerm_network_security_group" "cloudshell" { + name = "nsg-cloudshell" + location = var.location + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_network_security_rule" "cloudshell_outbound_internet" { + name = "AllowOutboundInternet" + priority = 1000 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Internet" + resource_group_name = var.resource_group_name + network_security_group_name = azurerm_network_security_group.cloudshell.name +} + +resource "azurerm_subnet_network_security_group_association" "cloudshell" { + subnet_id = azurerm_subnet.cloudshell_container.id + network_security_group_id = azurerm_network_security_group.cloudshell.id +} diff --git a/testing/prereq-network/variables.tf b/testing/prereq-network/variables.tf new file mode 100644 index 0000000..7c3f3e0 --- /dev/null +++ b/testing/prereq-network/variables.tf @@ -0,0 +1,19 @@ +variable "application_name" { + type = string +} +variable "environment_name" { + type = string +} +variable "resource_group_name" { + type = string +} +variable "location" { + type = string +} +variable "tags" { + type = map(string) + default = {} +} +variable "vnet_address_space" { + type = string +} diff --git a/testing/prereq-network/versions.tf b/testing/prereq-network/versions.tf new file mode 100644 index 0000000..e10d47e --- /dev/null +++ b/testing/prereq-network/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.35.0" + } + azapi = { + source = "azure/azapi" + version = "~> 2.2.0" + } + } +} diff --git a/testing/prereq-rg/main.tf b/testing/prereq-rg/main.tf new file mode 100644 index 0000000..279c3a8 --- /dev/null +++ b/testing/prereq-rg/main.tf @@ -0,0 +1,5 @@ +data "azurerm_client_config" "current" {} + +data "azurerm_resource_group" "main" { + name = "rg-pvt-aks-cluster-tf-test" +} diff --git a/testing/prereq-rg/outputs.tf b/testing/prereq-rg/outputs.tf new file mode 100644 index 0000000..a722519 --- /dev/null +++ b/testing/prereq-rg/outputs.tf @@ -0,0 +1,6 @@ +output "resource_group_name" { + value = data.azurerm_resource_group.main.name +} +output "location" { + value = data.azurerm_resource_group.main.location +} diff --git a/testing/prereq-rg/versions.tf b/testing/prereq-rg/versions.tf new file mode 100644 index 0000000..79cbff3 --- /dev/null +++ b/testing/prereq-rg/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.35.0" + } + } + + required_version = "~> 1.2" +} diff --git a/testing/setup/compute-skus.tf b/testing/prereq-vm-size/compute-skus.tf similarity index 100% rename from testing/setup/compute-skus.tf rename to testing/prereq-vm-size/compute-skus.tf diff --git a/testing/setup/outputs.tf b/testing/prereq-vm-size/outputs.tf similarity index 100% rename from testing/setup/outputs.tf rename to testing/prereq-vm-size/outputs.tf diff --git a/testing/setup/variables.tf b/testing/prereq-vm-size/variables.tf similarity index 100% rename from testing/setup/variables.tf rename to testing/prereq-vm-size/variables.tf diff --git a/testing/prereq-vm-size/versions.tf b/testing/prereq-vm-size/versions.tf new file mode 100644 index 0000000..ac00b6a --- /dev/null +++ b/testing/prereq-vm-size/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.35.0" + } + azapi = { + source = "azure/azapi" + version = "~> 2.2.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.7.2" + } + } +} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..ea59686 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,6 @@ +## Getting Started + +1. Run `terraform init` +2. Run all tests `terraform test -verbose` +3. Run a specific test `terraform test -filter=./tests/aks-managed-id.tftest.hcl` + diff --git a/tests/aks-baseline.tftest.hcl b/tests/aks-baseline.tftest.hcl index 6b6341b..4df4244 100644 --- a/tests/aks-baseline.tftest.hcl +++ b/tests/aks-baseline.tftest.hcl @@ -5,16 +5,23 @@ provider "azurerm" { } variables { - application_name = "aks-tf-tests" - environment_name = "test" - location = "westus3" } -// Sample Setup. This could setup any pre-requisites needed for the test. -// Perhaps stage some data or files in a storage account. -run "setup" { +run "name" { module { - source = "./testing/setup" + source = "./testing/prereq-name" + } +} + +run "resource_group" { + module { + source = "./testing/prereq-rg" + } +} + +run "vm_size" { + module { + source = "./testing/prereq-vm-size" } variables { @@ -42,7 +49,11 @@ run "provision" { } variables { - vm_size = run.setup.candidate_sku + resource_group_name = run.resource_group.resource_group_name + location = "westus3" + application_name = "tft-${run.name.suffix}" + environment_name = "test" + vm_size = run.vm_size.candidate_sku } providers = { @@ -50,8 +61,8 @@ run "provision" { } assert { - condition = length(data.azurerm_resource_group.main.name) > 0 - error_message = "Must have a valid Resource Group Name" + condition = length(data.azurerm_kubernetes_cluster.main.name) > 0 + error_message = "Must have a valid AKS Cluster Name" } } diff --git a/tests/aks-managed-id.tftest.hcl b/tests/aks-managed-id.tftest.hcl new file mode 100644 index 0000000..f0f7d91 --- /dev/null +++ b/tests/aks-managed-id.tftest.hcl @@ -0,0 +1,87 @@ +provider "azurerm" { + features {} + resource_provider_registrations = "none" + subscription_id = "24a4c592-bfaf-492f-beaf-f10b3b67f03f" +} + +variables { + location = "westus3" +} + +run "name" { + module { + source = "./testing/prereq-name" + } +} + +run "resource_group" { + module { + source = "./testing/prereq-rg" + } +} + +run "network" { + module { + source = "./testing/prereq-network" + } + + variables { + resource_group_name = run.resource_group.resource_group_name + location = var.location + application_name = "tft-${run.name.suffix}" + environment_name = "test" + vnet_address_space = "10.1.0.0/16" + } +} + +run "vm_size" { + module { + source = "./testing/prereq-vm-size" + } + + variables { + location = var.location + vcpu_min = 2 + vcpu_max = 8 + memory_gb_min = 4 + memory_gb_max = 8 + name_filter = "D" + } + + providers = { + azurerm = azurerm + } + +} + +# Provision the AKS Cluster +run "provision" { + + command = apply + + module { + source = "./src/terraform/aks-baseline" + } + + variables { + resource_group_name = run.resource_group.resource_group_name + location = var.location + application_name = "tft-${run.name.suffix}" + environment_name = "test" + vm_size = run.vm_size.candidate_sku + aks_subnet_id = run.network.aks_subnet_id + aks_api_subnet_id = run.network.api_server_subnet_id + pod_cidr = "10.244.0.0/16" + service_cidr = "10.0.0.0/16" + dns_service_ip = "10.0.0.10" + } + + providers = { + azurerm = azurerm + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.name) > 0 + error_message = "Must have a valid AKS Cluster Name" + } +} From dad6bd1a8b71cd80fc4f106891e7f8d1538cb95e Mon Sep 17 00:00:00 2001 From: Mark Tinderholt Date: Thu, 11 Dec 2025 12:28:05 -0500 Subject: [PATCH 2/2] acr subnet --- src/terraform/aks-managed-id/acr.tf | 6 +++--- src/terraform/aks-managed-id/variables.tf | 3 +++ tests/aks-managed-id.tftest.hcl | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/terraform/aks-managed-id/acr.tf b/src/terraform/aks-managed-id/acr.tf index 876f13c..b662094 100644 --- a/src/terraform/aks-managed-id/acr.tf +++ b/src/terraform/aks-managed-id/acr.tf @@ -35,9 +35,9 @@ resource "azurerm_container_registry_cache_rule" "aks_managed" { resource "azurerm_private_endpoint" "acr" { name = "pe-acr-${local.acr_name}" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - subnet_id = azurerm_subnet.acr_subnet.id + location = var.location + resource_group_name = var.resource_group_name + subnet_id = var.acr_subnet_id tags = var.tags private_service_connection { diff --git a/src/terraform/aks-managed-id/variables.tf b/src/terraform/aks-managed-id/variables.tf index 0a9390b..8ae9a0d 100644 --- a/src/terraform/aks-managed-id/variables.tf +++ b/src/terraform/aks-managed-id/variables.tf @@ -40,6 +40,9 @@ variable "default_node_count" { variable "aks_subnet_id" { type = string } +variable "acr_subnet_id" { + type = string +} variable "pod_cidr" { type = string } diff --git a/tests/aks-managed-id.tftest.hcl b/tests/aks-managed-id.tftest.hcl index f0f7d91..b6caf9f 100644 --- a/tests/aks-managed-id.tftest.hcl +++ b/tests/aks-managed-id.tftest.hcl @@ -60,7 +60,7 @@ run "provision" { command = apply module { - source = "./src/terraform/aks-baseline" + source = "./src/terraform/aks-managed-id" } variables { @@ -71,6 +71,7 @@ run "provision" { vm_size = run.vm_size.candidate_sku aks_subnet_id = run.network.aks_subnet_id aks_api_subnet_id = run.network.api_server_subnet_id + acr_subnet_id = run.network.acr_subnet_id pod_cidr = "10.244.0.0/16" service_cidr = "10.0.0.0/16" dns_service_ip = "10.0.0.10"