From 0ae82c68f31448edbdb5e6e83b257b0a7d495648 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 10 Nov 2025 15:41:07 -0500 Subject: [PATCH 01/10] terraform/azure: Finish accelerated networking set-up Signed-off-by: Chuck Lever --- .../roles/gen_tfvars/templates/azure/terraform.tfvars.j2 | 1 + terraform/azure/main.tf | 2 +- terraform/azure/vars.tf | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 b/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 index 9c3ac0a0f..e75b85180 100644 --- a/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 +++ b/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 @@ -9,6 +9,7 @@ azure_image_sku = "{{ terraform_azure_image_sku }}" azure_managed_disks_per_instance = {{ terraform_azure_managed_disks_per_instance }} azure_managed_disks_size = {{ terraform_azure_managed_disks_size }} azure_managed_disks_tier = "{{ terraform_azure_managed_disks_tier }}" +azure_accelerated_networking_enabled = {{ terraform_azure_accelerated_networking_enabled | lower }} ssh_config_pubkey_file = "{{ kdevops_terraform_ssh_config_pubkey_file }}" ssh_config_user = "{{ kdevops_terraform_ssh_config_user }}" diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index eb609933f..683263715 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -57,7 +57,7 @@ resource "azurerm_network_interface_security_group_association" "kdevops_sg_asso resource "azurerm_network_interface" "kdevops_nic" { count = local.kdevops_num_boxes - accelerated_networking_enabled = true + accelerated_networking_enabled = var.azure_accelerated_networking_enabled name = format("kdevops_nic_%02d", count.index + 1) location = var.azure_location resource_group_name = azurerm_resource_group.kdevops_group.name diff --git a/terraform/azure/vars.tf b/terraform/azure/vars.tf index dd3c20ed0..b1aec690c 100644 --- a/terraform/azure/vars.tf +++ b/terraform/azure/vars.tf @@ -47,3 +47,8 @@ variable "azure_vmsize" { description = "VM size" type = string } + +variable "azure_accelerated_networking_enabled" { + description = "Enable accelerated networking for network interfaces" + type = bool +} From 460213137bad48f3b8f02cd6812fa6794582d3b9 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 10 Nov 2025 20:13:21 -0500 Subject: [PATCH 02/10] terraform/azure: Update the slug name for the Azure menu Match the format of the other cloud provider menu entries. Signed-off-by: Chuck Lever --- terraform/Kconfig.providers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/Kconfig.providers b/terraform/Kconfig.providers index 7813c52fa..985dd8b3b 100644 --- a/terraform/Kconfig.providers +++ b/terraform/Kconfig.providers @@ -16,7 +16,7 @@ config TERRAFORM_AWS Enabling this means you are going to use AWS for your cloud solution. config TERRAFORM_AZURE - bool "Azure" + bool "Azure - Microsoft Azure" depends on TARGET_ARCH_X86_64 select TERRAFORM_PRIVATE_NET help From bcecc8f6c945be88649c8017ae7492efa112ed68 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 1 Nov 2025 15:20:53 -0400 Subject: [PATCH 03/10] terraform/azure: Re-organize Kconfig.compute Split the VM size menu and the OS image menu into separate Kconfig files, as these two will each be generated by different scripts. Signed-off-by: Chuck Lever --- terraform/azure/Kconfig | 5 +- terraform/azure/kconfigs/Kconfig.image | 150 ++++++++++++++++++ .../{Kconfig.compute => Kconfig.size} | 57 ++++--- .../azure/kconfigs/publishers/Kconfig.debian | 60 ------- .../azure/kconfigs/publishers/Kconfig.rhel | 64 -------- 5 files changed, 185 insertions(+), 151 deletions(-) create mode 100644 terraform/azure/kconfigs/Kconfig.image rename terraform/azure/kconfigs/{Kconfig.compute => Kconfig.size} (66%) delete mode 100644 terraform/azure/kconfigs/publishers/Kconfig.debian delete mode 100644 terraform/azure/kconfigs/publishers/Kconfig.rhel diff --git a/terraform/azure/Kconfig b/terraform/azure/Kconfig index 9ab8f4c85..2326df696 100644 --- a/terraform/azure/Kconfig +++ b/terraform/azure/Kconfig @@ -4,7 +4,10 @@ menu "Resource Location" source "terraform/azure/kconfigs/Kconfig.location" endmenu menu "Compute" -source "terraform/azure/kconfigs/Kconfig.compute" +comment "Size selection" +source "terraform/azure/kconfigs/Kconfig.size" +comment "OS image selection" +source "terraform/azure/kconfigs/Kconfig.image" endmenu menu "Storage" source "terraform/azure/kconfigs/Kconfig.storage" diff --git a/terraform/azure/kconfigs/Kconfig.image b/terraform/azure/kconfigs/Kconfig.image new file mode 100644 index 000000000..a5b7fe2fe --- /dev/null +++ b/terraform/azure/kconfigs/Kconfig.image @@ -0,0 +1,150 @@ +choice + prompt "Azure image publisher" + default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + help + This option specifies the publisher of the boot image used to + create the kdevops target nodes. + +config TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + bool "Debian" + help + This option sets the boot image publisher to "Debian". + +config TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + bool "Red Hat" + help + This option sets the boot image publisher to "RedHat". + +endchoice + +config TERRAFORM_AZURE_IMAGE_PUBLISHER + string + output yaml + default "Debian" if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + default "RedHat" if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + +if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + +if TARGET_ARCH_X86_64 + +choice + prompt "Debian release" + default TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + bool "debian-10 (x86)" + help + This option sets the OS image to Debian 10 (Buster). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + bool "debian-11 (x86)" + help + This option sets the OS image to Debian 11 (Bullseye). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + bool "debian-12 (x86)" + help + This option sets the OS image to Debian 12 (Bookworm). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + bool "debian-13-daily (x86)" + help + This option sets the OS image to Debian 13's daily build. + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + bool "debian-sid-daily (x86)" + help + This option sets the OS image to the Debian unstable daily + build. + +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml + default "debian-10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + default "debian-11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + default "debian-12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + default "debian-13-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + default "debian-sid-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml + default "10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + default "11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + default "12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + default "13" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + default "sid" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + +endif # TARGET_ARCH_X86_64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + +if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + +if TARGET_ARCH_X86_64 + +choice + prompt "Red Hat Enterprise Linux release" + default TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + bool "RHEL 7.9 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 7 update 9. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + bool "RHEL 8.9 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 8 update 9. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + bool "RHEL 8.10 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 8 update 10. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + bool "RHEL 9.5 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 9 update 5. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + bool "RHEL 9.6 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 9 update 6. + +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml + default "7_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + default "8_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + default "8_10" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + default "9_5" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + default "9_6" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + +endif # TARGET_ARCH_X86_64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT diff --git a/terraform/azure/kconfigs/Kconfig.compute b/terraform/azure/kconfigs/Kconfig.size similarity index 66% rename from terraform/azure/kconfigs/Kconfig.compute rename to terraform/azure/kconfigs/Kconfig.size index 3b09891b9..758ec8ce7 100644 --- a/terraform/azure/kconfigs/Kconfig.compute +++ b/terraform/azure/kconfigs/Kconfig.size @@ -94,30 +94,35 @@ config TERRAFORM_AZURE_VM_SIZE default "Standard_D4s_v6" if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V6 default "Standard_D8s_v6" if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V6 -choice - prompt "Azure image publisher" - default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - help - This option specifies the publisher of the boot image used to - create the kdevops target nodes. - -config TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - bool "Debian" - help - This option sets the boot image publisher to "Debian". - -config TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - bool "Red Hat" - help - This option sets the boot image publisher to "RedHat". - -endchoice - -config TERRAFORM_AZURE_IMAGE_PUBLISHER - string +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool output yaml - default "Debian" if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - default "RedHat" if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - -source "terraform/azure/kconfigs/publishers/Kconfig.debian" -source "terraform/azure/kconfigs/publishers/Kconfig.rhel" + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B1S + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2S + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B12MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B20MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2TS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B32LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B32S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_DS1_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_DS3_V2 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V6 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V6 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V6 diff --git a/terraform/azure/kconfigs/publishers/Kconfig.debian b/terraform/azure/kconfigs/publishers/Kconfig.debian deleted file mode 100644 index 372a809dc..000000000 --- a/terraform/azure/kconfigs/publishers/Kconfig.debian +++ /dev/null @@ -1,60 +0,0 @@ -if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - -if TARGET_ARCH_X86_64 - -choice - prompt "Debian release" - default TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - help - This option specifies which of a publisher's offers to use - when creating kdevops compute instances. - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - bool "debian-10 (x86)" - help - This option sets the OS image to Debian 10 (Buster). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - bool "debian-11 (x86)" - help - This option sets the OS image to Debian 11 (Bullseye). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - bool "debian-12 (x86)" - help - This option sets the OS image to Debian 12 (Bookworm). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - bool "debian-13-daily (x86)" - help - This option sets the OS image to Debian 13's daily build. - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - bool "debian-sid-daily (x86)" - help - This option sets the OS image to the Debian unstable daily - build. - -endchoice - -config TERRAFORM_AZURE_IMAGE_OFFER - string - output yaml - default "debian-10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - default "debian-11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - default "debian-12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - default "debian-13-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - default "debian-sid-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - -config TERRAFORM_AZURE_IMAGE_SKU - string - output yaml - default "10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - default "11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - default "12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - default "13" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - default "sid" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - -endif # TARGET_ARCH_X86_64 - -endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN diff --git a/terraform/azure/kconfigs/publishers/Kconfig.rhel b/terraform/azure/kconfigs/publishers/Kconfig.rhel deleted file mode 100644 index 3a6a7d9a1..000000000 --- a/terraform/azure/kconfigs/publishers/Kconfig.rhel +++ /dev/null @@ -1,64 +0,0 @@ -if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - -if TARGET_ARCH_X86_64 - -choice - prompt "Red Hat Enterprise Linux release" - default TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - help - This option specifies which of a publisher's offers to use - when creating kdevops compute instances. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - bool "RHEL 7.9 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 7 update 9. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - bool "RHEL 8.9 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 8 update 9. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - bool "RHEL 8.10 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 8 update 10. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - bool "RHEL 9.5 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 9 update 5. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - bool "RHEL 9.6 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 9 update 6. - -endchoice - -config TERRAFORM_AZURE_IMAGE_OFFER - string - output yaml - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - -config TERRAFORM_AZURE_IMAGE_SKU - string - output yaml - default "7_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - default "8_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - default "8_10" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - default "9_5" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - default "9_6" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - -endif # TARGET_ARCH_X86_64 - -endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT From b9c20ee93fe3fa9b9975d77539ca705a8da20ce8 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 1 Nov 2025 13:11:43 -0400 Subject: [PATCH 04/10] terraform/azure: Generate Kconfig.location dynamically This adds a Python script that queries the Azure API to retrieve available regions and generates the Kconfig.location file dynamically. This ensures the region list is always up-to-date and includes only regions accessible to the user's subscription. The implementation follows the same pattern as AWS and OCI, providing: - Dynamic region discovery via Azure CLI/SDK - Friendly display names for each region - Automatic filtering of inaccessible regions - Support for both command-line querying and Kconfig generation Generated-by: Claude AI Signed-off-by: Chuck Lever --- .gitignore | 2 + terraform/azure/scripts/azure_common.py | 173 ++++++++++++++ terraform/azure/scripts/gen_kconfig_location | 235 +++++++++++++++++++ terraform/azure/scripts/regions.j2 | 52 ++++ 4 files changed, 462 insertions(+) create mode 100644 terraform/azure/scripts/azure_common.py create mode 100755 terraform/azure/scripts/gen_kconfig_location create mode 100644 terraform/azure/scripts/regions.j2 diff --git a/.gitignore b/.gitignore index 7472860ee..aa0aafa41 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,8 @@ terraform/aws/kconfigs/Kconfig.instance.generated terraform/aws/kconfigs/Kconfig.location.generated terraform/aws/scripts/__pycache__/ +terraform/azure/scripts/__pycache__/ + terraform/oci/kconfigs/Kconfig.image.generated terraform/oci/kconfigs/Kconfig.location.generated terraform/oci/kconfigs/Kconfig.shape.generated diff --git a/terraform/azure/scripts/azure_common.py b/terraform/azure/scripts/azure_common.py new file mode 100644 index 000000000..49aad8599 --- /dev/null +++ b/terraform/azure/scripts/azure_common.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Common utilities for Azure Kconfig generation scripts. + +This module provides shared functionality for Azure-specific Kconfig +generation, including region discovery, default region detection, and +Jinja2 template management. +""" + +import os +import sys +import json +from configparser import ConfigParser + +from jinja2 import Environment, FileSystemLoader + + +def get_default_region(): + """ + Get the default Azure region from Azure configuration. + + Returns: + str: Default region, or 'westus' if no default is found. + """ + try: + from azure.common.credentials import get_cli_profile + + # Check if user is authenticated by getting the CLI profile + # This reuses the 'az login' session without subprocess calls + profile = get_cli_profile() + _, _, _ = profile.get_login_credentials(resource="https://management.azure.com") + + # Azure doesn't have a per-account default region like AWS + # Check environment variable first + if "AZURE_DEFAULTS_LOCATION" in os.environ: + return os.environ["AZURE_DEFAULTS_LOCATION"] + + # Try to read from azure config + config_path = os.path.expanduser("~/.azure/config") + if os.path.exists(config_path): + try: + config = ConfigParser() + config.read(config_path) + if "defaults" in config and "location" in config["defaults"]: + return config["defaults"]["location"] + except Exception: + pass + + # Default fallback + return "westus" + except Exception as e: + print(f"Warning: Error reading Azure config: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return "westus" + + +def get_jinja2_environment(template_path=None): + """ + Create a standardized Jinja2 environment for template rendering. + + Args: + template_path (str): Path to template directory. If None, uses caller's directory. + + Returns: + Environment: Configured Jinja2 Environment object + """ + if template_path is None: + template_path = sys.path[0] + + return Environment( + loader=FileSystemLoader(template_path), + trim_blocks=True, + lstrip_blocks=True, + ) + + +def get_all_regions(quiet=False): + """ + Retrieve the list of all Azure regions using the Azure SDK. + + Returns: + list: List of region dictionaries with name, displayName, and metadata + """ + if not quiet: + print("Querying Azure for available regions...", file=sys.stderr) + + try: + from azure.common.credentials import get_cli_profile + from azure.mgmt.resource import SubscriptionClient + + # Get credentials from Azure CLI profile (reuses 'az login' session) + profile = get_cli_profile() + credentials, subscription_id, _ = profile.get_login_credentials( + resource="https://management.azure.com" + ) + + # Query regions using SDK + subscription_client = SubscriptionClient(credentials) + locations = subscription_client.subscriptions.list_locations(subscription_id) + + # Convert SDK objects to dict format + region_list = [] + for location in locations: + # Skip logical regions (not for general use) + if location.metadata and location.metadata.region_type == "Logical": + continue + + # Convert metadata to dict with camelCase keys + # Only convert the fields we actually use to minimize overhead + metadata = {} + if location.metadata: + meta = location.metadata + metadata = { + "physicalLocation": meta.physical_location, + "pairedRegion": meta.paired_region, + } + + region_list.append( + { + "name": location.name or "", + "displayName": location.display_name or "", + "regionalDisplayName": location.regional_display_name or "", + "metadata": metadata, + } + ) + + return sorted(region_list, key=lambda x: x["name"]) + + except Exception as e: + if not quiet: + print(f"Error: Failed to query Azure regions: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return [] + + +def get_region_kconfig_name(region_name): + """ + Convert an Azure region name to a Kconfig variable name. + + Args: + region_name (str): Azure region name (e.g., 'westus', 'eastus2') + + Returns: + str: Kconfig-safe name (e.g., 'WESTUS', 'EASTUS2') + """ + return region_name.upper().replace("-", "_") + + +def exit_on_empty_result(result, context, quiet=False): + """ + Exit with error if result is empty or None. + + This consolidates the common pattern of checking if an API query + returned results and exiting with appropriate error messaging if not. + + Args: + result: Result from API query (list, dict, or other iterable) + context (str): Description of what operation failed + quiet (bool): Suppress error messages + + Returns: + Does not return; exits the process with status 1 + """ + if not result: + if not quiet: + print( + f"Error: Cannot perform {context}. Check Azure authentication status.", + file=sys.stderr, + ) + print("Run 'az login' to authenticate with Azure.", file=sys.stderr) + sys.exit(1) diff --git a/terraform/azure/scripts/gen_kconfig_location b/terraform/azure/scripts/gen_kconfig_location new file mode 100755 index 000000000..287fb8941 --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_location @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve region information from Azure. Use it to construct the +"locations" Kconfig menu. + +This script queries the Azure API using the Azure CLI to dynamically +generate a Kconfig file containing all available Azure regions. Unlike +OCI, Azure provides rich metadata including physical locations and +display names directly through the API, so no separate mapping file +is needed. + +Usage: + # Generate complete Kconfig.location file + ./gen_kconfig_location > ../kconfigs/Kconfig.location + + # List all available regions + ./gen_kconfig_location --regions + + # Get details for a specific region + ./gen_kconfig_location westus +""" + +import sys +import argparse + +from azure_common import ( + get_default_region, + get_all_regions, + get_jinja2_environment, + get_region_kconfig_name, + exit_on_empty_result, +) + + +def get_region_friendly_name(region): + """ + Get a friendly display name for a region using Azure API metadata. + + Args: + region (dict): Region dictionary from Azure API + + Returns: + str: Friendly name suitable for display in Kconfig + """ + display_name = region.get("displayName", region["name"]) + physical_location = region.get("metadata", {}).get("physicalLocation") + + # Combine display name with physical location if available + if physical_location: + return f"{display_name} ({physical_location})" + + # Fall back to just the display name + return display_name + + +def get_region_info(regions, region_name, quiet=False): + """ + Get detailed information about a specific region. + + Args: + regions (list): List of all available regions + region_name (str): Azure region name (e.g., 'westus', 'eastus2') + quiet (bool): Suppress debug messages + + Returns: + dict: Dictionary containing region information + """ + if not quiet: + print(f"Querying information for region {region_name}...", file=sys.stderr) + + region_info = next((r for r in regions if r["name"] == region_name), None) + + if not region_info: + if not quiet: + print(f"Region {region_name} was not found", file=sys.stderr) + return None + + return region_info + + +def output_region_kconfig(region_info): + """Output region information in Kconfig format.""" + region_name = region_info["name"] + friendly_name = get_region_friendly_name(region_info) + kconfig_name = get_region_kconfig_name(region_name) + + print(f"config TERRAFORM_AZURE_REGION_{kconfig_name}") + print(f'\tbool "{friendly_name}"') + print("\thelp") + print(f"\t This option selects the {friendly_name} region.") + + # Add paired region info if available + metadata = region_info.get("metadata", {}) + paired_regions = metadata.get("pairedRegion", []) + if paired_regions: + paired_name = paired_regions[0].get("name", "") + if paired_name: + print(f"\t This region is paired with {paired_name}.") + + print() + + +def output_region_raw(region_info, quiet=False): + """Output region information in table format.""" + if not quiet: + print(f"Region: {region_info['name']}") + print(f"Display Name: {region_info['displayName']}") + print(f"Regional Name: {region_info.get('regionalDisplayName', 'N/A')}") + + metadata = region_info.get("metadata", {}) + if metadata: + print("\nMetadata:") + for key, value in metadata.items(): + if key == "pairedRegion" and value: + print(f" {key}: {value[0].get('name', 'N/A')}") + else: + print(f" {key}: {value}") + + +def output_regions_kconfig(regions): + """Output available regions in Kconfig format.""" + environment = get_jinja2_environment() + template = environment.get_template("regions.j2") + + default_region = get_default_region() + default_kconfig = get_region_kconfig_name(default_region) + + # Enhance region data with friendly names for template + enhanced_regions = [] + for region in regions: + enhanced_region = region.copy() + enhanced_region["friendly_name"] = get_region_friendly_name(region) + enhanced_regions.append(enhanced_region) + + print( + template.render( + default_region=default_kconfig, + regions=enhanced_regions, + ) + ) + + +def output_regions_raw(regions, quiet=False): + """Output available regions in table format.""" + if not quiet: + print(f"Available Azure regions ({len(regions)}):\n") + print(f"{'Region Name':<25} {'Display Name':<40}") + print("-" * 67) + + for region in regions: + print(f"{region['name']:<25} {region['displayName']:<40}") + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure region information", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python %(prog)s --regions + python %(prog)s westus + python %(prog)s eastus2 --quiet + """, + ) + parser.add_argument( + "region_name", + nargs="?", + help="Azure region name (e.g., westus, eastus2, westeurope)", + ) + + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--regions", action="store_true", help="List all available Azure regions" + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + if not args.quiet: + print("Fetching list of all Azure regions...", file=sys.stderr) + + regions = get_all_regions(args.quiet) + exit_on_empty_result(regions, "Azure region query", args.quiet) + + if args.regions: + if args.format == "kconfig": + output_regions_kconfig(regions) + else: + output_regions_raw(regions, args.quiet) + return + + if args.region_name: + if not args.quiet: + print( + f"Fetching information for region {args.region_name}...", + file=sys.stderr, + ) + + region_info = get_region_info(regions, args.region_name, args.quiet) + if region_info: + if args.format == "kconfig": + output_region_kconfig(region_info) + else: + output_region_raw(region_info, args.quiet) + else: + print( + f"Could not retrieve information for region '{args.region_name}'.", + file=sys.stderr, + ) + print( + "Try running with --regions to see available regions.", file=sys.stderr + ) + sys.exit(1) + return + + output_regions_kconfig(regions) + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/regions.j2 b/terraform/azure/scripts/regions.j2 new file mode 100644 index 000000000..cd421028d --- /dev/null +++ b/terraform/azure/scripts/regions.j2 @@ -0,0 +1,52 @@ +choice + prompt "Azure resource location" + default TERRAFORM_AZURE_REGION_{{ default_region }} + help + Choose the region and data center which will host your + kdevops resources. The list below is dynamically generated + from the Azure API and includes all regions available to + your subscription. + + Azure automatically chooses availability zones for your + resources. You can query available locations using: + + az account list-locations -o table + + For more information about Azure regions and geographies: + + https://azure.microsoft.com/en-us/explore/global-infrastructure/geographies/ + +{% for region in regions %} +config TERRAFORM_AZURE_REGION_{{ region['name'].upper().replace('-', '_') }} + bool "{{ region['friendly_name'] }}" + help + This option selects the {{ region['friendly_name'] }} region. +{% if region.get('metadata', {}).get('pairedRegion') %} + This region is paired with {{ region['metadata']['pairedRegion'][0]['name'] }}. +{% endif %} + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_LOCATION + string + output yaml +{% for region in regions %} + default "{{ region['name'] }}" if TERRAFORM_AZURE_REGION_{{ region['name'].upper().replace('-', '_') }} +{% endfor %} + +config TERRAFORM_AZURE_RESOURCE_GROUP_NAME + string "Azure resource group name" + output yaml + default "kdevops_resource_group" + help + An Azure resource group is a container that holds related + resources so they can be managed as a single unit. These + resources share the same life cycle and are deployed, + updated, and deleted together. Resource groups are global + to your subscription. + + To run concurrent kdevops jobs in Azure, each run must + have a unique resource group name. + + https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups From 1643a09a751d64b35feb449cde5618dc4a264845 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 1 Nov 2025 15:36:01 -0400 Subject: [PATCH 05/10] terraform/azure: Generate Kconfig.size dynamically Similar to commit 0e572ac21f4d ("terraform/oci: Generate Kconfig.shapes dynamically"), this patch adds a Python script that queries the Azure API to dynamically generate a Kconfig menu for VM sizes. The script generates Kconfig entries with complete hardware specifications including: - Number of vCPUs - Memory capacity - Maximum data disk count - Temporary storage size - CPU architecture (x86_64 or ARM64) The generated menu can include over 1000 VM size options depending on region availability, providing a comprehensive selection compared to the handful of sizes in the static Kconfig.size file. Usage: ./gen_kconfig_size > ../kconfigs/Kconfig.size.generated ./gen_kconfig_size --all-regions > ../kconfigs/Kconfig.size.generated Generated-by: Claude AI Signed-off-by: Chuck Lever --- terraform/azure/scripts/azure_common.py | 107 +++ terraform/azure/scripts/families.j2 | 81 +++ terraform/azure/scripts/gen_kconfig_size | 653 ++++++++++++++++++ terraform/azure/scripts/sizes.j2 | 48 ++ .../azure/scripts/vm_family_metadata.yml | 253 +++++++ 5 files changed, 1142 insertions(+) create mode 100644 terraform/azure/scripts/families.j2 create mode 100755 terraform/azure/scripts/gen_kconfig_size create mode 100644 terraform/azure/scripts/sizes.j2 create mode 100644 terraform/azure/scripts/vm_family_metadata.yml diff --git a/terraform/azure/scripts/azure_common.py b/terraform/azure/scripts/azure_common.py index 49aad8599..d41ff55b7 100644 --- a/terraform/azure/scripts/azure_common.py +++ b/terraform/azure/scripts/azure_common.py @@ -148,6 +148,113 @@ def get_region_kconfig_name(region_name): return region_name.upper().replace("-", "_") +def get_compute_client(): + """ + Get an authenticated Azure Compute Management client. + + Returns: + tuple: (ComputeManagementClient, subscription_id) + + Raises: + Exception: If authentication fails or SDK is not available + """ + from azure.common.credentials import get_cli_profile + from azure.mgmt.compute import ComputeManagementClient + + # Get credentials from Azure CLI profile (reuses 'az login' session) + profile = get_cli_profile() + credentials, subscription_id, _ = profile.get_login_credentials( + resource="https://management.azure.com" + ) + + client = ComputeManagementClient(credentials, subscription_id) + return client, subscription_id + + +def get_vm_sizes_and_skus(region, quiet=False): + """ + Get all VM sizes and capabilities for a region using a single Azure SDK call. + + This function uses the resource SKUs API which provides all the information + from both VM sizes and SKU capabilities in a single efficient API call. + + Args: + region (str): Azure region name + quiet (bool): Suppress debug messages + + Returns: + tuple: (sizes_list, capabilities_dict) where: + - sizes_list: List of VM size dictionaries in CLI-compatible format + - capabilities_dict: Dict mapping VM size names to their capabilities + """ + if not quiet: + print(f"Fetching VM sizes and capabilities from {region}...", file=sys.stderr) + + try: + client, _ = get_compute_client() + + # Query resource SKUs using SDK with location filter + # This single API call provides all information we need + skus = list(client.resource_skus.list(filter=f"location eq '{region}'")) + + # Filter to VM SKUs only + vm_skus = [s for s in skus if s.resource_type == "virtualMachines"] + + # Build both data structures + size_list = [] + sku_capabilities = {} + + for sku in vm_skus: + if not sku.capabilities: + continue + + # Convert capabilities list to dictionary for easy lookup + caps = {cap.name: cap.value for cap in sku.capabilities} + + # Extract size information from capabilities + # The resource SKUs API provides the same data as the VM sizes API + try: + cores = int(caps.get("vCPUs", 0)) + memory_mb = int(float(caps.get("MemoryGB", 0)) * 1024) + max_disks = int(caps.get("MaxDataDiskCount", 0)) + resource_disk_mb = int(caps.get("MaxResourceVolumeMB", 0)) + os_disk_mb = int(caps.get("OSVhdSizeMB", 0)) + + # Build VM size dict in CLI-compatible format + size_list.append( + { + "name": sku.name, + "numberOfCores": cores, + "memoryInMB": memory_mb, + "maxDataDiskCount": max_disks, + "resourceDiskSizeInMB": resource_disk_mb, + "osDiskSizeInMB": os_disk_mb, + } + ) + + # Store capabilities for this size + sku_capabilities[sku.name] = caps + + except (ValueError, TypeError) as e: + if not quiet: + print( + f"Warning: Could not parse capabilities for {sku.name}: {e}", + file=sys.stderr, + ) + continue + + if not quiet: + print(f" Found {len(size_list)} VM sizes in {region}", file=sys.stderr) + + return size_list, sku_capabilities + + except Exception as e: + if not quiet: + print(f"Error: Failed to query VM sizes and SKUs: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return [], {} + + def exit_on_empty_result(result, context, quiet=False): """ Exit with error if result is empty or None. diff --git a/terraform/azure/scripts/families.j2 b/terraform/azure/scripts/families.j2 new file mode 100644 index 000000000..2fbfb633c --- /dev/null +++ b/terraform/azure/scripts/families.j2 @@ -0,0 +1,81 @@ +# Azure VM Size Selection +# This file is auto-generated. Do not edit manually. +# Generated by: terraform/azure/scripts/gen_kconfig_size + +choice + prompt "VM size family" + default {{ default_family_config }} + help + Select the Azure VM size family for your target nodes. + Each family is optimized for different workloads. + + VM size availability varies by region. You can verify + available VM sizes for your region using: + + az vm list-sizes --location -o table + + For detailed information about Azure VM families: + + https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview + +{% for family in families %} +config TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + bool "{{ family.family_name }} - {{ family.description }}" +{% if family.architecture == 'arm64' %} + depends on TARGET_ARCH_ARM64 +{% else %} + depends on TARGET_ARCH_X86_64 +{% endif %} + help + {{ family.help_text | replace('\n', '\n\t ') }} + + This family contains {{ family.size_count }} VM size{{ 's' if family.size_count > 1 else '' }}: + - CPU cores: {{ family.min_cores }}-{{ family.max_cores }} + - Memory: {{ family.min_memory_gb|round(1) }}-{{ family.max_memory_gb|round(1) }}GB + +{% endfor %} +endchoice + +{% for family in families %} +if TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + +choice + prompt "{{ family.family_name }} VM size" + default {{ family.default_size_config }} + help + Select a specific VM size from the {{ family.family_name }} family. + +{% for size in family.sizes %} +config TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} + bool "{{ size.name }}" + help + This virtual machine size has {{ size.cores }} vCPU{{ 's' if size.cores > 1 else '' }} and {{ size.memory_gb|round(1) }}GB of memory. +{% if size.resource_disk_gb > 0 %} + Includes {{ size.resource_disk_gb|round(1) }}GB temporary storage. +{% endif %} + Supports up to {{ size.max_data_disks }} data disk{{ 's' if size.max_data_disks > 1 else '' }}. + +{% endfor %} +endchoice + +endif # TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + +{% endfor %} + +config TERRAFORM_AZURE_VM_SIZE + string + output yaml +{% for family in families %} +{% for size in family.sizes %} + default "{{ size.name }}" if TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} +{% endfor %} +{% endfor %} + +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool + output yaml +{% for family in families %} +{% for size in family.sizes %} + default {{ 'y' if size.accelerated_networking else 'n' }} if TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} +{% endfor %} +{% endfor %} diff --git a/terraform/azure/scripts/gen_kconfig_size b/terraform/azure/scripts/gen_kconfig_size new file mode 100755 index 000000000..bdaaa55fc --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_size @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve VM size information from Azure. Use it to construct the +"VM sizes" Kconfig menu. + +Azure VM sizes represent compute instance configurations with varying +combinations of CPU, memory, storage, and networking capacity. This script +queries the Azure API to discover available VM sizes and generates Kconfig +menu entries for them. + +VM Size Discovery: + By default, this script queries VM sizes from the default region configured + in Azure CLI. The --all-regions option aggregates VM sizes from all regions + to provide a comprehensive list. + + Note: VM size availability varies by region. Not all sizes listed may be + available in your selected region. + +Usage: + # Generate VM sizes Kconfig from default region + ./gen_kconfig_size > ../kconfigs/Kconfig.size.generated + + # Include VM sizes from all regions + ./gen_kconfig_size --all-regions > ../kconfigs/Kconfig.size.generated + + # List all available VM size families + ./gen_kconfig_size --families + + # Get details for a specific VM size family + ./gen_kconfig_size Standard_D +""" + +import sys +import argparse +import re +import os +import yaml + +from azure_common import ( + get_default_region, + get_all_regions, + get_jinja2_environment, + get_vm_sizes_and_skus, + exit_on_empty_result, +) + + +def get_all_vm_sizes_and_capabilities(regions=None, quiet=False): + """ + Get all available VM sizes and capabilities across specified regions. + + Filters out Gen1-only VM sizes to ensure compatibility with Gen2 OS images. + Gen2 images work on both Gen2-only and Gen1+Gen2 VM sizes (94% of all sizes), + while Gen1 images only work on Gen1-only and Gen1+Gen2 sizes (59% of all sizes). + This filtering maximizes VM size availability while using modern Gen2 images. + + Args: + regions (list): List of region names to query. If None, uses default region. + quiet (bool): Suppress debug messages + + Returns: + tuple: (sizes_list, capabilities_dict) where: + - sizes_list: List of VM size dictionaries (deduplicated by name) + - capabilities_dict: Dict mapping VM size names to capabilities + """ + if regions is None: + default_region = get_default_region() + regions = [default_region] + + all_sizes = {} # Use dict to deduplicate by VM size name + all_capabilities = {} # Capabilities from first region where size is found + + for region_info in regions: + region = region_info if isinstance(region_info, str) else region_info["name"] + + sizes, capabilities = get_vm_sizes_and_skus(region, quiet) + + # Add sizes to dict, using size name as key to deduplicate + # Filter out Gen1-only VM sizes to ensure compatibility with Gen2 images + for size in sizes: + if size["name"] not in all_sizes: + # Check HyperV generation support + size_caps = capabilities.get(size["name"], {}) + hyperv_gens = size_caps.get("HyperVGenerations", "") + + # Skip Gen1-only VM sizes (keep Gen2-only and Gen1+Gen2) + if hyperv_gens == "V1": + continue + + all_sizes[size["name"]] = size + # Store capabilities from first region where we see this size + if size["name"] in capabilities: + all_capabilities[size["name"]] = capabilities[size["name"]] + + if not quiet: + print( + f"\nTotal unique VM sizes across all regions: {len(all_sizes)}", + file=sys.stderr, + ) + + return list(all_sizes.values()), all_capabilities + + +def extract_vm_size_family(size_name): + """ + Extract VM size family from the VM size name. + + Azure VM size names follow patterns like: + - Standard_D2s_v3 -> Standard_D (family) + - Standard_DS3_v2 -> Standard_DS (family) + - Standard_B1ls -> Standard_B (family) + - Standard_NC6s_v3 -> Standard_NC (family) + + Args: + size_name (str): Azure VM size name + + Returns: + str: Family name + """ + # Match patterns like Standard_D, Standard_DS, Standard_B, etc. + match = re.match(r"^(Standard_[A-Z]+)", size_name) + if match: + return match.group(1) + + # Handle special cases or return the full name if pattern doesn't match + parts = size_name.split("_") + if len(parts) >= 2: + return f"{parts[0]}_{parts[1][0]}" + + return size_name + + +def parse_vm_size_families(sizes, quiet=False): + """ + Extract VM size families from the list of sizes. + + Args: + sizes (list): List of VM size dictionaries + quiet (bool): Suppress debug messages + + Returns: + dict: Dictionary with family info including count of sizes per family + """ + families = {} + seen_sizes = set() + + for size in sizes: + size_name = size["name"] + + # Skip duplicate size names + if size_name in seen_sizes: + continue + seen_sizes.add(size_name) + + # Parse VM size family from name + family = extract_vm_size_family(size_name) + + if family not in families: + families[family] = { + "family_name": family, + "size_count": 0, + "min_cores": float("inf"), + "max_cores": 0, + "min_memory_gb": float("inf"), + "max_memory_gb": 0, + } + + families[family]["size_count"] += 1 + + # Track min/max cores and memory + cores = size["numberOfCores"] + memory_gb = size["memoryInMB"] / 1024.0 + + families[family]["min_cores"] = min(families[family]["min_cores"], cores) + families[family]["max_cores"] = max(families[family]["max_cores"], cores) + families[family]["min_memory_gb"] = min( + families[family]["min_memory_gb"], memory_gb + ) + families[family]["max_memory_gb"] = max( + families[family]["max_memory_gb"], memory_gb + ) + + return families + + +def get_vm_size_family_info(family_name, sizes, quiet=False): + """ + Get VM size information for a specific family. + + Args: + family_name (str): VM size family name (e.g., 'Standard_D', 'Standard_DS') + sizes (list): List of all VM sizes + quiet (bool): Suppress debug messages + + Returns: + list: List of dictionaries containing VM size information + """ + family_sizes = [] + + for size in sizes: + size_name = size["name"] + size_family = extract_vm_size_family(size_name) + + # Match sizes that belong to this family + if size_family == family_name: + family_sizes.append(size) + + if not family_sizes: + if not quiet: + print(f"No VM sizes found in family '{family_name}'.", file=sys.stderr) + return [] + + if not quiet: + print( + f"Found {len(family_sizes)} VM sizes in family '{family_name}'", + file=sys.stderr, + ) + + # Extract detailed information + size_info = [] + for size in family_sizes: + size_info.append( + { + "name": size["name"], + "cores": size["numberOfCores"], + "memory_gb": size["memoryInMB"] / 1024.0, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0, + } + ) + + # Sort by cores, then memory + size_info.sort(key=lambda x: (x["cores"], x["memory_gb"])) + + return size_info + + +def determine_architecture(size_name): + """ + Determine CPU architecture from VM size name. + + Most Azure VM sizes are x86_64. ARM64-based sizes are in the + Dps and Dpds families (Ampere Altra processors). + + Args: + size_name (str): Azure VM size name + + Returns: + str: CPU architecture ('arm64' or 'x86_64') + """ + # ARM64 VM families + if re.match(r"Standard_Dp[sd]", size_name): + return "arm64" + + # Default to x86_64 + return "x86_64" + + +def natural_sort_key(vm_size_name): + """ + Generate a sort key for natural (numeric) ordering of VM size names. + + Converts VM size names like Standard_DS11_v2 into a tuple that sorts + naturally: Standard_DS1, DS2, DS3, ..., DS10, DS11, DS12 instead of + the alphabetic ordering DS1, DS11, DS12, DS2, DS3. + + Args: + vm_size_name (str): Azure VM size name (e.g., "Standard_DS3_v2") + + Returns: + tuple: Sort key tuple with string and integer components + """ + parts = re.split(r'(\d+)', vm_size_name) + key = [] + for part in parts: + if part.isdigit(): + key.append(int(part)) + else: + key.append(part) + return tuple(key) + + +def load_family_metadata(quiet=False): + """ + Load VM family metadata from YAML file. + + Returns: + dict: Dictionary mapping family prefixes to metadata + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + metadata_file = os.path.join(script_dir, "vm_family_metadata.yml") + + try: + with open(metadata_file, "r") as f: + metadata = yaml.safe_load(f) + if not quiet: + print( + f"Loaded metadata for {len(metadata)} VM families", file=sys.stderr + ) + return metadata + except FileNotFoundError: + if not quiet: + print( + f"Warning: Family metadata file not found: {metadata_file}", + file=sys.stderr, + ) + return {} + except Exception as e: + if not quiet: + print(f"Warning: Error loading family metadata: {e}", file=sys.stderr) + return {} + + +def output_sizes_kconfig(sizes, sku_capabilities=None, default_size="Standard_DS3_v2"): + """ + Output VM sizes menu in Kconfig format. + + Args: + sizes (list): List of VM size dictionaries + sku_capabilities (dict): Dictionary mapping VM size names to capabilities + default_size (str): Default VM size to select + """ + if sku_capabilities is None: + sku_capabilities = {} + + environment = get_jinja2_environment() + template = environment.get_template("sizes.j2") + + # Enhance size data for template + enhanced_sizes = [] + for size in sizes: + memory_gb = size["memoryInMB"] / 1024.0 + resource_disk_gb = ( + size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0 + ) + arch = determine_architecture(size["name"]) + + # Get accelerated networking capability + size_name = size["name"] + caps = sku_capabilities.get(size_name, {}) + accel_networking = caps.get("AcceleratedNetworkingEnabled", "False") == "True" + + enhanced_sizes.append( + { + "name": size_name, + "cores": size["numberOfCores"], + "memory_gb": memory_gb, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": resource_disk_gb, + "architecture": arch, + "accelerated_networking": accel_networking, + } + ) + + # Sort sizes by family, then cores, then memory + enhanced_sizes.sort( + key=lambda x: (extract_vm_size_family(x["name"]), x["cores"], x["memory_gb"]) + ) + + # Find default size config name + default_config = f"TERRAFORM_AZURE_VM_SIZE_{default_size.upper().replace('.', '_').replace('-', '_')}" + + print( + template.render( + sizes=enhanced_sizes, + default_config=default_config, + ) + ) + + +def output_families_kconfig(sizes, sku_capabilities=None, default_size="Standard_DS3_v2", quiet=False): + """ + Output VM sizes in family-based hierarchical Kconfig format. + + Args: + sizes (list): List of VM size dictionaries + sku_capabilities (dict): Dictionary mapping VM size names to capabilities + default_size (str): Default VM size to select + quiet (bool): Suppress informational messages + """ + if sku_capabilities is None: + sku_capabilities = {} + + # Load family metadata from YAML + family_metadata = load_family_metadata(quiet) + + # Group sizes by family + family_groups = {} + for size in sizes: + size_name = size["name"] + family_prefix = extract_vm_size_family(size_name) + + if family_prefix not in family_groups: + family_groups[family_prefix] = [] + + family_groups[family_prefix].append(size) + + # Build family data for template + families = [] + default_family = None + + for family_prefix in sorted(family_groups.keys()): + family_sizes = family_groups[family_prefix] + + # Get metadata from YAML or use defaults + metadata = family_metadata.get(family_prefix, {}) + description = metadata.get("description", "Azure VM family") + help_text = metadata.get("help_text", f"Virtual machines in the {family_prefix} family.") + + # Calculate family statistics + min_cores = min(s["numberOfCores"] for s in family_sizes) + max_cores = max(s["numberOfCores"] for s in family_sizes) + min_memory_gb = min(s["memoryInMB"] / 1024.0 for s in family_sizes) + max_memory_gb = max(s["memoryInMB"] / 1024.0 for s in family_sizes) + + # Determine family architecture + family_arch = determine_architecture(family_sizes[0]["name"]) + + # Build enhanced size list for this family + enhanced_sizes = [] + for size in sorted(family_sizes, key=lambda x: natural_sort_key(x["name"])): + size_name = size["name"] + memory_gb = size["memoryInMB"] / 1024.0 + resource_disk_gb = ( + size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0 + ) + + # Get accelerated networking capability + caps = sku_capabilities.get(size_name, {}) + accel_networking = caps.get("AcceleratedNetworkingEnabled", "False") == "True" + + config_name = size_name.upper().replace(".", "_").replace("-", "_") + enhanced_sizes.append( + { + "name": size_name, + "config_name": config_name, + "cores": size["numberOfCores"], + "memory_gb": memory_gb, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": resource_disk_gb, + "accelerated_networking": accel_networking, + } + ) + + # Check if default size is in this family + is_default_family = any(s["name"] == default_size for s in family_sizes) + if is_default_family: + default_family = family_prefix + + # Find default size config within family + default_size_config = None + for enhanced_size in enhanced_sizes: + if enhanced_size["name"] == default_size: + default_size_config = f"TERRAFORM_AZURE_VM_SIZE_{enhanced_size['config_name']}" + break + if default_size_config is None: + default_size_config = f"TERRAFORM_AZURE_VM_SIZE_{enhanced_sizes[0]['config_name']}" + + config_name = family_prefix.upper().replace(".", "_").replace("-", "_") + families.append( + { + "family_name": family_prefix, + "config_name": config_name, + "description": description, + "help_text": help_text, + "architecture": family_arch, + "size_count": len(family_sizes), + "min_cores": min_cores, + "max_cores": max_cores, + "min_memory_gb": min_memory_gb, + "max_memory_gb": max_memory_gb, + "sizes": enhanced_sizes, + "default_size_config": default_size_config, + } + ) + + # Set default family config + if default_family: + default_family_config = f"TERRAFORM_AZURE_VM_FAMILY_{default_family.upper().replace('.', '_').replace('-', '_')}" + else: + default_family_config = f"TERRAFORM_AZURE_VM_FAMILY_{families[0]['config_name']}" + + # Render template + environment = get_jinja2_environment() + template = environment.get_template("families.j2") + + print( + template.render( + families=families, + default_family_config=default_family_config, + ) + ) + + +def output_families_raw(sizes, quiet=False): + """Output available VM size families in table format.""" + families = parse_vm_size_families(sizes) + + if not quiet: + print(f"Available VM size families ({len(families)}):\n") + + print(f"{'Family':<20} {'Count':<6} {'Cores Range':<15} {'Memory Range (GB)':<20}") + print("-" * 65) + + sorted_families = sorted(families.values(), key=lambda x: x["family_name"]) + for family in sorted_families: + cores_range = f"{family['min_cores']}-{family['max_cores']}" + memory_range = f"{family['min_memory_gb']:.1f}-{family['max_memory_gb']:.1f}" + + print( + f"{family['family_name']:<20} " + f"{family['size_count']:<6} " + f"{cores_range:<15} " + f"{memory_range:<20}" + ) + + +def output_family_raw(sizes, quiet=False): + """Output VM size family information in table format.""" + if not quiet: + print(f"Found {len(sizes)} VM sizes:\n") + + print( + f"{'VM Size Name':<30} {'Cores':<8} {'Memory (GB)':<15} {'Max Disks':<12} {'Temp Storage (GB)':<18}" + ) + print("-" * 90) + + for size in sizes: + print( + f"{size['name']:<30} " + f"{size['cores']:<8} " + f"{size['memory_gb']:<15.1f} " + f"{size['max_data_disks']:<12} " + f"{size['resource_disk_gb']:<18.1f}" + ) + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure VM size information including hardware specs", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate Kconfig from default region + python %(prog)s > ../kconfigs/Kconfig.size.generated + + # Include sizes from all regions + python %(prog)s --all-regions > ../kconfigs/Kconfig.size.generated + + # Query specific VM size family + python %(prog)s Standard_D + + # List all available families + python %(prog)s --families --format raw + + # Query specific region only + python %(prog)s --region westus2 --families + """, + ) + parser.add_argument( + "family_name", + nargs="?", + help="VM size family name (e.g., Standard_D, Standard_DS, Standard_B)", + ) + + parser.add_argument( + "--families", action="store_true", help="List all available VM size families" + ) + parser.add_argument( + "--all-regions", + action="store_true", + help="Include VM sizes from all regions (default: default region only)", + ) + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--region", + "-r", + help="Query specific region only (default: default configured region)", + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + # Determine which regions to query + if args.region: + # Query specific region only + regions = [args.region] + elif args.all_regions: + # Query all regions + regions = get_all_regions(args.quiet) + exit_on_empty_result(regions, "Azure region query", args.quiet) + else: + # Query default region only + regions = [get_default_region()] + + # Get VM sizes and capabilities in a single API call + sizes, sku_capabilities = get_all_vm_sizes_and_capabilities(regions, args.quiet) + exit_on_empty_result(sizes, "Azure VM size query", args.quiet) + + if args.families: + output_families_raw(sizes, args.quiet) + return + + if args.family_name: + if not args.quiet: + print( + f"Fetching information for the {args.family_name} family...", + file=sys.stderr, + ) + + family_sizes = get_vm_size_family_info(args.family_name, sizes, args.quiet) + + if not family_sizes: + print( + f"No VM sizes found for family '{args.family_name}'.", file=sys.stderr + ) + print( + "Try running with --families to see available VM size families.", + file=sys.stderr, + ) + sys.exit(1) + + # Output raw format for family queries + output_family_raw(family_sizes, args.quiet) + return + + # Output family-based hierarchical Kconfig menu + output_families_kconfig(sizes, sku_capabilities, quiet=args.quiet) + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/sizes.j2 b/terraform/azure/scripts/sizes.j2 new file mode 100644 index 000000000..b9eb51446 --- /dev/null +++ b/terraform/azure/scripts/sizes.j2 @@ -0,0 +1,48 @@ +choice + prompt "Azure VM size" + default {{ default_config }} + help + This option chooses the size of the virtual machine instances + to be created for the kdevops target nodes. + + VM size availability varies by region. Not all sizes listed + may be available in your selected region. You can query + available VM sizes for a specific region using: + + az vm list-sizes --location -o table + + For more information about Azure VM sizes and families: + + https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview + +{% for size in sizes %} +config TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} + bool "{{ size['name'] }}" +{% if size['architecture'] == 'arm64' %} + depends on TARGET_ARCH_ARM64 +{% else %} + depends on TARGET_ARCH_X86_64 +{% endif %} + help + This virtual machine size has {{ size['cores'] }} vCPU{{ 's' if size['cores'] > 1 else '' }} and {{ size['memory_gb']|round(1) }}GB of memory. +{% if size['resource_disk_gb'] > 0 %} + Includes {{ size['resource_disk_gb']|round(1) }}GB temporary storage. +{% endif %} + Supports up to {{ size['max_data_disks'] }} data disk{{ 's' if size['max_data_disks'] > 1 else '' }}. + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_VM_SIZE + string + output yaml +{% for size in sizes %} + default "{{ size['name'] }}" if TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} +{% endfor %} + +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool + output yaml +{% for size in sizes %} + default {{ 'y' if size['accelerated_networking'] else 'n' }} if TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} +{% endfor %} diff --git a/terraform/azure/scripts/vm_family_metadata.yml b/terraform/azure/scripts/vm_family_metadata.yml new file mode 100644 index 000000000..613208146 --- /dev/null +++ b/terraform/azure/scripts/vm_family_metadata.yml @@ -0,0 +1,253 @@ +# Azure VM Family Metadata +# +# This file contains human-readable descriptions and metadata for Azure VM +# size families to support hierarchical Kconfig menu generation. +# +# Purpose: +# The gen_kconfig_size script uses this metadata to generate a two-layer +# Kconfig menu: first select a VM family, then select a specific size +# within that family. +# +# Canonical Sources: +# https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview +# (Official Azure VM sizes documentation) +# +# Last Updated: 2025-11-10 +# +# Format: +# family-prefix: +# description: "Short one-line description" +# help_text: "Detailed multi-line help text describing use cases" +# workloads: +# - "Example workload 1" +# - "Example workload 2" +# +# Family Naming Convention: +# Azure VM families are identified by the prefix pattern in VM size names. +# For example: +# - Standard_D2s_v3 → Standard_D family +# - Standard_E16_v4 → Standard_E family +# - Standard_F8s_v2 → Standard_F family +# +# How to update this file: +# ----------------------- +# When Azure introduces new VM families or you need to update descriptions: +# +# 1. Check which families are discovered by the API: +# $ cd terraform/azure/scripts +# $ ./gen_kconfig_size --families +# +# 2. Verify family details in Azure documentation: +# https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview +# +# 3. Add or update the family entry following the format above. +# +# 4. Update the "Last Updated" date above. +# +# 5. Test that families render correctly: +# $ ./gen_kconfig_size > /tmp/Kconfig.size.test 2>&1 +# $ python3 ../../../scripts/detect_indentation_issues.py /tmp/Kconfig.size.test +# +# 6. Regenerate Kconfig.size: +# $ ./gen_kconfig_size > ../kconfigs/Kconfig.size.generated +# + +# General Purpose Families +Standard_A: + description: "Basic compute, entry-level" + help_text: | + Entry-level VMs for development and test workloads with basic CPU + performance and economical pricing. Suitable for learning, development, + and non-production environments. + workloads: + - "Development and testing" + - "Low-traffic web servers" + - "Small databases" + +Standard_B: + description: "Burstable, cost-effective" + help_text: | + Burstable CPU performance for workloads that don't need continuous full + CPU. Accumulates CPU credits during idle periods and uses them during + bursts. Ideal for workloads with variable CPU usage patterns. + workloads: + - "Development and test environments" + - "Low-traffic web servers" + - "Small databases" + - "Microservices" + +Standard_D: + description: "General purpose, balanced" + help_text: | + Balanced CPU-to-memory ratio suitable for most production workloads. + Provides consistent performance with local SSD temporary storage. + Most versatile family for general compute needs. + workloads: + - "Web servers and application servers" + - "Medium-traffic websites" + - "Small to medium databases" + - "Cache servers" + +Standard_DS: + description: "General purpose with premium storage" + help_text: | + D-series with premium SSD storage support. Balanced compute and storage + performance for production workloads requiring fast, low-latency storage. + workloads: + - "Production web applications" + - "Application servers" + - "Small to medium databases with SSD requirements" + +# Memory Optimized Families +Standard_E: + description: "Memory optimized, high RAM" + help_text: | + High memory-to-CPU ratio optimized for memory-intensive workloads. + Provides 8GB RAM per vCPU, ideal for applications that require large + amounts of memory relative to CPU. + workloads: + - "Relational databases (SQL Server, MySQL, PostgreSQL)" + - "In-memory analytics" + - "Large caches (Redis, Memcached)" + - "SAP applications" + +Standard_ES: + description: "Memory optimized with premium storage" + help_text: | + E-series with premium SSD storage support. High memory and fast storage + for databases and memory-intensive applications requiring low-latency disk. + workloads: + - "Large relational databases" + - "In-memory databases with persistent storage" + - "Data warehousing" + +Standard_M: + description: "Memory optimized, very large" + help_text: | + Largest memory VMs in Azure with up to 4TB of RAM. Designed for extremely + memory-intensive enterprise workloads. Supports premium storage and + write accelerator for optimal database performance. + workloads: + - "SAP HANA" + - "Very large SQL Server instances" + - "In-memory databases" + - "Large-scale analytics" + +# Compute Optimized Families +Standard_F: + description: "Compute optimized, high CPU" + help_text: | + High CPU-to-memory ratio optimized for compute-intensive workloads. + Provides 2GB RAM per vCPU, ideal for CPU-bound applications. + workloads: + - "Batch processing" + - "Web servers with high traffic" + - "Analytics and data processing" + - "Gaming servers" + - "Machine learning inference" + +Standard_FS: + description: "Compute optimized with premium storage" + help_text: | + F-series with premium SSD storage support. High CPU performance combined + with fast storage for compute and I/O intensive workloads. + workloads: + - "High-performance batch processing" + - "Analytics with large datasets" + - "Gaming servers with fast storage" + +# Storage Optimized Families +Standard_L: + description: "Storage optimized, high I/O" + help_text: | + High disk throughput and I/O optimized for storage-intensive workloads. + Features large local NVMe disks with very high IOPS and throughput. + workloads: + - "NoSQL databases (Cassandra, MongoDB, Couchbase)" + - "Data warehousing" + - "Large transactional databases" + - "Distributed file systems" + +Standard_LS: + description: "Storage optimized with premium storage" + help_text: | + L-series with premium SSD storage support. Maximum storage throughput + for data-intensive applications requiring both local and remote storage. + workloads: + - "Large-scale NoSQL databases" + - "Big data analytics" + - "Data lake applications" + +# GPU Families +Standard_N: + description: "GPU-enabled compute" + help_text: | + GPU-enabled VMs with NVIDIA GPUs for AI/ML training and inference, + graphics rendering, video processing, and other GPU-accelerated workloads. + workloads: + - "Deep learning training" + - "Machine learning inference" + - "Graphics rendering" + - "Video encoding/transcoding" + - "Scientific simulations" + +Standard_NC: + description: "GPU compute, NVIDIA" + help_text: | + NVIDIA GPU VMs for AI/ML workloads and GPU-accelerated compute. + Optimized for deep learning training with high GPU memory bandwidth. + workloads: + - "Deep learning model training" + - "AI/ML workloads" + - "GPU-accelerated HPC" + +Standard_ND: + description: "GPU deep learning, NVIDIA" + help_text: | + NVIDIA GPU VMs optimized for deep learning training with very high + GPU-to-GPU interconnect bandwidth (NVLink). Ideal for distributed + training of large models. + workloads: + - "Large-scale distributed deep learning" + - "Training large language models" + - "Multi-GPU AI/ML workloads" + +Standard_NV: + description: "GPU visualization, NVIDIA" + help_text: | + NVIDIA GPU VMs for remote visualization, streaming, gaming, and + OpenGL/DirectX applications. Optimized for graphics workloads. + workloads: + - "Remote visualization" + - "CAD/CAM applications" + - "3D rendering" + - "Cloud gaming" + - "Video streaming" + +# High Performance Compute +Standard_H: + description: "HPC, high performance" + help_text: | + High Performance Computing VMs with high CPU performance and optional + RDMA networking for MPI workloads. Designed for compute-intensive HPC + and scientific computing. + workloads: + - "Scientific simulations" + - "Computational fluid dynamics" + - "Weather modeling" + - "Molecular dynamics" + - "MPI-based parallel computing" + +# ARM64 Families +Standard_Dp: + description: "ARM64 general purpose (Ampere Altra)" + help_text: | + ARM64-based VMs using Ampere Altra processors. General purpose workloads + with excellent performance-per-dollar ratio and energy efficiency. + Compatible with ARM64 Linux distributions. + workloads: + - "Cloud-native applications" + - "Web servers" + - "Application servers" + - "Containerized workloads" + - "Microservices" From 3f2ff6381af032b2d3f8e9bd1072ba9a5e13900b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 1 Nov 2025 16:50:29 -0400 Subject: [PATCH 06/10] terraform/azure: Generate Kconfig.image dynamically Similar to commit dab6b487bfae for OCI, add a Python script to dynamically generate Azure VM image Kconfig configuration by querying the Azure API for available publishers, offers, and SKUs. The gen_kconfig_image script queries Azure for available VM images and generates Kconfig menu entries for them. Publisher definitions are maintained in publisher_definitions.yml to make it easy to add new distributions or customize display names and priorities. The script: - Queries Azure API for available publishers, offers, and SKUs - Generates separate menus for x86_64 and ARM64 architectures - Filters out test/preview/experimental offers - Supports both well-known publishers (Debian, Red Hat, Ubuntu) and marketplace publishers (AlmaLinux, Rocky Linux) - Provides --publishers flag to list available publishers - Supports querying specific regions or publishers Usage: cd terraform/azure/scripts && ./gen_kconfig_image --quiet Generated-by: Claude AI Signed-off-by: Chuck Lever --- terraform/azure/scripts/azure_common.py | 93 +++ terraform/azure/scripts/gen_kconfig_image | 671 ++++++++++++++++++ .../azure/scripts/image_distributions.j2 | 35 + terraform/azure/scripts/image_publisher.j2 | 110 +++ .../azure/scripts/publisher_definitions.yml | 96 +++ 5 files changed, 1005 insertions(+) create mode 100755 terraform/azure/scripts/gen_kconfig_image create mode 100644 terraform/azure/scripts/image_distributions.j2 create mode 100644 terraform/azure/scripts/image_publisher.j2 create mode 100644 terraform/azure/scripts/publisher_definitions.yml diff --git a/terraform/azure/scripts/azure_common.py b/terraform/azure/scripts/azure_common.py index d41ff55b7..de9abb915 100644 --- a/terraform/azure/scripts/azure_common.py +++ b/terraform/azure/scripts/azure_common.py @@ -255,6 +255,99 @@ def get_vm_sizes_and_skus(region, quiet=False): return [], {} +def get_all_offers_and_skus(publisher_id, region, quiet=False, max_workers=10): + """ + Get all offers and SKUs for a publisher using Azure SDK with parallel execution. + + This function uses parallel SKU fetching for optimal performance with + publishers that have many offers. The parallelization is safe because + each SKU query is independent and the Azure API supports concurrent requests. + + Args: + publisher_id (str): Azure publisher identifier (e.g., "Debian", "RedHat") + region (str): Azure region name + quiet (bool): Suppress debug messages + max_workers (int): Maximum concurrent workers for parallel SKU fetching + + Returns: + dict: Dictionary mapping offer names to lists of SKU names + Example: {"debian-12": ["12", "12-arm64"], ...} + """ + from concurrent.futures import ThreadPoolExecutor + + if not quiet: + print(f"Querying offers for {publisher_id} in {region}...", file=sys.stderr) + + try: + client, _ = get_compute_client() + + # Get all offers (single fast API call) + offers = list(client.virtual_machine_images.list_offers(region, publisher_id)) + + if not offers: + return {} + + if not quiet: + print( + f" Found {len(offers)} offers, fetching SKUs in parallel...", + file=sys.stderr, + ) + + def fetch_skus_for_offer(offer): + """Helper function to fetch SKUs for a single offer.""" + offer_name = offer.name + + # Skip test offers and other non-production offers + offer_lower = offer_name.lower() + if any( + skip in offer_lower + for skip in ["test", "preview", "experimental", "-dev", "staging"] + ): + return (offer_name, []) + + try: + skus = list( + client.virtual_machine_images.list_skus( + region, publisher_id, offer_name + ) + ) + sku_names = [sku.name for sku in skus if sku.name] + return (offer_name, sku_names) + except Exception as e: + if not quiet: + print( + f" Warning: Failed to get SKUs for {offer_name}: {e}", + file=sys.stderr, + ) + return (offer_name, []) + + # Fetch SKUs for all offers in parallel + offers_dict = {} + with ThreadPoolExecutor(max_workers=max_workers) as executor: + results = executor.map(fetch_skus_for_offer, offers) + + for offer_name, sku_names in results: + if sku_names: + offers_dict[offer_name] = sku_names + + if not quiet: + print( + f" Found {len(offers_dict)} offers with available SKUs", + file=sys.stderr, + ) + + return offers_dict + + except Exception as e: + if not quiet: + print( + f"Error: Failed to query offers and SKUs for {publisher_id}: {e}", + file=sys.stderr, + ) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return {} + + def exit_on_empty_result(result, context, quiet=False): """ Exit with error if result is empty or None. diff --git a/terraform/azure/scripts/gen_kconfig_image b/terraform/azure/scripts/gen_kconfig_image new file mode 100755 index 000000000..d8b39a72f --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_image @@ -0,0 +1,671 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve VM image information from Azure. Use it to construct the "images" +Kconfig menu. + +Azure VM images are OS templates that can be used to launch compute instances. +This script queries the Azure API to discover available images and generates +Kconfig menu entries for them. + +Publisher Definitions: + Publisher definitions (e.g., "Debian", "Red Hat") are maintained + in the publisher_definitions.yml file in the same directory as this + script. This makes it easier to update publisher information when + Microsoft adds new distributions or when you want to customize display + names and priorities. + + To update publisher definitions: + 1. Edit terraform/azure/scripts/publisher_definitions.yml + 2. Add or modify entries following the existing format + 3. Run this script to regenerate Kconfig.image + + The YAML file contains detailed instructions for how to add new + publisher definitions. If the YAML file is not found, the script + falls back to dynamic publisher discovery by analyzing image names. + +Image Discovery: + By default, this script aggregates images from all Azure regions to + provide a comprehensive mapping of image URNs across regions. + + This script focuses on well-known Linux distribution publishers. + The automated discovery identifies available offers and SKUs for + each publisher. + +Usage: + # Generate images Kconfig from default region + ./gen_kconfig_image > ../kconfigs/Kconfig.image.generated + + # List all available publishers + ./gen_kconfig_image --publishers + + # Get details for a specific publisher + ./gen_kconfig_image debian + + # Query specific region + ./gen_kconfig_image --region eastus +""" + +import sys +import os +import argparse +import re +from collections import defaultdict +from functools import lru_cache + +from azure_common import ( + get_default_region, + get_jinja2_environment, + get_all_regions, + get_region_kconfig_name, + get_all_offers_and_skus, +) + + +def load_yaml_config(filename, quiet=False): + """ + Load YAML configuration file from the same directory as this script. + + Args: + filename (str): Name of the YAML file to load + quiet (bool): Suppress warning messages + + Returns: + dict: Parsed YAML content, or None if file not found or parse error + """ + try: + import yaml + except ImportError: + if not quiet: + print( + "Warning: PyYAML not installed. Install with: pip install pyyaml", + file=sys.stderr, + ) + return None + + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, filename) + + if not os.path.exists(config_path): + if not quiet: + print(f"Warning: {filename} not found at {config_path}", file=sys.stderr) + return None + + try: + with open(config_path, "r") as f: + return yaml.safe_load(f) + except yaml.YAMLError as e: + if not quiet: + print(f"Error parsing {filename}: {e}", file=sys.stderr) + return None + except Exception as e: + if not quiet: + print(f"Error reading {filename}: {e}", file=sys.stderr) + return None + + +@lru_cache(maxsize=1) +def get_known_publishers(): + """ + Get dictionary of known OS image publishers in Azure. + + Publisher definitions are loaded from publisher_definitions.yml to make + it easier to update when Microsoft adds new Linux distributions or when + you want to customize publisher priorities and naming. + + The YAML file contains publisher information including: + - publisher_name: Display name for the publisher + - description: Full description used in Kconfig help text + - offer_patterns: List of regex patterns to match offer names + - priority: Display order (lower numbers appear first) + + If the YAML file is not found or fails to parse, a minimal set of + hardcoded publishers is returned. + + Results are cached using @lru_cache to avoid repeated file I/O. + + To update the publisher definitions: + 1. Edit terraform/azure/scripts/publisher_definitions.yml + 2. Add or modify entries following the existing format + 3. Regenerate Kconfig.image by running this script + + See publisher_definitions.yml for detailed update instructions. + + Returns: + dict: Dictionary mapping publisher keys to publisher information + """ + publishers = load_yaml_config("publisher_definitions.yml", quiet=True) + if not publishers: + print( + "Warning: publisher_definitions.yml not found or empty. " + "Using fallback publisher list.", + file=sys.stderr, + ) + return get_fallback_publishers() + + return publishers + + +def get_fallback_publishers(): + """ + Get minimal fallback set of known Azure publishers. + + This is used when publisher_definitions.yml is not available. + + Returns: + dict: Dictionary of publisher information + """ + return { + "debian": { + "publisher_id": "Debian", + "publisher_name": "Debian", + "description": "Debian GNU/Linux", + "priority": 10, + }, + "redhat": { + "publisher_id": "RedHat", + "publisher_name": "Red Hat", + "description": "Red Hat Enterprise Linux", + "priority": 20, + }, + } + + +def classify_offer_sku( + publisher_key, publisher_id, offer_name, sku_name, publisher_info +): + """ + Classify an offer/SKU combination into a version and architecture. + + Args: + publisher_key (str): Publisher key (e.g., "debian", "redhat") + publisher_id (str): Azure publisher ID (e.g., "Debian", "RedHat") + offer_name (str): Offer name (e.g., "debian-12", "RHEL") + sku_name (str): SKU name (e.g., "12", "12-arm64", "9_6") + publisher_info (dict): Publisher information from definitions + + Returns: + tuple: (version_key, friendly_name, architecture, offer, sku) + or (None, None, None, None, None) if not classifiable + """ + # Detect architecture from SKU name + sku_lower = sku_name.lower() + if "arm64" in sku_lower or "aarch64" in sku_lower: + arch = "arm64" + arch_suffix = "_ARM64" + arch_display = "(arm64)" + else: + arch = "x86_64" + arch_suffix = "_X86" + arch_display = "(x86)" + + # Extract version information + # Try to extract version from offer or SKU name + version_match = None + + # Pattern 1: debian-12, debian-11 + if re.match(r"^debian-\d+", offer_name, re.IGNORECASE): + version_match = re.search(r"(\d+)", offer_name) + # Pattern 2: debian-sid (Debian unstable) + elif publisher_key == "debian" and "sid" in offer_name.lower(): + # Debian sid is the unstable rolling release + version_key = f"DEBIAN_SID{arch_suffix}" + friendly_name = f"Debian sid unstable {arch_display}" + if "daily" in offer_name.lower() or "daily" in sku_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 3: RHEL with SKU like "9_6", "8_10" + elif publisher_key == "redhat" and re.match(r"^\d+_\d+$", sku_name): + version_match = re.match(r"^(\d+)_(\d+)$", sku_name) + if version_match: + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"RHEL {major}.{minor} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 3b: RHEL Gen2 with condensed version numbers + # Examples: "74-gen2" → 7.4, "810-gen2" → 8.10, "100-gen2" → 10.0 + # Also handles SAP variants: "74sap-gen2" → 7.4, "82sapapps-gen2" → 8.2, "82sapha-gen2" → 8.2 + # Handles CI variants: "76-ci-gen2" → 7.6, "81-ci-gen2" → 8.1 + # Excludes SKUs with intermediate separators like "10-lvm-gen2" + elif publisher_key == "redhat" and re.match(r"^\d{2,3}(sap(-gen2)?|sapapps(-gen2)?|sapha(-gen2)?|_gen2|-gen2|gen2|-ci-gen2|-ci)$", sku_name): + version_digits = re.match(r"^(\d{2,3})", sku_name).group(1) + if len(version_digits) == 2: + # Two digits: split as X.Y (e.g., "74" → "7.4") + major = version_digits[0] + minor = version_digits[1] + elif version_digits.startswith("10"): + # Three digits starting with "10": RHEL 10.0 + major = "10" + minor = "0" + else: + # Three digits: first digit is major, rest is minor (e.g., "810" → "8.10") + major = version_digits[0] + minor = version_digits[1:] + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"RHEL {major}.{minor} {arch_display} Gen2" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 3c: Oracle Linux with condensed version numbers + # Examples: "ol82-gen2" → 8.2, "77" → 7.7, "ol810-lvm" → 8.10, "79-gen2" → 7.9 + # Handles with/without "ol" prefix, Gen2, LVM, and CI variants + elif publisher_key == "oracle" and re.match(r"^(ol)?\d{2,3}(-gen2|-lvm|-lvm-gen2|-ci-gen2|-arm64-lvm-gen2)?$", sku_name): + version_match = re.match(r"^(ol)?(\d{2,3})", sku_name) + version_digits = version_match.group(2) + if len(version_digits) == 2: + # Two digits: split as X.Y (e.g., "77" → "7.7", "ol82" → "8.2") + major = version_digits[0] + minor = version_digits[1] + elif version_digits.startswith("10"): + # Three digits starting with "10": Oracle Linux 10.0 + major = "10" + minor = "0" + else: + # Three digits: first digit is major, rest is minor (e.g., "810" → "8.10") + major = version_digits[0] + minor = version_digits[1:] + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"Oracle Linux {major}.{minor} {arch_display}" + if "-gen2" in sku_name.lower(): + friendly_name += " Gen2" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 4: Ubuntu LTS from offer name + # Examples: "ubuntu-22_04-lts" → 22.04, "ubuntu-24_04-lts-daily" → 24.04 + # Skip ubuntu-pro SKUs as these are commercial support variants + elif publisher_key == "canonical" and re.match(r"^ubuntu-(\d+)_(\d+)-lts", offer_name): + if "ubuntu-pro" in sku_name.lower(): + # Skip Ubuntu Pro SKUs - these are commercial variants + return (None, None, None, None, None) + version_match = re.match(r"^ubuntu-(\d+)_(\d+)-lts", offer_name) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"Ubuntu Linux {major}.{minor} {arch_display}" + if "daily" in offer_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 5: Ubuntu non-LTS from offer name + # Examples: "ubuntu-24_10" → 24.10, "ubuntu-25_04" → 25.04, "ubuntu-25_04-daily" → 25.04 (daily) + elif publisher_key == "canonical" and re.match(r"^ubuntu-(\d+)_(\d+)", offer_name): + version_match = re.match(r"^ubuntu-(\d+)_(\d+)", offer_name) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"Ubuntu Linux {major}.{minor} {arch_display}" + if "daily" in offer_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 6: openSUSE Leap from offer name + # Examples: "opensuse-leap-15-6" → 15.6, "opensuse-leap-15-6-arm64" → 15.6 (arm64) + # Also "openSUSE-Leap" with SKU "15-2" → 15.2 + elif publisher_key == "suse" and re.match(r"^opensuse-leap", offer_name, re.IGNORECASE): + # Check if version is in offer name (e.g., opensuse-leap-15-6) + offer_version_match = re.match(r"^opensuse-leap-(\d+)-(\d+)", offer_name, re.IGNORECASE) + if offer_version_match: + major = offer_version_match.group(1) + minor = offer_version_match.group(2) + else: + # Version must be in SKU (e.g., openSUSE-Leap with SKU 15-2) + sku_version_match = re.match(r"^(\d+)-(\d+)$", sku_name) + if not sku_version_match: + return (None, None, None, None, None) + major = sku_version_match.group(1) + minor = sku_version_match.group(2) + version_key = f"OPENSUSE_LEAP_{major}_{minor}{arch_suffix}" + friendly_name = f"openSUSE Leap {major}.{minor} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 7: SLE Micro from offer name (uses X-Y format, not SP) + # Examples: "sle-micro-5-1-byos" → 5.1, "sle-micro-6-0-byos" → 6.0 + elif publisher_key == "suse" and re.match(r"^sle-micro-(\d+)-(\d+)", offer_name, re.IGNORECASE): + version_match = re.match(r"^sle-micro-(\d+)-(\d+)", offer_name, re.IGNORECASE) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"SLE_MICRO_{major}_{minor}{arch_suffix}" + friendly_name = f"SUSE Linux Enterprise Micro {major}.{minor} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 8: SLES/SLE-HPC from offer name with SP (Service Pack) + # Examples: "sles-15-sp6" → 15 SP6, "sle-hpc-15-sp4-byos" → 15 SP4 + elif publisher_key == "suse" and re.match(r"^(sles|sle-hpc)-(\d+)-sp(\d+)", offer_name, re.IGNORECASE): + version_match = re.match(r"^(sles|sle-hpc)-(\d+)-sp(\d+)", offer_name, re.IGNORECASE) + product_type = version_match.group(1).upper() + major = version_match.group(2) + sp = version_match.group(3) + version_key = f"SUSE_{major}_SP{sp}{arch_suffix}" + if "hpc" in product_type.lower(): + friendly_name = f"SUSE Linux Enterprise HPC {major} SP{sp} {arch_display}" + else: + friendly_name = f"SUSE Linux Enterprise {major} SP{sp} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 6: Generic version extraction + elif not version_match: + version_match = re.search(r"(\d+)(?:[_\.](\d+))?", sku_name) + + if version_match: + if len(version_match.groups()) >= 2 and version_match.group(2): + # Has minor version + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = ( + f"{publisher_info.get('description', publisher_key)} " + f"{major}.{minor} {arch_display}" + ) + else: + # Only major version + major = version_match.group(1) + version_key = f"{publisher_key.upper()}_{major}{arch_suffix}" + friendly_name = ( + f"{publisher_info.get('description', publisher_key)} " + f"{major} {arch_display}" + ) + + # Add special notes for daily/backports variants + if "daily" in offer_name.lower() or "daily" in sku_name.lower(): + friendly_name += " (daily)" + elif "backports" in sku_name.lower(): + friendly_name += " (backports)" + elif "gen2" in sku_name.lower() and "backports" not in sku_name.lower(): + # Don't add gen2 suffix if it's already a backports variant + if arch == "x86_64": + friendly_name += " Gen2" + + return (version_key, friendly_name, arch, offer_name, sku_name) + + return (None, None, None, None, None) + + +def organize_images_by_publisher(publishers, region=None, quiet=False): + """ + Organize Azure images by publisher and version. + + Args: + publishers (dict): Dictionary of publisher information + region (str): Azure region to query (default: get_default_region()) + quiet (bool): Suppress debug messages + + Returns: + dict: Organized structure {publisher: {version_key: {offer, sku, ...}}} + """ + if region is None: + region = get_default_region() + + organized = defaultdict(lambda: defaultdict(dict)) + + for publisher_key, publisher_info in publishers.items(): + publisher_id = publisher_info.get("publisher_id") + if not publisher_id: + continue + + if not quiet: + print(f"\nProcessing {publisher_id}...", file=sys.stderr) + + offers_dict = get_all_offers_and_skus(publisher_id, region, quiet) + + # Get offer patterns for filtering (if defined) + offer_patterns = publisher_info.get("offer_patterns", []) + offer_regexes = [re.compile(pattern) for pattern in offer_patterns] if offer_patterns else [] + + for offer_name, skus in offers_dict.items(): + # Skip offers that don't match any pattern (if patterns are defined) + if offer_regexes and not any(regex.match(offer_name) for regex in offer_regexes): + continue + for sku_name in skus: + ( + version_key, + friendly_name, + arch, + offer, + sku, + ) = classify_offer_sku( + publisher_key, publisher_id, offer_name, sku_name, publisher_info + ) + + if not version_key: + continue + + # Store image information + # For simplicity, we keep the first occurrence + if version_key not in organized[publisher_key]: + organized[publisher_key][version_key] = { + "friendly_name": friendly_name, + "architecture": arch, + "offer": offer, + "sku": sku, + "publisher_id": publisher_id, + } + + if not quiet: + print("\n--- Summary ---", file=sys.stderr) + for publisher, versions in organized.items(): + print(f"Publisher '{publisher}': {len(versions)} versions", file=sys.stderr) + + return organized + + +def output_images_kconfig(organized_images, publishers): + """ + Output images menu in Kconfig format. + + Args: + organized_images (dict): Organized images from organize_images_by_publisher() + publishers (dict): Dictionary of all publishers + """ + print("# This file was auto-generated by gen_kconfig_image") + print("#") + print("# To regenerate: cd terraform/azure/scripts && ./gen_kconfig_image") + print() + + environment = get_jinja2_environment() + + # Sort publishers by priority for consistent ordering + sorted_publishers = sorted( + [(k, v) for k, v in publishers.items() if k in organized_images], + key=lambda x: (x[1].get("priority", 100), x[0]), + ) + + # Output the top-level distribution choice menu + template = environment.get_template("image_distributions.j2") + print( + template.render( + publishers=[pub[0] for pub in sorted_publishers], + ) + ) + print() + + def version_sort_key(version_item): + """ + Extract numeric version from version_key for proper chronological sorting. + + Version keys are like: DEBIAN_12_X86, RHEL_9_6_ARM64 + Sort by numeric version (oldest to newest), not alphabetic. + """ + version_key, version_data = version_item + parts = version_key.split("_") + + # Remove publisher prefix + version_parts = parts[1:] + + major = 0 + minor = 0 + + # The last part is usually the architecture (X86, ARM64) + # Everything before that is version numbers + if len(version_parts) >= 2: + try: + major = int(version_parts[0]) + # Try to parse second part as minor version + if len(version_parts) >= 3: + try: + minor = int(version_parts[1]) + except ValueError: + # Second part is arch, no minor version + minor = 0 + except ValueError: + pass + + # Architecture preference: X86, ARM64 for consistent ordering + arch_order = {"X86": 0, "ARM64": 1} + arch = version_parts[-1] if version_parts else "X86" + arch_priority = arch_order.get(arch, 99) + + return (major, minor, arch_priority) + + # Output each publisher's images + template = environment.get_template("image_publisher.j2") + for publisher_key, publisher_info in sorted_publishers: + versions = organized_images.get(publisher_key, {}) + + # Sort versions numerically (oldest to newest) + sorted_versions = sorted(versions.items(), key=version_sort_key) + + print( + template.render( + publisher_key=publisher_key, + publisher_name=publisher_info.get("publisher_name", publisher_key), + publisher_description=publisher_info.get("description", publisher_key), + versions=sorted_versions, + ) + ) + print() + + +def output_publishers_raw(quiet=False): + """Output available publishers in table format.""" + publishers = get_known_publishers() + + if not quiet: + print(f"Known OS image publishers ({len(publishers)}):\n") + + print(f"{'Publisher Key':<15} {'Publisher Name':<20} {'Description':<30}") + print("-" * 70) + + for key, info in publishers.items(): + print( + f"{key:<15} " + f"{info['publisher_name']:<20} " + f"{info['description']:<30}" + ) + + +def output_publisher_raw(publisher_key, organized_images, quiet=False): + """Output publisher image information in table format.""" + publishers = get_known_publishers() + publisher_info = publishers.get(publisher_key, {}) + + if not quiet: + print(f"Images for {publisher_info.get('publisher_name', publisher_key)}") + print(f"Description: {publisher_info.get('description', '')}\n") + + versions = organized_images.get(publisher_key, {}) + if not versions: + print(f"No images found for publisher '{publisher_key}'.") + return + + print(f"{'Version':<50} {'Offer':<20} {'SKU':<15}") + print("-" * 90) + + for version_key, version_info in sorted(versions.items()): + print( + f"{version_info['friendly_name']:<50} " + f"{version_info['offer']:<20} " + f"{version_info['sku']:<15}" + ) + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure VM image information and generate Kconfig", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate Kconfig from default region + python %(prog)s > ../kconfigs/Kconfig.image.generated + + # List available publishers + python %(prog)s --publishers + + # Get details for a specific publisher + python %(prog)s debian --format raw + + # Query specific region + python %(prog)s --region eastus + """, + ) + parser.add_argument( + "publisher_key", nargs="?", help="Publisher key (e.g., debian, redhat)" + ) + + parser.add_argument( + "--publishers", action="store_true", help="List all known publishers" + ) + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--region", + "-r", + help="Query specific region (default: from Azure CLI config or westus)", + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + if args.publishers: + output_publishers_raw(args.quiet) + return + + publishers = get_known_publishers() + + # Filter to specific publisher if requested + if args.publisher_key: + if args.publisher_key not in publishers: + print( + f"Error: Unknown publisher '{args.publisher_key}'. " + f"Use --publishers to list available publishers.", + file=sys.stderr, + ) + sys.exit(1) + publishers = {args.publisher_key: publishers[args.publisher_key]} + + # Determine region + region = args.region if args.region else get_default_region() + + if not args.quiet: + print(f"Using region: {region}", file=sys.stderr) + + # Organize images + organized_images = organize_images_by_publisher(publishers, region, args.quiet) + + # Output based on format + if args.publisher_key: + if args.format == "raw": + output_publisher_raw(args.publisher_key, organized_images, args.quiet) + else: + print( + "Error: Kconfig format not supported for single publisher query", + file=sys.stderr, + ) + sys.exit(1) + else: + if args.format == "kconfig": + output_images_kconfig(organized_images, publishers) + else: + # Raw format for all publishers + for publisher_key in sorted(organized_images.keys()): + output_publisher_raw(publisher_key, organized_images, args.quiet) + print() + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/image_distributions.j2 b/terraform/azure/scripts/image_distributions.j2 new file mode 100644 index 000000000..93e98d7f6 --- /dev/null +++ b/terraform/azure/scripts/image_distributions.j2 @@ -0,0 +1,35 @@ +choice + prompt "Azure image publisher" +{% if 'debian' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN +{% elif 'redhat' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT +{% elif 'canonical' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_CANONICAL +{% elif publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publishers[0] | upper }} +{% endif %} + help + This option specifies the publisher of the boot image used to + create the kdevops target nodes. + + The distributions listed here are official Azure platform images + provided by various publishers. These images are well-tested and + actively maintained. + +{% for publisher_key in publishers %} +config TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} + bool "{{ publisher_key | title }}" + help + Select this if you want to use images from the {{ publisher_key | title }} + publisher. + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_PUBLISHER + string + output yaml +{% for publisher_key in publishers %} + default "{{ publisher_key }}" if TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} +{% endfor %} diff --git a/terraform/azure/scripts/image_publisher.j2 b/terraform/azure/scripts/image_publisher.j2 new file mode 100644 index 000000000..01b818b14 --- /dev/null +++ b/terraform/azure/scripts/image_publisher.j2 @@ -0,0 +1,110 @@ +{# + This template is rendered by gen_kconfig_image script. + + Template variables: + - publisher_key: str (e.g., 'debian', 'redhat') + - publisher_name: str (e.g., 'Debian') + - publisher_description: str (full description) + - versions: list of (version_key, version_info) tuples (pre-sorted) +#} +if TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} + +if TARGET_ARCH_X86_64 + +choice + prompt "{{ publisher_description }} release" +{# versions is a list of (version_key, version_info) tuples from sorted_versions in gen_kconfig_image #} +{% for version_key, version_info in versions | reverse %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} if TARGET_ARCH_X86_64 +{% endif %} +{% endfor %} + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} +config TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} + bool "{{ version_info['friendly_name'] }}" + help + This option sets the OS image to {{ version_info['friendly_name'] }}. + + Publisher: {{ version_info['publisher_id'] }} + Offer: {{ version_info['offer'] }} + SKU: {{ version_info['sku'] }} + +{% endif %} +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default "{{ version_info['offer'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default "{{ version_info['sku'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +endif # TARGET_ARCH_X86_64 + +if TARGET_ARCH_ARM64 + +choice + prompt "{{ publisher_description }} release" +{# versions is a list of (version_key, version_info) tuples from sorted_versions in gen_kconfig_image #} +{% for version_key, version_info in versions | reverse %} +{% if "ARM64" in version_key %} + default TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} if TARGET_ARCH_ARM64 +{% endif %} +{% endfor %} + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} +config TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} + bool "{{ version_info['friendly_name'] }}" + help + This option sets the OS image to {{ version_info['friendly_name'] }}. + + Publisher: {{ version_info['publisher_id'] }} + Offer: {{ version_info['offer'] }} + SKU: {{ version_info['sku'] }} + +{% endif %} +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} + default "{{ version_info['offer'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} + default "{{ version_info['sku'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +endif # TARGET_ARCH_ARM64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} diff --git a/terraform/azure/scripts/publisher_definitions.yml b/terraform/azure/scripts/publisher_definitions.yml new file mode 100644 index 000000000..966a52073 --- /dev/null +++ b/terraform/azure/scripts/publisher_definitions.yml @@ -0,0 +1,96 @@ +# Azure VM Image Publisher Definitions +# +# This file defines the known publishers of Azure VM images that kdevops +# supports. It is used by the gen_kconfig_image script to generate the +# Kconfig.image configuration file. +# +# To add a new publisher: +# 1. Add a new entry with a unique key (lowercase, alphanumeric + underscores) +# 2. Specify the Azure publisher_id (exact match from Azure API) +# 3. Provide a user-friendly publisher_name and description +# 4. Set a priority (lower numbers appear first in menus) +# 5. Run: cd terraform/azure/scripts && ./gen_kconfig_image +# +# Fields: +# publisher_id: The exact publisher identifier used in Azure CLI +# (e.g., "Debian", "RedHat", "Canonical") +# Note: Some marketplace publishers have auto-generated IDs +# publisher_name: User-friendly display name +# description: Full description used in Kconfig help text +# priority: Display order (lower = higher priority, appears first) +# +# Note: The script will automatically discover offers and SKUs for each +# publisher by querying the Azure API. +# +# To find the correct publisher_id for a new publisher, use: +# az vm image list-publishers --location westus -o json | grep -i + +debian: + publisher_id: "Debian" + publisher_name: "Debian" + description: "Debian GNU/Linux" + priority: 10 + +redhat: + publisher_id: "RedHat" + publisher_name: "Red Hat" + description: "Red Hat Enterprise Linux" + priority: 20 + offer_patterns: + - "^RHEL$" + - "^rhel-arm64$" + - "^rh-rhel$" + - "^RHEL-HA$" + - "^RHEL-SAP$" + - "^RHEL-SAP-APPS$" + - "^RHEL-SAP-HA$" + +canonical: + publisher_id: "Canonical" + publisher_name: "Canonical" + description: "Ubuntu Linux" + priority: 30 + offer_patterns: + - "^0001-com-ubuntu-server-.*$" + - "^ubuntu-[0-9]{2}_[0-9]{2}-lts.*$" + - "^ubuntu-[0-9]{2}_[0-9]{2}.*$" + - "^UbuntuServer$" + - "^ubuntu$" + - "^0001-com-ubuntu-minimal-.*$" + +oracle: + publisher_id: "Oracle" + publisher_name: "Oracle" + description: "Oracle Linux" + priority: 40 + offer_patterns: + - "^Oracle-Linux$" + +suse: + publisher_id: "SUSE" + publisher_name: "SUSE" + description: "SUSE Linux Enterprise" + priority: 50 + offer_patterns: + - "^sles-.*$" + - "^SLES.*$" + - "^opensuse-leap-.*$" + - "^openSUSE-Leap$" + - "^sle-hpc-.*$" + - "^sle-micro-.*$" + +# Note: AlmaLinux and Rocky Linux use marketplace publisher IDs which +# include timestamps. These IDs may change over time as the publishers +# update their marketplace presence. + +almalinux: + publisher_id: "almalinuxosfoundation1628089859865" + publisher_name: "AlmaLinux" + description: "AlmaLinux" + priority: 60 + +rockylinux: + publisher_id: "erockyenterprisesoftwarefoundationinc1653071250513" + publisher_name: "Rocky Linux" + description: "Rocky Linux" + priority: 70 From 9a04dc4b075f3cd2e5a053451b55d4ebd82bda1d Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 1 Nov 2025 13:28:30 -0400 Subject: [PATCH 07/10] scripts: Add Azure location support to generate_cloud_configs.py Following the pattern established in commit 41d3f694bee8 for OCI, add Azure region generation support to the cloud configuration script. This change enables dynamic Azure region discovery and Kconfig generation via "make cloud-config" or "make cloud-config-azure". Changes made: - Add generate_azure_kconfig() function to generate_cloud_configs.py - Update process_azure() to generate Kconfig files - Add Azure targets to dynamic-cloud-kconfig.Makefile - Update terraform/azure/Kconfig to source generated file - Add Azure generated files to .gitignore Generated-by: Claude AI Signed-off-by: Chuck Lever --- .gitignore | 3 ++ scripts/dynamic-cloud-kconfig.Makefile | 30 ++++++++++++-- scripts/generate_cloud_configs.py | 55 +++++++++++++++++++++++++- terraform/azure/Kconfig | 6 +-- 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index aa0aafa41..bc52f4046 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,9 @@ terraform/aws/kconfigs/Kconfig.instance.generated terraform/aws/kconfigs/Kconfig.location.generated terraform/aws/scripts/__pycache__/ +terraform/azure/kconfigs/Kconfig.image.generated +terraform/azure/kconfigs/Kconfig.location.generated +terraform/azure/kconfigs/Kconfig.size.generated terraform/azure/scripts/__pycache__/ terraform/oci/kconfigs/Kconfig.image.generated diff --git a/scripts/dynamic-cloud-kconfig.Makefile b/scripts/dynamic-cloud-kconfig.Makefile index 840f7d805..a9dd895b2 100644 --- a/scripts/dynamic-cloud-kconfig.Makefile +++ b/scripts/dynamic-cloud-kconfig.Makefile @@ -20,6 +20,14 @@ AWS_KCONFIG_LOCATION := $(AWS_KCONFIG_DIR)/Kconfig.location.generated AWS_KCONFIGS := $(AWS_KCONFIG_AMI) $(AWS_KCONFIG_INSTANCE) $(AWS_KCONFIG_LOCATION) +# Azure dynamic configuration +AZURE_KCONFIG_DIR := terraform/azure/kconfigs +AZURE_KCONFIG_IMAGE := $(AZURE_KCONFIG_DIR)/Kconfig.image.generated +AZURE_KCONFIG_LOCATION := $(AZURE_KCONFIG_DIR)/Kconfig.location.generated +AZURE_KCONFIG_SIZE := $(AZURE_KCONFIG_DIR)/Kconfig.size.generated + +AZURE_KCONFIGS := $(AZURE_KCONFIG_LOCATION) $(AZURE_KCONFIG_SIZE) $(AZURE_KCONFIG_IMAGE) + # OCI dynamic configuration OCI_KCONFIG_DIR := terraform/oci/kconfigs OCI_KCONFIG_IMAGE := $(OCI_KCONFIG_DIR)/Kconfig.image.generated @@ -29,7 +37,7 @@ OCI_KCONFIG_SHAPE := $(OCI_KCONFIG_DIR)/Kconfig.shape.generated OCI_KCONFIGS := $(OCI_KCONFIG_IMAGE) $(OCI_KCONFIG_LOCATION) $(OCI_KCONFIG_SHAPE) # Add generated files to mrproper clean list -KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS) $(AWS_KCONFIGS) $(OCI_KCONFIGS) +KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS) $(AWS_KCONFIGS) $(AZURE_KCONFIGS) $(OCI_KCONFIGS) # Touch Lambda Labs generated files so Kconfig can source them # This ensures the files exist (even if empty) before Kconfig runs @@ -40,11 +48,15 @@ dynamic_lambdalabs_kconfig_touch: dynamic_aws_kconfig_touch: $(Q)touch $(AWS_KCONFIGS) +# Touch Azure generated files so Kconfig can source them +dynamic_azure_kconfig_touch: + $(Q)touch $(AZURE_KCONFIGS) + # Touch OCI generated files so Kconfig can source them dynamic_oci_kconfig_touch: $(Q)touch $(OCI_KCONFIGS) -DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch dynamic_aws_kconfig_touch dynamic_oci_kconfig_touch +DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch dynamic_aws_kconfig_touch dynamic_azure_kconfig_touch dynamic_oci_kconfig_touch # Lambda Labs targets use --provider argument for efficiency cloud-config-lambdalabs: @@ -54,6 +66,10 @@ cloud-config-lambdalabs: cloud-config-aws: $(Q)python3 scripts/generate_cloud_configs.py --provider aws +# Azure targets use --provider argument for efficiency +cloud-config-azure: + $(Q)python3 scripts/generate_cloud_configs.py --provider azure + # OCI targets use --provider argument for efficiency cloud-config-oci: $(Q)python3 scripts/generate_cloud_configs.py --provider oci @@ -66,17 +82,22 @@ clean-cloud-config-lambdalabs: clean-cloud-config-aws: $(Q)rm -f $(AWS_KCONFIGS) +# Clean Azure generated files +clean-cloud-config-azure: + $(Q)rm -f $(AZURE_KCONFIGS) + # Clean OCI generated files clean-cloud-config-oci: $(Q)rm -f $(OCI_KCONFIGS) -DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs cloud-config-aws cloud-config-oci +DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs cloud-config-aws cloud-config-azure cloud-config-oci cloud-config-help: @echo "Cloud-specific dynamic kconfig targets:" @echo "cloud-config - generates all cloud provider dynamic kconfig content" @echo "cloud-config-lambdalabs - generates Lambda Labs dynamic kconfig content" @echo "cloud-config-aws - generates AWS dynamic kconfig content" + @echo "cloud-config-azure - generates Azure dynamic kconfig content" @echo "cloud-config-oci - generates OCI dynamic kconfig content" @echo "clean-cloud-config - removes all generated cloud kconfig files" @echo "cloud-list-all - list all cloud instances for configured provider" @@ -86,7 +107,7 @@ HELP_TARGETS += cloud-config-help cloud-config: $(Q)python3 scripts/generate_cloud_configs.py -clean-cloud-config: clean-cloud-config-lambdalabs clean-cloud-config-aws clean-cloud-config-oci +clean-cloud-config: clean-cloud-config-lambdalabs clean-cloud-config-aws clean-cloud-config-azure clean-cloud-config-oci $(Q)echo "Cleaned all cloud provider dynamic Kconfig files." cloud-list-all: @@ -95,5 +116,6 @@ cloud-list-all: PHONY += cloud-config clean-cloud-config cloud-config-help cloud-list-all PHONY += cloud-config-aws clean-cloud-config-aws +PHONY += cloud-config-azure clean-cloud-config-azure PHONY += cloud-config-lambdalabs clean-cloud-config-lambdalabs PHONY += cloud-config-oci clean-cloud-config-oci diff --git a/scripts/generate_cloud_configs.py b/scripts/generate_cloud_configs.py index d14b56d20..4cefd5d7a 100755 --- a/scripts/generate_cloud_configs.py +++ b/scripts/generate_cloud_configs.py @@ -147,6 +147,52 @@ def generate_aws_kconfig() -> bool: return all_success +def generate_azure_kconfig() -> bool: + """ + Generate Azure Kconfig files. + Returns True on success, False on failure. + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = os.path.dirname(script_dir) + azure_scripts_dir = os.path.join(project_root, "terraform", "azure", "scripts") + azure_kconfigs_dir = os.path.join(project_root, "terraform", "azure", "kconfigs") + + # Define the script-to-output mapping + scripts_to_run = [ + ("gen_kconfig_image", "Kconfig.image.generated"), + ("gen_kconfig_location", "Kconfig.location.generated"), + ("gen_kconfig_size", "Kconfig.size.generated"), + ] + + all_success = True + + for script_name, kconfig_file in scripts_to_run: + script_path = os.path.join(azure_scripts_dir, script_name) + output_path = os.path.join(azure_kconfigs_dir, kconfig_file) + + # Run the script and capture its output + result = subprocess.run( + [script_path], + capture_output=True, + text=True, + check=False, + ) + + if result.returncode == 0: + # Write the output to the corresponding Kconfig file + try: + with open(output_path, "w") as f: + f.write(result.stdout) + except IOError as e: + print(f"Error writing {kconfig_file}: {e}", file=sys.stderr) + all_success = False + else: + print(f"Error running {script_name}: {result.stderr}", file=sys.stderr) + all_success = False + + return all_success + + def generate_oci_kconfig() -> bool: """ Generate OCI Kconfig files. @@ -222,8 +268,13 @@ def process_aws(): def process_azure(): - """Process Azure configuration (placeholder).""" - print("⚠ Azure: Dynamic configuration not yet implemented") + """Process Azure configuration.""" + kconfig_generated = generate_azure_kconfig() + if kconfig_generated: + print("✓ Azure: Kconfig files generated successfully") + else: + print("⚠ Azure: Failed to generate Kconfig files - using defaults") + print() def process_gce(): diff --git a/terraform/azure/Kconfig b/terraform/azure/Kconfig index 2326df696..5c4e86362 100644 --- a/terraform/azure/Kconfig +++ b/terraform/azure/Kconfig @@ -1,13 +1,13 @@ if TERRAFORM_AZURE menu "Resource Location" -source "terraform/azure/kconfigs/Kconfig.location" +source "terraform/azure/kconfigs/Kconfig.location.generated" endmenu menu "Compute" comment "Size selection" -source "terraform/azure/kconfigs/Kconfig.size" +source "terraform/azure/kconfigs/Kconfig.size.generated" comment "OS image selection" -source "terraform/azure/kconfigs/Kconfig.image" +source "terraform/azure/kconfigs/Kconfig.image.generated" endmenu menu "Storage" source "terraform/azure/kconfigs/Kconfig.storage" From 600729e55710ae506cd77be60bbf54075421ba0f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 11 Nov 2025 14:38:10 -0500 Subject: [PATCH 08/10] azure/gen_kconfig_image: Simplify Oracle pattern, add smart SKU selection, and filter cloud-init-incompatible versions Simplify Pattern 3c for Oracle Linux SKU classification with a more flexible regex that handles edge cases missed by the previous implementation. The new pattern handles single-digit versions (8, ol9), underscore format (ol8_2-gen2), and all suffix variants without hardcoding them. Add intelligent SKU selection when multiple variants exist for the same OS version. The scoring system prefers Gen2 VMs (UEFI boot, better performance) and LVM-based images (flexible partitioning) while hiding implementation details from users. This ensures users get the best available variant without seeing confusing technical suffixes in the Kconfig menu. Filter out distribution versions that lack cloud-init support, which kdevops terraform provisioning requires. Oracle Linux and RHEL versions before 7.7 are excluded as they predate Azure's adoption of cloud-init as the standard provisioning method. This prevents provisioning failures from incompatible images appearing in the Kconfig menu. The friendly names now show clean version strings like 'Oracle Linux 8.9 (x86)' instead of 'Oracle Linux 8.9 (x86) Gen2 LVM', while internally selecting the optimal SKU (ol89-lvm-gen2 instead of ol89-lvm). Generated-by: Claude AI Signed-off-by: Chuck Lever --- terraform/azure/scripts/gen_kconfig_image | 156 ++++++++++++++++++---- 1 file changed, 133 insertions(+), 23 deletions(-) diff --git a/terraform/azure/scripts/gen_kconfig_image b/terraform/azure/scripts/gen_kconfig_image index d8b39a72f..216acc0a0 100755 --- a/terraform/azure/scripts/gen_kconfig_image +++ b/terraform/azure/scripts/gen_kconfig_image @@ -172,6 +172,77 @@ def get_fallback_publishers(): } +def score_sku_quality(sku_name): + """ + Score SKU by quality - higher score means better/more modern variant. + + This helps automatically select the best SKU when multiple variants exist + for the same OS version. Prefers Gen2 VMs and LVM-based images. + + Args: + sku_name (str): SKU name (e.g., "ol89-lvm-gen2", "79-gen2") + + Returns: + int: Quality score (higher is better) + """ + sku_lower = sku_name.lower() + score = 0 + + # Prefer Gen2 (UEFI boot, larger VM sizes, better performance) + if "gen2" in sku_lower: + score += 100 + + # Prefer LVM (flexible disk partitioning) + if "lvm" in sku_lower: + score += 50 + + # Avoid CI variants (legacy cloud-init explicit variants, not needed for 7.7+) + if "ci" in sku_lower: + score -= 10 + + # Avoid underscore format (legacy naming convention) + if "_" in sku_name and not sku_name.startswith("ol"): + score -= 5 + + return score + + +def supports_cloud_init(publisher_key, major, minor): + """ + Check if a distribution version supports cloud-init by default. + + kdevops terraform provisioning depends on cloud-init. Only include + versions that have cloud-init support built-in. + + Args: + publisher_key (str): Publisher key (e.g., "oracle", "redhat") + major (str): Major version number + minor (str): Minor version number + + Returns: + bool: True if version supports cloud-init, False otherwise + """ + try: + major_int = int(major) + minor_int = int(minor) + except (ValueError, TypeError): + # If we can't parse the version, assume it's too old + return False + + # Per Microsoft docs: Oracle Linux 7.7+, RHEL 7.7+ have cloud-init by default + # https://learn.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init + if publisher_key in ("oracle", "redhat"): + if major_int > 7: + return True + if major_int == 7 and minor_int >= 7: + return True + return False + + # Other distributions: assume modern versions have cloud-init + # (Debian, Ubuntu, SUSE, etc. have had cloud-init for longer) + return True + + def classify_offer_sku( publisher_key, publisher_id, offer_name, sku_name, publisher_info ): @@ -223,6 +294,11 @@ def classify_offer_sku( minor = version_match.group(2) version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" friendly_name = f"RHEL {major}.{minor} {arch_display}" + + # Filter out versions without cloud-init support (kdevops requirement) + if not supports_cloud_init(publisher_key, major, minor): + return (None, None, None, None, None) + return (version_key, friendly_name, arch, offer_name, sku_name) # Pattern 3b: RHEL Gen2 with condensed version numbers # Examples: "74-gen2" → 7.4, "810-gen2" → 8.10, "100-gen2" → 10.0 @@ -245,30 +321,48 @@ def classify_offer_sku( minor = version_digits[1:] version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" friendly_name = f"RHEL {major}.{minor} {arch_display} Gen2" + + # Filter out versions without cloud-init support (kdevops requirement) + if not supports_cloud_init(publisher_key, major, minor): + return (None, None, None, None, None) + return (version_key, friendly_name, arch, offer_name, sku_name) # Pattern 3c: Oracle Linux with condensed version numbers # Examples: "ol82-gen2" → 8.2, "77" → 7.7, "ol810-lvm" → 8.10, "79-gen2" → 7.9 - # Handles with/without "ol" prefix, Gen2, LVM, and CI variants - elif publisher_key == "oracle" and re.match(r"^(ol)?\d{2,3}(-gen2|-lvm|-lvm-gen2|-ci-gen2|-arm64-lvm-gen2)?$", sku_name): - version_match = re.match(r"^(ol)?(\d{2,3})", sku_name) - version_digits = version_match.group(2) - if len(version_digits) == 2: - # Two digits: split as X.Y (e.g., "77" → "7.7", "ol82" → "8.2") - major = version_digits[0] - minor = version_digits[1] - elif version_digits.startswith("10"): - # Three digits starting with "10": Oracle Linux 10.0 - major = "10" - minor = "0" - else: - # Three digits: first digit is major, rest is minor (e.g., "810" → "8.10") - major = version_digits[0] - minor = version_digits[1:] - version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" - friendly_name = f"Oracle Linux {major}.{minor} {arch_display}" - if "-gen2" in sku_name.lower(): - friendly_name += " Gen2" - return (version_key, friendly_name, arch, offer_name, sku_name) + # Handles: optional "ol" prefix, condensed versions, underscore format (ol8_2-gen2) + elif publisher_key == "oracle": + # Flexible regex: optional prefix, version digits, optional underscore minor, optional suffix + match = re.match(r"^(ol)?(\d{1,3})(?:_(\d+))?(?:-(.+))?$", sku_name) + if match: + major_digits = match.group(2) + underscore_minor = match.group(3) + + # Parse version into major.minor + if underscore_minor: + # Underscore format: ol8_2 → 8.2 + major, minor = major_digits, underscore_minor + elif len(major_digits) == 1: + # Single digit: 8 → 8.0, ol9 → 9.0 + major, minor = major_digits, "0" + elif len(major_digits) == 2: + # Two digits: 77 → 7.7, ol82 → 8.2 + major, minor = major_digits[0], major_digits[1] + elif major_digits.startswith("10"): + # Three digits starting with 10: ol100 → 10.0 + major, minor = "10", "0" + else: + # Three digits: ol810 → 8.10 + major, minor = major_digits[0], major_digits[1:] + + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + # Clean friendly name without implementation details (gen2/lvm/ci) + friendly_name = f"Oracle Linux {major}.{minor} {arch_display}" + + # Filter out versions without cloud-init support (kdevops requirement) + if not supports_cloud_init(publisher_key, major, minor): + return (None, None, None, None, None) + + return (version_key, friendly_name, arch, offer_name, sku_name) # Pattern 4: Ubuntu LTS from offer name # Examples: "ubuntu-22_04-lts" → 22.04, "ubuntu-24_04-lts-daily" → 24.04 # Skip ubuntu-pro SKUs as these are commercial support variants @@ -423,16 +517,32 @@ def organize_images_by_publisher(publishers, region=None, quiet=False): if not version_key: continue - # Store image information - # For simplicity, we keep the first occurrence + # Store image information - select best SKU when multiple variants exist + # Use score_sku_quality() to prefer Gen2 and LVM variants if version_key not in organized[publisher_key]: + # First occurrence for this version organized[publisher_key][version_key] = { "friendly_name": friendly_name, "architecture": arch, "offer": offer, "sku": sku, "publisher_id": publisher_id, + "_score": score_sku_quality(sku), } + else: + # Multiple SKUs for same version - keep the better one + current_score = organized[publisher_key][version_key].get("_score", 0) + new_score = score_sku_quality(sku) + if new_score > current_score: + # This SKU is better, replace it + organized[publisher_key][version_key] = { + "friendly_name": friendly_name, + "architecture": arch, + "offer": offer, + "sku": sku, + "publisher_id": publisher_id, + "_score": new_score, + } if not quiet: print("\n--- Summary ---", file=sys.stderr) From 65bc1a1aedd42c44df0063662568b336673cb5bc Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Nov 2025 14:52:56 -0500 Subject: [PATCH 09/10] azure/gen_kconfig_image: Simplify RHEL OS image pattern Similar to the Oracle Linux improvements, consolidate RHEL SKU parsing into a single flexible pattern that handles all format variants. The previous Pattern 3 and Pattern 3b required separate regex patterns for underscore format (9_6) and condensed Gen2 format (810-gen2), with a complex hardcoded regex that excluded LVM variants. The new unified Pattern 3 uses a flexible regex that handles underscore format (9_6, 10_1), condensed versions (77, 810), LVM variants (7lvm-gen2, 8-lvm-gen2), and all suffix combinations (gen2, -gen2, _gen2, sap variants, ci variants). This eliminates the need for maintaining explicit suffix lists in the regex. Smart SKU selection via score_sku_quality() now automatically chooses the best variant when multiple SKUs exist for the same version. This prefers Gen2 and LVM variants, so RHEL 8.0 now selects 8-lvm-gen2 and RHEL 9.0 selects 9-lvm-gen2 instead of basic Gen2 SKUs. Friendly names are simplified to hide implementation details. Instead of "RHEL 8.0 (x86) Gen2", users see "RHEL 8.0 (x86)", while the system internally selects the optimal SKU. This reduces the number of visible RHEL versions from 49 to 38 by consolidating duplicate variants. Generated-by: Claude AI Signed-off-by: Chuck Lever --- terraform/azure/scripts/gen_kconfig_image | 65 +++++++++++------------ 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/terraform/azure/scripts/gen_kconfig_image b/terraform/azure/scripts/gen_kconfig_image index 216acc0a0..bfb9c338b 100755 --- a/terraform/azure/scripts/gen_kconfig_image +++ b/terraform/azure/scripts/gen_kconfig_image @@ -286,13 +286,39 @@ def classify_offer_sku( if "daily" in offer_name.lower() or "daily" in sku_name.lower(): friendly_name += " (daily)" return (version_key, friendly_name, arch, offer_name, sku_name) - # Pattern 3: RHEL with SKU like "9_6", "8_10" - elif publisher_key == "redhat" and re.match(r"^\d+_\d+$", sku_name): - version_match = re.match(r"^(\d+)_(\d+)$", sku_name) - if version_match: - major = version_match.group(1) - minor = version_match.group(2) + # Pattern 3: RHEL with flexible version and suffix handling + # Examples: "9_6" → 9.6, "810-gen2" → 8.10, "8-lvm-gen2" → 8.0, "7lvm-gen2" → 7.0 + # Handles: underscore format, condensed versions, all suffix variants + elif publisher_key == "redhat": + # Flexible regex: version digits, optional underscore minor, optional suffix + # Handles both "9_6" and "810-gen2" and "7lvm-gen2" and "81gen2" patterns + match = re.match(r"^(\d{1,3})(?:_(\d+)|([a-z]+))?(?:-(.+)|(.+))?$", sku_name) + if match: + major_digits = match.group(1) + underscore_minor = match.group(2) + embedded_suffix = match.group(3) # e.g., "lvm" in "7lvm-gen2" + hyphen_suffix = match.group(4) # e.g., "gen2" in "810-gen2" + direct_suffix = match.group(5) # e.g., "gen2" in "81gen2" + + # Parse version into major.minor + if underscore_minor: + # Underscore format: 9_6 → 9.6, 10_1 → 10.1 + major, minor = major_digits, underscore_minor + elif len(major_digits) == 1: + # Single digit: 8 → 8.0 + major, minor = major_digits, "0" + elif len(major_digits) == 2: + # Two digits: 77 → 7.7, 81 → 8.1 + major, minor = major_digits[0], major_digits[1] + elif major_digits.startswith("10"): + # Three digits starting with 10: 100 → 10.0 + major, minor = "10", "0" + else: + # Three digits: 810 → 8.10 + major, minor = major_digits[0], major_digits[1:] + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + # Clean friendly name without implementation details (gen2/lvm/sap/ci) friendly_name = f"RHEL {major}.{minor} {arch_display}" # Filter out versions without cloud-init support (kdevops requirement) @@ -300,33 +326,6 @@ def classify_offer_sku( return (None, None, None, None, None) return (version_key, friendly_name, arch, offer_name, sku_name) - # Pattern 3b: RHEL Gen2 with condensed version numbers - # Examples: "74-gen2" → 7.4, "810-gen2" → 8.10, "100-gen2" → 10.0 - # Also handles SAP variants: "74sap-gen2" → 7.4, "82sapapps-gen2" → 8.2, "82sapha-gen2" → 8.2 - # Handles CI variants: "76-ci-gen2" → 7.6, "81-ci-gen2" → 8.1 - # Excludes SKUs with intermediate separators like "10-lvm-gen2" - elif publisher_key == "redhat" and re.match(r"^\d{2,3}(sap(-gen2)?|sapapps(-gen2)?|sapha(-gen2)?|_gen2|-gen2|gen2|-ci-gen2|-ci)$", sku_name): - version_digits = re.match(r"^(\d{2,3})", sku_name).group(1) - if len(version_digits) == 2: - # Two digits: split as X.Y (e.g., "74" → "7.4") - major = version_digits[0] - minor = version_digits[1] - elif version_digits.startswith("10"): - # Three digits starting with "10": RHEL 10.0 - major = "10" - minor = "0" - else: - # Three digits: first digit is major, rest is minor (e.g., "810" → "8.10") - major = version_digits[0] - minor = version_digits[1:] - version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" - friendly_name = f"RHEL {major}.{minor} {arch_display} Gen2" - - # Filter out versions without cloud-init support (kdevops requirement) - if not supports_cloud_init(publisher_key, major, minor): - return (None, None, None, None, None) - - return (version_key, friendly_name, arch, offer_name, sku_name) # Pattern 3c: Oracle Linux with condensed version numbers # Examples: "ol82-gen2" → 8.2, "77" → 7.7, "ol810-lvm" → 8.10, "79-gen2" → 7.9 # Handles: optional "ol" prefix, condensed versions, underscore format (ol8_2-gen2) From 686290e81fdce8c757f0f3837b1a6e07e0f47b15 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 21 Nov 2025 15:16:20 -0500 Subject: [PATCH 10/10] azure/gen_kconfig_image: Simplify SUSE friendly names Similar to the RHEL and Oracle Linux improvements, remove Gen2 implementation details from SUSE friendly names. The previous generic version extraction code added Gen2 suffixes to friendly names inconsistently, causing some SUSE versions to display Gen2 while others did not. The generic Pattern 6 code had conditional logic that added Gen2 to friendly names for x86_64 gen2 SKUs, but only when processing offers that fell through to the generic handler. SUSE-specific patterns for SLES and openSUSE did not add Gen2, creating inconsistencies where SUSE 12 and 15 showed Gen2 but SUSE 12 SP5 and 15 SP2 did not. Removing the Gen2 suffix from the generic code ensures all SUSE versions have consistent friendly names. Instead of SUSE Linux Enterprise 12 Gen2 and SUSE Linux Enterprise 15 Gen2, users now see simplified names without implementation details. The system continues to select optimal Gen2 SKUs via score_sku_quality, but hides these details from user-facing names. This maintains consistency with the RHEL and Oracle Linux improvements where Gen2, LVM, and other SKU variants are selected automatically but not exposed in friendly names. Generated-by: Claude AI Signed-off-by: Chuck Lever --- terraform/azure/scripts/gen_kconfig_image | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/terraform/azure/scripts/gen_kconfig_image b/terraform/azure/scripts/gen_kconfig_image index bfb9c338b..673474ce3 100755 --- a/terraform/azure/scripts/gen_kconfig_image +++ b/terraform/azure/scripts/gen_kconfig_image @@ -452,15 +452,11 @@ def classify_offer_sku( f"{major} {arch_display}" ) - # Add special notes for daily/backports variants + # Add special notes for daily/backports variants (but not gen2 - implementation detail) if "daily" in offer_name.lower() or "daily" in sku_name.lower(): friendly_name += " (daily)" elif "backports" in sku_name.lower(): friendly_name += " (backports)" - elif "gen2" in sku_name.lower() and "backports" not in sku_name.lower(): - # Don't add gen2 suffix if it's already a backports variant - if arch == "x86_64": - friendly_name += " Gen2" return (version_key, friendly_name, arch, offer_name, sku_name)