diff --git a/.gitignore b/.gitignore index 7472860ee..bc52f4046 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,11 @@ terraform/aws/kconfigs/Kconfig.instance.generated terraform/aws/kconfigs/Kconfig.location.generated terraform/aws/scripts/__pycache__/ +terraform/azure/kconfigs/Kconfig.image.generated +terraform/azure/kconfigs/Kconfig.location.generated +terraform/azure/kconfigs/Kconfig.size.generated +terraform/azure/scripts/__pycache__/ + terraform/oci/kconfigs/Kconfig.image.generated terraform/oci/kconfigs/Kconfig.location.generated terraform/oci/kconfigs/Kconfig.shape.generated diff --git a/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 b/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 index 9c3ac0a0f..e75b85180 100644 --- a/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 +++ b/playbooks/roles/gen_tfvars/templates/azure/terraform.tfvars.j2 @@ -9,6 +9,7 @@ azure_image_sku = "{{ terraform_azure_image_sku }}" azure_managed_disks_per_instance = {{ terraform_azure_managed_disks_per_instance }} azure_managed_disks_size = {{ terraform_azure_managed_disks_size }} azure_managed_disks_tier = "{{ terraform_azure_managed_disks_tier }}" +azure_accelerated_networking_enabled = {{ terraform_azure_accelerated_networking_enabled | lower }} ssh_config_pubkey_file = "{{ kdevops_terraform_ssh_config_pubkey_file }}" ssh_config_user = "{{ kdevops_terraform_ssh_config_user }}" diff --git a/scripts/dynamic-cloud-kconfig.Makefile b/scripts/dynamic-cloud-kconfig.Makefile index 840f7d805..a9dd895b2 100644 --- a/scripts/dynamic-cloud-kconfig.Makefile +++ b/scripts/dynamic-cloud-kconfig.Makefile @@ -20,6 +20,14 @@ AWS_KCONFIG_LOCATION := $(AWS_KCONFIG_DIR)/Kconfig.location.generated AWS_KCONFIGS := $(AWS_KCONFIG_AMI) $(AWS_KCONFIG_INSTANCE) $(AWS_KCONFIG_LOCATION) +# Azure dynamic configuration +AZURE_KCONFIG_DIR := terraform/azure/kconfigs +AZURE_KCONFIG_IMAGE := $(AZURE_KCONFIG_DIR)/Kconfig.image.generated +AZURE_KCONFIG_LOCATION := $(AZURE_KCONFIG_DIR)/Kconfig.location.generated +AZURE_KCONFIG_SIZE := $(AZURE_KCONFIG_DIR)/Kconfig.size.generated + +AZURE_KCONFIGS := $(AZURE_KCONFIG_LOCATION) $(AZURE_KCONFIG_SIZE) $(AZURE_KCONFIG_IMAGE) + # OCI dynamic configuration OCI_KCONFIG_DIR := terraform/oci/kconfigs OCI_KCONFIG_IMAGE := $(OCI_KCONFIG_DIR)/Kconfig.image.generated @@ -29,7 +37,7 @@ OCI_KCONFIG_SHAPE := $(OCI_KCONFIG_DIR)/Kconfig.shape.generated OCI_KCONFIGS := $(OCI_KCONFIG_IMAGE) $(OCI_KCONFIG_LOCATION) $(OCI_KCONFIG_SHAPE) # Add generated files to mrproper clean list -KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS) $(AWS_KCONFIGS) $(OCI_KCONFIGS) +KDEVOPS_MRPROPER += $(LAMBDALABS_KCONFIGS) $(AWS_KCONFIGS) $(AZURE_KCONFIGS) $(OCI_KCONFIGS) # Touch Lambda Labs generated files so Kconfig can source them # This ensures the files exist (even if empty) before Kconfig runs @@ -40,11 +48,15 @@ dynamic_lambdalabs_kconfig_touch: dynamic_aws_kconfig_touch: $(Q)touch $(AWS_KCONFIGS) +# Touch Azure generated files so Kconfig can source them +dynamic_azure_kconfig_touch: + $(Q)touch $(AZURE_KCONFIGS) + # Touch OCI generated files so Kconfig can source them dynamic_oci_kconfig_touch: $(Q)touch $(OCI_KCONFIGS) -DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch dynamic_aws_kconfig_touch dynamic_oci_kconfig_touch +DYNAMIC_KCONFIG += dynamic_lambdalabs_kconfig_touch dynamic_aws_kconfig_touch dynamic_azure_kconfig_touch dynamic_oci_kconfig_touch # Lambda Labs targets use --provider argument for efficiency cloud-config-lambdalabs: @@ -54,6 +66,10 @@ cloud-config-lambdalabs: cloud-config-aws: $(Q)python3 scripts/generate_cloud_configs.py --provider aws +# Azure targets use --provider argument for efficiency +cloud-config-azure: + $(Q)python3 scripts/generate_cloud_configs.py --provider azure + # OCI targets use --provider argument for efficiency cloud-config-oci: $(Q)python3 scripts/generate_cloud_configs.py --provider oci @@ -66,17 +82,22 @@ clean-cloud-config-lambdalabs: clean-cloud-config-aws: $(Q)rm -f $(AWS_KCONFIGS) +# Clean Azure generated files +clean-cloud-config-azure: + $(Q)rm -f $(AZURE_KCONFIGS) + # Clean OCI generated files clean-cloud-config-oci: $(Q)rm -f $(OCI_KCONFIGS) -DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs cloud-config-aws cloud-config-oci +DYNAMIC_CLOUD_KCONFIG += cloud-config-lambdalabs cloud-config-aws cloud-config-azure cloud-config-oci cloud-config-help: @echo "Cloud-specific dynamic kconfig targets:" @echo "cloud-config - generates all cloud provider dynamic kconfig content" @echo "cloud-config-lambdalabs - generates Lambda Labs dynamic kconfig content" @echo "cloud-config-aws - generates AWS dynamic kconfig content" + @echo "cloud-config-azure - generates Azure dynamic kconfig content" @echo "cloud-config-oci - generates OCI dynamic kconfig content" @echo "clean-cloud-config - removes all generated cloud kconfig files" @echo "cloud-list-all - list all cloud instances for configured provider" @@ -86,7 +107,7 @@ HELP_TARGETS += cloud-config-help cloud-config: $(Q)python3 scripts/generate_cloud_configs.py -clean-cloud-config: clean-cloud-config-lambdalabs clean-cloud-config-aws clean-cloud-config-oci +clean-cloud-config: clean-cloud-config-lambdalabs clean-cloud-config-aws clean-cloud-config-azure clean-cloud-config-oci $(Q)echo "Cleaned all cloud provider dynamic Kconfig files." cloud-list-all: @@ -95,5 +116,6 @@ cloud-list-all: PHONY += cloud-config clean-cloud-config cloud-config-help cloud-list-all PHONY += cloud-config-aws clean-cloud-config-aws +PHONY += cloud-config-azure clean-cloud-config-azure PHONY += cloud-config-lambdalabs clean-cloud-config-lambdalabs PHONY += cloud-config-oci clean-cloud-config-oci diff --git a/scripts/generate_cloud_configs.py b/scripts/generate_cloud_configs.py index d14b56d20..4cefd5d7a 100755 --- a/scripts/generate_cloud_configs.py +++ b/scripts/generate_cloud_configs.py @@ -147,6 +147,52 @@ def generate_aws_kconfig() -> bool: return all_success +def generate_azure_kconfig() -> bool: + """ + Generate Azure Kconfig files. + Returns True on success, False on failure. + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = os.path.dirname(script_dir) + azure_scripts_dir = os.path.join(project_root, "terraform", "azure", "scripts") + azure_kconfigs_dir = os.path.join(project_root, "terraform", "azure", "kconfigs") + + # Define the script-to-output mapping + scripts_to_run = [ + ("gen_kconfig_image", "Kconfig.image.generated"), + ("gen_kconfig_location", "Kconfig.location.generated"), + ("gen_kconfig_size", "Kconfig.size.generated"), + ] + + all_success = True + + for script_name, kconfig_file in scripts_to_run: + script_path = os.path.join(azure_scripts_dir, script_name) + output_path = os.path.join(azure_kconfigs_dir, kconfig_file) + + # Run the script and capture its output + result = subprocess.run( + [script_path], + capture_output=True, + text=True, + check=False, + ) + + if result.returncode == 0: + # Write the output to the corresponding Kconfig file + try: + with open(output_path, "w") as f: + f.write(result.stdout) + except IOError as e: + print(f"Error writing {kconfig_file}: {e}", file=sys.stderr) + all_success = False + else: + print(f"Error running {script_name}: {result.stderr}", file=sys.stderr) + all_success = False + + return all_success + + def generate_oci_kconfig() -> bool: """ Generate OCI Kconfig files. @@ -222,8 +268,13 @@ def process_aws(): def process_azure(): - """Process Azure configuration (placeholder).""" - print("⚠ Azure: Dynamic configuration not yet implemented") + """Process Azure configuration.""" + kconfig_generated = generate_azure_kconfig() + if kconfig_generated: + print("✓ Azure: Kconfig files generated successfully") + else: + print("⚠ Azure: Failed to generate Kconfig files - using defaults") + print() def process_gce(): diff --git a/terraform/Kconfig.providers b/terraform/Kconfig.providers index 7813c52fa..985dd8b3b 100644 --- a/terraform/Kconfig.providers +++ b/terraform/Kconfig.providers @@ -16,7 +16,7 @@ config TERRAFORM_AWS Enabling this means you are going to use AWS for your cloud solution. config TERRAFORM_AZURE - bool "Azure" + bool "Azure - Microsoft Azure" depends on TARGET_ARCH_X86_64 select TERRAFORM_PRIVATE_NET help diff --git a/terraform/azure/Kconfig b/terraform/azure/Kconfig index 9ab8f4c85..5c4e86362 100644 --- a/terraform/azure/Kconfig +++ b/terraform/azure/Kconfig @@ -1,10 +1,13 @@ if TERRAFORM_AZURE menu "Resource Location" -source "terraform/azure/kconfigs/Kconfig.location" +source "terraform/azure/kconfigs/Kconfig.location.generated" endmenu menu "Compute" -source "terraform/azure/kconfigs/Kconfig.compute" +comment "Size selection" +source "terraform/azure/kconfigs/Kconfig.size.generated" +comment "OS image selection" +source "terraform/azure/kconfigs/Kconfig.image.generated" endmenu menu "Storage" source "terraform/azure/kconfigs/Kconfig.storage" diff --git a/terraform/azure/kconfigs/Kconfig.image b/terraform/azure/kconfigs/Kconfig.image new file mode 100644 index 000000000..a5b7fe2fe --- /dev/null +++ b/terraform/azure/kconfigs/Kconfig.image @@ -0,0 +1,150 @@ +choice + prompt "Azure image publisher" + default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + help + This option specifies the publisher of the boot image used to + create the kdevops target nodes. + +config TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + bool "Debian" + help + This option sets the boot image publisher to "Debian". + +config TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + bool "Red Hat" + help + This option sets the boot image publisher to "RedHat". + +endchoice + +config TERRAFORM_AZURE_IMAGE_PUBLISHER + string + output yaml + default "Debian" if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + default "RedHat" if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + +if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + +if TARGET_ARCH_X86_64 + +choice + prompt "Debian release" + default TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + bool "debian-10 (x86)" + help + This option sets the OS image to Debian 10 (Buster). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + bool "debian-11 (x86)" + help + This option sets the OS image to Debian 11 (Bullseye). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + bool "debian-12 (x86)" + help + This option sets the OS image to Debian 12 (Bookworm). + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + bool "debian-13-daily (x86)" + help + This option sets the OS image to Debian 13's daily build. + +config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + bool "debian-sid-daily (x86)" + help + This option sets the OS image to the Debian unstable daily + build. + +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml + default "debian-10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + default "debian-11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + default "debian-12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + default "debian-13-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + default "debian-sid-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml + default "10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 + default "11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 + default "12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 + default "13" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY + default "sid" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY + +endif # TARGET_ARCH_X86_64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN + +if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT + +if TARGET_ARCH_X86_64 + +choice + prompt "Red Hat Enterprise Linux release" + default TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + bool "RHEL 7.9 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 7 update 9. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + bool "RHEL 8.9 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 8 update 9. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + bool "RHEL 8.10 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 8 update 10. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + bool "RHEL 9.5 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 9 update 5. + +config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + bool "RHEL 9.6 x64" + help + This option sets the OS image to Red Hat Enterprise Linux + release 9 update 6. + +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml + default "7_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 + default "8_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 + default "8_10" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 + default "9_5" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 + default "9_6" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 + +endif # TARGET_ARCH_X86_64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT diff --git a/terraform/azure/kconfigs/Kconfig.compute b/terraform/azure/kconfigs/Kconfig.size similarity index 66% rename from terraform/azure/kconfigs/Kconfig.compute rename to terraform/azure/kconfigs/Kconfig.size index 3b09891b9..758ec8ce7 100644 --- a/terraform/azure/kconfigs/Kconfig.compute +++ b/terraform/azure/kconfigs/Kconfig.size @@ -94,30 +94,35 @@ config TERRAFORM_AZURE_VM_SIZE default "Standard_D4s_v6" if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V6 default "Standard_D8s_v6" if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V6 -choice - prompt "Azure image publisher" - default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - help - This option specifies the publisher of the boot image used to - create the kdevops target nodes. - -config TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - bool "Debian" - help - This option sets the boot image publisher to "Debian". - -config TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - bool "Red Hat" - help - This option sets the boot image publisher to "RedHat". - -endchoice - -config TERRAFORM_AZURE_IMAGE_PUBLISHER - string +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool output yaml - default "Debian" if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - default "RedHat" if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - -source "terraform/azure/kconfigs/publishers/Kconfig.debian" -source "terraform/azure/kconfigs/publishers/Kconfig.rhel" + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B1S + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2S + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B12MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B20MS + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2TS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B2S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B4S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B8S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B16S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B32LS_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_B32S_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_DS1_V2 + default n if TERRAFORM_AZURE_VM_SIZE_STANDARD_DS3_V2 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V3 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V4 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D2S_V6 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D4S_V6 + default y if TERRAFORM_AZURE_VM_SIZE_STANDARD_D8S_V6 diff --git a/terraform/azure/kconfigs/publishers/Kconfig.debian b/terraform/azure/kconfigs/publishers/Kconfig.debian deleted file mode 100644 index 372a809dc..000000000 --- a/terraform/azure/kconfigs/publishers/Kconfig.debian +++ /dev/null @@ -1,60 +0,0 @@ -if TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN - -if TARGET_ARCH_X86_64 - -choice - prompt "Debian release" - default TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - help - This option specifies which of a publisher's offers to use - when creating kdevops compute instances. - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - bool "debian-10 (x86)" - help - This option sets the OS image to Debian 10 (Buster). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - bool "debian-11 (x86)" - help - This option sets the OS image to Debian 11 (Bullseye). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - bool "debian-12 (x86)" - help - This option sets the OS image to Debian 12 (Bookworm). - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - bool "debian-13-daily (x86)" - help - This option sets the OS image to Debian 13's daily build. - -config TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - bool "debian-sid-daily (x86)" - help - This option sets the OS image to the Debian unstable daily - build. - -endchoice - -config TERRAFORM_AZURE_IMAGE_OFFER - string - output yaml - default "debian-10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - default "debian-11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - default "debian-12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - default "debian-13-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - default "debian-sid-daily" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - -config TERRAFORM_AZURE_IMAGE_SKU - string - output yaml - default "10" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_10 - default "11" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_11 - default "12" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_12 - default "13" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_13_DAILY - default "sid" if TERRAFORM_AZURE_IMAGE_LINUX_DEBIAN_SID_DAILY - -endif # TARGET_ARCH_X86_64 - -endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN diff --git a/terraform/azure/kconfigs/publishers/Kconfig.rhel b/terraform/azure/kconfigs/publishers/Kconfig.rhel deleted file mode 100644 index 3a6a7d9a1..000000000 --- a/terraform/azure/kconfigs/publishers/Kconfig.rhel +++ /dev/null @@ -1,64 +0,0 @@ -if TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT - -if TARGET_ARCH_X86_64 - -choice - prompt "Red Hat Enterprise Linux release" - default TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - help - This option specifies which of a publisher's offers to use - when creating kdevops compute instances. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - bool "RHEL 7.9 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 7 update 9. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - bool "RHEL 8.9 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 8 update 9. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - bool "RHEL 8.10 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 8 update 10. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - bool "RHEL 9.5 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 9 update 5. - -config TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - bool "RHEL 9.6 x64" - help - This option sets the OS image to Red Hat Enterprise Linux - release 9 update 6. - -endchoice - -config TERRAFORM_AZURE_IMAGE_OFFER - string - output yaml - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - default "RHEL" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - -config TERRAFORM_AZURE_IMAGE_SKU - string - output yaml - default "7_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_7_9 - default "8_9" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_9 - default "8_10" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_8_10 - default "9_5" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_5 - default "9_6" if TERRAFORM_AZURE_IMAGE_LINUX_RHEL_9_6 - -endif # TARGET_ARCH_X86_64 - -endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index eb609933f..683263715 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -57,7 +57,7 @@ resource "azurerm_network_interface_security_group_association" "kdevops_sg_asso resource "azurerm_network_interface" "kdevops_nic" { count = local.kdevops_num_boxes - accelerated_networking_enabled = true + accelerated_networking_enabled = var.azure_accelerated_networking_enabled name = format("kdevops_nic_%02d", count.index + 1) location = var.azure_location resource_group_name = azurerm_resource_group.kdevops_group.name diff --git a/terraform/azure/scripts/azure_common.py b/terraform/azure/scripts/azure_common.py new file mode 100644 index 000000000..de9abb915 --- /dev/null +++ b/terraform/azure/scripts/azure_common.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Common utilities for Azure Kconfig generation scripts. + +This module provides shared functionality for Azure-specific Kconfig +generation, including region discovery, default region detection, and +Jinja2 template management. +""" + +import os +import sys +import json +from configparser import ConfigParser + +from jinja2 import Environment, FileSystemLoader + + +def get_default_region(): + """ + Get the default Azure region from Azure configuration. + + Returns: + str: Default region, or 'westus' if no default is found. + """ + try: + from azure.common.credentials import get_cli_profile + + # Check if user is authenticated by getting the CLI profile + # This reuses the 'az login' session without subprocess calls + profile = get_cli_profile() + _, _, _ = profile.get_login_credentials(resource="https://management.azure.com") + + # Azure doesn't have a per-account default region like AWS + # Check environment variable first + if "AZURE_DEFAULTS_LOCATION" in os.environ: + return os.environ["AZURE_DEFAULTS_LOCATION"] + + # Try to read from azure config + config_path = os.path.expanduser("~/.azure/config") + if os.path.exists(config_path): + try: + config = ConfigParser() + config.read(config_path) + if "defaults" in config and "location" in config["defaults"]: + return config["defaults"]["location"] + except Exception: + pass + + # Default fallback + return "westus" + except Exception as e: + print(f"Warning: Error reading Azure config: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return "westus" + + +def get_jinja2_environment(template_path=None): + """ + Create a standardized Jinja2 environment for template rendering. + + Args: + template_path (str): Path to template directory. If None, uses caller's directory. + + Returns: + Environment: Configured Jinja2 Environment object + """ + if template_path is None: + template_path = sys.path[0] + + return Environment( + loader=FileSystemLoader(template_path), + trim_blocks=True, + lstrip_blocks=True, + ) + + +def get_all_regions(quiet=False): + """ + Retrieve the list of all Azure regions using the Azure SDK. + + Returns: + list: List of region dictionaries with name, displayName, and metadata + """ + if not quiet: + print("Querying Azure for available regions...", file=sys.stderr) + + try: + from azure.common.credentials import get_cli_profile + from azure.mgmt.resource import SubscriptionClient + + # Get credentials from Azure CLI profile (reuses 'az login' session) + profile = get_cli_profile() + credentials, subscription_id, _ = profile.get_login_credentials( + resource="https://management.azure.com" + ) + + # Query regions using SDK + subscription_client = SubscriptionClient(credentials) + locations = subscription_client.subscriptions.list_locations(subscription_id) + + # Convert SDK objects to dict format + region_list = [] + for location in locations: + # Skip logical regions (not for general use) + if location.metadata and location.metadata.region_type == "Logical": + continue + + # Convert metadata to dict with camelCase keys + # Only convert the fields we actually use to minimize overhead + metadata = {} + if location.metadata: + meta = location.metadata + metadata = { + "physicalLocation": meta.physical_location, + "pairedRegion": meta.paired_region, + } + + region_list.append( + { + "name": location.name or "", + "displayName": location.display_name or "", + "regionalDisplayName": location.regional_display_name or "", + "metadata": metadata, + } + ) + + return sorted(region_list, key=lambda x: x["name"]) + + except Exception as e: + if not quiet: + print(f"Error: Failed to query Azure regions: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return [] + + +def get_region_kconfig_name(region_name): + """ + Convert an Azure region name to a Kconfig variable name. + + Args: + region_name (str): Azure region name (e.g., 'westus', 'eastus2') + + Returns: + str: Kconfig-safe name (e.g., 'WESTUS', 'EASTUS2') + """ + return region_name.upper().replace("-", "_") + + +def get_compute_client(): + """ + Get an authenticated Azure Compute Management client. + + Returns: + tuple: (ComputeManagementClient, subscription_id) + + Raises: + Exception: If authentication fails or SDK is not available + """ + from azure.common.credentials import get_cli_profile + from azure.mgmt.compute import ComputeManagementClient + + # Get credentials from Azure CLI profile (reuses 'az login' session) + profile = get_cli_profile() + credentials, subscription_id, _ = profile.get_login_credentials( + resource="https://management.azure.com" + ) + + client = ComputeManagementClient(credentials, subscription_id) + return client, subscription_id + + +def get_vm_sizes_and_skus(region, quiet=False): + """ + Get all VM sizes and capabilities for a region using a single Azure SDK call. + + This function uses the resource SKUs API which provides all the information + from both VM sizes and SKU capabilities in a single efficient API call. + + Args: + region (str): Azure region name + quiet (bool): Suppress debug messages + + Returns: + tuple: (sizes_list, capabilities_dict) where: + - sizes_list: List of VM size dictionaries in CLI-compatible format + - capabilities_dict: Dict mapping VM size names to their capabilities + """ + if not quiet: + print(f"Fetching VM sizes and capabilities from {region}...", file=sys.stderr) + + try: + client, _ = get_compute_client() + + # Query resource SKUs using SDK with location filter + # This single API call provides all information we need + skus = list(client.resource_skus.list(filter=f"location eq '{region}'")) + + # Filter to VM SKUs only + vm_skus = [s for s in skus if s.resource_type == "virtualMachines"] + + # Build both data structures + size_list = [] + sku_capabilities = {} + + for sku in vm_skus: + if not sku.capabilities: + continue + + # Convert capabilities list to dictionary for easy lookup + caps = {cap.name: cap.value for cap in sku.capabilities} + + # Extract size information from capabilities + # The resource SKUs API provides the same data as the VM sizes API + try: + cores = int(caps.get("vCPUs", 0)) + memory_mb = int(float(caps.get("MemoryGB", 0)) * 1024) + max_disks = int(caps.get("MaxDataDiskCount", 0)) + resource_disk_mb = int(caps.get("MaxResourceVolumeMB", 0)) + os_disk_mb = int(caps.get("OSVhdSizeMB", 0)) + + # Build VM size dict in CLI-compatible format + size_list.append( + { + "name": sku.name, + "numberOfCores": cores, + "memoryInMB": memory_mb, + "maxDataDiskCount": max_disks, + "resourceDiskSizeInMB": resource_disk_mb, + "osDiskSizeInMB": os_disk_mb, + } + ) + + # Store capabilities for this size + sku_capabilities[sku.name] = caps + + except (ValueError, TypeError) as e: + if not quiet: + print( + f"Warning: Could not parse capabilities for {sku.name}: {e}", + file=sys.stderr, + ) + continue + + if not quiet: + print(f" Found {len(size_list)} VM sizes in {region}", file=sys.stderr) + + return size_list, sku_capabilities + + except Exception as e: + if not quiet: + print(f"Error: Failed to query VM sizes and SKUs: {e}", file=sys.stderr) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return [], {} + + +def get_all_offers_and_skus(publisher_id, region, quiet=False, max_workers=10): + """ + Get all offers and SKUs for a publisher using Azure SDK with parallel execution. + + This function uses parallel SKU fetching for optimal performance with + publishers that have many offers. The parallelization is safe because + each SKU query is independent and the Azure API supports concurrent requests. + + Args: + publisher_id (str): Azure publisher identifier (e.g., "Debian", "RedHat") + region (str): Azure region name + quiet (bool): Suppress debug messages + max_workers (int): Maximum concurrent workers for parallel SKU fetching + + Returns: + dict: Dictionary mapping offer names to lists of SKU names + Example: {"debian-12": ["12", "12-arm64"], ...} + """ + from concurrent.futures import ThreadPoolExecutor + + if not quiet: + print(f"Querying offers for {publisher_id} in {region}...", file=sys.stderr) + + try: + client, _ = get_compute_client() + + # Get all offers (single fast API call) + offers = list(client.virtual_machine_images.list_offers(region, publisher_id)) + + if not offers: + return {} + + if not quiet: + print( + f" Found {len(offers)} offers, fetching SKUs in parallel...", + file=sys.stderr, + ) + + def fetch_skus_for_offer(offer): + """Helper function to fetch SKUs for a single offer.""" + offer_name = offer.name + + # Skip test offers and other non-production offers + offer_lower = offer_name.lower() + if any( + skip in offer_lower + for skip in ["test", "preview", "experimental", "-dev", "staging"] + ): + return (offer_name, []) + + try: + skus = list( + client.virtual_machine_images.list_skus( + region, publisher_id, offer_name + ) + ) + sku_names = [sku.name for sku in skus if sku.name] + return (offer_name, sku_names) + except Exception as e: + if not quiet: + print( + f" Warning: Failed to get SKUs for {offer_name}: {e}", + file=sys.stderr, + ) + return (offer_name, []) + + # Fetch SKUs for all offers in parallel + offers_dict = {} + with ThreadPoolExecutor(max_workers=max_workers) as executor: + results = executor.map(fetch_skus_for_offer, offers) + + for offer_name, sku_names in results: + if sku_names: + offers_dict[offer_name] = sku_names + + if not quiet: + print( + f" Found {len(offers_dict)} offers with available SKUs", + file=sys.stderr, + ) + + return offers_dict + + except Exception as e: + if not quiet: + print( + f"Error: Failed to query offers and SKUs for {publisher_id}: {e}", + file=sys.stderr, + ) + print("Ensure you are logged in with 'az login'", file=sys.stderr) + return {} + + +def exit_on_empty_result(result, context, quiet=False): + """ + Exit with error if result is empty or None. + + This consolidates the common pattern of checking if an API query + returned results and exiting with appropriate error messaging if not. + + Args: + result: Result from API query (list, dict, or other iterable) + context (str): Description of what operation failed + quiet (bool): Suppress error messages + + Returns: + Does not return; exits the process with status 1 + """ + if not result: + if not quiet: + print( + f"Error: Cannot perform {context}. Check Azure authentication status.", + file=sys.stderr, + ) + print("Run 'az login' to authenticate with Azure.", file=sys.stderr) + sys.exit(1) diff --git a/terraform/azure/scripts/families.j2 b/terraform/azure/scripts/families.j2 new file mode 100644 index 000000000..2fbfb633c --- /dev/null +++ b/terraform/azure/scripts/families.j2 @@ -0,0 +1,81 @@ +# Azure VM Size Selection +# This file is auto-generated. Do not edit manually. +# Generated by: terraform/azure/scripts/gen_kconfig_size + +choice + prompt "VM size family" + default {{ default_family_config }} + help + Select the Azure VM size family for your target nodes. + Each family is optimized for different workloads. + + VM size availability varies by region. You can verify + available VM sizes for your region using: + + az vm list-sizes --location -o table + + For detailed information about Azure VM families: + + https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview + +{% for family in families %} +config TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + bool "{{ family.family_name }} - {{ family.description }}" +{% if family.architecture == 'arm64' %} + depends on TARGET_ARCH_ARM64 +{% else %} + depends on TARGET_ARCH_X86_64 +{% endif %} + help + {{ family.help_text | replace('\n', '\n\t ') }} + + This family contains {{ family.size_count }} VM size{{ 's' if family.size_count > 1 else '' }}: + - CPU cores: {{ family.min_cores }}-{{ family.max_cores }} + - Memory: {{ family.min_memory_gb|round(1) }}-{{ family.max_memory_gb|round(1) }}GB + +{% endfor %} +endchoice + +{% for family in families %} +if TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + +choice + prompt "{{ family.family_name }} VM size" + default {{ family.default_size_config }} + help + Select a specific VM size from the {{ family.family_name }} family. + +{% for size in family.sizes %} +config TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} + bool "{{ size.name }}" + help + This virtual machine size has {{ size.cores }} vCPU{{ 's' if size.cores > 1 else '' }} and {{ size.memory_gb|round(1) }}GB of memory. +{% if size.resource_disk_gb > 0 %} + Includes {{ size.resource_disk_gb|round(1) }}GB temporary storage. +{% endif %} + Supports up to {{ size.max_data_disks }} data disk{{ 's' if size.max_data_disks > 1 else '' }}. + +{% endfor %} +endchoice + +endif # TERRAFORM_AZURE_VM_FAMILY_{{ family.config_name }} + +{% endfor %} + +config TERRAFORM_AZURE_VM_SIZE + string + output yaml +{% for family in families %} +{% for size in family.sizes %} + default "{{ size.name }}" if TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} +{% endfor %} +{% endfor %} + +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool + output yaml +{% for family in families %} +{% for size in family.sizes %} + default {{ 'y' if size.accelerated_networking else 'n' }} if TERRAFORM_AZURE_VM_SIZE_{{ size.config_name }} +{% endfor %} +{% endfor %} diff --git a/terraform/azure/scripts/gen_kconfig_image b/terraform/azure/scripts/gen_kconfig_image new file mode 100755 index 000000000..673474ce3 --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_image @@ -0,0 +1,776 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve VM image information from Azure. Use it to construct the "images" +Kconfig menu. + +Azure VM images are OS templates that can be used to launch compute instances. +This script queries the Azure API to discover available images and generates +Kconfig menu entries for them. + +Publisher Definitions: + Publisher definitions (e.g., "Debian", "Red Hat") are maintained + in the publisher_definitions.yml file in the same directory as this + script. This makes it easier to update publisher information when + Microsoft adds new distributions or when you want to customize display + names and priorities. + + To update publisher definitions: + 1. Edit terraform/azure/scripts/publisher_definitions.yml + 2. Add or modify entries following the existing format + 3. Run this script to regenerate Kconfig.image + + The YAML file contains detailed instructions for how to add new + publisher definitions. If the YAML file is not found, the script + falls back to dynamic publisher discovery by analyzing image names. + +Image Discovery: + By default, this script aggregates images from all Azure regions to + provide a comprehensive mapping of image URNs across regions. + + This script focuses on well-known Linux distribution publishers. + The automated discovery identifies available offers and SKUs for + each publisher. + +Usage: + # Generate images Kconfig from default region + ./gen_kconfig_image > ../kconfigs/Kconfig.image.generated + + # List all available publishers + ./gen_kconfig_image --publishers + + # Get details for a specific publisher + ./gen_kconfig_image debian + + # Query specific region + ./gen_kconfig_image --region eastus +""" + +import sys +import os +import argparse +import re +from collections import defaultdict +from functools import lru_cache + +from azure_common import ( + get_default_region, + get_jinja2_environment, + get_all_regions, + get_region_kconfig_name, + get_all_offers_and_skus, +) + + +def load_yaml_config(filename, quiet=False): + """ + Load YAML configuration file from the same directory as this script. + + Args: + filename (str): Name of the YAML file to load + quiet (bool): Suppress warning messages + + Returns: + dict: Parsed YAML content, or None if file not found or parse error + """ + try: + import yaml + except ImportError: + if not quiet: + print( + "Warning: PyYAML not installed. Install with: pip install pyyaml", + file=sys.stderr, + ) + return None + + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(script_dir, filename) + + if not os.path.exists(config_path): + if not quiet: + print(f"Warning: {filename} not found at {config_path}", file=sys.stderr) + return None + + try: + with open(config_path, "r") as f: + return yaml.safe_load(f) + except yaml.YAMLError as e: + if not quiet: + print(f"Error parsing {filename}: {e}", file=sys.stderr) + return None + except Exception as e: + if not quiet: + print(f"Error reading {filename}: {e}", file=sys.stderr) + return None + + +@lru_cache(maxsize=1) +def get_known_publishers(): + """ + Get dictionary of known OS image publishers in Azure. + + Publisher definitions are loaded from publisher_definitions.yml to make + it easier to update when Microsoft adds new Linux distributions or when + you want to customize publisher priorities and naming. + + The YAML file contains publisher information including: + - publisher_name: Display name for the publisher + - description: Full description used in Kconfig help text + - offer_patterns: List of regex patterns to match offer names + - priority: Display order (lower numbers appear first) + + If the YAML file is not found or fails to parse, a minimal set of + hardcoded publishers is returned. + + Results are cached using @lru_cache to avoid repeated file I/O. + + To update the publisher definitions: + 1. Edit terraform/azure/scripts/publisher_definitions.yml + 2. Add or modify entries following the existing format + 3. Regenerate Kconfig.image by running this script + + See publisher_definitions.yml for detailed update instructions. + + Returns: + dict: Dictionary mapping publisher keys to publisher information + """ + publishers = load_yaml_config("publisher_definitions.yml", quiet=True) + if not publishers: + print( + "Warning: publisher_definitions.yml not found or empty. " + "Using fallback publisher list.", + file=sys.stderr, + ) + return get_fallback_publishers() + + return publishers + + +def get_fallback_publishers(): + """ + Get minimal fallback set of known Azure publishers. + + This is used when publisher_definitions.yml is not available. + + Returns: + dict: Dictionary of publisher information + """ + return { + "debian": { + "publisher_id": "Debian", + "publisher_name": "Debian", + "description": "Debian GNU/Linux", + "priority": 10, + }, + "redhat": { + "publisher_id": "RedHat", + "publisher_name": "Red Hat", + "description": "Red Hat Enterprise Linux", + "priority": 20, + }, + } + + +def score_sku_quality(sku_name): + """ + Score SKU by quality - higher score means better/more modern variant. + + This helps automatically select the best SKU when multiple variants exist + for the same OS version. Prefers Gen2 VMs and LVM-based images. + + Args: + sku_name (str): SKU name (e.g., "ol89-lvm-gen2", "79-gen2") + + Returns: + int: Quality score (higher is better) + """ + sku_lower = sku_name.lower() + score = 0 + + # Prefer Gen2 (UEFI boot, larger VM sizes, better performance) + if "gen2" in sku_lower: + score += 100 + + # Prefer LVM (flexible disk partitioning) + if "lvm" in sku_lower: + score += 50 + + # Avoid CI variants (legacy cloud-init explicit variants, not needed for 7.7+) + if "ci" in sku_lower: + score -= 10 + + # Avoid underscore format (legacy naming convention) + if "_" in sku_name and not sku_name.startswith("ol"): + score -= 5 + + return score + + +def supports_cloud_init(publisher_key, major, minor): + """ + Check if a distribution version supports cloud-init by default. + + kdevops terraform provisioning depends on cloud-init. Only include + versions that have cloud-init support built-in. + + Args: + publisher_key (str): Publisher key (e.g., "oracle", "redhat") + major (str): Major version number + minor (str): Minor version number + + Returns: + bool: True if version supports cloud-init, False otherwise + """ + try: + major_int = int(major) + minor_int = int(minor) + except (ValueError, TypeError): + # If we can't parse the version, assume it's too old + return False + + # Per Microsoft docs: Oracle Linux 7.7+, RHEL 7.7+ have cloud-init by default + # https://learn.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init + if publisher_key in ("oracle", "redhat"): + if major_int > 7: + return True + if major_int == 7 and minor_int >= 7: + return True + return False + + # Other distributions: assume modern versions have cloud-init + # (Debian, Ubuntu, SUSE, etc. have had cloud-init for longer) + return True + + +def classify_offer_sku( + publisher_key, publisher_id, offer_name, sku_name, publisher_info +): + """ + Classify an offer/SKU combination into a version and architecture. + + Args: + publisher_key (str): Publisher key (e.g., "debian", "redhat") + publisher_id (str): Azure publisher ID (e.g., "Debian", "RedHat") + offer_name (str): Offer name (e.g., "debian-12", "RHEL") + sku_name (str): SKU name (e.g., "12", "12-arm64", "9_6") + publisher_info (dict): Publisher information from definitions + + Returns: + tuple: (version_key, friendly_name, architecture, offer, sku) + or (None, None, None, None, None) if not classifiable + """ + # Detect architecture from SKU name + sku_lower = sku_name.lower() + if "arm64" in sku_lower or "aarch64" in sku_lower: + arch = "arm64" + arch_suffix = "_ARM64" + arch_display = "(arm64)" + else: + arch = "x86_64" + arch_suffix = "_X86" + arch_display = "(x86)" + + # Extract version information + # Try to extract version from offer or SKU name + version_match = None + + # Pattern 1: debian-12, debian-11 + if re.match(r"^debian-\d+", offer_name, re.IGNORECASE): + version_match = re.search(r"(\d+)", offer_name) + # Pattern 2: debian-sid (Debian unstable) + elif publisher_key == "debian" and "sid" in offer_name.lower(): + # Debian sid is the unstable rolling release + version_key = f"DEBIAN_SID{arch_suffix}" + friendly_name = f"Debian sid unstable {arch_display}" + if "daily" in offer_name.lower() or "daily" in sku_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 3: RHEL with flexible version and suffix handling + # Examples: "9_6" → 9.6, "810-gen2" → 8.10, "8-lvm-gen2" → 8.0, "7lvm-gen2" → 7.0 + # Handles: underscore format, condensed versions, all suffix variants + elif publisher_key == "redhat": + # Flexible regex: version digits, optional underscore minor, optional suffix + # Handles both "9_6" and "810-gen2" and "7lvm-gen2" and "81gen2" patterns + match = re.match(r"^(\d{1,3})(?:_(\d+)|([a-z]+))?(?:-(.+)|(.+))?$", sku_name) + if match: + major_digits = match.group(1) + underscore_minor = match.group(2) + embedded_suffix = match.group(3) # e.g., "lvm" in "7lvm-gen2" + hyphen_suffix = match.group(4) # e.g., "gen2" in "810-gen2" + direct_suffix = match.group(5) # e.g., "gen2" in "81gen2" + + # Parse version into major.minor + if underscore_minor: + # Underscore format: 9_6 → 9.6, 10_1 → 10.1 + major, minor = major_digits, underscore_minor + elif len(major_digits) == 1: + # Single digit: 8 → 8.0 + major, minor = major_digits, "0" + elif len(major_digits) == 2: + # Two digits: 77 → 7.7, 81 → 8.1 + major, minor = major_digits[0], major_digits[1] + elif major_digits.startswith("10"): + # Three digits starting with 10: 100 → 10.0 + major, minor = "10", "0" + else: + # Three digits: 810 → 8.10 + major, minor = major_digits[0], major_digits[1:] + + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + # Clean friendly name without implementation details (gen2/lvm/sap/ci) + friendly_name = f"RHEL {major}.{minor} {arch_display}" + + # Filter out versions without cloud-init support (kdevops requirement) + if not supports_cloud_init(publisher_key, major, minor): + return (None, None, None, None, None) + + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 3c: Oracle Linux with condensed version numbers + # Examples: "ol82-gen2" → 8.2, "77" → 7.7, "ol810-lvm" → 8.10, "79-gen2" → 7.9 + # Handles: optional "ol" prefix, condensed versions, underscore format (ol8_2-gen2) + elif publisher_key == "oracle": + # Flexible regex: optional prefix, version digits, optional underscore minor, optional suffix + match = re.match(r"^(ol)?(\d{1,3})(?:_(\d+))?(?:-(.+))?$", sku_name) + if match: + major_digits = match.group(2) + underscore_minor = match.group(3) + + # Parse version into major.minor + if underscore_minor: + # Underscore format: ol8_2 → 8.2 + major, minor = major_digits, underscore_minor + elif len(major_digits) == 1: + # Single digit: 8 → 8.0, ol9 → 9.0 + major, minor = major_digits, "0" + elif len(major_digits) == 2: + # Two digits: 77 → 7.7, ol82 → 8.2 + major, minor = major_digits[0], major_digits[1] + elif major_digits.startswith("10"): + # Three digits starting with 10: ol100 → 10.0 + major, minor = "10", "0" + else: + # Three digits: ol810 → 8.10 + major, minor = major_digits[0], major_digits[1:] + + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + # Clean friendly name without implementation details (gen2/lvm/ci) + friendly_name = f"Oracle Linux {major}.{minor} {arch_display}" + + # Filter out versions without cloud-init support (kdevops requirement) + if not supports_cloud_init(publisher_key, major, minor): + return (None, None, None, None, None) + + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 4: Ubuntu LTS from offer name + # Examples: "ubuntu-22_04-lts" → 22.04, "ubuntu-24_04-lts-daily" → 24.04 + # Skip ubuntu-pro SKUs as these are commercial support variants + elif publisher_key == "canonical" and re.match(r"^ubuntu-(\d+)_(\d+)-lts", offer_name): + if "ubuntu-pro" in sku_name.lower(): + # Skip Ubuntu Pro SKUs - these are commercial variants + return (None, None, None, None, None) + version_match = re.match(r"^ubuntu-(\d+)_(\d+)-lts", offer_name) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"Ubuntu Linux {major}.{minor} {arch_display}" + if "daily" in offer_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 5: Ubuntu non-LTS from offer name + # Examples: "ubuntu-24_10" → 24.10, "ubuntu-25_04" → 25.04, "ubuntu-25_04-daily" → 25.04 (daily) + elif publisher_key == "canonical" and re.match(r"^ubuntu-(\d+)_(\d+)", offer_name): + version_match = re.match(r"^ubuntu-(\d+)_(\d+)", offer_name) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = f"Ubuntu Linux {major}.{minor} {arch_display}" + if "daily" in offer_name.lower(): + friendly_name += " (daily)" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 6: openSUSE Leap from offer name + # Examples: "opensuse-leap-15-6" → 15.6, "opensuse-leap-15-6-arm64" → 15.6 (arm64) + # Also "openSUSE-Leap" with SKU "15-2" → 15.2 + elif publisher_key == "suse" and re.match(r"^opensuse-leap", offer_name, re.IGNORECASE): + # Check if version is in offer name (e.g., opensuse-leap-15-6) + offer_version_match = re.match(r"^opensuse-leap-(\d+)-(\d+)", offer_name, re.IGNORECASE) + if offer_version_match: + major = offer_version_match.group(1) + minor = offer_version_match.group(2) + else: + # Version must be in SKU (e.g., openSUSE-Leap with SKU 15-2) + sku_version_match = re.match(r"^(\d+)-(\d+)$", sku_name) + if not sku_version_match: + return (None, None, None, None, None) + major = sku_version_match.group(1) + minor = sku_version_match.group(2) + version_key = f"OPENSUSE_LEAP_{major}_{minor}{arch_suffix}" + friendly_name = f"openSUSE Leap {major}.{minor} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 7: SLE Micro from offer name (uses X-Y format, not SP) + # Examples: "sle-micro-5-1-byos" → 5.1, "sle-micro-6-0-byos" → 6.0 + elif publisher_key == "suse" and re.match(r"^sle-micro-(\d+)-(\d+)", offer_name, re.IGNORECASE): + version_match = re.match(r"^sle-micro-(\d+)-(\d+)", offer_name, re.IGNORECASE) + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"SLE_MICRO_{major}_{minor}{arch_suffix}" + friendly_name = f"SUSE Linux Enterprise Micro {major}.{minor} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 8: SLES/SLE-HPC from offer name with SP (Service Pack) + # Examples: "sles-15-sp6" → 15 SP6, "sle-hpc-15-sp4-byos" → 15 SP4 + elif publisher_key == "suse" and re.match(r"^(sles|sle-hpc)-(\d+)-sp(\d+)", offer_name, re.IGNORECASE): + version_match = re.match(r"^(sles|sle-hpc)-(\d+)-sp(\d+)", offer_name, re.IGNORECASE) + product_type = version_match.group(1).upper() + major = version_match.group(2) + sp = version_match.group(3) + version_key = f"SUSE_{major}_SP{sp}{arch_suffix}" + if "hpc" in product_type.lower(): + friendly_name = f"SUSE Linux Enterprise HPC {major} SP{sp} {arch_display}" + else: + friendly_name = f"SUSE Linux Enterprise {major} SP{sp} {arch_display}" + return (version_key, friendly_name, arch, offer_name, sku_name) + # Pattern 6: Generic version extraction + elif not version_match: + version_match = re.search(r"(\d+)(?:[_\.](\d+))?", sku_name) + + if version_match: + if len(version_match.groups()) >= 2 and version_match.group(2): + # Has minor version + major = version_match.group(1) + minor = version_match.group(2) + version_key = f"{publisher_key.upper()}_{major}_{minor}{arch_suffix}" + friendly_name = ( + f"{publisher_info.get('description', publisher_key)} " + f"{major}.{minor} {arch_display}" + ) + else: + # Only major version + major = version_match.group(1) + version_key = f"{publisher_key.upper()}_{major}{arch_suffix}" + friendly_name = ( + f"{publisher_info.get('description', publisher_key)} " + f"{major} {arch_display}" + ) + + # Add special notes for daily/backports variants (but not gen2 - implementation detail) + if "daily" in offer_name.lower() or "daily" in sku_name.lower(): + friendly_name += " (daily)" + elif "backports" in sku_name.lower(): + friendly_name += " (backports)" + + return (version_key, friendly_name, arch, offer_name, sku_name) + + return (None, None, None, None, None) + + +def organize_images_by_publisher(publishers, region=None, quiet=False): + """ + Organize Azure images by publisher and version. + + Args: + publishers (dict): Dictionary of publisher information + region (str): Azure region to query (default: get_default_region()) + quiet (bool): Suppress debug messages + + Returns: + dict: Organized structure {publisher: {version_key: {offer, sku, ...}}} + """ + if region is None: + region = get_default_region() + + organized = defaultdict(lambda: defaultdict(dict)) + + for publisher_key, publisher_info in publishers.items(): + publisher_id = publisher_info.get("publisher_id") + if not publisher_id: + continue + + if not quiet: + print(f"\nProcessing {publisher_id}...", file=sys.stderr) + + offers_dict = get_all_offers_and_skus(publisher_id, region, quiet) + + # Get offer patterns for filtering (if defined) + offer_patterns = publisher_info.get("offer_patterns", []) + offer_regexes = [re.compile(pattern) for pattern in offer_patterns] if offer_patterns else [] + + for offer_name, skus in offers_dict.items(): + # Skip offers that don't match any pattern (if patterns are defined) + if offer_regexes and not any(regex.match(offer_name) for regex in offer_regexes): + continue + for sku_name in skus: + ( + version_key, + friendly_name, + arch, + offer, + sku, + ) = classify_offer_sku( + publisher_key, publisher_id, offer_name, sku_name, publisher_info + ) + + if not version_key: + continue + + # Store image information - select best SKU when multiple variants exist + # Use score_sku_quality() to prefer Gen2 and LVM variants + if version_key not in organized[publisher_key]: + # First occurrence for this version + organized[publisher_key][version_key] = { + "friendly_name": friendly_name, + "architecture": arch, + "offer": offer, + "sku": sku, + "publisher_id": publisher_id, + "_score": score_sku_quality(sku), + } + else: + # Multiple SKUs for same version - keep the better one + current_score = organized[publisher_key][version_key].get("_score", 0) + new_score = score_sku_quality(sku) + if new_score > current_score: + # This SKU is better, replace it + organized[publisher_key][version_key] = { + "friendly_name": friendly_name, + "architecture": arch, + "offer": offer, + "sku": sku, + "publisher_id": publisher_id, + "_score": new_score, + } + + if not quiet: + print("\n--- Summary ---", file=sys.stderr) + for publisher, versions in organized.items(): + print(f"Publisher '{publisher}': {len(versions)} versions", file=sys.stderr) + + return organized + + +def output_images_kconfig(organized_images, publishers): + """ + Output images menu in Kconfig format. + + Args: + organized_images (dict): Organized images from organize_images_by_publisher() + publishers (dict): Dictionary of all publishers + """ + print("# This file was auto-generated by gen_kconfig_image") + print("#") + print("# To regenerate: cd terraform/azure/scripts && ./gen_kconfig_image") + print() + + environment = get_jinja2_environment() + + # Sort publishers by priority for consistent ordering + sorted_publishers = sorted( + [(k, v) for k, v in publishers.items() if k in organized_images], + key=lambda x: (x[1].get("priority", 100), x[0]), + ) + + # Output the top-level distribution choice menu + template = environment.get_template("image_distributions.j2") + print( + template.render( + publishers=[pub[0] for pub in sorted_publishers], + ) + ) + print() + + def version_sort_key(version_item): + """ + Extract numeric version from version_key for proper chronological sorting. + + Version keys are like: DEBIAN_12_X86, RHEL_9_6_ARM64 + Sort by numeric version (oldest to newest), not alphabetic. + """ + version_key, version_data = version_item + parts = version_key.split("_") + + # Remove publisher prefix + version_parts = parts[1:] + + major = 0 + minor = 0 + + # The last part is usually the architecture (X86, ARM64) + # Everything before that is version numbers + if len(version_parts) >= 2: + try: + major = int(version_parts[0]) + # Try to parse second part as minor version + if len(version_parts) >= 3: + try: + minor = int(version_parts[1]) + except ValueError: + # Second part is arch, no minor version + minor = 0 + except ValueError: + pass + + # Architecture preference: X86, ARM64 for consistent ordering + arch_order = {"X86": 0, "ARM64": 1} + arch = version_parts[-1] if version_parts else "X86" + arch_priority = arch_order.get(arch, 99) + + return (major, minor, arch_priority) + + # Output each publisher's images + template = environment.get_template("image_publisher.j2") + for publisher_key, publisher_info in sorted_publishers: + versions = organized_images.get(publisher_key, {}) + + # Sort versions numerically (oldest to newest) + sorted_versions = sorted(versions.items(), key=version_sort_key) + + print( + template.render( + publisher_key=publisher_key, + publisher_name=publisher_info.get("publisher_name", publisher_key), + publisher_description=publisher_info.get("description", publisher_key), + versions=sorted_versions, + ) + ) + print() + + +def output_publishers_raw(quiet=False): + """Output available publishers in table format.""" + publishers = get_known_publishers() + + if not quiet: + print(f"Known OS image publishers ({len(publishers)}):\n") + + print(f"{'Publisher Key':<15} {'Publisher Name':<20} {'Description':<30}") + print("-" * 70) + + for key, info in publishers.items(): + print( + f"{key:<15} " + f"{info['publisher_name']:<20} " + f"{info['description']:<30}" + ) + + +def output_publisher_raw(publisher_key, organized_images, quiet=False): + """Output publisher image information in table format.""" + publishers = get_known_publishers() + publisher_info = publishers.get(publisher_key, {}) + + if not quiet: + print(f"Images for {publisher_info.get('publisher_name', publisher_key)}") + print(f"Description: {publisher_info.get('description', '')}\n") + + versions = organized_images.get(publisher_key, {}) + if not versions: + print(f"No images found for publisher '{publisher_key}'.") + return + + print(f"{'Version':<50} {'Offer':<20} {'SKU':<15}") + print("-" * 90) + + for version_key, version_info in sorted(versions.items()): + print( + f"{version_info['friendly_name']:<50} " + f"{version_info['offer']:<20} " + f"{version_info['sku']:<15}" + ) + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure VM image information and generate Kconfig", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate Kconfig from default region + python %(prog)s > ../kconfigs/Kconfig.image.generated + + # List available publishers + python %(prog)s --publishers + + # Get details for a specific publisher + python %(prog)s debian --format raw + + # Query specific region + python %(prog)s --region eastus + """, + ) + parser.add_argument( + "publisher_key", nargs="?", help="Publisher key (e.g., debian, redhat)" + ) + + parser.add_argument( + "--publishers", action="store_true", help="List all known publishers" + ) + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--region", + "-r", + help="Query specific region (default: from Azure CLI config or westus)", + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + if args.publishers: + output_publishers_raw(args.quiet) + return + + publishers = get_known_publishers() + + # Filter to specific publisher if requested + if args.publisher_key: + if args.publisher_key not in publishers: + print( + f"Error: Unknown publisher '{args.publisher_key}'. " + f"Use --publishers to list available publishers.", + file=sys.stderr, + ) + sys.exit(1) + publishers = {args.publisher_key: publishers[args.publisher_key]} + + # Determine region + region = args.region if args.region else get_default_region() + + if not args.quiet: + print(f"Using region: {region}", file=sys.stderr) + + # Organize images + organized_images = organize_images_by_publisher(publishers, region, args.quiet) + + # Output based on format + if args.publisher_key: + if args.format == "raw": + output_publisher_raw(args.publisher_key, organized_images, args.quiet) + else: + print( + "Error: Kconfig format not supported for single publisher query", + file=sys.stderr, + ) + sys.exit(1) + else: + if args.format == "kconfig": + output_images_kconfig(organized_images, publishers) + else: + # Raw format for all publishers + for publisher_key in sorted(organized_images.keys()): + output_publisher_raw(publisher_key, organized_images, args.quiet) + print() + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/gen_kconfig_location b/terraform/azure/scripts/gen_kconfig_location new file mode 100755 index 000000000..287fb8941 --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_location @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve region information from Azure. Use it to construct the +"locations" Kconfig menu. + +This script queries the Azure API using the Azure CLI to dynamically +generate a Kconfig file containing all available Azure regions. Unlike +OCI, Azure provides rich metadata including physical locations and +display names directly through the API, so no separate mapping file +is needed. + +Usage: + # Generate complete Kconfig.location file + ./gen_kconfig_location > ../kconfigs/Kconfig.location + + # List all available regions + ./gen_kconfig_location --regions + + # Get details for a specific region + ./gen_kconfig_location westus +""" + +import sys +import argparse + +from azure_common import ( + get_default_region, + get_all_regions, + get_jinja2_environment, + get_region_kconfig_name, + exit_on_empty_result, +) + + +def get_region_friendly_name(region): + """ + Get a friendly display name for a region using Azure API metadata. + + Args: + region (dict): Region dictionary from Azure API + + Returns: + str: Friendly name suitable for display in Kconfig + """ + display_name = region.get("displayName", region["name"]) + physical_location = region.get("metadata", {}).get("physicalLocation") + + # Combine display name with physical location if available + if physical_location: + return f"{display_name} ({physical_location})" + + # Fall back to just the display name + return display_name + + +def get_region_info(regions, region_name, quiet=False): + """ + Get detailed information about a specific region. + + Args: + regions (list): List of all available regions + region_name (str): Azure region name (e.g., 'westus', 'eastus2') + quiet (bool): Suppress debug messages + + Returns: + dict: Dictionary containing region information + """ + if not quiet: + print(f"Querying information for region {region_name}...", file=sys.stderr) + + region_info = next((r for r in regions if r["name"] == region_name), None) + + if not region_info: + if not quiet: + print(f"Region {region_name} was not found", file=sys.stderr) + return None + + return region_info + + +def output_region_kconfig(region_info): + """Output region information in Kconfig format.""" + region_name = region_info["name"] + friendly_name = get_region_friendly_name(region_info) + kconfig_name = get_region_kconfig_name(region_name) + + print(f"config TERRAFORM_AZURE_REGION_{kconfig_name}") + print(f'\tbool "{friendly_name}"') + print("\thelp") + print(f"\t This option selects the {friendly_name} region.") + + # Add paired region info if available + metadata = region_info.get("metadata", {}) + paired_regions = metadata.get("pairedRegion", []) + if paired_regions: + paired_name = paired_regions[0].get("name", "") + if paired_name: + print(f"\t This region is paired with {paired_name}.") + + print() + + +def output_region_raw(region_info, quiet=False): + """Output region information in table format.""" + if not quiet: + print(f"Region: {region_info['name']}") + print(f"Display Name: {region_info['displayName']}") + print(f"Regional Name: {region_info.get('regionalDisplayName', 'N/A')}") + + metadata = region_info.get("metadata", {}) + if metadata: + print("\nMetadata:") + for key, value in metadata.items(): + if key == "pairedRegion" and value: + print(f" {key}: {value[0].get('name', 'N/A')}") + else: + print(f" {key}: {value}") + + +def output_regions_kconfig(regions): + """Output available regions in Kconfig format.""" + environment = get_jinja2_environment() + template = environment.get_template("regions.j2") + + default_region = get_default_region() + default_kconfig = get_region_kconfig_name(default_region) + + # Enhance region data with friendly names for template + enhanced_regions = [] + for region in regions: + enhanced_region = region.copy() + enhanced_region["friendly_name"] = get_region_friendly_name(region) + enhanced_regions.append(enhanced_region) + + print( + template.render( + default_region=default_kconfig, + regions=enhanced_regions, + ) + ) + + +def output_regions_raw(regions, quiet=False): + """Output available regions in table format.""" + if not quiet: + print(f"Available Azure regions ({len(regions)}):\n") + print(f"{'Region Name':<25} {'Display Name':<40}") + print("-" * 67) + + for region in regions: + print(f"{region['name']:<25} {region['displayName']:<40}") + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure region information", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python %(prog)s --regions + python %(prog)s westus + python %(prog)s eastus2 --quiet + """, + ) + parser.add_argument( + "region_name", + nargs="?", + help="Azure region name (e.g., westus, eastus2, westeurope)", + ) + + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--regions", action="store_true", help="List all available Azure regions" + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + if not args.quiet: + print("Fetching list of all Azure regions...", file=sys.stderr) + + regions = get_all_regions(args.quiet) + exit_on_empty_result(regions, "Azure region query", args.quiet) + + if args.regions: + if args.format == "kconfig": + output_regions_kconfig(regions) + else: + output_regions_raw(regions, args.quiet) + return + + if args.region_name: + if not args.quiet: + print( + f"Fetching information for region {args.region_name}...", + file=sys.stderr, + ) + + region_info = get_region_info(regions, args.region_name, args.quiet) + if region_info: + if args.format == "kconfig": + output_region_kconfig(region_info) + else: + output_region_raw(region_info, args.quiet) + else: + print( + f"Could not retrieve information for region '{args.region_name}'.", + file=sys.stderr, + ) + print( + "Try running with --regions to see available regions.", file=sys.stderr + ) + sys.exit(1) + return + + output_regions_kconfig(regions) + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/gen_kconfig_size b/terraform/azure/scripts/gen_kconfig_size new file mode 100755 index 000000000..bdaaa55fc --- /dev/null +++ b/terraform/azure/scripts/gen_kconfig_size @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 +# ex: set filetype=python: + +""" +Retrieve VM size information from Azure. Use it to construct the +"VM sizes" Kconfig menu. + +Azure VM sizes represent compute instance configurations with varying +combinations of CPU, memory, storage, and networking capacity. This script +queries the Azure API to discover available VM sizes and generates Kconfig +menu entries for them. + +VM Size Discovery: + By default, this script queries VM sizes from the default region configured + in Azure CLI. The --all-regions option aggregates VM sizes from all regions + to provide a comprehensive list. + + Note: VM size availability varies by region. Not all sizes listed may be + available in your selected region. + +Usage: + # Generate VM sizes Kconfig from default region + ./gen_kconfig_size > ../kconfigs/Kconfig.size.generated + + # Include VM sizes from all regions + ./gen_kconfig_size --all-regions > ../kconfigs/Kconfig.size.generated + + # List all available VM size families + ./gen_kconfig_size --families + + # Get details for a specific VM size family + ./gen_kconfig_size Standard_D +""" + +import sys +import argparse +import re +import os +import yaml + +from azure_common import ( + get_default_region, + get_all_regions, + get_jinja2_environment, + get_vm_sizes_and_skus, + exit_on_empty_result, +) + + +def get_all_vm_sizes_and_capabilities(regions=None, quiet=False): + """ + Get all available VM sizes and capabilities across specified regions. + + Filters out Gen1-only VM sizes to ensure compatibility with Gen2 OS images. + Gen2 images work on both Gen2-only and Gen1+Gen2 VM sizes (94% of all sizes), + while Gen1 images only work on Gen1-only and Gen1+Gen2 sizes (59% of all sizes). + This filtering maximizes VM size availability while using modern Gen2 images. + + Args: + regions (list): List of region names to query. If None, uses default region. + quiet (bool): Suppress debug messages + + Returns: + tuple: (sizes_list, capabilities_dict) where: + - sizes_list: List of VM size dictionaries (deduplicated by name) + - capabilities_dict: Dict mapping VM size names to capabilities + """ + if regions is None: + default_region = get_default_region() + regions = [default_region] + + all_sizes = {} # Use dict to deduplicate by VM size name + all_capabilities = {} # Capabilities from first region where size is found + + for region_info in regions: + region = region_info if isinstance(region_info, str) else region_info["name"] + + sizes, capabilities = get_vm_sizes_and_skus(region, quiet) + + # Add sizes to dict, using size name as key to deduplicate + # Filter out Gen1-only VM sizes to ensure compatibility with Gen2 images + for size in sizes: + if size["name"] not in all_sizes: + # Check HyperV generation support + size_caps = capabilities.get(size["name"], {}) + hyperv_gens = size_caps.get("HyperVGenerations", "") + + # Skip Gen1-only VM sizes (keep Gen2-only and Gen1+Gen2) + if hyperv_gens == "V1": + continue + + all_sizes[size["name"]] = size + # Store capabilities from first region where we see this size + if size["name"] in capabilities: + all_capabilities[size["name"]] = capabilities[size["name"]] + + if not quiet: + print( + f"\nTotal unique VM sizes across all regions: {len(all_sizes)}", + file=sys.stderr, + ) + + return list(all_sizes.values()), all_capabilities + + +def extract_vm_size_family(size_name): + """ + Extract VM size family from the VM size name. + + Azure VM size names follow patterns like: + - Standard_D2s_v3 -> Standard_D (family) + - Standard_DS3_v2 -> Standard_DS (family) + - Standard_B1ls -> Standard_B (family) + - Standard_NC6s_v3 -> Standard_NC (family) + + Args: + size_name (str): Azure VM size name + + Returns: + str: Family name + """ + # Match patterns like Standard_D, Standard_DS, Standard_B, etc. + match = re.match(r"^(Standard_[A-Z]+)", size_name) + if match: + return match.group(1) + + # Handle special cases or return the full name if pattern doesn't match + parts = size_name.split("_") + if len(parts) >= 2: + return f"{parts[0]}_{parts[1][0]}" + + return size_name + + +def parse_vm_size_families(sizes, quiet=False): + """ + Extract VM size families from the list of sizes. + + Args: + sizes (list): List of VM size dictionaries + quiet (bool): Suppress debug messages + + Returns: + dict: Dictionary with family info including count of sizes per family + """ + families = {} + seen_sizes = set() + + for size in sizes: + size_name = size["name"] + + # Skip duplicate size names + if size_name in seen_sizes: + continue + seen_sizes.add(size_name) + + # Parse VM size family from name + family = extract_vm_size_family(size_name) + + if family not in families: + families[family] = { + "family_name": family, + "size_count": 0, + "min_cores": float("inf"), + "max_cores": 0, + "min_memory_gb": float("inf"), + "max_memory_gb": 0, + } + + families[family]["size_count"] += 1 + + # Track min/max cores and memory + cores = size["numberOfCores"] + memory_gb = size["memoryInMB"] / 1024.0 + + families[family]["min_cores"] = min(families[family]["min_cores"], cores) + families[family]["max_cores"] = max(families[family]["max_cores"], cores) + families[family]["min_memory_gb"] = min( + families[family]["min_memory_gb"], memory_gb + ) + families[family]["max_memory_gb"] = max( + families[family]["max_memory_gb"], memory_gb + ) + + return families + + +def get_vm_size_family_info(family_name, sizes, quiet=False): + """ + Get VM size information for a specific family. + + Args: + family_name (str): VM size family name (e.g., 'Standard_D', 'Standard_DS') + sizes (list): List of all VM sizes + quiet (bool): Suppress debug messages + + Returns: + list: List of dictionaries containing VM size information + """ + family_sizes = [] + + for size in sizes: + size_name = size["name"] + size_family = extract_vm_size_family(size_name) + + # Match sizes that belong to this family + if size_family == family_name: + family_sizes.append(size) + + if not family_sizes: + if not quiet: + print(f"No VM sizes found in family '{family_name}'.", file=sys.stderr) + return [] + + if not quiet: + print( + f"Found {len(family_sizes)} VM sizes in family '{family_name}'", + file=sys.stderr, + ) + + # Extract detailed information + size_info = [] + for size in family_sizes: + size_info.append( + { + "name": size["name"], + "cores": size["numberOfCores"], + "memory_gb": size["memoryInMB"] / 1024.0, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0, + } + ) + + # Sort by cores, then memory + size_info.sort(key=lambda x: (x["cores"], x["memory_gb"])) + + return size_info + + +def determine_architecture(size_name): + """ + Determine CPU architecture from VM size name. + + Most Azure VM sizes are x86_64. ARM64-based sizes are in the + Dps and Dpds families (Ampere Altra processors). + + Args: + size_name (str): Azure VM size name + + Returns: + str: CPU architecture ('arm64' or 'x86_64') + """ + # ARM64 VM families + if re.match(r"Standard_Dp[sd]", size_name): + return "arm64" + + # Default to x86_64 + return "x86_64" + + +def natural_sort_key(vm_size_name): + """ + Generate a sort key for natural (numeric) ordering of VM size names. + + Converts VM size names like Standard_DS11_v2 into a tuple that sorts + naturally: Standard_DS1, DS2, DS3, ..., DS10, DS11, DS12 instead of + the alphabetic ordering DS1, DS11, DS12, DS2, DS3. + + Args: + vm_size_name (str): Azure VM size name (e.g., "Standard_DS3_v2") + + Returns: + tuple: Sort key tuple with string and integer components + """ + parts = re.split(r'(\d+)', vm_size_name) + key = [] + for part in parts: + if part.isdigit(): + key.append(int(part)) + else: + key.append(part) + return tuple(key) + + +def load_family_metadata(quiet=False): + """ + Load VM family metadata from YAML file. + + Returns: + dict: Dictionary mapping family prefixes to metadata + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + metadata_file = os.path.join(script_dir, "vm_family_metadata.yml") + + try: + with open(metadata_file, "r") as f: + metadata = yaml.safe_load(f) + if not quiet: + print( + f"Loaded metadata for {len(metadata)} VM families", file=sys.stderr + ) + return metadata + except FileNotFoundError: + if not quiet: + print( + f"Warning: Family metadata file not found: {metadata_file}", + file=sys.stderr, + ) + return {} + except Exception as e: + if not quiet: + print(f"Warning: Error loading family metadata: {e}", file=sys.stderr) + return {} + + +def output_sizes_kconfig(sizes, sku_capabilities=None, default_size="Standard_DS3_v2"): + """ + Output VM sizes menu in Kconfig format. + + Args: + sizes (list): List of VM size dictionaries + sku_capabilities (dict): Dictionary mapping VM size names to capabilities + default_size (str): Default VM size to select + """ + if sku_capabilities is None: + sku_capabilities = {} + + environment = get_jinja2_environment() + template = environment.get_template("sizes.j2") + + # Enhance size data for template + enhanced_sizes = [] + for size in sizes: + memory_gb = size["memoryInMB"] / 1024.0 + resource_disk_gb = ( + size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0 + ) + arch = determine_architecture(size["name"]) + + # Get accelerated networking capability + size_name = size["name"] + caps = sku_capabilities.get(size_name, {}) + accel_networking = caps.get("AcceleratedNetworkingEnabled", "False") == "True" + + enhanced_sizes.append( + { + "name": size_name, + "cores": size["numberOfCores"], + "memory_gb": memory_gb, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": resource_disk_gb, + "architecture": arch, + "accelerated_networking": accel_networking, + } + ) + + # Sort sizes by family, then cores, then memory + enhanced_sizes.sort( + key=lambda x: (extract_vm_size_family(x["name"]), x["cores"], x["memory_gb"]) + ) + + # Find default size config name + default_config = f"TERRAFORM_AZURE_VM_SIZE_{default_size.upper().replace('.', '_').replace('-', '_')}" + + print( + template.render( + sizes=enhanced_sizes, + default_config=default_config, + ) + ) + + +def output_families_kconfig(sizes, sku_capabilities=None, default_size="Standard_DS3_v2", quiet=False): + """ + Output VM sizes in family-based hierarchical Kconfig format. + + Args: + sizes (list): List of VM size dictionaries + sku_capabilities (dict): Dictionary mapping VM size names to capabilities + default_size (str): Default VM size to select + quiet (bool): Suppress informational messages + """ + if sku_capabilities is None: + sku_capabilities = {} + + # Load family metadata from YAML + family_metadata = load_family_metadata(quiet) + + # Group sizes by family + family_groups = {} + for size in sizes: + size_name = size["name"] + family_prefix = extract_vm_size_family(size_name) + + if family_prefix not in family_groups: + family_groups[family_prefix] = [] + + family_groups[family_prefix].append(size) + + # Build family data for template + families = [] + default_family = None + + for family_prefix in sorted(family_groups.keys()): + family_sizes = family_groups[family_prefix] + + # Get metadata from YAML or use defaults + metadata = family_metadata.get(family_prefix, {}) + description = metadata.get("description", "Azure VM family") + help_text = metadata.get("help_text", f"Virtual machines in the {family_prefix} family.") + + # Calculate family statistics + min_cores = min(s["numberOfCores"] for s in family_sizes) + max_cores = max(s["numberOfCores"] for s in family_sizes) + min_memory_gb = min(s["memoryInMB"] / 1024.0 for s in family_sizes) + max_memory_gb = max(s["memoryInMB"] / 1024.0 for s in family_sizes) + + # Determine family architecture + family_arch = determine_architecture(family_sizes[0]["name"]) + + # Build enhanced size list for this family + enhanced_sizes = [] + for size in sorted(family_sizes, key=lambda x: natural_sort_key(x["name"])): + size_name = size["name"] + memory_gb = size["memoryInMB"] / 1024.0 + resource_disk_gb = ( + size["resourceDiskSizeInMB"] / 1024.0 + if size["resourceDiskSizeInMB"] > 0 + else 0 + ) + + # Get accelerated networking capability + caps = sku_capabilities.get(size_name, {}) + accel_networking = caps.get("AcceleratedNetworkingEnabled", "False") == "True" + + config_name = size_name.upper().replace(".", "_").replace("-", "_") + enhanced_sizes.append( + { + "name": size_name, + "config_name": config_name, + "cores": size["numberOfCores"], + "memory_gb": memory_gb, + "max_data_disks": size["maxDataDiskCount"], + "resource_disk_gb": resource_disk_gb, + "accelerated_networking": accel_networking, + } + ) + + # Check if default size is in this family + is_default_family = any(s["name"] == default_size for s in family_sizes) + if is_default_family: + default_family = family_prefix + + # Find default size config within family + default_size_config = None + for enhanced_size in enhanced_sizes: + if enhanced_size["name"] == default_size: + default_size_config = f"TERRAFORM_AZURE_VM_SIZE_{enhanced_size['config_name']}" + break + if default_size_config is None: + default_size_config = f"TERRAFORM_AZURE_VM_SIZE_{enhanced_sizes[0]['config_name']}" + + config_name = family_prefix.upper().replace(".", "_").replace("-", "_") + families.append( + { + "family_name": family_prefix, + "config_name": config_name, + "description": description, + "help_text": help_text, + "architecture": family_arch, + "size_count": len(family_sizes), + "min_cores": min_cores, + "max_cores": max_cores, + "min_memory_gb": min_memory_gb, + "max_memory_gb": max_memory_gb, + "sizes": enhanced_sizes, + "default_size_config": default_size_config, + } + ) + + # Set default family config + if default_family: + default_family_config = f"TERRAFORM_AZURE_VM_FAMILY_{default_family.upper().replace('.', '_').replace('-', '_')}" + else: + default_family_config = f"TERRAFORM_AZURE_VM_FAMILY_{families[0]['config_name']}" + + # Render template + environment = get_jinja2_environment() + template = environment.get_template("families.j2") + + print( + template.render( + families=families, + default_family_config=default_family_config, + ) + ) + + +def output_families_raw(sizes, quiet=False): + """Output available VM size families in table format.""" + families = parse_vm_size_families(sizes) + + if not quiet: + print(f"Available VM size families ({len(families)}):\n") + + print(f"{'Family':<20} {'Count':<6} {'Cores Range':<15} {'Memory Range (GB)':<20}") + print("-" * 65) + + sorted_families = sorted(families.values(), key=lambda x: x["family_name"]) + for family in sorted_families: + cores_range = f"{family['min_cores']}-{family['max_cores']}" + memory_range = f"{family['min_memory_gb']:.1f}-{family['max_memory_gb']:.1f}" + + print( + f"{family['family_name']:<20} " + f"{family['size_count']:<6} " + f"{cores_range:<15} " + f"{memory_range:<20}" + ) + + +def output_family_raw(sizes, quiet=False): + """Output VM size family information in table format.""" + if not quiet: + print(f"Found {len(sizes)} VM sizes:\n") + + print( + f"{'VM Size Name':<30} {'Cores':<8} {'Memory (GB)':<15} {'Max Disks':<12} {'Temp Storage (GB)':<18}" + ) + print("-" * 90) + + for size in sizes: + print( + f"{size['name']:<30} " + f"{size['cores']:<8} " + f"{size['memory_gb']:<15.1f} " + f"{size['max_data_disks']:<12} " + f"{size['resource_disk_gb']:<18.1f}" + ) + + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Get Azure VM size information including hardware specs", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate Kconfig from default region + python %(prog)s > ../kconfigs/Kconfig.size.generated + + # Include sizes from all regions + python %(prog)s --all-regions > ../kconfigs/Kconfig.size.generated + + # Query specific VM size family + python %(prog)s Standard_D + + # List all available families + python %(prog)s --families --format raw + + # Query specific region only + python %(prog)s --region westus2 --families + """, + ) + parser.add_argument( + "family_name", + nargs="?", + help="VM size family name (e.g., Standard_D, Standard_DS, Standard_B)", + ) + + parser.add_argument( + "--families", action="store_true", help="List all available VM size families" + ) + parser.add_argument( + "--all-regions", + action="store_true", + help="Include VM sizes from all regions (default: default region only)", + ) + parser.add_argument( + "--format", + "-f", + choices=["raw", "kconfig"], + default="kconfig", + help="Output format (default: kconfig)", + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Suppress informational messages" + ) + parser.add_argument( + "--region", + "-r", + help="Query specific region only (default: default configured region)", + ) + return parser.parse_args() + + +def main(): + """Main function to run the program.""" + args = parse_arguments() + + # Determine which regions to query + if args.region: + # Query specific region only + regions = [args.region] + elif args.all_regions: + # Query all regions + regions = get_all_regions(args.quiet) + exit_on_empty_result(regions, "Azure region query", args.quiet) + else: + # Query default region only + regions = [get_default_region()] + + # Get VM sizes and capabilities in a single API call + sizes, sku_capabilities = get_all_vm_sizes_and_capabilities(regions, args.quiet) + exit_on_empty_result(sizes, "Azure VM size query", args.quiet) + + if args.families: + output_families_raw(sizes, args.quiet) + return + + if args.family_name: + if not args.quiet: + print( + f"Fetching information for the {args.family_name} family...", + file=sys.stderr, + ) + + family_sizes = get_vm_size_family_info(args.family_name, sizes, args.quiet) + + if not family_sizes: + print( + f"No VM sizes found for family '{args.family_name}'.", file=sys.stderr + ) + print( + "Try running with --families to see available VM size families.", + file=sys.stderr, + ) + sys.exit(1) + + # Output raw format for family queries + output_family_raw(family_sizes, args.quiet) + return + + # Output family-based hierarchical Kconfig menu + output_families_kconfig(sizes, sku_capabilities, quiet=args.quiet) + + +if __name__ == "__main__": + main() diff --git a/terraform/azure/scripts/image_distributions.j2 b/terraform/azure/scripts/image_distributions.j2 new file mode 100644 index 000000000..93e98d7f6 --- /dev/null +++ b/terraform/azure/scripts/image_distributions.j2 @@ -0,0 +1,35 @@ +choice + prompt "Azure image publisher" +{% if 'debian' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_DEBIAN +{% elif 'redhat' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_REDHAT +{% elif 'canonical' in publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_CANONICAL +{% elif publishers %} + default TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publishers[0] | upper }} +{% endif %} + help + This option specifies the publisher of the boot image used to + create the kdevops target nodes. + + The distributions listed here are official Azure platform images + provided by various publishers. These images are well-tested and + actively maintained. + +{% for publisher_key in publishers %} +config TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} + bool "{{ publisher_key | title }}" + help + Select this if you want to use images from the {{ publisher_key | title }} + publisher. + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_PUBLISHER + string + output yaml +{% for publisher_key in publishers %} + default "{{ publisher_key }}" if TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} +{% endfor %} diff --git a/terraform/azure/scripts/image_publisher.j2 b/terraform/azure/scripts/image_publisher.j2 new file mode 100644 index 000000000..01b818b14 --- /dev/null +++ b/terraform/azure/scripts/image_publisher.j2 @@ -0,0 +1,110 @@ +{# + This template is rendered by gen_kconfig_image script. + + Template variables: + - publisher_key: str (e.g., 'debian', 'redhat') + - publisher_name: str (e.g., 'Debian') + - publisher_description: str (full description) + - versions: list of (version_key, version_info) tuples (pre-sorted) +#} +if TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} + +if TARGET_ARCH_X86_64 + +choice + prompt "{{ publisher_description }} release" +{# versions is a list of (version_key, version_info) tuples from sorted_versions in gen_kconfig_image #} +{% for version_key, version_info in versions | reverse %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} if TARGET_ARCH_X86_64 +{% endif %} +{% endfor %} + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} +config TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} + bool "{{ version_info['friendly_name'] }}" + help + This option sets the OS image to {{ version_info['friendly_name'] }}. + + Publisher: {{ version_info['publisher_id'] }} + Offer: {{ version_info['offer'] }} + SKU: {{ version_info['sku'] }} + +{% endif %} +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default "{{ version_info['offer'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml +{% for version_key, version_info in versions %} +{% if "X86" in version_key and "ARM64" not in version_key %} + default "{{ version_info['sku'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +endif # TARGET_ARCH_X86_64 + +if TARGET_ARCH_ARM64 + +choice + prompt "{{ publisher_description }} release" +{# versions is a list of (version_key, version_info) tuples from sorted_versions in gen_kconfig_image #} +{% for version_key, version_info in versions | reverse %} +{% if "ARM64" in version_key %} + default TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} if TARGET_ARCH_ARM64 +{% endif %} +{% endfor %} + help + This option specifies which of a publisher's offers to use + when creating kdevops compute instances. + +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} +config TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} + bool "{{ version_info['friendly_name'] }}" + help + This option sets the OS image to {{ version_info['friendly_name'] }}. + + Publisher: {{ version_info['publisher_id'] }} + Offer: {{ version_info['offer'] }} + SKU: {{ version_info['sku'] }} + +{% endif %} +{% endfor %} +endchoice + +config TERRAFORM_AZURE_IMAGE_OFFER + string + output yaml +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} + default "{{ version_info['offer'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +config TERRAFORM_AZURE_IMAGE_SKU + string + output yaml +{% for version_key, version_info in versions %} +{% if "ARM64" in version_key %} + default "{{ version_info['sku'] }}" if TERRAFORM_AZURE_IMAGE_LINUX_{{ version_key }} +{% endif %} +{% endfor %} + +endif # TARGET_ARCH_ARM64 + +endif # TERRAFORM_AZURE_IMAGE_PUBLISHER_{{ publisher_key | upper }} diff --git a/terraform/azure/scripts/publisher_definitions.yml b/terraform/azure/scripts/publisher_definitions.yml new file mode 100644 index 000000000..966a52073 --- /dev/null +++ b/terraform/azure/scripts/publisher_definitions.yml @@ -0,0 +1,96 @@ +# Azure VM Image Publisher Definitions +# +# This file defines the known publishers of Azure VM images that kdevops +# supports. It is used by the gen_kconfig_image script to generate the +# Kconfig.image configuration file. +# +# To add a new publisher: +# 1. Add a new entry with a unique key (lowercase, alphanumeric + underscores) +# 2. Specify the Azure publisher_id (exact match from Azure API) +# 3. Provide a user-friendly publisher_name and description +# 4. Set a priority (lower numbers appear first in menus) +# 5. Run: cd terraform/azure/scripts && ./gen_kconfig_image +# +# Fields: +# publisher_id: The exact publisher identifier used in Azure CLI +# (e.g., "Debian", "RedHat", "Canonical") +# Note: Some marketplace publishers have auto-generated IDs +# publisher_name: User-friendly display name +# description: Full description used in Kconfig help text +# priority: Display order (lower = higher priority, appears first) +# +# Note: The script will automatically discover offers and SKUs for each +# publisher by querying the Azure API. +# +# To find the correct publisher_id for a new publisher, use: +# az vm image list-publishers --location westus -o json | grep -i + +debian: + publisher_id: "Debian" + publisher_name: "Debian" + description: "Debian GNU/Linux" + priority: 10 + +redhat: + publisher_id: "RedHat" + publisher_name: "Red Hat" + description: "Red Hat Enterprise Linux" + priority: 20 + offer_patterns: + - "^RHEL$" + - "^rhel-arm64$" + - "^rh-rhel$" + - "^RHEL-HA$" + - "^RHEL-SAP$" + - "^RHEL-SAP-APPS$" + - "^RHEL-SAP-HA$" + +canonical: + publisher_id: "Canonical" + publisher_name: "Canonical" + description: "Ubuntu Linux" + priority: 30 + offer_patterns: + - "^0001-com-ubuntu-server-.*$" + - "^ubuntu-[0-9]{2}_[0-9]{2}-lts.*$" + - "^ubuntu-[0-9]{2}_[0-9]{2}.*$" + - "^UbuntuServer$" + - "^ubuntu$" + - "^0001-com-ubuntu-minimal-.*$" + +oracle: + publisher_id: "Oracle" + publisher_name: "Oracle" + description: "Oracle Linux" + priority: 40 + offer_patterns: + - "^Oracle-Linux$" + +suse: + publisher_id: "SUSE" + publisher_name: "SUSE" + description: "SUSE Linux Enterprise" + priority: 50 + offer_patterns: + - "^sles-.*$" + - "^SLES.*$" + - "^opensuse-leap-.*$" + - "^openSUSE-Leap$" + - "^sle-hpc-.*$" + - "^sle-micro-.*$" + +# Note: AlmaLinux and Rocky Linux use marketplace publisher IDs which +# include timestamps. These IDs may change over time as the publishers +# update their marketplace presence. + +almalinux: + publisher_id: "almalinuxosfoundation1628089859865" + publisher_name: "AlmaLinux" + description: "AlmaLinux" + priority: 60 + +rockylinux: + publisher_id: "erockyenterprisesoftwarefoundationinc1653071250513" + publisher_name: "Rocky Linux" + description: "Rocky Linux" + priority: 70 diff --git a/terraform/azure/scripts/regions.j2 b/terraform/azure/scripts/regions.j2 new file mode 100644 index 000000000..cd421028d --- /dev/null +++ b/terraform/azure/scripts/regions.j2 @@ -0,0 +1,52 @@ +choice + prompt "Azure resource location" + default TERRAFORM_AZURE_REGION_{{ default_region }} + help + Choose the region and data center which will host your + kdevops resources. The list below is dynamically generated + from the Azure API and includes all regions available to + your subscription. + + Azure automatically chooses availability zones for your + resources. You can query available locations using: + + az account list-locations -o table + + For more information about Azure regions and geographies: + + https://azure.microsoft.com/en-us/explore/global-infrastructure/geographies/ + +{% for region in regions %} +config TERRAFORM_AZURE_REGION_{{ region['name'].upper().replace('-', '_') }} + bool "{{ region['friendly_name'] }}" + help + This option selects the {{ region['friendly_name'] }} region. +{% if region.get('metadata', {}).get('pairedRegion') %} + This region is paired with {{ region['metadata']['pairedRegion'][0]['name'] }}. +{% endif %} + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_LOCATION + string + output yaml +{% for region in regions %} + default "{{ region['name'] }}" if TERRAFORM_AZURE_REGION_{{ region['name'].upper().replace('-', '_') }} +{% endfor %} + +config TERRAFORM_AZURE_RESOURCE_GROUP_NAME + string "Azure resource group name" + output yaml + default "kdevops_resource_group" + help + An Azure resource group is a container that holds related + resources so they can be managed as a single unit. These + resources share the same life cycle and are deployed, + updated, and deleted together. Resource groups are global + to your subscription. + + To run concurrent kdevops jobs in Azure, each run must + have a unique resource group name. + + https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups diff --git a/terraform/azure/scripts/sizes.j2 b/terraform/azure/scripts/sizes.j2 new file mode 100644 index 000000000..b9eb51446 --- /dev/null +++ b/terraform/azure/scripts/sizes.j2 @@ -0,0 +1,48 @@ +choice + prompt "Azure VM size" + default {{ default_config }} + help + This option chooses the size of the virtual machine instances + to be created for the kdevops target nodes. + + VM size availability varies by region. Not all sizes listed + may be available in your selected region. You can query + available VM sizes for a specific region using: + + az vm list-sizes --location -o table + + For more information about Azure VM sizes and families: + + https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview + +{% for size in sizes %} +config TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} + bool "{{ size['name'] }}" +{% if size['architecture'] == 'arm64' %} + depends on TARGET_ARCH_ARM64 +{% else %} + depends on TARGET_ARCH_X86_64 +{% endif %} + help + This virtual machine size has {{ size['cores'] }} vCPU{{ 's' if size['cores'] > 1 else '' }} and {{ size['memory_gb']|round(1) }}GB of memory. +{% if size['resource_disk_gb'] > 0 %} + Includes {{ size['resource_disk_gb']|round(1) }}GB temporary storage. +{% endif %} + Supports up to {{ size['max_data_disks'] }} data disk{{ 's' if size['max_data_disks'] > 1 else '' }}. + +{% endfor %} +endchoice + +config TERRAFORM_AZURE_VM_SIZE + string + output yaml +{% for size in sizes %} + default "{{ size['name'] }}" if TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} +{% endfor %} + +config TERRAFORM_AZURE_ACCELERATED_NETWORKING_ENABLED + bool + output yaml +{% for size in sizes %} + default {{ 'y' if size['accelerated_networking'] else 'n' }} if TERRAFORM_AZURE_VM_SIZE_{{ size['name'].upper().replace('.', '_').replace('-', '_') }} +{% endfor %} diff --git a/terraform/azure/scripts/vm_family_metadata.yml b/terraform/azure/scripts/vm_family_metadata.yml new file mode 100644 index 000000000..613208146 --- /dev/null +++ b/terraform/azure/scripts/vm_family_metadata.yml @@ -0,0 +1,253 @@ +# Azure VM Family Metadata +# +# This file contains human-readable descriptions and metadata for Azure VM +# size families to support hierarchical Kconfig menu generation. +# +# Purpose: +# The gen_kconfig_size script uses this metadata to generate a two-layer +# Kconfig menu: first select a VM family, then select a specific size +# within that family. +# +# Canonical Sources: +# https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview +# (Official Azure VM sizes documentation) +# +# Last Updated: 2025-11-10 +# +# Format: +# family-prefix: +# description: "Short one-line description" +# help_text: "Detailed multi-line help text describing use cases" +# workloads: +# - "Example workload 1" +# - "Example workload 2" +# +# Family Naming Convention: +# Azure VM families are identified by the prefix pattern in VM size names. +# For example: +# - Standard_D2s_v3 → Standard_D family +# - Standard_E16_v4 → Standard_E family +# - Standard_F8s_v2 → Standard_F family +# +# How to update this file: +# ----------------------- +# When Azure introduces new VM families or you need to update descriptions: +# +# 1. Check which families are discovered by the API: +# $ cd terraform/azure/scripts +# $ ./gen_kconfig_size --families +# +# 2. Verify family details in Azure documentation: +# https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/overview +# +# 3. Add or update the family entry following the format above. +# +# 4. Update the "Last Updated" date above. +# +# 5. Test that families render correctly: +# $ ./gen_kconfig_size > /tmp/Kconfig.size.test 2>&1 +# $ python3 ../../../scripts/detect_indentation_issues.py /tmp/Kconfig.size.test +# +# 6. Regenerate Kconfig.size: +# $ ./gen_kconfig_size > ../kconfigs/Kconfig.size.generated +# + +# General Purpose Families +Standard_A: + description: "Basic compute, entry-level" + help_text: | + Entry-level VMs for development and test workloads with basic CPU + performance and economical pricing. Suitable for learning, development, + and non-production environments. + workloads: + - "Development and testing" + - "Low-traffic web servers" + - "Small databases" + +Standard_B: + description: "Burstable, cost-effective" + help_text: | + Burstable CPU performance for workloads that don't need continuous full + CPU. Accumulates CPU credits during idle periods and uses them during + bursts. Ideal for workloads with variable CPU usage patterns. + workloads: + - "Development and test environments" + - "Low-traffic web servers" + - "Small databases" + - "Microservices" + +Standard_D: + description: "General purpose, balanced" + help_text: | + Balanced CPU-to-memory ratio suitable for most production workloads. + Provides consistent performance with local SSD temporary storage. + Most versatile family for general compute needs. + workloads: + - "Web servers and application servers" + - "Medium-traffic websites" + - "Small to medium databases" + - "Cache servers" + +Standard_DS: + description: "General purpose with premium storage" + help_text: | + D-series with premium SSD storage support. Balanced compute and storage + performance for production workloads requiring fast, low-latency storage. + workloads: + - "Production web applications" + - "Application servers" + - "Small to medium databases with SSD requirements" + +# Memory Optimized Families +Standard_E: + description: "Memory optimized, high RAM" + help_text: | + High memory-to-CPU ratio optimized for memory-intensive workloads. + Provides 8GB RAM per vCPU, ideal for applications that require large + amounts of memory relative to CPU. + workloads: + - "Relational databases (SQL Server, MySQL, PostgreSQL)" + - "In-memory analytics" + - "Large caches (Redis, Memcached)" + - "SAP applications" + +Standard_ES: + description: "Memory optimized with premium storage" + help_text: | + E-series with premium SSD storage support. High memory and fast storage + for databases and memory-intensive applications requiring low-latency disk. + workloads: + - "Large relational databases" + - "In-memory databases with persistent storage" + - "Data warehousing" + +Standard_M: + description: "Memory optimized, very large" + help_text: | + Largest memory VMs in Azure with up to 4TB of RAM. Designed for extremely + memory-intensive enterprise workloads. Supports premium storage and + write accelerator for optimal database performance. + workloads: + - "SAP HANA" + - "Very large SQL Server instances" + - "In-memory databases" + - "Large-scale analytics" + +# Compute Optimized Families +Standard_F: + description: "Compute optimized, high CPU" + help_text: | + High CPU-to-memory ratio optimized for compute-intensive workloads. + Provides 2GB RAM per vCPU, ideal for CPU-bound applications. + workloads: + - "Batch processing" + - "Web servers with high traffic" + - "Analytics and data processing" + - "Gaming servers" + - "Machine learning inference" + +Standard_FS: + description: "Compute optimized with premium storage" + help_text: | + F-series with premium SSD storage support. High CPU performance combined + with fast storage for compute and I/O intensive workloads. + workloads: + - "High-performance batch processing" + - "Analytics with large datasets" + - "Gaming servers with fast storage" + +# Storage Optimized Families +Standard_L: + description: "Storage optimized, high I/O" + help_text: | + High disk throughput and I/O optimized for storage-intensive workloads. + Features large local NVMe disks with very high IOPS and throughput. + workloads: + - "NoSQL databases (Cassandra, MongoDB, Couchbase)" + - "Data warehousing" + - "Large transactional databases" + - "Distributed file systems" + +Standard_LS: + description: "Storage optimized with premium storage" + help_text: | + L-series with premium SSD storage support. Maximum storage throughput + for data-intensive applications requiring both local and remote storage. + workloads: + - "Large-scale NoSQL databases" + - "Big data analytics" + - "Data lake applications" + +# GPU Families +Standard_N: + description: "GPU-enabled compute" + help_text: | + GPU-enabled VMs with NVIDIA GPUs for AI/ML training and inference, + graphics rendering, video processing, and other GPU-accelerated workloads. + workloads: + - "Deep learning training" + - "Machine learning inference" + - "Graphics rendering" + - "Video encoding/transcoding" + - "Scientific simulations" + +Standard_NC: + description: "GPU compute, NVIDIA" + help_text: | + NVIDIA GPU VMs for AI/ML workloads and GPU-accelerated compute. + Optimized for deep learning training with high GPU memory bandwidth. + workloads: + - "Deep learning model training" + - "AI/ML workloads" + - "GPU-accelerated HPC" + +Standard_ND: + description: "GPU deep learning, NVIDIA" + help_text: | + NVIDIA GPU VMs optimized for deep learning training with very high + GPU-to-GPU interconnect bandwidth (NVLink). Ideal for distributed + training of large models. + workloads: + - "Large-scale distributed deep learning" + - "Training large language models" + - "Multi-GPU AI/ML workloads" + +Standard_NV: + description: "GPU visualization, NVIDIA" + help_text: | + NVIDIA GPU VMs for remote visualization, streaming, gaming, and + OpenGL/DirectX applications. Optimized for graphics workloads. + workloads: + - "Remote visualization" + - "CAD/CAM applications" + - "3D rendering" + - "Cloud gaming" + - "Video streaming" + +# High Performance Compute +Standard_H: + description: "HPC, high performance" + help_text: | + High Performance Computing VMs with high CPU performance and optional + RDMA networking for MPI workloads. Designed for compute-intensive HPC + and scientific computing. + workloads: + - "Scientific simulations" + - "Computational fluid dynamics" + - "Weather modeling" + - "Molecular dynamics" + - "MPI-based parallel computing" + +# ARM64 Families +Standard_Dp: + description: "ARM64 general purpose (Ampere Altra)" + help_text: | + ARM64-based VMs using Ampere Altra processors. General purpose workloads + with excellent performance-per-dollar ratio and energy efficiency. + Compatible with ARM64 Linux distributions. + workloads: + - "Cloud-native applications" + - "Web servers" + - "Application servers" + - "Containerized workloads" + - "Microservices" diff --git a/terraform/azure/vars.tf b/terraform/azure/vars.tf index dd3c20ed0..b1aec690c 100644 --- a/terraform/azure/vars.tf +++ b/terraform/azure/vars.tf @@ -47,3 +47,8 @@ variable "azure_vmsize" { description = "VM size" type = string } + +variable "azure_accelerated_networking_enabled" { + description = "Enable accelerated networking for network interfaces" + type = bool +}