From 5e5fcf3efd43b4471146437620353cc13e352e3d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 12:57:49 +0000 Subject: [PATCH 01/19] feat: add unified airgap RKE2 infra pipeline (Jenkinsfile.airgap-rke2-infra) Create a single Declarative Pipeline that replaces both Jenkinsfile.setup.airgap.rke2 and Jenkinsfile.destroy.airgap.rke2, controlled by an ACTION parameter (setup/destroy). Consumes shared functions from qa-jenkins-library (#589): - airgap.standardCheckout, airgap.configureAnsible, airgap.deployRKE2, airgap.deployRancher, airgap.teardownInfrastructure - s3.uploadArtifact, s3.downloadArtifact, s3.deleteArtifact - tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs Original Jenkinsfiles remain untouched for parallel coexistence. Refs #590 --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 512 ++++++++++++++++++ 1 file changed, 512 insertions(+) create mode 100644 validation/pipeline/Jenkinsfile.airgap-rke2-infra diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra new file mode 100644 index 000000000..1ebef7656 --- /dev/null +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -0,0 +1,512 @@ +#!groovy +/** + * Unified airgap RKE2 infrastructure pipeline. + * + * Replaces Jenkinsfile.setup.airgap.rke2 and Jenkinsfile.destroy.airgap.rke2 + * with a single Declarative Pipeline controlled by the ACTION parameter. + * + * Actions: + * setup - Provision AWS infrastructure, deploy RKE2, optionally deploy Rancher + * destroy - Tear down infrastructure for a given workspace + * + * Consumes shared functions from qa-jenkins-library: + * airgap.standardCheckout, airgap.configureAnsible, airgap.deployRKE2, + * airgap.deployRancher, airgap.teardownInfrastructure, + * s3.uploadArtifact, s3.downloadArtifact, s3.deleteArtifact, + * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs + */ + +def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +pipeline { + agent any + + options { + ansiColor('xterm') + timeout(time: 180, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + choice( + name: 'ACTION', + choices: ['setup', 'destroy'], + description: 'Infrastructure action to perform' + ) + booleanParam( + name: 'DEPLOY_RANCHER', + defaultValue: true, + description: 'Deploy Rancher via helm after RKE2 setup (setup action only)' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Tear down infrastructure if setup fails' + ) + string( + name: 'TARGET_WORKSPACE', + defaultValue: '', + description: 'Workspace name to destroy (required for destroy action)' + ) + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/qa-infra-automation repository' + ) + } + + environment { + WORKSPACE_NAME = '' + } + + stages { + // ── Shared stages (both actions) ────────────────────────────── + + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [branch: params.TESTS_BRANCH], + infraRepo: [branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + } + } + } + + stage('Build Infrastructure Tools Image') { + steps { + sh "docker build --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + } + } + + // ── Setup action ────────────────────────────────────────────── + + stage('Setup: Initialize Tofu Backend') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + env.TOFU_MODULE_PATH = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + tofu.initBackend( + dir: env.TOFU_MODULE_PATH, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + } + } + } + + stage('Setup: Create Workspace') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + def wsName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_airgap_ansible_workspace', + suffix: env.HOSTNAME_PREFIX, + includeTimestamp: false + ) + env.WORKSPACE_NAME = wsName + + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + tofu.createWorkspace( + dir: env.TOFU_MODULE_PATH, + name: wsName + ) + } + + infrastructure.archiveWorkspaceName(workspaceName: wsName) + } + } + } + + stage('Setup: Configure SSH Key') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + property.useWithProperties(['AWS_SSH_PEM_KEY']) { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + } + } + } + } + + stage('Setup: Configure Tofu Variables') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + + infrastructure.writeConfig( + path: "${env.TOFU_MODULE_PATH}/terraform.tfvars", + content: terraformConfig + ) + } + } + } + } + + stage('Setup: Apply Tofu') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + tofu.apply( + dir: env.TOFU_MODULE_PATH, + varFile: 'terraform.tfvars', + autoApprove: true + ) + } + + def inventoryPath = "${env.INFRA_DIR}/ansible/rke2/airgap/inventory/inventory.yml" + if (fileExists(inventoryPath)) { + archiveArtifacts artifacts: inventoryPath, fingerprint: true + } else { + echo "Warning: Inventory file not found at ${inventoryPath}" + } + } + } + } + + stage('Setup: Upload Terraform Variables to S3') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + s3.uploadArtifact( + workspaceName: env.WORKSPACE_NAME, + localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars", + s3Key: 'terraform.tfvars' + ) + } + } + } + } + + stage('Setup: Configure Ansible Variables') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + + property.useWithProperties([ + 'PRIVATE_REGISTRY_URL', + 'PRIVATE_REGISTRY_USERNAME', + 'PRIVATE_REGISTRY_PASSWORD' + ]) { + airgap.configureAnsible( + sshKey: [ + content: env.AWS_SSH_PEM_KEY, + name: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ], + inventoryVars: [ + content: env.ANSIBLE_VARIABLES, + path: "${ansiblePath}/inventory/group_vars/all.yml", + envVars: [ + 'RKE2_VERSION': env.RKE2_VERSION, + 'RANCHER_VERSION': env.RANCHER_VERSION, + 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', + 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', + 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' + ] + ], + ansibleDir: ansiblePath, + inventoryFile: 'inventory/inventory.yml', + validate: false + ) + } + } + } + } + + stage('Setup: Deploy RKE2 Cluster') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + + airgap.deployRKE2( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rke2-tarball-playbook.yml' + ) + } + } + } + + stage('Setup: Configure Private Registry') { + when { + allOf { + expression { params.ACTION == 'setup' } + expression { env.PRIVATE_REGISTRY_URL?.trim() } + } + } + steps { + script { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + + ansible.runPlaybook( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rke2-registry-config-playbook.yml' + ) + } + } + } + + stage('Setup: Deploy Rancher') { + when { + allOf { + expression { params.ACTION == 'setup' } + expression { params.DEPLOY_RANCHER } + } + } + steps { + script { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + + airgap.deployRancher( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rancher-helm-deploy-playbook.yml', + enabled: params.DEPLOY_RANCHER + ) + } + } + } + + stage('Setup: Output Infrastructure Details') { + when { + expression { params.ACTION == 'setup' } + } + steps { + script { + echo '=== Infrastructure Setup Complete ===' + echo "Workspace Name: ${env.WORKSPACE_NAME}" + + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + try { + def bastionDns = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'bastion_public_dns' + ) + echo "Bastion Host: ${bastionDns}" + } catch (e) { + echo "Could not retrieve bastion DNS: ${e.message}" + } + + try { + def rancherHostname = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'external_lb_hostname' + ) + echo "Rancher Hostname (External): ${rancherHostname}" + currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" + } catch (e) { + echo "Could not retrieve external LB hostname: ${e.message}" + } + + try { + def internalLb = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'internal_lb_hostname' + ) + echo "Internal LB Hostname: ${internalLb}" + } catch (e) { + echo "Could not retrieve internal LB hostname: ${e.message}" + } + } + + echo '========================================' + } + } + } + + // ── Destroy action ──────────────────────────────────────────── + + stage('Destroy: Validate Parameters') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + if (!params.TARGET_WORKSPACE?.trim()) { + error 'TARGET_WORKSPACE parameter is required for destroy action' + } + echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" + env.WORKSPACE_NAME = params.TARGET_WORKSPACE + env.TOFU_MODULE_PATH = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + } + } + } + + stage('Destroy: Initialize Tofu Backend') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + tofu.initBackend( + dir: env.TOFU_MODULE_PATH, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION ?: env.S3_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + } + } + } + + stage('Destroy: Download Terraform Variables from S3') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + s3.downloadArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars', + localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars" + ) + } + } + } + } + + stage('Destroy: Teardown Infrastructure') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + airgap.teardownInfrastructure( + dir: env.TOFU_MODULE_PATH, + name: params.TARGET_WORKSPACE, + varFile: 'terraform.tfvars' + ) + } + } + } + } + + stage('Destroy: Delete S3 Artifacts') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + s3.deleteArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars' + ) + } + } + } + } + + stage('Destroy: Cleanup Local Artifacts') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + script { + infrastructure.cleanupArtifacts( + paths: [ + "${env.TOFU_MODULE_PATH}/.terraform", + "${env.TOFU_MODULE_PATH}/terraform.tfstate*", + "${env.TOFU_MODULE_PATH}/*.tfvars" + ], + force: true + ) + } + } + } + + stage('Destroy: Summary') { + when { + expression { params.ACTION == 'destroy' } + } + steps { + echo '=== Infrastructure Destruction Complete ===' + echo "Workspace: ${params.TARGET_WORKSPACE}" + echo 'All resources have been destroyed and cleaned up' + echo '===========================================' + } + } + } + + post { + failure { + script { + if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && env.WORKSPACE_NAME) { + echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' + + try { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + airgap.teardownInfrastructure( + dir: env.TOFU_MODULE_PATH, + name: env.WORKSPACE_NAME, + varFile: 'terraform.tfvars' + ) + } + } catch (cleanupErr) { + echo "Cleanup failed: ${cleanupErr.message}" + } + } + } + } + } +} From 83c2c752713ae24982c11fb90d91527f0cdf79db Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 20:38:49 +0000 Subject: [PATCH 02/19] fix: add repo URL parameters to standardCheckout Pass TESTS_REPO_URL and QA_INFRA_REPO_URL to airgap.standardCheckout() so the pipeline can be pointed at a fork for testing. Refs #590 --- validation/pipeline/Jenkinsfile.airgap-rke2-infra | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 1ebef7656..686f89335 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -54,11 +54,21 @@ pipeline { defaultValue: 'main', description: 'Branch of qa-jenkins-library to use' ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) string( name: 'TESTS_BRANCH', defaultValue: 'main', description: 'Branch of rancher/tests repository' ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) string( name: 'QA_INFRA_BRANCH', defaultValue: 'main', @@ -77,8 +87,8 @@ pipeline { steps { script { def dirs = airgap.standardCheckout( - testsRepo: [branch: params.TESTS_BRANCH], - infraRepo: [branch: params.QA_INFRA_BRANCH] + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] ) env.TESTS_DIR = dirs.testsDir env.INFRA_DIR = dirs.infraDir From 3d703e3048409c41140f8af04d469713a8ecd76a Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 21:04:30 +0000 Subject: [PATCH 03/19] fix: load all credentials once at stages level The per-stage useWithProperties wrapping caused AWS_SSH_PEM_KEY_NAME to be unavailable when only AWS_SSH_PEM_KEY was loaded. Now all credentials are loaded in a single wrapper around the stages block, matching the original pipeline's approach. Refs #590 --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 264 ++++++++---------- 1 file changed, 117 insertions(+), 147 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 686f89335..0c6deb5a3 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -112,15 +112,13 @@ pipeline { script { env.TOFU_MODULE_PATH = "${env.INFRA_DIR}/tofu/aws/modules/airgap" - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - tofu.initBackend( - dir: env.TOFU_MODULE_PATH, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } + tofu.initBackend( + dir: env.TOFU_MODULE_PATH, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) } } } @@ -138,12 +136,10 @@ pipeline { ) env.WORKSPACE_NAME = wsName - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - tofu.createWorkspace( - dir: env.TOFU_MODULE_PATH, - name: wsName - ) - } + tofu.createWorkspace( + dir: env.TOFU_MODULE_PATH, + name: wsName + ) infrastructure.archiveWorkspaceName(workspaceName: wsName) } @@ -156,13 +152,11 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_SSH_PEM_KEY']) { - infrastructure.writeSshKey( - keyContent: env.AWS_SSH_PEM_KEY, - keyName: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ) - } + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) } } } @@ -173,22 +167,20 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - def terraformConfig = infrastructure.parseAndSubstituteVars( - content: env.TERRAFORM_CONFIG, - envVars: [ - 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, - 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME - ] - ) + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) - infrastructure.writeConfig( - path: "${env.TOFU_MODULE_PATH}/terraform.tfvars", - content: terraformConfig - ) - } + infrastructure.writeConfig( + path: "${env.TOFU_MODULE_PATH}/terraform.tfvars", + content: terraformConfig + ) } } } @@ -199,13 +191,11 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - tofu.apply( - dir: env.TOFU_MODULE_PATH, - varFile: 'terraform.tfvars', - autoApprove: true - ) - } + tofu.apply( + dir: env.TOFU_MODULE_PATH, + varFile: 'terraform.tfvars', + autoApprove: true + ) def inventoryPath = "${env.INFRA_DIR}/ansible/rke2/airgap/inventory/inventory.yml" if (fileExists(inventoryPath)) { @@ -223,13 +213,11 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - s3.uploadArtifact( - workspaceName: env.WORKSPACE_NAME, - localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars", - s3Key: 'terraform.tfvars' - ) - } + s3.uploadArtifact( + workspaceName: env.WORKSPACE_NAME, + localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars", + s3Key: 'terraform.tfvars' + ) } } } @@ -242,35 +230,29 @@ pipeline { script { def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - property.useWithProperties([ - 'PRIVATE_REGISTRY_URL', - 'PRIVATE_REGISTRY_USERNAME', - 'PRIVATE_REGISTRY_PASSWORD' - ]) { - airgap.configureAnsible( - sshKey: [ - content: env.AWS_SSH_PEM_KEY, - name: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ], - inventoryVars: [ - content: env.ANSIBLE_VARIABLES, - path: "${ansiblePath}/inventory/group_vars/all.yml", - envVars: [ - 'RKE2_VERSION': env.RKE2_VERSION, - 'RANCHER_VERSION': env.RANCHER_VERSION, - 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', - 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', - 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' - ] - ], - ansibleDir: ansiblePath, - inventoryFile: 'inventory/inventory.yml', - validate: false - ) - } + airgap.configureAnsible( + sshKey: [ + content: env.AWS_SSH_PEM_KEY, + name: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ], + inventoryVars: [ + content: env.ANSIBLE_VARIABLES, + path: "${ansiblePath}/inventory/group_vars/all.yml", + envVars: [ + 'RKE2_VERSION': env.RKE2_VERSION, + 'RANCHER_VERSION': env.RANCHER_VERSION, + 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', + 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', + 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' + ] + ], + ansibleDir: ansiblePath, + inventoryFile: 'inventory/inventory.yml', + validate: false + ) } } } @@ -342,37 +324,35 @@ pipeline { echo '=== Infrastructure Setup Complete ===' echo "Workspace Name: ${env.WORKSPACE_NAME}" - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - try { - def bastionDns = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'bastion_public_dns' - ) - echo "Bastion Host: ${bastionDns}" - } catch (e) { - echo "Could not retrieve bastion DNS: ${e.message}" - } - - try { - def rancherHostname = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'external_lb_hostname' - ) - echo "Rancher Hostname (External): ${rancherHostname}" - currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" - } catch (e) { - echo "Could not retrieve external LB hostname: ${e.message}" - } - - try { - def internalLb = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'internal_lb_hostname' - ) - echo "Internal LB Hostname: ${internalLb}" - } catch (e) { - echo "Could not retrieve internal LB hostname: ${e.message}" - } + try { + def bastionDns = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'bastion_public_dns' + ) + echo "Bastion Host: ${bastionDns}" + } catch (e) { + echo "Could not retrieve bastion DNS: ${e.message}" + } + + try { + def rancherHostname = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'external_lb_hostname' + ) + echo "Rancher Hostname (External): ${rancherHostname}" + currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" + } catch (e) { + echo "Could not retrieve external LB hostname: ${e.message}" + } + + try { + def internalLb = tofu.getOutputs( + dir: env.TOFU_MODULE_PATH, + output: 'internal_lb_hostname' + ) + echo "Internal LB Hostname: ${internalLb}" + } catch (e) { + echo "Could not retrieve internal LB hostname: ${e.message}" } echo '========================================' @@ -404,15 +384,13 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - tofu.initBackend( - dir: env.TOFU_MODULE_PATH, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION ?: env.S3_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } + tofu.initBackend( + dir: env.TOFU_MODULE_PATH, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION ?: env.S3_REGION, + backendInitScript: './scripts/init-backend.sh' + ) } } } @@ -423,13 +401,11 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - s3.downloadArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars', - localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars" - ) - } + s3.downloadArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars', + localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars" + ) } } } @@ -440,13 +416,11 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - airgap.teardownInfrastructure( - dir: env.TOFU_MODULE_PATH, - name: params.TARGET_WORKSPACE, - varFile: 'terraform.tfvars' - ) - } + airgap.teardownInfrastructure( + dir: env.TOFU_MODULE_PATH, + name: params.TARGET_WORKSPACE, + varFile: 'terraform.tfvars' + ) } } } @@ -457,12 +431,10 @@ pipeline { } steps { script { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - s3.deleteArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars' - ) - } + s3.deleteArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars' + ) } } } @@ -505,13 +477,11 @@ pipeline { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' try { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - airgap.teardownInfrastructure( - dir: env.TOFU_MODULE_PATH, - name: env.WORKSPACE_NAME, - varFile: 'terraform.tfvars' - ) - } + airgap.teardownInfrastructure( + dir: env.TOFU_MODULE_PATH, + name: env.WORKSPACE_NAME, + varFile: 'terraform.tfvars' + ) } catch (cleanupErr) { echo "Cleanup failed: ${cleanupErr.message}" } From fc4912f01cd4a0522f72efe0de18dc1b676481ed Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 21:39:50 +0000 Subject: [PATCH 04/19] fix: wrap all stages in single property.useWithProperties block AWS credentials were unavailable inside Docker containers because the property.useWithProperties wrapper was removed. Restructured to use a parent stage with nested stages inside a single credentials wrapper, ensuring all credentials are available throughout the pipeline. Refs #590 --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 620 +++++++----------- 1 file changed, 250 insertions(+), 370 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 0c6deb5a3..b9818c64b 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -81,8 +81,6 @@ pipeline { } stages { - // ── Shared stages (both actions) ────────────────────────────── - stage('Checkout') { steps { script { @@ -102,372 +100,252 @@ pipeline { } } - // ── Setup action ────────────────────────────────────────────── - - stage('Setup: Initialize Tofu Backend') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - env.TOFU_MODULE_PATH = "${env.INFRA_DIR}/tofu/aws/modules/airgap" - - tofu.initBackend( - dir: env.TOFU_MODULE_PATH, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } - } - } - - stage('Setup: Create Workspace') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - def wsName = infrastructure.generateWorkspaceName( - prefix: 'jenkins_airgap_ansible_workspace', - suffix: env.HOSTNAME_PREFIX, - includeTimestamp: false - ) - env.WORKSPACE_NAME = wsName - - tofu.createWorkspace( - dir: env.TOFU_MODULE_PATH, - name: wsName - ) - - infrastructure.archiveWorkspaceName(workspaceName: wsName) - } - } - } - - stage('Setup: Configure SSH Key') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - infrastructure.writeSshKey( - keyContent: env.AWS_SSH_PEM_KEY, - keyName: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ) - } - } - } - - stage('Setup: Configure Tofu Variables') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - def terraformConfig = infrastructure.parseAndSubstituteVars( - content: env.TERRAFORM_CONFIG, - envVars: [ - 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, - 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME - ] - ) - - infrastructure.writeConfig( - path: "${env.TOFU_MODULE_PATH}/terraform.tfvars", - content: terraformConfig - ) - } - } - } - - stage('Setup: Apply Tofu') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - tofu.apply( - dir: env.TOFU_MODULE_PATH, - varFile: 'terraform.tfvars', - autoApprove: true - ) - - def inventoryPath = "${env.INFRA_DIR}/ansible/rke2/airgap/inventory/inventory.yml" - if (fileExists(inventoryPath)) { - archiveArtifacts artifacts: inventoryPath, fingerprint: true - } else { - echo "Warning: Inventory file not found at ${inventoryPath}" - } - } - } - } - - stage('Setup: Upload Terraform Variables to S3') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - s3.uploadArtifact( - workspaceName: env.WORKSPACE_NAME, - localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars", - s3Key: 'terraform.tfvars' - ) - } - } - } - - stage('Setup: Configure Ansible Variables') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - - airgap.configureAnsible( - sshKey: [ - content: env.AWS_SSH_PEM_KEY, - name: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ], - inventoryVars: [ - content: env.ANSIBLE_VARIABLES, - path: "${ansiblePath}/inventory/group_vars/all.yml", - envVars: [ - 'RKE2_VERSION': env.RKE2_VERSION, - 'RANCHER_VERSION': env.RANCHER_VERSION, - 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', - 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', - 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' - ] - ], - ansibleDir: ansiblePath, - inventoryFile: 'inventory/inventory.yml', - validate: false - ) - } - } - } - - stage('Setup: Deploy RKE2 Cluster') { - when { - expression { params.ACTION == 'setup' } - } - steps { - script { - def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - - airgap.deployRKE2( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rke2-tarball-playbook.yml' - ) - } - } - } - - stage('Setup: Configure Private Registry') { - when { - allOf { - expression { params.ACTION == 'setup' } - expression { env.PRIVATE_REGISTRY_URL?.trim() } - } - } - steps { - script { - def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - - ansible.runPlaybook( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rke2-registry-config-playbook.yml' - ) - } - } - } - - stage('Setup: Deploy Rancher') { - when { - allOf { - expression { params.ACTION == 'setup' } - expression { params.DEPLOY_RANCHER } - } - } - steps { - script { - def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - - airgap.deployRancher( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rancher-helm-deploy-playbook.yml', - enabled: params.DEPLOY_RANCHER - ) - } - } - } - - stage('Setup: Output Infrastructure Details') { - when { - expression { params.ACTION == 'setup' } - } + stage('Provision Infrastructure') { steps { script { - echo '=== Infrastructure Setup Complete ===' - echo "Workspace Name: ${env.WORKSPACE_NAME}" - - try { - def bastionDns = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'bastion_public_dns' - ) - echo "Bastion Host: ${bastionDns}" - } catch (e) { - echo "Could not retrieve bastion DNS: ${e.message}" - } - - try { - def rancherHostname = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'external_lb_hostname' - ) - echo "Rancher Hostname (External): ${rancherHostname}" - currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" - } catch (e) { - echo "Could not retrieve external LB hostname: ${e.message}" + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME', + 'PRIVATE_REGISTRY_URL', + 'PRIVATE_REGISTRY_USERNAME', + 'PRIVATE_REGISTRY_PASSWORD' + ]) { + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + + if (params.ACTION == 'setup') { + // ── Setup ──────────────────────────────────────── + stage('Setup: Initialize Tofu Backend') { + tofu.initBackend( + dir: tofuModulePath, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + + stage('Setup: Create Workspace') { + def wsName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_airgap_ansible_workspace', + suffix: env.HOSTNAME_PREFIX, + includeTimestamp: false + ) + env.WORKSPACE_NAME = wsName + + tofu.createWorkspace(dir: tofuModulePath, name: wsName) + infrastructure.archiveWorkspaceName(workspaceName: wsName) + } + + stage('Setup: Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + } + + stage('Setup: Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + stage('Setup: Apply Tofu') { + tofu.apply( + dir: tofuModulePath, + varFile: 'terraform.tfvars', + autoApprove: true + ) + + def inventoryPath = "${ansiblePath}/inventory/inventory.yml" + if (fileExists(inventoryPath)) { + archiveArtifacts artifacts: inventoryPath, fingerprint: true + } else { + echo "Warning: Inventory file not found at ${inventoryPath}" + } + } + + stage('Setup: Upload Terraform Variables to S3') { + s3.uploadArtifact( + workspaceName: env.WORKSPACE_NAME, + localPath: "${tofuModulePath}/terraform.tfvars", + s3Key: 'terraform.tfvars' + ) + } + + stage('Setup: Configure Ansible Variables') { + airgap.configureAnsible( + sshKey: [ + content: env.AWS_SSH_PEM_KEY, + name: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ], + inventoryVars: [ + content: env.ANSIBLE_VARIABLES, + path: "${ansiblePath}/inventory/group_vars/all.yml", + envVars: [ + 'RKE2_VERSION': env.RKE2_VERSION, + 'RANCHER_VERSION': env.RANCHER_VERSION, + 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', + 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', + 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' + ] + ], + ansibleDir: ansiblePath, + inventoryFile: 'inventory/inventory.yml', + validate: false + ) + } + + stage('Setup: Deploy RKE2 Cluster') { + airgap.deployRKE2( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rke2-tarball-playbook.yml' + ) + } + + stage('Setup: Configure Private Registry') { + if (env.PRIVATE_REGISTRY_URL?.trim()) { + ansible.runPlaybook( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rke2-registry-config-playbook.yml' + ) + } else { + echo 'Skipping private registry configuration (PRIVATE_REGISTRY_URL not set)' + } + } + + stage('Setup: Deploy Rancher') { + if (params.DEPLOY_RANCHER) { + airgap.deployRancher( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/deploy/rancher-helm-deploy-playbook.yml', + enabled: true + ) + } else { + echo 'Skipping Rancher deployment (DEPLOY_RANCHER not checked)' + } + } + + stage('Setup: Output Infrastructure Details') { + echo '=== Infrastructure Setup Complete ===' + echo "Workspace Name: ${env.WORKSPACE_NAME}" + + try { + def bastionDns = tofu.getOutputs( + dir: tofuModulePath, output: 'bastion_public_dns' + ) + echo "Bastion Host: ${bastionDns}" + } catch (e) { + echo "Could not retrieve bastion DNS: ${e.message}" + } + + try { + def rancherHostname = tofu.getOutputs( + dir: tofuModulePath, output: 'external_lb_hostname' + ) + echo "Rancher Hostname (External): ${rancherHostname}" + currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" + } catch (e) { + echo "Could not retrieve external LB hostname: ${e.message}" + } + + try { + def internalLb = tofu.getOutputs( + dir: tofuModulePath, output: 'internal_lb_hostname' + ) + echo "Internal LB Hostname: ${internalLb}" + } catch (e) { + echo "Could not retrieve internal LB hostname: ${e.message}" + } + + echo '========================================' + } + + } else { + // ── Destroy ───────────────────────────────────── + stage('Destroy: Validate Parameters') { + if (!params.TARGET_WORKSPACE?.trim()) { + error 'TARGET_WORKSPACE parameter is required for destroy action' + } + echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" + env.WORKSPACE_NAME = params.TARGET_WORKSPACE + } + + stage('Destroy: Initialize Tofu Backend') { + tofu.initBackend( + dir: tofuModulePath, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION ?: env.S3_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + + stage('Destroy: Download Terraform Variables from S3') { + s3.downloadArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars', + localPath: "${tofuModulePath}/terraform.tfvars" + ) + } + + stage('Destroy: Teardown Infrastructure') { + airgap.teardownInfrastructure( + dir: tofuModulePath, + name: params.TARGET_WORKSPACE, + varFile: 'terraform.tfvars' + ) + } + + stage('Destroy: Delete S3 Artifacts') { + s3.deleteArtifact( + workspaceName: params.TARGET_WORKSPACE, + s3Key: 'terraform.tfvars' + ) + } + + stage('Destroy: Cleanup Local Artifacts') { + infrastructure.cleanupArtifacts( + paths: [ + "${tofuModulePath}/.terraform", + "${tofuModulePath}/terraform.tfstate*", + "${tofuModulePath}/*.tfvars" + ], + force: true + ) + } + + stage('Destroy: Summary') { + echo '=== Infrastructure Destruction Complete ===' + echo "Workspace: ${params.TARGET_WORKSPACE}" + echo 'All resources have been destroyed and cleaned up' + echo '===========================================' + } + } } - - try { - def internalLb = tofu.getOutputs( - dir: env.TOFU_MODULE_PATH, - output: 'internal_lb_hostname' - ) - echo "Internal LB Hostname: ${internalLb}" - } catch (e) { - echo "Could not retrieve internal LB hostname: ${e.message}" - } - - echo '========================================' } } } - - // ── Destroy action ──────────────────────────────────────────── - - stage('Destroy: Validate Parameters') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - if (!params.TARGET_WORKSPACE?.trim()) { - error 'TARGET_WORKSPACE parameter is required for destroy action' - } - echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" - env.WORKSPACE_NAME = params.TARGET_WORKSPACE - env.TOFU_MODULE_PATH = "${env.INFRA_DIR}/tofu/aws/modules/airgap" - } - } - } - - stage('Destroy: Initialize Tofu Backend') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - tofu.initBackend( - dir: env.TOFU_MODULE_PATH, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION ?: env.S3_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } - } - } - - stage('Destroy: Download Terraform Variables from S3') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - s3.downloadArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars', - localPath: "${env.TOFU_MODULE_PATH}/terraform.tfvars" - ) - } - } - } - - stage('Destroy: Teardown Infrastructure') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - airgap.teardownInfrastructure( - dir: env.TOFU_MODULE_PATH, - name: params.TARGET_WORKSPACE, - varFile: 'terraform.tfvars' - ) - } - } - } - - stage('Destroy: Delete S3 Artifacts') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - s3.deleteArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars' - ) - } - } - } - - stage('Destroy: Cleanup Local Artifacts') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - script { - infrastructure.cleanupArtifacts( - paths: [ - "${env.TOFU_MODULE_PATH}/.terraform", - "${env.TOFU_MODULE_PATH}/terraform.tfstate*", - "${env.TOFU_MODULE_PATH}/*.tfvars" - ], - force: true - ) - } - } - } - - stage('Destroy: Summary') { - when { - expression { params.ACTION == 'destroy' } - } - steps { - echo '=== Infrastructure Destruction Complete ===' - echo "Workspace: ${params.TARGET_WORKSPACE}" - echo 'All resources have been destroyed and cleaned up' - echo '===========================================' - } - } } post { @@ -476,14 +354,16 @@ pipeline { if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && env.WORKSPACE_NAME) { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' - try { - airgap.teardownInfrastructure( - dir: env.TOFU_MODULE_PATH, - name: env.WORKSPACE_NAME, - varFile: 'terraform.tfvars' - ) - } catch (cleanupErr) { - echo "Cleanup failed: ${cleanupErr.message}" + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + try { + airgap.teardownInfrastructure( + dir: "${env.INFRA_DIR}/tofu/aws/modules/airgap", + name: env.WORKSPACE_NAME, + varFile: 'terraform.tfvars' + ) + } catch (cleanupErr) { + echo "Cleanup failed: ${cleanupErr.message}" + } } } } From fe058f44bda1b78f765fe39746a6e217a358ef5d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 22:22:55 +0000 Subject: [PATCH 05/19] debug: add logging before S3 upload to diagnose empty params Refs #590 --- validation/pipeline/Jenkinsfile.airgap-rke2-infra | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index b9818c64b..b7dfe7ea2 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -180,9 +180,12 @@ pipeline { } stage('Setup: Upload Terraform Variables to S3') { + def wsName = env.WORKSPACE_NAME + def tfvarsPath = "${tofuModulePath}/terraform.tfvars" + echo "DEBUG: workspaceName=${wsName}, localPath=${tfvarsPath}, s3Key=terraform.tfvars" s3.uploadArtifact( - workspaceName: env.WORKSPACE_NAME, - localPath: "${tofuModulePath}/terraform.tfvars", + workspaceName: wsName, + localPath: tfvarsPath, s3Key: 'terraform.tfvars' ) } From 0b3d102017954a5d258ea8030230317c9afdd6b1 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 22:33:07 +0000 Subject: [PATCH 06/19] fix: use local variable for workspaceName instead of env env.WORKSPACE_NAME was null inside nested closures (script > useWithProperties > stage). Use a local variable which persists correctly across all nested stages within the same script block. Refs #590 --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index b7dfe7ea2..751c48755 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -114,6 +114,7 @@ pipeline { ]) { def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + def workspaceName = '' if (params.ACTION == 'setup') { // ── Setup ──────────────────────────────────────── @@ -128,15 +129,15 @@ pipeline { } stage('Setup: Create Workspace') { - def wsName = infrastructure.generateWorkspaceName( + workspaceName = infrastructure.generateWorkspaceName( prefix: 'jenkins_airgap_ansible_workspace', suffix: env.HOSTNAME_PREFIX, includeTimestamp: false ) - env.WORKSPACE_NAME = wsName + env.WORKSPACE_NAME = workspaceName - tofu.createWorkspace(dir: tofuModulePath, name: wsName) - infrastructure.archiveWorkspaceName(workspaceName: wsName) + tofu.createWorkspace(dir: tofuModulePath, name: workspaceName) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) } stage('Setup: Configure SSH Key') { @@ -180,12 +181,9 @@ pipeline { } stage('Setup: Upload Terraform Variables to S3') { - def wsName = env.WORKSPACE_NAME - def tfvarsPath = "${tofuModulePath}/terraform.tfvars" - echo "DEBUG: workspaceName=${wsName}, localPath=${tfvarsPath}, s3Key=terraform.tfvars" s3.uploadArtifact( - workspaceName: wsName, - localPath: tfvarsPath, + workspaceName: workspaceName, + localPath: "${tofuModulePath}/terraform.tfvars", s3Key: 'terraform.tfvars' ) } @@ -251,7 +249,7 @@ pipeline { stage('Setup: Output Infrastructure Details') { echo '=== Infrastructure Setup Complete ===' - echo "Workspace Name: ${env.WORKSPACE_NAME}" + echo "Workspace Name: ${workspaceName}" try { def bastionDns = tofu.getOutputs( @@ -267,7 +265,7 @@ pipeline { dir: tofuModulePath, output: 'external_lb_hostname' ) echo "Rancher Hostname (External): ${rancherHostname}" - currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${env.WORKSPACE_NAME}" + currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${workspaceName}" } catch (e) { echo "Could not retrieve external LB hostname: ${e.message}" } @@ -291,7 +289,7 @@ pipeline { error 'TARGET_WORKSPACE parameter is required for destroy action' } echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" - env.WORKSPACE_NAME = params.TARGET_WORKSPACE + workspaceName = params.TARGET_WORKSPACE } stage('Destroy: Initialize Tofu Backend') { @@ -357,16 +355,16 @@ pipeline { if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && env.WORKSPACE_NAME) { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { - try { + try { + property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { airgap.teardownInfrastructure( dir: "${env.INFRA_DIR}/tofu/aws/modules/airgap", name: env.WORKSPACE_NAME, varFile: 'terraform.tfvars' ) - } catch (cleanupErr) { - echo "Cleanup failed: ${cleanupErr.message}" } + } catch (cleanupErr) { + echo "Cleanup failed: ${cleanupErr.message}" } } } From 3753779e450a2a3032aca323635ae39aa11b4ad6 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 31 Mar 2026 22:48:02 +0000 Subject: [PATCH 07/19] fix: pass S3 bucket/region overrides to shared s3 functions The s3.groovy shared functions default to config.groovy values (rancher-qa-artifacts, us-east-1) but the pipeline uses S3_BUCKET_NAME and S3_BUCKET_REGION from Jenkins job parameters. Pass these as explicit overrides to uploadArtifact, downloadArtifact, and deleteArtifact. Also move env.WORKSPACE_NAME assignment after tofu.createWorkspace succeeds so the post-failure cleanup block has a valid value. --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 751c48755..efd4525ee 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -134,10 +134,12 @@ pipeline { suffix: env.HOSTNAME_PREFIX, includeTimestamp: false ) - env.WORKSPACE_NAME = workspaceName tofu.createWorkspace(dir: tofuModulePath, name: workspaceName) infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + + // Set env after all calls succeed so post-failure cleanup can use it + env.WORKSPACE_NAME = workspaceName } stage('Setup: Configure SSH Key') { @@ -184,7 +186,9 @@ pipeline { s3.uploadArtifact( workspaceName: workspaceName, localPath: "${tofuModulePath}/terraform.tfvars", - s3Key: 'terraform.tfvars' + s3Key: 'terraform.tfvars', + bucket: env.S3_BUCKET_NAME, + region: env.S3_BUCKET_REGION ) } @@ -306,7 +310,9 @@ pipeline { s3.downloadArtifact( workspaceName: params.TARGET_WORKSPACE, s3Key: 'terraform.tfvars', - localPath: "${tofuModulePath}/terraform.tfvars" + localPath: "${tofuModulePath}/terraform.tfvars", + bucket: env.S3_BUCKET_NAME, + region: env.S3_BUCKET_REGION ) } @@ -321,7 +327,9 @@ pipeline { stage('Destroy: Delete S3 Artifacts') { s3.deleteArtifact( workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars' + s3Key: 'terraform.tfvars', + bucket: env.S3_BUCKET_NAME, + region: env.S3_BUCKET_REGION ) } From f96e2e8c97ae33ff9b3d5d8371d1939db91b3ed9 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 02:39:17 +0000 Subject: [PATCH 08/19] feat: generate Ansible inventory from tofu airgap_inventory_json output The feature/decouple-tofu branch removed the local_file resource that previously wrote inventory.yml during tofu apply. The pipeline now generates the inventory by fetching the airgap_inventory_json output, parsing it, and rendering the YAML inventory file before Ansible stages. --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index efd4525ee..54af529fd 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -173,13 +173,75 @@ pipeline { varFile: 'terraform.tfvars', autoApprove: true ) + } - def inventoryPath = "${ansiblePath}/inventory/inventory.yml" - if (fileExists(inventoryPath)) { - archiveArtifacts artifacts: inventoryPath, fingerprint: true - } else { - echo "Warning: Inventory file not found at ${inventoryPath}" + stage('Setup: Generate Inventory') { + // The tofu module outputs inventory data as JSON; + // the pipeline generates the Ansible inventory file. + def inventoryJson = tofu.getOutputs( + dir: tofuModulePath, output: 'airgap_inventory_json' + ) + def inventory = readJSON text: inventoryJson + + // Build inventory YAML from tofu output + def sshKey = inventory.ssh_key + def sshUser = inventory.ssh_user + def bastion = inventory.bastion_host + def extLb = inventory.external_lb_hostname + def intLb = inventory.internal_lb_hostname + def regHost = inventory.registry_host + def groups = inventory.node_groups + + def yaml = """all: + vars: + ssh_private_key_file: ${sshKey} + ansible_ssh_private_key_file: "{{ ssh_private_key_file }}" + bastion_user: ${sshUser} + bastion_host: ${bastion} + external_lb_hostname: ${extLb} + internal_lb_hostname: ${intLb}""" + if (regHost) { + yaml += "\n registry_host: ${regHost}" } + yaml += """ + children: + bastion: + hosts: + bastion-node: + ansible_host: "{{ bastion_host }}" + ansible_user: "{{ bastion_user }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +""" + if (regHost) { + yaml += """ registry: + hosts: + registry-node: + ansible_host: "{{ registry_host }}" + ansible_user: "{{ bastion_user }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +""" + } + yaml += """ airgap_nodes: + vars: + ansible_user: ${sshUser} + ansible_ssh_common_args: "-o ProxyCommand='ssh -i {{ ssh_private_key_file }} -W %h:%p {{ bastion_user }}@{{ bastion_host }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + bastion_ip: "{{ bastion_host }}" + children: +""" + def groupNames = groups.keySet() as List + for (int gi = 0; gi < groupNames.size(); gi++) { + def groupName = groupNames[gi] + def addresses = groups[groupName] + yaml += " ${groupName}:\n hosts:\n" + for (int ai = 0; ai < addresses.size(); ai++) { + yaml += " ${groupName}_node_${ai + 1}:\n ansible_host: ${addresses[ai]}\n" + } + } + + def inventoryDir = "${ansiblePath}/inventory" + sh "mkdir -p ${inventoryDir}" + writeFile file: "${inventoryDir}/inventory.yml", text: yaml + echo "Inventory file written to ${inventoryDir}/inventory.yml" } stage('Setup: Upload Terraform Variables to S3') { From cf2f1266a85311672a36af4d17080506390b52be Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 02:50:24 +0000 Subject: [PATCH 09/19] refactor: use generate_inventory.py instead of inline Groovy inventory Replace the hand-rolled Groovy inventory generation with qa-infra-automation's scripts/generate_inventory.py, which is the canonical inventory renderer. Runs inside the Docker container that already has Python and PyYAML. --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 80 ++++--------------- 1 file changed, 14 insertions(+), 66 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 54af529fd..49b4472ee 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -176,72 +176,20 @@ pipeline { } stage('Setup: Generate Inventory') { - // The tofu module outputs inventory data as JSON; - // the pipeline generates the Ansible inventory file. - def inventoryJson = tofu.getOutputs( - dir: tofuModulePath, output: 'airgap_inventory_json' - ) - def inventory = readJSON text: inventoryJson - - // Build inventory YAML from tofu output - def sshKey = inventory.ssh_key - def sshUser = inventory.ssh_user - def bastion = inventory.bastion_host - def extLb = inventory.external_lb_hostname - def intLb = inventory.internal_lb_hostname - def regHost = inventory.registry_host - def groups = inventory.node_groups - - def yaml = """all: - vars: - ssh_private_key_file: ${sshKey} - ansible_ssh_private_key_file: "{{ ssh_private_key_file }}" - bastion_user: ${sshUser} - bastion_host: ${bastion} - external_lb_hostname: ${extLb} - internal_lb_hostname: ${intLb}""" - if (regHost) { - yaml += "\n registry_host: ${regHost}" - } - yaml += """ - children: - bastion: - hosts: - bastion-node: - ansible_host: "{{ bastion_host }}" - ansible_user: "{{ bastion_user }}" - ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -""" - if (regHost) { - yaml += """ registry: - hosts: - registry-node: - ansible_host: "{{ registry_host }}" - ansible_user: "{{ bastion_user }}" - ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -""" - } - yaml += """ airgap_nodes: - vars: - ansible_user: ${sshUser} - ansible_ssh_common_args: "-o ProxyCommand='ssh -i {{ ssh_private_key_file }} -W %h:%p {{ bastion_user }}@{{ bastion_host }} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - bastion_ip: "{{ bastion_host }}" - children: -""" - def groupNames = groups.keySet() as List - for (int gi = 0; gi < groupNames.size(); gi++) { - def groupName = groupNames[gi] - def addresses = groups[groupName] - yaml += " ${groupName}:\n hosts:\n" - for (int ai = 0; ai < addresses.size(); ai++) { - yaml += " ${groupName}_node_${ai + 1}:\n ansible_host: ${addresses[ai]}\n" - } - } - - def inventoryDir = "${ansiblePath}/inventory" - sh "mkdir -p ${inventoryDir}" - writeFile file: "${inventoryDir}/inventory.yml", text: yaml - echo "Inventory file written to ${inventoryDir}/inventory.yml" + // Use qa-infra-automation's generate_inventory.py to render + // the Ansible inventory from the tofu JSON output. + sh """ + docker run --rm --platform linux/amd64 \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -v \$(pwd):/workspace -w /workspace \ + rancher-infra-tools:latest sh -c ' + tofu -chdir=${tofuModulePath} output -raw airgap_inventory_json > /tmp/airgap.json && + python3 ./qa-infra-automation/scripts/generate_inventory.py \ + --input /tmp/airgap.json \ + --distro rke2 --env airgap \ + --output-dir ${ansiblePath}/inventory + ' + """ } stage('Setup: Upload Terraform Variables to S3') { From de2d73e6a4529da2685536098b36e73c34ee98f1 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 03:44:47 +0000 Subject: [PATCH 10/19] refactor: eliminate S3 tfvars round-trip, reconstruct from job config Instead of uploading terraform.tfvars to S3 during setup and downloading during destroy, reconstruct the tfvars file from TERRAFORM_CONFIG environment variable (set via Jenkins folder properties) in both the destroy path and post-failure cleanup. Removes: - Setup: Upload Terraform Variables to S3 - Destroy: Download Terraform Variables from S3 - Destroy: Delete S3 Artifacts Adds: - Destroy: Configure Tofu Variables (reconstructs tfvars) - Post-failure cleanup now writes tfvars before teardown --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 49b4472ee..51cf6e6e3 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -12,8 +12,8 @@ * Consumes shared functions from qa-jenkins-library: * airgap.standardCheckout, airgap.configureAnsible, airgap.deployRKE2, * airgap.deployRancher, airgap.teardownInfrastructure, - * s3.uploadArtifact, s3.downloadArtifact, s3.deleteArtifact, - * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs + * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs, + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig */ def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' @@ -186,22 +186,13 @@ pipeline { tofu -chdir=${tofuModulePath} output -raw airgap_inventory_json > /tmp/airgap.json && python3 ./qa-infra-automation/scripts/generate_inventory.py \ --input /tmp/airgap.json \ + --schema ./qa-infra-automation/ansible/_inventory-schema.yaml \ --distro rke2 --env airgap \ --output-dir ${ansiblePath}/inventory ' """ } - stage('Setup: Upload Terraform Variables to S3') { - s3.uploadArtifact( - workspaceName: workspaceName, - localPath: "${tofuModulePath}/terraform.tfvars", - s3Key: 'terraform.tfvars', - bucket: env.S3_BUCKET_NAME, - region: env.S3_BUCKET_REGION - ) - } - stage('Setup: Configure Ansible Variables') { airgap.configureAnsible( sshKey: [ @@ -316,13 +307,20 @@ pipeline { ) } - stage('Destroy: Download Terraform Variables from S3') { - s3.downloadArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars', - localPath: "${tofuModulePath}/terraform.tfvars", - bucket: env.S3_BUCKET_NAME, - region: env.S3_BUCKET_REGION + stage('Destroy: Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig ) } @@ -334,15 +332,6 @@ pipeline { ) } - stage('Destroy: Delete S3 Artifacts') { - s3.deleteArtifact( - workspaceName: params.TARGET_WORKSPACE, - s3Key: 'terraform.tfvars', - bucket: env.S3_BUCKET_NAME, - region: env.S3_BUCKET_REGION - ) - } - stage('Destroy: Cleanup Local Artifacts') { infrastructure.cleanupArtifacts( paths: [ @@ -374,9 +363,29 @@ pipeline { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' try { - property.useWithProperties(['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']) { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + + // Reconstruct tfvars from the Jenkins job parameters + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + airgap.teardownInfrastructure( - dir: "${env.INFRA_DIR}/tofu/aws/modules/airgap", + dir: tofuDir, name: env.WORKSPACE_NAME, varFile: 'terraform.tfvars' ) From e5befd9c9cdf890e9dda748cef1871e6a5a2a94c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 12:08:56 +0000 Subject: [PATCH 11/19] fix: add SSH key copy stage before RKE2 deploy The bastion node needs the SSH key to scp bundles to airgap nodes. The original setup pipeline had this stage but it was missing from the unified pipeline. Runs setup-ssh-keys.yml playbook after Ansible configuration and before RKE2 deployment. --- validation/pipeline/Jenkinsfile.airgap-rke2-infra | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 51cf6e6e3..b614d7316 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -219,6 +219,14 @@ pipeline { ) } + stage('Setup: Copy SSH Keys to Nodes') { + ansible.runPlaybook( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/setup/setup-ssh-keys.yml' + ) + } + stage('Setup: Deploy RKE2 Cluster') { airgap.deployRKE2( dir: ansiblePath, From dbc343bf404bd980c89d1d66cf2655a2456a6074 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 12:20:46 +0000 Subject: [PATCH 12/19] fix: add debug logging to post-failure teardown Add echo statements showing which conditions pass/fail so we can diagnose why teardown isn't triggering when the build fails. --- validation/pipeline/Jenkinsfile.airgap-rke2-infra | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index b614d7316..848999f35 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -367,6 +367,8 @@ pipeline { post { failure { script { + echo "DEBUG: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${env.WORKSPACE_NAME}', INFRA_DIR='${env.INFRA_DIR}'" + if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && env.WORKSPACE_NAME) { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' @@ -401,6 +403,8 @@ pipeline { } catch (cleanupErr) { echo "Cleanup failed: ${cleanupErr.message}" } + } else { + echo "Skipping teardown: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${env.WORKSPACE_NAME}'" } } } From 4cd926bc4ca6fef63b6da622342ced2480d99141 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 13:16:54 +0000 Subject: [PATCH 13/19] fix: read workspace name from file in post-failure teardown env.WORKSPACE_NAME doesn't persist from inside the property.useWithProperties closure to the post block (shows 'null'). Instead, read workspace_name.txt which is written by infrastructure.archiveWorkspaceName during the setup stages. --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra index 848999f35..a8a97496b 100644 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.airgap-rke2-infra @@ -367,9 +367,20 @@ pipeline { post { failure { script { - echo "DEBUG: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${env.WORKSPACE_NAME}', INFRA_DIR='${env.INFRA_DIR}'" + // Read workspace name from the file written by archiveWorkspaceName + // (env.WORKSPACE_NAME doesn't persist from inside useWithProperties) + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + echo "DEBUG: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}', INFRA_DIR='${env.INFRA_DIR}'" - if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && env.WORKSPACE_NAME) { + if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && wsName) { echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' try { @@ -396,7 +407,7 @@ pipeline { airgap.teardownInfrastructure( dir: tofuDir, - name: env.WORKSPACE_NAME, + name: wsName, varFile: 'terraform.tfvars' ) } @@ -404,7 +415,7 @@ pipeline { echo "Cleanup failed: ${cleanupErr.message}" } } else { - echo "Skipping teardown: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${env.WORKSPACE_NAME}'" + echo "Skipping teardown: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}'" } } } From 2e227af3aa292ded39a0df70eaeb674037eaaa5b Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 17:05:56 +0000 Subject: [PATCH 14/19] feat: split airgap RKE2 infra pipeline into setup/destroy jobs Split unified Jenkinsfile.airgap-rke2-infra into separate setup and destroy pipelines. Setup uses make.runTarget() for cluster/registry/ rancher Makefile targets; destroy uses Jenkins library directly. Also adds make to Dockerfile.infra and archives Rancher admin token. --- validation/pipeline/Dockerfile.infra | 1 + .../Jenkinsfile.destroy.airgap-rke2-infra | 203 +++++++ .../Jenkinsfile.setup.airgap-rke2-infra | 508 ++++++++++++++++++ 3 files changed, 712 insertions(+) create mode 100644 validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra create mode 100644 validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra diff --git a/validation/pipeline/Dockerfile.infra b/validation/pipeline/Dockerfile.infra index 65a7cfc38..e9b5ef4f8 100644 --- a/validation/pipeline/Dockerfile.infra +++ b/validation/pipeline/Dockerfile.infra @@ -12,6 +12,7 @@ RUN apk add --no-cache \ bash \ curl \ git \ + make \ openssh-client \ python3 \ py3-pip \ diff --git a/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra new file mode 100644 index 000000000..539f20c13 --- /dev/null +++ b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra @@ -0,0 +1,203 @@ +#!groovy +/** + * Airgap RKE2 infrastructure destroy pipeline. + * + * Tears down AWS infrastructure provisioned by the setup pipeline. + * Uses the Jenkins library directly (NOT the Makefile) because: + * - The Makefile's infra-down target has an interactive confirmation prompt + * - The Makefile has no workspace management (select/delete) + * - The Jenkins library's tofu.teardownInfrastructure handles the full + * sequence: selectWorkspace → destroy → deleteWorkspace + * + * Consumes shared functions from qa-jenkins-library: + * airgap.standardCheckout, airgap.teardownInfrastructure, + * tofu.initBackend, infrastructure.parseAndSubstituteVars, + * infrastructure.writeConfig, infrastructure.cleanupArtifacts + */ + +def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +pipeline { + agent any + + options { + ansiColor('xterm') + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + string( + name: 'TARGET_WORKSPACE', + defaultValue: '', + description: 'Terraform workspace to destroy (e.g. jenkins_airgap_ansible_workspace_42_hostname)' + ) + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-infra-automation repository' + ) + string( + name: 'HOSTNAME_PREFIX', + defaultValue: '', + description: 'Hostname prefix used during setup (for tfvars substitution)' + ) + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform config values (same as used during setup)', + default: | + aws_access_key = "${AWS_ACCESS_KEY_ID}" + aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" + aws_ami = "ami-09457fad1d2c34c31" + instance_type = "t3a.xlarge" + aws_security_group = ["sg-08e8243a8cfbea8a0"] + aws_subnet = "subnet-ee8cac86" + aws_volume_size = 100 + aws_hostname_prefix = "${HOSTNAME_PREFIX}" + aws_region = "us-east-2" + aws_route53_zone = "qa.rancher.space" + aws_ssh_user = "ubuntu" + aws_vpc = "vpc-bfccf4d7" + user_id = "ubuntu" + ssh_key = "/root/.ssh/jenkins-elliptic-validation.pem" + ssh_key_name = "jenkins-elliptic-validation" + provision_registry = false + ) + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + } + } + } + + stage('Build Infrastructure Tools Image') { + steps { + sh "docker build --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + } + } + + stage('Destroy Infrastructure') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + + // ── Validate parameters ──────────────────────────── + stage('Validate Parameters') { + if (!params.TARGET_WORKSPACE?.trim()) { + error 'TARGET_WORKSPACE parameter is required for destroy action' + } + echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" + } + + // ── Initialize Tofu backend ──────────────────────── + stage('Initialize Tofu Backend') { + tofu.initBackend( + dir: tofuModulePath, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + + // ── Configure Tofu variables ─────────────────────── + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Teardown infrastructure ──────────────────────── + stage('Teardown Infrastructure') { + airgap.teardownInfrastructure( + dir: tofuModulePath, + name: params.TARGET_WORKSPACE, + varFile: 'terraform.tfvars' + ) + } + + // ── Cleanup local artifacts ──────────────────────── + stage('Cleanup Local Artifacts') { + infrastructure.cleanupArtifacts( + paths: [ + "${tofuModulePath}/.terraform", + "${tofuModulePath}/terraform.tfstate*", + "${tofuModulePath}/*.tfvars" + ], + force: true + ) + } + + // ── Summary ──────────────────────────────────────── + stage('Summary') { + echo '=== Infrastructure Destruction Complete ===' + echo "Workspace: ${params.TARGET_WORKSPACE}" + echo 'All resources have been destroyed and cleaned up' + echo '===========================================' + } + } + } + } + } + } +} diff --git a/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra new file mode 100644 index 000000000..3aafed9e7 --- /dev/null +++ b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra @@ -0,0 +1,508 @@ +#!groovy +/** + * Airgap RKE2 infrastructure setup pipeline. + * + * Provisions AWS infrastructure via OpenTofu, deploys an airgapped RKE2 + * cluster, optionally configures a private registry, and deploys Rancher. + * + * Consumes shared functions from qa-jenkins-library: + * airgap.standardCheckout, airgap.configureAnsible, airgap.deployRKE2, + * airgap.deployRancher, airgap.teardownInfrastructure, + * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs, + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * make.runTarget + * + * Make targets (run via make.runTarget in Docker): + * cluster — Deploy RKE2 via Ansible tarball playbook + * registry — Configure private registry on cluster nodes + * rancher — Deploy Rancher via Helm + */ + +def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +pipeline { + agent any + + options { + ansiColor('xterm') + timeout(time: 180, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + booleanParam( + name: 'DEPLOY_RANCHER', + defaultValue: true, + description: 'Deploy Rancher via helm after RKE2 setup' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Tear down infrastructure if setup fails' + ) + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-infra-automation repository' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.33.6+rke2r1', + description: 'RKE2 version to deploy' + ) + string( + name: 'RANCHER_VERSION', + defaultValue: 'v2.13.0', + description: 'Rancher version to deploy' + ) + string( + name: 'HOSTNAME_PREFIX', + defaultValue: '', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'PRIVATE_REGISTRY_URL', + defaultValue: '', + description: 'Private registry URL' + ) + string( + name: 'PRIVATE_REGISTRY_USERNAME', + defaultValue: '', + description: 'Private registry username' + ) + string( + name: 'PRIVATE_REGISTRY_PASSWORD', + defaultValue: '', + description: 'Private registry password' + ) + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform config values for the VM instances', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-09457fad1d2c34c31" +instance_type = "t3a.xlarge" +aws_security_group = ["sg-08e8243a8cfbea8a0"] +aws_subnet = "subnet-ee8cac86" +aws_volume_size = 100 +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +aws_vpc = "vpc-bfccf4d7" +user_id = "ubuntu" +ssh_key = "/root/.ssh/jenkins-elliptic-validation.pem" +ssh_key_name = "jenkins-elliptic-validation" +provision_registry = false''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible config values for the RKE2 airgap deployment', + defaultValue: '''--- +# Global variables for RKE2 airgap deployment with tarball installation + +# RKE2 Configuration +rke2_version: ${RKE2_VERSION} +rke2_server_options: | + cluster-cidr: {{ cluster_cidr }} + service-cidr: {{ service_cidr }} + cluster-dns: {{ cluster_dns }} + disable: + - rke2-snapshot-controller + - rke2-snapshot-controller-crd + - rke2-snapshot-validation-webhook +rke2_agent_options: "" +installation_method: "tarball" + +# Network Configuration +cluster_cidr: "10.42.0.0/16" +service_cidr: "10.43.0.0/16" +cluster_dns: "10.43.0.10" + +# CNI Configuration +cni: "calico" + +# Logging and Monitoring +enable_audit_log: false +audit_log_path: "/var/lib/rancher/rke2/server/logs/audit.log" +audit_log_maxage: 30 +audit_log_maxbackup: 10 +audit_log_maxsize: 100 + +# Registry mirrors configuration +private_registry_mirrors: + - registry: "docker.io" + endpoints: + - "https://privateregistry.qa.rancher.space" + rewrite: + - pattern: "^(.*)" + replacement: "proxycache/$1" + - registry: "quay.io" + endpoints: + - "https://privateregistry.qa.rancher.space" + rewrite: + - pattern: "^(.*)" + replacement: "quaycache/$1" + +# Registry authentication and TLS configuration +private_registry_configs: + - registry: ${PRIVATE_REGISTRY_URL} + auth: + username: ${PRIVATE_REGISTRY_USERNAME} + password: ${PRIVATE_REGISTRY_PASSWORD} + tls: + insecure_skip_verify: true + +# Whether to enable private registry configuration +enable_private_registry: true + +# Deploy Rancher +deploy_rancher: true +install_helm: true + +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "rancherrocks" +rancher_admin_password: "rancherrocks" +rancher_image_tag: ${RANCHER_VERSION} +rancher_use_bundled_system_charts: true''' + ) + } + + environment { + WORKSPACE_NAME = '' + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + } + } + } + + stage('Build Infrastructure Tools Image') { + steps { + sh "docker build --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + } + } + + stage('Provision Infrastructure') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME', + 'PRIVATE_REGISTRY_URL', + 'PRIVATE_REGISTRY_USERNAME', + 'PRIVATE_REGISTRY_PASSWORD' + ]) { + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + def workspaceName = '' + + // ── Tofu: init backend ───────────────────────────── + stage('Initialize Tofu Backend') { + tofu.initBackend( + dir: tofuModulePath, + bucket: env.S3_BUCKET_NAME, + key: env.S3_KEY_PREFIX, + region: env.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + + // ── Tofu: create workspace ───────────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_airgap_ansible_workspace', + suffix: env.HOSTNAME_PREFIX, + includeTimestamp: false + ) + + tofu.createWorkspace(dir: tofuModulePath, name: workspaceName) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── SSH key setup ─────────────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + } + + // ── Tofu: configure variables ────────────────────── + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Tofu: apply ───────────────────────────────────── + stage('Apply Tofu') { + tofu.apply( + dir: tofuModulePath, + varFile: 'terraform.tfvars', + autoApprove: true + ) + } + + // ── Inventory generation ──────────────────────────── + stage('Generate Inventory') { + sh """ + docker run --rm --platform linux/amd64 \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -v \$(pwd):/workspace -w /workspace \ + rancher-infra-tools:latest sh -c ' + tofu -chdir=${tofuModulePath} output -raw airgap_inventory_json > /tmp/airgap.json && + python3 ./qa-infra-automation/scripts/generate_inventory.py \ + --input /tmp/airgap.json \ + --schema ./qa-infra-automation/ansible/_inventory-schema.yaml \ + --distro rke2 --env airgap \ + --output-dir ${ansiblePath}/inventory + ' + """ + } + + // ── Ansible: configure variables ──────────────────── + stage('Configure Ansible Variables') { + airgap.configureAnsible( + sshKey: [ + content: env.AWS_SSH_PEM_KEY, + name: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ], + inventoryVars: [ + content: env.ANSIBLE_VARIABLES, + path: "${ansiblePath}/inventory/group_vars/all.yml", + envVars: [ + 'RKE2_VERSION': env.RKE2_VERSION, + 'RANCHER_VERSION': env.RANCHER_VERSION, + 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', + 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', + 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' + ] + ], + ansibleDir: ansiblePath, + inventoryFile: 'inventory/inventory.yml', + validate: false + ) + } + + // ── Ansible: copy SSH keys to nodes ───────────────── + stage('Copy SSH Keys to Nodes') { + ansible.runPlaybook( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/setup/setup-ssh-keys.yml' + ) + } + + // ── Make: deploy RKE2 cluster ─────────────────────── + stage('Deploy RKE2 Cluster') { + def mk = new make() + mk.runTarget( + target: 'cluster', + dir: 'qa-infra-automation', + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } + + // ── Make: configure private registry ──────────────── + stage('Configure Private Registry') { + if (env.PRIVATE_REGISTRY_URL?.trim()) { + def mk = new make() + mk.runTarget( + target: 'registry', + dir: 'qa-infra-automation', + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } else { + echo 'Skipping private registry configuration (PRIVATE_REGISTRY_URL not set)' + } + } + + // ── Make: deploy Rancher ──────────────────────────── + stage('Deploy Rancher') { + if (params.DEPLOY_RANCHER) { + def mk = new make() + mk.runTarget( + target: 'rancher', + dir: 'qa-infra-automation', + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } else { + echo 'Skipping Rancher deployment (DEPLOY_RANCHER not checked)' + } + } + + // ── Archive Rancher admin token ───────────────────── + stage('Archive Rancher Admin Token') { + if (params.DEPLOY_RANCHER) { + def tokenPath = 'qa-infra-automation/tmp/rancher-admin-token.json' + if (fileExists(tokenPath)) { + archiveArtifacts artifacts: tokenPath, fingerprint: true + echo "Archived Rancher admin token from ${tokenPath}" + } else { + echo "Warning: Rancher admin token file not found at ${tokenPath}" + } + } + } + + // ── Output infrastructure details ─────────────────── + stage('Output Infrastructure Details') { + echo '=== Infrastructure Setup Complete ===' + echo "Workspace Name: ${workspaceName}" + + try { + def bastionDns = tofu.getOutputs( + dir: tofuModulePath, output: 'bastion_public_dns' + ) + echo "Bastion Host: ${bastionDns}" + } catch (e) { + echo "Could not retrieve bastion DNS: ${e.message}" + } + + try { + def rancherHostname = tofu.getOutputs( + dir: tofuModulePath, output: 'external_lb_hostname' + ) + echo "Rancher Hostname (External): ${rancherHostname}" + currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${workspaceName}" + } catch (e) { + echo "Could not retrieve external LB hostname: ${e.message}" + } + + try { + def internalLb = tofu.getOutputs( + dir: tofuModulePath, output: 'internal_lb_hostname' + ) + echo "Internal LB Hostname: ${internalLb}" + } catch (e) { + echo "Could not retrieve internal LB hostname: ${e.message}" + } + + echo '========================================' + } + } + } + } + } + } + + post { + failure { + script { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + echo "DEBUG: DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}', INFRA_DIR='${env.INFRA_DIR}'" + + if (params.DESTROY_ON_FAILURE && wsName) { + echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: env.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + + airgap.teardownInfrastructure( + dir: tofuDir, + name: wsName, + varFile: 'terraform.tfvars' + ) + } + } catch (cleanupErr) { + echo "Cleanup failed: ${cleanupErr.message}" + } + } else { + echo "Skipping teardown: DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}'" + } + } + } + } +} From 8d2c8fb86b08ec6ebf7055ed69534fd1469143ef Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 17:16:23 +0000 Subject: [PATCH 15/19] fix: call make.runTarget() directly instead of new make() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Jenkinsfiles, shared library vars are global variables — use make.runTarget() directly, not new make(). The new X() pattern is only needed inside other vars/*.groovy files. --- validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra index 3aafed9e7..6beda7e0a 100644 --- a/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra @@ -359,8 +359,7 @@ rancher_use_bundled_system_charts: true''' // ── Make: deploy RKE2 cluster ─────────────────────── stage('Deploy RKE2 Cluster') { - def mk = new make() - mk.runTarget( + make.runTarget( target: 'cluster', dir: 'qa-infra-automation', makeArgs: 'ENV=airgap', @@ -371,8 +370,7 @@ rancher_use_bundled_system_charts: true''' // ── Make: configure private registry ──────────────── stage('Configure Private Registry') { if (env.PRIVATE_REGISTRY_URL?.trim()) { - def mk = new make() - mk.runTarget( + make.runTarget( target: 'registry', dir: 'qa-infra-automation', makeArgs: 'ENV=airgap', @@ -386,8 +384,7 @@ rancher_use_bundled_system_charts: true''' // ── Make: deploy Rancher ──────────────────────────── stage('Deploy Rancher') { if (params.DEPLOY_RANCHER) { - def mk = new make() - mk.runTarget( + make.runTarget( target: 'rancher', dir: 'qa-infra-automation', makeArgs: 'ENV=airgap', From 832530ecee703ce6aeb3c1b9cc071d6ea01745ae Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 20:19:44 +0000 Subject: [PATCH 16/19] fix: replace YAML pipe syntax with Groovy triple-quoted string in destroy Jenkinsfile The `text` parameter `default: |` is YAML multiline syntax, not valid Groovy. Replaced with `defaultValue: '''...'''` to fix the compilation error: "unexpected token: | @ line 84". --- .../Jenkinsfile.destroy.airgap-rke2-infra | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra index 539f20c13..3edfa0521 100644 --- a/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra @@ -81,23 +81,22 @@ pipeline { text( name: 'TERRAFORM_CONFIG', description: 'Terraform config values (same as used during setup)', - default: | - aws_access_key = "${AWS_ACCESS_KEY_ID}" - aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" - aws_ami = "ami-09457fad1d2c34c31" - instance_type = "t3a.xlarge" - aws_security_group = ["sg-08e8243a8cfbea8a0"] - aws_subnet = "subnet-ee8cac86" - aws_volume_size = 100 - aws_hostname_prefix = "${HOSTNAME_PREFIX}" - aws_region = "us-east-2" - aws_route53_zone = "qa.rancher.space" - aws_ssh_user = "ubuntu" - aws_vpc = "vpc-bfccf4d7" - user_id = "ubuntu" - ssh_key = "/root/.ssh/jenkins-elliptic-validation.pem" - ssh_key_name = "jenkins-elliptic-validation" - provision_registry = false + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-09457fad1d2c34c31" +instance_type = "t3a.xlarge" +aws_security_group = ["sg-08e8243a8cfbea8a0"] +aws_subnet = "subnet-ee8cac86" +aws_volume_size = 100 +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +aws_vpc = "vpc-bfccf4d7" +user_id = "ubuntu" +ssh_key = "/root/.ssh/jenkins-elliptic-validation.pem" +ssh_key_name = "jenkins-elliptic-validation" +provision_registry = false''' ) } From 1df97644d6dd2c48b1ee9fb2b7d434d20f6b135d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 20:44:03 +0000 Subject: [PATCH 17/19] chore: remove unified Jenkinsfile.airgap-rke2-infra (replaced by split jobs) The setup and destroy pipelines are now separate: - Jenkinsfile.setup.airgap-rke2-infra - Jenkinsfile.destroy.airgap-rke2-infra --- .../pipeline/Jenkinsfile.airgap-rke2-infra | 423 ------------------ 1 file changed, 423 deletions(-) delete mode 100644 validation/pipeline/Jenkinsfile.airgap-rke2-infra diff --git a/validation/pipeline/Jenkinsfile.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.airgap-rke2-infra deleted file mode 100644 index a8a97496b..000000000 --- a/validation/pipeline/Jenkinsfile.airgap-rke2-infra +++ /dev/null @@ -1,423 +0,0 @@ -#!groovy -/** - * Unified airgap RKE2 infrastructure pipeline. - * - * Replaces Jenkinsfile.setup.airgap.rke2 and Jenkinsfile.destroy.airgap.rke2 - * with a single Declarative Pipeline controlled by the ACTION parameter. - * - * Actions: - * setup - Provision AWS infrastructure, deploy RKE2, optionally deploy Rancher - * destroy - Tear down infrastructure for a given workspace - * - * Consumes shared functions from qa-jenkins-library: - * airgap.standardCheckout, airgap.configureAnsible, airgap.deployRKE2, - * airgap.deployRancher, airgap.teardownInfrastructure, - * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs, - * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig - */ - -def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' -library "qa-jenkins-library@${libraryBranch}" - -pipeline { - agent any - - options { - ansiColor('xterm') - timeout(time: 180, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '30')) - } - - parameters { - choice( - name: 'ACTION', - choices: ['setup', 'destroy'], - description: 'Infrastructure action to perform' - ) - booleanParam( - name: 'DEPLOY_RANCHER', - defaultValue: true, - description: 'Deploy Rancher via helm after RKE2 setup (setup action only)' - ) - booleanParam( - name: 'DESTROY_ON_FAILURE', - defaultValue: true, - description: 'Tear down infrastructure if setup fails' - ) - string( - name: 'TARGET_WORKSPACE', - defaultValue: '', - description: 'Workspace name to destroy (required for destroy action)' - ) - string( - name: 'QA_JENKINS_LIBRARY_BRANCH', - defaultValue: 'main', - description: 'Branch of qa-jenkins-library to use' - ) - string( - name: 'TESTS_REPO_URL', - defaultValue: 'https://github.com/rancher/tests', - description: 'URL of rancher/tests repository' - ) - string( - name: 'TESTS_BRANCH', - defaultValue: 'main', - description: 'Branch of rancher/tests repository' - ) - string( - name: 'QA_INFRA_REPO_URL', - defaultValue: 'https://github.com/rancher/qa-infra-automation', - description: 'URL of qa-infra-automation repository' - ) - string( - name: 'QA_INFRA_BRANCH', - defaultValue: 'main', - description: 'Branch of rancher/qa-infra-automation repository' - ) - } - - environment { - WORKSPACE_NAME = '' - } - - stages { - stage('Checkout') { - steps { - script { - def dirs = airgap.standardCheckout( - testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], - infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] - ) - env.TESTS_DIR = dirs.testsDir - env.INFRA_DIR = dirs.infraDir - } - } - } - - stage('Build Infrastructure Tools Image') { - steps { - sh "docker build --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." - } - } - - stage('Provision Infrastructure') { - steps { - script { - property.useWithProperties([ - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY', - 'AWS_SSH_PEM_KEY', - 'AWS_SSH_PEM_KEY_NAME', - 'PRIVATE_REGISTRY_URL', - 'PRIVATE_REGISTRY_USERNAME', - 'PRIVATE_REGISTRY_PASSWORD' - ]) { - def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" - def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" - def workspaceName = '' - - if (params.ACTION == 'setup') { - // ── Setup ──────────────────────────────────────── - stage('Setup: Initialize Tofu Backend') { - tofu.initBackend( - dir: tofuModulePath, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } - - stage('Setup: Create Workspace') { - workspaceName = infrastructure.generateWorkspaceName( - prefix: 'jenkins_airgap_ansible_workspace', - suffix: env.HOSTNAME_PREFIX, - includeTimestamp: false - ) - - tofu.createWorkspace(dir: tofuModulePath, name: workspaceName) - infrastructure.archiveWorkspaceName(workspaceName: workspaceName) - - // Set env after all calls succeed so post-failure cleanup can use it - env.WORKSPACE_NAME = workspaceName - } - - stage('Setup: Configure SSH Key') { - infrastructure.writeSshKey( - keyContent: env.AWS_SSH_PEM_KEY, - keyName: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ) - } - - stage('Setup: Configure Tofu Variables') { - def terraformConfig = infrastructure.parseAndSubstituteVars( - content: env.TERRAFORM_CONFIG, - envVars: [ - 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, - 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME - ] - ) - - infrastructure.writeConfig( - path: "${tofuModulePath}/terraform.tfvars", - content: terraformConfig - ) - } - - stage('Setup: Apply Tofu') { - tofu.apply( - dir: tofuModulePath, - varFile: 'terraform.tfvars', - autoApprove: true - ) - } - - stage('Setup: Generate Inventory') { - // Use qa-infra-automation's generate_inventory.py to render - // the Ansible inventory from the tofu JSON output. - sh """ - docker run --rm --platform linux/amd64 \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ - -v \$(pwd):/workspace -w /workspace \ - rancher-infra-tools:latest sh -c ' - tofu -chdir=${tofuModulePath} output -raw airgap_inventory_json > /tmp/airgap.json && - python3 ./qa-infra-automation/scripts/generate_inventory.py \ - --input /tmp/airgap.json \ - --schema ./qa-infra-automation/ansible/_inventory-schema.yaml \ - --distro rke2 --env airgap \ - --output-dir ${ansiblePath}/inventory - ' - """ - } - - stage('Setup: Configure Ansible Variables') { - airgap.configureAnsible( - sshKey: [ - content: env.AWS_SSH_PEM_KEY, - name: env.AWS_SSH_PEM_KEY_NAME, - dir: '.ssh' - ], - inventoryVars: [ - content: env.ANSIBLE_VARIABLES, - path: "${ansiblePath}/inventory/group_vars/all.yml", - envVars: [ - 'RKE2_VERSION': env.RKE2_VERSION, - 'RANCHER_VERSION': env.RANCHER_VERSION, - 'CERT_MANAGER_VERSION': env.CERT_MANAGER_VERSION ?: '', - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', - 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', - 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' - ] - ], - ansibleDir: ansiblePath, - inventoryFile: 'inventory/inventory.yml', - validate: false - ) - } - - stage('Setup: Copy SSH Keys to Nodes') { - ansible.runPlaybook( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/setup/setup-ssh-keys.yml' - ) - } - - stage('Setup: Deploy RKE2 Cluster') { - airgap.deployRKE2( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rke2-tarball-playbook.yml' - ) - } - - stage('Setup: Configure Private Registry') { - if (env.PRIVATE_REGISTRY_URL?.trim()) { - ansible.runPlaybook( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rke2-registry-config-playbook.yml' - ) - } else { - echo 'Skipping private registry configuration (PRIVATE_REGISTRY_URL not set)' - } - } - - stage('Setup: Deploy Rancher') { - if (params.DEPLOY_RANCHER) { - airgap.deployRancher( - dir: ansiblePath, - inventory: 'inventory/inventory.yml', - playbook: 'playbooks/deploy/rancher-helm-deploy-playbook.yml', - enabled: true - ) - } else { - echo 'Skipping Rancher deployment (DEPLOY_RANCHER not checked)' - } - } - - stage('Setup: Output Infrastructure Details') { - echo '=== Infrastructure Setup Complete ===' - echo "Workspace Name: ${workspaceName}" - - try { - def bastionDns = tofu.getOutputs( - dir: tofuModulePath, output: 'bastion_public_dns' - ) - echo "Bastion Host: ${bastionDns}" - } catch (e) { - echo "Could not retrieve bastion DNS: ${e.message}" - } - - try { - def rancherHostname = tofu.getOutputs( - dir: tofuModulePath, output: 'external_lb_hostname' - ) - echo "Rancher Hostname (External): ${rancherHostname}" - currentBuild.description = "Deployed: https://${rancherHostname} | Workspace: ${workspaceName}" - } catch (e) { - echo "Could not retrieve external LB hostname: ${e.message}" - } - - try { - def internalLb = tofu.getOutputs( - dir: tofuModulePath, output: 'internal_lb_hostname' - ) - echo "Internal LB Hostname: ${internalLb}" - } catch (e) { - echo "Could not retrieve internal LB hostname: ${e.message}" - } - - echo '========================================' - } - - } else { - // ── Destroy ───────────────────────────────────── - stage('Destroy: Validate Parameters') { - if (!params.TARGET_WORKSPACE?.trim()) { - error 'TARGET_WORKSPACE parameter is required for destroy action' - } - echo "Target workspace for destruction: ${params.TARGET_WORKSPACE}" - workspaceName = params.TARGET_WORKSPACE - } - - stage('Destroy: Initialize Tofu Backend') { - tofu.initBackend( - dir: tofuModulePath, - bucket: env.S3_BUCKET_NAME, - key: env.S3_KEY_PREFIX, - region: env.S3_BUCKET_REGION ?: env.S3_REGION, - backendInitScript: './scripts/init-backend.sh' - ) - } - - stage('Destroy: Configure Tofu Variables') { - def terraformConfig = infrastructure.parseAndSubstituteVars( - content: env.TERRAFORM_CONFIG, - envVars: [ - 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, - 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME - ] - ) - - infrastructure.writeConfig( - path: "${tofuModulePath}/terraform.tfvars", - content: terraformConfig - ) - } - - stage('Destroy: Teardown Infrastructure') { - airgap.teardownInfrastructure( - dir: tofuModulePath, - name: params.TARGET_WORKSPACE, - varFile: 'terraform.tfvars' - ) - } - - stage('Destroy: Cleanup Local Artifacts') { - infrastructure.cleanupArtifacts( - paths: [ - "${tofuModulePath}/.terraform", - "${tofuModulePath}/terraform.tfstate*", - "${tofuModulePath}/*.tfvars" - ], - force: true - ) - } - - stage('Destroy: Summary') { - echo '=== Infrastructure Destruction Complete ===' - echo "Workspace: ${params.TARGET_WORKSPACE}" - echo 'All resources have been destroyed and cleaned up' - echo '===========================================' - } - } - } - } - } - } - } - - post { - failure { - script { - // Read workspace name from the file written by archiveWorkspaceName - // (env.WORKSPACE_NAME doesn't persist from inside useWithProperties) - def wsName = '' - try { - if (fileExists('workspace_name.txt')) { - wsName = readFile('workspace_name.txt')?.trim() - } - } catch (e) { - echo "Could not read workspace_name.txt: ${e.message}" - } - - echo "DEBUG: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}', INFRA_DIR='${env.INFRA_DIR}'" - - if (params.ACTION == 'setup' && params.DESTROY_ON_FAILURE && wsName) { - echo 'DESTROY_ON_FAILURE is enabled. Cleaning up infrastructure...' - - try { - property.useWithProperties([ - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY' - ]) { - def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/airgap" - - // Reconstruct tfvars from the Jenkins job parameters - def terraformConfig = infrastructure.parseAndSubstituteVars( - content: env.TERRAFORM_CONFIG, - envVars: [ - 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, - 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME - ] - ) - infrastructure.writeConfig( - path: "${tofuDir}/terraform.tfvars", - content: terraformConfig - ) - - airgap.teardownInfrastructure( - dir: tofuDir, - name: wsName, - varFile: 'terraform.tfvars' - ) - } - } catch (cleanupErr) { - echo "Cleanup failed: ${cleanupErr.message}" - } - } else { - echo "Skipping teardown: ACTION=${params.ACTION}, DESTROY_ON_FAILURE=${params.DESTROY_ON_FAILURE}, WORKSPACE_NAME='${wsName}'" - } - } - } - } -} From 883dddfd8790600044bfbdcef98aa6f8506d2e3c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 21:07:59 +0000 Subject: [PATCH 18/19] fix: address PR #595 review comments - Use env instead of params for library branch resolution - Change PRIVATE_REGISTRY_PASSWORD to password type parameter - Template rancher bootstrap/admin passwords from parameters - Template enable_private_registry and deploy_rancher from params - Use env.INFRA_DIR consistently in make.runTarget and archiveArtifacts - Remove fingerprint from archiveArtifacts for token file - Add AWS_SSH_PEM_KEY_NAME to cleanup useWithProperties block - Remove unused AWS_SSH_PEM_KEY_NAME from destroy substitution map --- .../Jenkinsfile.destroy.airgap-rke2-infra | 5 +-- .../Jenkinsfile.setup.airgap-rke2-infra | 41 +++++++++++++------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra index 3edfa0521..00f6c0f5f 100644 --- a/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.destroy.airgap-rke2-infra @@ -15,7 +15,7 @@ * infrastructure.writeConfig, infrastructure.cleanupArtifacts */ -def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' library "qa-jenkins-library@${libraryBranch}" pipeline { @@ -155,8 +155,7 @@ provision_registry = false''' envVars: [ 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, - 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, - 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX ] ) diff --git a/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra index 6beda7e0a..837b4b8f1 100644 --- a/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra +++ b/validation/pipeline/Jenkinsfile.setup.airgap-rke2-infra @@ -18,7 +18,7 @@ * rancher — Deploy Rancher via Helm */ -def libraryBranch = params.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' library "qa-jenkins-library@${libraryBranch}" pipeline { @@ -91,11 +91,21 @@ pipeline { defaultValue: '', description: 'Private registry username' ) - string( + password( name: 'PRIVATE_REGISTRY_PASSWORD', defaultValue: '', description: 'Private registry password' ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: 'rancherrocks', + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: 'rancherrocks', + description: 'Rancher admin password after bootstrap' + ) string( name: 'S3_BUCKET_NAME', defaultValue: 'jenkins-terraform-state-storage', @@ -190,15 +200,15 @@ private_registry_configs: insecure_skip_verify: true # Whether to enable private registry configuration -enable_private_registry: true +enable_private_registry: ${ENABLE_PRIVATE_REGISTRY} # Deploy Rancher -deploy_rancher: true +deploy_rancher: ${DEPLOY_RANCHER_ANSIBLE} install_helm: true rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" -rancher_bootstrap_password: "rancherrocks" -rancher_admin_password: "rancherrocks" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" rancher_image_tag: ${RANCHER_VERSION} rancher_use_bundled_system_charts: true''' ) @@ -339,7 +349,11 @@ rancher_use_bundled_system_charts: true''' 'HOSTNAME_PREFIX': env.HOSTNAME_PREFIX, 'PRIVATE_REGISTRY_URL': env.PRIVATE_REGISTRY_URL ?: '', 'PRIVATE_REGISTRY_USERNAME': env.PRIVATE_REGISTRY_USERNAME ?: '', - 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '' + 'PRIVATE_REGISTRY_PASSWORD': env.PRIVATE_REGISTRY_PASSWORD ?: '', + 'RANCHER_BOOTSTRAP_PASSWORD': env.RANCHER_BOOTSTRAP_PASSWORD ?: '', + 'RANCHER_ADMIN_PASSWORD': env.RANCHER_ADMIN_PASSWORD ?: '', + 'ENABLE_PRIVATE_REGISTRY': env.PRIVATE_REGISTRY_URL ? 'true' : 'false', + 'DEPLOY_RANCHER_ANSIBLE': params.DEPLOY_RANCHER ? 'true' : 'false' ] ], ansibleDir: ansiblePath, @@ -361,7 +375,7 @@ rancher_use_bundled_system_charts: true''' stage('Deploy RKE2 Cluster') { make.runTarget( target: 'cluster', - dir: 'qa-infra-automation', + dir: env.INFRA_DIR, makeArgs: 'ENV=airgap', passAwsCreds: false ) @@ -372,7 +386,7 @@ rancher_use_bundled_system_charts: true''' if (env.PRIVATE_REGISTRY_URL?.trim()) { make.runTarget( target: 'registry', - dir: 'qa-infra-automation', + dir: env.INFRA_DIR, makeArgs: 'ENV=airgap', passAwsCreds: false ) @@ -386,7 +400,7 @@ rancher_use_bundled_system_charts: true''' if (params.DEPLOY_RANCHER) { make.runTarget( target: 'rancher', - dir: 'qa-infra-automation', + dir: env.INFRA_DIR, makeArgs: 'ENV=airgap', passAwsCreds: false ) @@ -398,9 +412,9 @@ rancher_use_bundled_system_charts: true''' // ── Archive Rancher admin token ───────────────────── stage('Archive Rancher Admin Token') { if (params.DEPLOY_RANCHER) { - def tokenPath = 'qa-infra-automation/tmp/rancher-admin-token.json' + def tokenPath = "${env.INFRA_DIR}/tmp/rancher-admin-token.json" if (fileExists(tokenPath)) { - archiveArtifacts artifacts: tokenPath, fingerprint: true + archiveArtifacts artifacts: tokenPath echo "Archived Rancher admin token from ${tokenPath}" } else { echo "Warning: Rancher admin token file not found at ${tokenPath}" @@ -469,7 +483,8 @@ rancher_use_bundled_system_charts: true''' try { property.useWithProperties([ 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY' + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' ]) { def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/airgap" From 1ae5f9663a0d0c2c89775294d806323403d913d9 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 1 Apr 2026 22:14:32 +0000 Subject: [PATCH 19/19] docs: add PR review guidelines to copilot-instructions Adds Jenkinsfile and Go test review patterns for Copilot to follow during PR reviews, covering security, pipeline correctness, and test quality checks specific to this repository. --- .github/copilot-instructions.md | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 885b1fc1c..6fe701b16 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -208,3 +208,43 @@ Always update relevant documentation when making changes: * return pointers to objects where possible in helper functions * if the case of ignoring errors arises, always justify via comment as to why it is ignored * strings should be a const wherever possible except for logs and error messages + +## Pull Request Review Guidelines + +When reviewing PRs, Copilot should check for the following patterns specific to this repository. + +### Jenkinsfile reviews (`validation/pipeline/Jenkinsfile.*`) + +**Security** +- No hardcoded secrets in `defaultValue` blocks — use `${VARIABLE}` substitution and `password` parameter types +- `password` type parameters for all credentials (registry passwords, Rancher passwords, API keys) +- `useWithProperties` blocks must include every credential referenced in their scope's substitution maps + +**Jenkins pipeline correctness** +- Use `env.X` not `params.X` for the `library` directive at the top of Jenkinsfiles — `params` is not resolved at library load time +- `text` parameters use `defaultValue: '''...'''` (Groovy triple-quoted), never `default: |` (YAML pipe syntax) +- Paths should use `env.*_DIR` variables (set by `airgap.standardCheckout`), not hardcoded directory names like `'qa-infra-automation'` +- `archiveArtifacts` should not use `fingerprint: true` for transient build artifacts + +**Parameterization** +- Ansible variable templates should reference pipeline parameters, not hardcode values (e.g., `${RANCHER_BOOTSTRAP_PASSWORD}` not `"rancherrocks"`) +- Boolean-like Ansible variables (`enable_private_registry`, `deploy_rancher`) should reflect pipeline parameters, not be hardcoded to `true` + +### Go test reviews + +**Build tags** +- Every test file must have a `//go:build` line with appropriate tags +- Version tags must follow the existing patterns in the codebase (e.g., `2.13`, `!2.8`) +- Do not add `pit` tags unless the test explicitly uses external non-Rancher APIs + +**Code organization** +- Helper functions that return errors are preferred over helpers accepting `testing.T` +- Use `logrus` for logging, never `testing.T.Log` +- Shared helpers belong in `actions/`, not duplicated across test packages +- Functions matching the extension criteria should target the [shepherd](https://github.com/rancher/shepherd) repo instead + +**Test quality** +- At least 2 tests per suite when possible: one dynamic (config-driven) and one static +- Feature name only in `SuiteName`, not in individual test names +- Validate in a separate function, not inline in the test +- Use `session.Cleanup()` in `TearDownSuite` for resource cleanup