Skip to content

PMM-14346: PMM HA EKS testing pipeline with ALB, Route53, and Access Entries #3693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

PMM-14346: PMM HA EKS testing pipeline with ALB, Route53, and Access Entries #3693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
b38da5e
feat(pmm-ha): add EKS testing pipeline with ALB, Route53, and Access …
nogueiraanderson Nov 29, 2025
f3ef4fc
refactor: derive AWS account ID and SSO role dynamically
nogueiraanderson Nov 29, 2025
f180443
fix: drop unsupported K8s version 1.34
nogueiraanderson Nov 29, 2025
06e1139
fix(pmm-ha): sanitize SSO role ARN from aws iam list-roles output
nogueiraanderson Nov 30, 2025
0ba9672
feat(pmm-ha): spread nodes across all available AZs dynamically
nogueiraanderson Nov 30, 2025
e4324e6
chore(pmm-ha): remove obvious comments
nogueiraanderson Nov 30, 2025
a3832ce
fix(pmm-ha): use agent-amd64-ol9 for cleanup pipeline
nogueiraanderson Nov 30, 2025
5d62e82
Revert "fix(pmm-ha): use agent-amd64-ol9 for cleanup pipeline"
nogueiraanderson Nov 30, 2025
89bad25
chore(pmm-ha): use cli agent for create pipeline
nogueiraanderson Nov 30, 2025
6f107ce
chore(pmm-ha): rename stage to 'Configure External PMM Access'
nogueiraanderson Nov 30, 2025
b496a5e
chore(pmm-ha): rename stage to 'Setup External Access'
nogueiraanderson Nov 30, 2025
3e61fce
fix(pmm-ha): clean kubeconfig directory before export
nogueiraanderson Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 105 additions & 61 deletions pmm/v3/pmm3-ha-eks-cleanup.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,31 @@
/**
* PMM HA EKS Cleanup Pipeline
*
* Manages cleanup of PMM HA test clusters. Supports manual and scheduled runs.
* Deletes Route53 records, ALB ingress, and EKS clusters.
*
* Actions:
* - LIST_ONLY: List all test clusters with age
* - DELETE_CLUSTER: Delete a specific cluster
* - DELETE_ALL: Delete all test clusters
* - DELETE_OLD (cron): Delete clusters older than 24 hours
*
* Related:
* - Create: pmm3-ha-eks.groovy
* - Shared library: vars/pmmHaEks.groovy
*/
library changelog: false, identifier: 'lib@fix/pmm-ha-eks-access-entries', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/Percona-Lab/jenkins-pipelines'
])

pipeline {
agent {
label 'agent-amd64-ol9'
label 'cli'
}

triggers {
cron('H 0,12 * * *') // Runs twice daily at 00:00 & 12:00
cron('H 0,12 * * *')
}

parameters {
Expand All @@ -23,11 +44,14 @@ pipeline {

options {
buildDiscarder(logRotator(numToKeepStr: '30'))
disableConcurrentBuilds()
timeout(time: 60, unit: 'MINUTES')
}

environment {
REGION = "us-east-2"
CLUSTER_PREFIX = "pmm-ha-test-"
REGION = 'us-east-2'
CLUSTER_PREFIX = 'pmm-ha-test-'
R53_ZONE_NAME = 'cd.percona.com'
}

stages {
Expand All @@ -36,14 +60,14 @@ pipeline {
script {
if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) {
env.ACTION = 'DELETE_OLD'
echo "Triggered by cron - will delete clusters older than 1 day."
echo 'Triggered by cron - will delete clusters older than 1 day.'
} else {
env.ACTION = params.ACTION
echo "Manual run with ACTION=${params.ACTION}"
}

if (env.ACTION == 'DELETE_CLUSTER' && !params.CLUSTER_NAME) {
error("CLUSTER_NAME is required for DELETE_CLUSTER.")
error('CLUSTER_NAME is required for DELETE_CLUSTER.')
}
if (params.CLUSTER_NAME && !params.CLUSTER_NAME.startsWith(env.CLUSTER_PREFIX)) {
error("Cluster name must start with ${env.CLUSTER_PREFIX}")
Expand All @@ -59,24 +83,24 @@ pipeline {
sh '''
set +x

CLUSTERS=$(aws eks list-clusters --region "$REGION" \
CLUSTERS=$(aws eks list-clusters --region "${REGION}" \
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" \
--output text)

if [ -z "$CLUSTERS" ]; then
if [ -z "${CLUSTERS}" ]; then
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
exit 0
fi

for c in $CLUSTERS; do
for cluster in ${CLUSTERS}; do
CREATED=$(aws eks describe-cluster \
--name "$c" --region "$REGION" \
--name "${cluster}" --region "${REGION}" \
--query "cluster.createdAt" --output text)

CREATED_EPOCH=$(date -d "$CREATED" +%s)
CREATED_EPOCH=$(date -d "${CREATED}" +%s)
AGE_HOURS=$(( ( $(date +%s) - CREATED_EPOCH ) / 3600 ))

echo "• $c | Created: $CREATED | Age: ${AGE_HOURS}h"
echo "* ${cluster} | Created: ${CREATED} | Age: ${AGE_HOURS}h"
done
'''
}
Expand All @@ -87,15 +111,22 @@ pipeline {
when { expression { env.ACTION == 'DELETE_CLUSTER' } }
steps {
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
sh '''
if ! aws eks describe-cluster --region "${REGION}" --name "${CLUSTER_NAME}" >/dev/null 2>&1; then
echo "Cluster '${CLUSTER_NAME}' not found in region '${REGION}'."
exit 0
fi

eksctl delete cluster --region "${REGION}" --name "${CLUSTER_NAME}" \
--disable-nodegroup-eviction --wait
'''
script {
def clusterExists = sh(
script: "aws eks describe-cluster --region ${REGION} --name ${params.CLUSTER_NAME} >/dev/null 2>&1",
returnStatus: true
) == 0

if (clusterExists) {
pmmHaEks.deleteCluster(
clusterName: params.CLUSTER_NAME,
region: env.REGION,
r53ZoneName: env.R53_ZONE_NAME
)
} else {
echo "Cluster '${params.CLUSTER_NAME}' not found in region '${REGION}'."
}
}
}
}
}
Expand All @@ -104,20 +135,25 @@ pipeline {
when { expression { env.ACTION == 'DELETE_ALL' } }
steps {
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
sh '''
CLUSTERS=$(aws eks list-clusters --region "$REGION" \
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
script {
def clusters = sh(
script: "aws eks list-clusters --region ${REGION} --query \"clusters[?starts_with(@, '${CLUSTER_PREFIX}')]\" --output text",
returnStdout: true
).trim()

if [ -z "$CLUSTERS" ]; then
if (!clusters) {
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
exit 0
fi

for c in $CLUSTERS; do
eksctl delete cluster --region "$REGION" --name "$c" \
--disable-nodegroup-eviction --wait
done
'''
return
}

clusters.split(/\s+/).each { clusterName ->
pmmHaEks.deleteCluster(
clusterName: clusterName,
region: env.REGION,
r53ZoneName: env.R53_ZONE_NAME
)
}
}
}
}
}
Expand All @@ -126,36 +162,44 @@ pipeline {
when { expression { env.ACTION == 'DELETE_OLD' } }
steps {
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
sh '''
CLUSTERS=$(aws eks list-clusters --region "$REGION" \
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
script {
def clusters = sh(
script: "aws eks list-clusters --region ${REGION} --query \"clusters[?starts_with(@, '${CLUSTER_PREFIX}')]\" --output text",
returnStdout: true
).trim()

if [ -z "$CLUSTERS" ]; then
if (!clusters) {
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
exit 0
fi

CUTOFF=$(date -d "1 day ago" +%s)

for c in $CLUSTERS; do
CREATED=$(aws eks describe-cluster --name "$c" --region "$REGION" \
--query "cluster.createdAt" --output text 2>/dev/null || true)

if [ -z "$CREATED" ] || [ "$CREATED" == "None" ]; then
echo "Unable to fetch creation time for $c — skipping."
continue
fi

CREATED_EPOCH=$(date -d "$CREATED" +%s)

if [ "$CREATED_EPOCH" -lt "$CUTOFF" ]; then
eksctl delete cluster --region "$REGION" --name "$c" \
--disable-nodegroup-eviction --wait
else
echo "Skipping recent cluster: $c (created within last 24h)"
fi
done
'''
return
}

def cutoffMs = System.currentTimeMillis() - (24 * 60 * 60 * 1000) // 1 day ago

clusters.split(/\s+/).each { clusterName ->
def createdAt = sh(
script: "aws eks describe-cluster --name ${clusterName} --region ${REGION} --query 'cluster.createdAt' --output text 2>/dev/null || echo ''",
returnStdout: true
).trim()

if (!createdAt || createdAt == 'None') {
echo "Unable to fetch creation time for ${clusterName} - skipping."
return // continue to next iteration
}

// Parse ISO 8601 timestamp
def createdMs = Date.parse("yyyy-MM-dd'T'HH:mm:ss", createdAt.take(19)).time

if (createdMs < cutoffMs) {
pmmHaEks.deleteCluster(
clusterName: clusterName,
region: env.REGION,
r53ZoneName: env.R53_ZONE_NAME
)
} else {
echo "Skipping recent cluster: ${clusterName} (created within last 24h)"
}
}
}
}
}
}
Expand Down
Loading