Skip to content

Commit b38da5e

Browse files
nogueiraandersonnogueiraanderson
committed
feat(pmm-ha): add EKS testing pipeline with ALB, Route53, and Access Entries
- Add AWS Load Balancer Controller with IRSA for ALB ingress - Add ALB Ingress with ACM certificate (*.cd.percona.com wildcard) - Add Route53 alias records for friendly URLs (pmm-ha-test-N.cd.percona.com) - Replace ConfigMap-based auth with EKS Access Entries API - Add pmm-eks-admins IAM group for kubectl access - Add SSO AdministratorAccess role support - Add cleanup job with Route53/ALB cleanup before cluster deletion - Extract shared library vars/pmmHaEks.groovy for reusable functions Jira: PMM-14346
1 parent 28d398e commit b38da5e

File tree

3 files changed

+856
-130
lines changed

3 files changed

+856
-130
lines changed

‎pmm/v3/pmm3-ha-eks-cleanup.groovy‎

Lines changed: 105 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,31 @@
1+
/**
2+
* PMM HA EKS Cleanup Pipeline
3+
*
4+
* Manages cleanup of PMM HA test clusters. Supports manual and scheduled runs.
5+
* Deletes Route53 records, ALB ingress, and EKS clusters.
6+
*
7+
* Actions:
8+
* - LIST_ONLY: List all test clusters with age
9+
* - DELETE_CLUSTER: Delete a specific cluster
10+
* - DELETE_ALL: Delete all test clusters
11+
* - DELETE_OLD (cron): Delete clusters older than 24 hours
12+
*
13+
* Related:
14+
* - Create: pmm3-ha-eks.groovy
15+
* - Shared library: vars/pmmHaEks.groovy
16+
*/
17+
library changelog: false, identifier: 'lib@master', retriever: modernSCM([
18+
$class: 'GitSCMSource',
19+
remote: 'https://github.com/Percona-Lab/jenkins-pipelines'
20+
])
21+
122
pipeline {
223
agent {
3-
label 'agent-amd64-ol9'
24+
label 'cli'
425
}
526

627
triggers {
7-
cron('H 0,12 * * *') // Runs twice daily at 00:00 & 12:00
28+
cron('H 0,12 * * *')
829
}
930

1031
parameters {
@@ -23,11 +44,14 @@ pipeline {
2344

2445
options {
2546
buildDiscarder(logRotator(numToKeepStr: '30'))
47+
disableConcurrentBuilds()
48+
timeout(time: 60, unit: 'MINUTES')
2649
}
2750

2851
environment {
29-
REGION = "us-east-2"
30-
CLUSTER_PREFIX = "pmm-ha-test-"
52+
REGION = 'us-east-2'
53+
CLUSTER_PREFIX = 'pmm-ha-test-'
54+
R53_ZONE_NAME = 'cd.percona.com'
3155
}
3256

3357
stages {
@@ -36,14 +60,14 @@ pipeline {
3660
script {
3761
if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) {
3862
env.ACTION = 'DELETE_OLD'
39-
echo "Triggered by cron - will delete clusters older than 1 day."
63+
echo 'Triggered by cron - will delete clusters older than 1 day.'
4064
} else {
4165
env.ACTION = params.ACTION
4266
echo "Manual run with ACTION=${params.ACTION}"
4367
}
4468

4569
if (env.ACTION == 'DELETE_CLUSTER' && !params.CLUSTER_NAME) {
46-
error("CLUSTER_NAME is required for DELETE_CLUSTER.")
70+
error('CLUSTER_NAME is required for DELETE_CLUSTER.')
4771
}
4872
if (params.CLUSTER_NAME && !params.CLUSTER_NAME.startsWith(env.CLUSTER_PREFIX)) {
4973
error("Cluster name must start with ${env.CLUSTER_PREFIX}")
@@ -59,24 +83,24 @@ pipeline {
5983
sh '''
6084
set +x
6185
62-
CLUSTERS=$(aws eks list-clusters --region "$REGION" \
86+
CLUSTERS=$(aws eks list-clusters --region "${REGION}" \
6387
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" \
6488
--output text)
6589
66-
if [ -z "$CLUSTERS" ]; then
90+
if [ -z "${CLUSTERS}" ]; then
6791
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
6892
exit 0
6993
fi
7094
71-
for c in $CLUSTERS; do
95+
for cluster in ${CLUSTERS}; do
7296
CREATED=$(aws eks describe-cluster \
73-
--name "$c" --region "$REGION" \
97+
--name "${cluster}" --region "${REGION}" \
7498
--query "cluster.createdAt" --output text)
7599
76-
CREATED_EPOCH=$(date -d "$CREATED" +%s)
100+
CREATED_EPOCH=$(date -d "${CREATED}" +%s)
77101
AGE_HOURS=$(( ( $(date +%s) - CREATED_EPOCH ) / 3600 ))
78102
79-
echo "• $c | Created: $CREATED | Age: ${AGE_HOURS}h"
103+
echo "* ${cluster} | Created: ${CREATED} | Age: ${AGE_HOURS}h"
80104
done
81105
'''
82106
}
@@ -87,15 +111,22 @@ pipeline {
87111
when { expression { env.ACTION == 'DELETE_CLUSTER' } }
88112
steps {
89113
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
90-
sh '''
91-
if ! aws eks describe-cluster --region "${REGION}" --name "${CLUSTER_NAME}" >/dev/null 2>&1; then
92-
echo "Cluster '${CLUSTER_NAME}' not found in region '${REGION}'."
93-
exit 0
94-
fi
95-
96-
eksctl delete cluster --region "${REGION}" --name "${CLUSTER_NAME}" \
97-
--disable-nodegroup-eviction --wait
98-
'''
114+
script {
115+
def clusterExists = sh(
116+
script: "aws eks describe-cluster --region ${REGION} --name ${params.CLUSTER_NAME} >/dev/null 2>&1",
117+
returnStatus: true
118+
) == 0
119+
120+
if (clusterExists) {
121+
pmmHaEks.deleteCluster(
122+
clusterName: params.CLUSTER_NAME,
123+
region: env.REGION,
124+
r53ZoneName: env.R53_ZONE_NAME
125+
)
126+
} else {
127+
echo "Cluster '${params.CLUSTER_NAME}' not found in region '${REGION}'."
128+
}
129+
}
99130
}
100131
}
101132
}
@@ -104,20 +135,25 @@ pipeline {
104135
when { expression { env.ACTION == 'DELETE_ALL' } }
105136
steps {
106137
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
107-
sh '''
108-
CLUSTERS=$(aws eks list-clusters --region "$REGION" \
109-
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
138+
script {
139+
def clusters = sh(
140+
script: "aws eks list-clusters --region ${REGION} --query \"clusters[?starts_with(@, '${CLUSTER_PREFIX}')]\" --output text",
141+
returnStdout: true
142+
).trim()
110143

111-
if [ -z "$CLUSTERS" ]; then
144+
if (!clusters) {
112145
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
113-
exit 0
114-
fi
115-
116-
for c in $CLUSTERS; do
117-
eksctl delete cluster --region "$REGION" --name "$c" \
118-
--disable-nodegroup-eviction --wait
119-
done
120-
'''
146+
return
147+
}
148+
149+
clusters.split(/\s+/).each { clusterName ->
150+
pmmHaEks.deleteCluster(
151+
clusterName: clusterName,
152+
region: env.REGION,
153+
r53ZoneName: env.R53_ZONE_NAME
154+
)
155+
}
156+
}
121157
}
122158
}
123159
}
@@ -126,36 +162,44 @@ pipeline {
126162
when { expression { env.ACTION == 'DELETE_OLD' } }
127163
steps {
128164
withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
129-
sh '''
130-
CLUSTERS=$(aws eks list-clusters --region "$REGION" \
131-
--query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
165+
script {
166+
def clusters = sh(
167+
script: "aws eks list-clusters --region ${REGION} --query \"clusters[?starts_with(@, '${CLUSTER_PREFIX}')]\" --output text",
168+
returnStdout: true
169+
).trim()
132170

133-
if [ -z "$CLUSTERS" ]; then
171+
if (!clusters) {
134172
echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
135-
exit 0
136-
fi
137-
138-
CUTOFF=$(date -d "1 day ago" +%s)
139-
140-
for c in $CLUSTERS; do
141-
CREATED=$(aws eks describe-cluster --name "$c" --region "$REGION" \
142-
--query "cluster.createdAt" --output text 2>/dev/null || true)
143-
144-
if [ -z "$CREATED" ] || [ "$CREATED" == "None" ]; then
145-
echo "Unable to fetch creation time for $c — skipping."
146-
continue
147-
fi
148-
149-
CREATED_EPOCH=$(date -d "$CREATED" +%s)
150-
151-
if [ "$CREATED_EPOCH" -lt "$CUTOFF" ]; then
152-
eksctl delete cluster --region "$REGION" --name "$c" \
153-
--disable-nodegroup-eviction --wait
154-
else
155-
echo "Skipping recent cluster: $c (created within last 24h)"
156-
fi
157-
done
158-
'''
173+
return
174+
}
175+
176+
def cutoffMs = System.currentTimeMillis() - (24 * 60 * 60 * 1000) // 1 day ago
177+
178+
clusters.split(/\s+/).each { clusterName ->
179+
def createdAt = sh(
180+
script: "aws eks describe-cluster --name ${clusterName} --region ${REGION} --query 'cluster.createdAt' --output text 2>/dev/null || echo ''",
181+
returnStdout: true
182+
).trim()
183+
184+
if (!createdAt || createdAt == 'None') {
185+
echo "Unable to fetch creation time for ${clusterName} - skipping."
186+
return // continue to next iteration
187+
}
188+
189+
// Parse ISO 8601 timestamp
190+
def createdMs = Date.parse("yyyy-MM-dd'T'HH:mm:ss", createdAt.take(19)).time
191+
192+
if (createdMs < cutoffMs) {
193+
pmmHaEks.deleteCluster(
194+
clusterName: clusterName,
195+
region: env.REGION,
196+
r53ZoneName: env.R53_ZONE_NAME
197+
)
198+
} else {
199+
echo "Skipping recent cluster: ${clusterName} (created within last 24h)"
200+
}
201+
}
202+
}
159203
}
160204
}
161205
}

0 commit comments

Comments
 (0)