Skip to content

Commit ff7ca56

Browse files
camilamacedo86camilamacedo86
committed
🐛 Fix e2e test flakes when webhooks are scaffolded
Projects with webhooks may experience flaky e2e tests due to webhook server not being ready when the metrics test creates the curl-metrics pod.
1 parent 9eee7a7 commit ff7ca56

File tree

8 files changed

+344
-257
lines changed

8 files changed

+344
-257
lines changed

‎docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,58 @@ var _ = Describe("Manager", Ordered, func() {
173173
Eventually(verifyControllerUp).Should(Succeed())
174174
})
175175

176+
It("should provisioned cert-manager", func() {
177+
By("validating that cert-manager has the certificate Secret")
178+
verifyCertManager := func(g Gomega) {
179+
cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace)
180+
_, err := utils.Run(cmd)
181+
g.Expect(err).NotTo(HaveOccurred())
182+
}
183+
Eventually(verifyCertManager).Should(Succeed())
184+
})
185+
186+
It("should have the webhook server ready", func() {
187+
By("waiting for the webhook server to be ready")
188+
verifyWebhookServerReady := func(g Gomega) {
189+
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
190+
output, err := utils.Run(cmd)
191+
g.Expect(err).NotTo(HaveOccurred())
192+
g.Expect(output).To(ContainSubstring("controller-runtime.webhook\tServing webhook server"),
193+
"Webhook server not yet started")
194+
}
195+
Eventually(verifyWebhookServerReady, 3*time.Minute).Should(Succeed())
196+
})
197+
198+
It("should have CA injection for mutating webhooks", func() {
199+
By("checking CA injection for mutating webhooks")
200+
verifyCAInjection := func(g Gomega) {
201+
cmd := exec.Command("kubectl", "get",
202+
"mutatingwebhookconfigurations.admissionregistration.k8s.io",
203+
"project-mutating-webhook-configuration",
204+
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
205+
mwhOutput, err := utils.Run(cmd)
206+
g.Expect(err).NotTo(HaveOccurred())
207+
g.Expect(len(mwhOutput)).To(BeNumerically(">", 10))
208+
}
209+
Eventually(verifyCAInjection).Should(Succeed())
210+
})
211+
212+
It("should have CA injection for validating webhooks", func() {
213+
By("checking CA injection for validating webhooks")
214+
verifyCAInjection := func(g Gomega) {
215+
cmd := exec.Command("kubectl", "get",
216+
"validatingwebhookconfigurations.admissionregistration.k8s.io",
217+
"project-validating-webhook-configuration",
218+
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
219+
vwhOutput, err := utils.Run(cmd)
220+
g.Expect(err).NotTo(HaveOccurred())
221+
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
222+
}
223+
Eventually(verifyCAInjection).Should(Succeed())
224+
})
225+
226+
// +kubebuilder:scaffold:e2e-webhooks-checks
227+
176228
It("should ensure the metrics endpoint is serving metrics", func() {
177229
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
178230
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
@@ -268,46 +320,6 @@ var _ = Describe("Manager", Ordered, func() {
268320
Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed())
269321
})
270322

271-
It("should provisioned cert-manager", func() {
272-
By("validating that cert-manager has the certificate Secret")
273-
verifyCertManager := func(g Gomega) {
274-
cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace)
275-
_, err := utils.Run(cmd)
276-
g.Expect(err).NotTo(HaveOccurred())
277-
}
278-
Eventually(verifyCertManager).Should(Succeed())
279-
})
280-
281-
It("should have CA injection for mutating webhooks", func() {
282-
By("checking CA injection for mutating webhooks")
283-
verifyCAInjection := func(g Gomega) {
284-
cmd := exec.Command("kubectl", "get",
285-
"mutatingwebhookconfigurations.admissionregistration.k8s.io",
286-
"project-mutating-webhook-configuration",
287-
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
288-
mwhOutput, err := utils.Run(cmd)
289-
g.Expect(err).NotTo(HaveOccurred())
290-
g.Expect(len(mwhOutput)).To(BeNumerically(">", 10))
291-
}
292-
Eventually(verifyCAInjection).Should(Succeed())
293-
})
294-
295-
It("should have CA injection for validating webhooks", func() {
296-
By("checking CA injection for validating webhooks")
297-
verifyCAInjection := func(g Gomega) {
298-
cmd := exec.Command("kubectl", "get",
299-
"validatingwebhookconfigurations.admissionregistration.k8s.io",
300-
"project-validating-webhook-configuration",
301-
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
302-
vwhOutput, err := utils.Run(cmd)
303-
g.Expect(err).NotTo(HaveOccurred())
304-
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
305-
}
306-
Eventually(verifyCAInjection).Should(Succeed())
307-
})
308-
309-
// +kubebuilder:scaffold:e2e-webhooks-checks
310-
311323
// TODO: Customize the e2e test suite with scenarios specific to your project.
312324
// Consider applying sample/CR(s) and check their status and/or verifying
313325
// the reconciliation by using the metrics, i.e.:

‎docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ var _ = Describe("Manager", Ordered, func() {
173173
Eventually(verifyControllerUp).Should(Succeed())
174174
})
175175

176+
// +kubebuilder:scaffold:e2e-webhooks-checks
177+
176178
It("should ensure the metrics endpoint is serving metrics", func() {
177179
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
178180
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
@@ -263,8 +265,6 @@ var _ = Describe("Manager", Ordered, func() {
263265
Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed())
264266
})
265267

266-
// +kubebuilder:scaffold:e2e-webhooks-checks
267-
268268
// TODO: Customize the e2e test suite with scenarios specific to your project.
269269
// Consider applying sample/CR(s) and check their status and/or verifying
270270
// the reconciliation by using the metrics, i.e.:

‎docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 66 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,72 @@ var _ = Describe("Manager", Ordered, func() {
180180
Eventually(verifyControllerUp).Should(Succeed())
181181
})
182182

183+
It("should provisioned cert-manager", func() {
184+
By("validating that cert-manager has the certificate Secret")
185+
verifyCertManager := func(g Gomega) {
186+
cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace)
187+
_, err := utils.Run(cmd)
188+
g.Expect(err).NotTo(HaveOccurred())
189+
}
190+
Eventually(verifyCertManager).Should(Succeed())
191+
})
192+
193+
It("should have the webhook server ready", func() {
194+
By("waiting for the webhook server to be ready")
195+
verifyWebhookServerReady := func(g Gomega) {
196+
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
197+
output, err := utils.Run(cmd)
198+
g.Expect(err).NotTo(HaveOccurred())
199+
g.Expect(output).To(ContainSubstring("controller-runtime.webhook\tServing webhook server"),
200+
"Webhook server not yet started")
201+
}
202+
Eventually(verifyWebhookServerReady, 3*time.Minute).Should(Succeed())
203+
})
204+
205+
It("should have CA injection for mutating webhooks", func() {
206+
By("checking CA injection for mutating webhooks")
207+
verifyCAInjection := func(g Gomega) {
208+
cmd := exec.Command("kubectl", "get",
209+
"mutatingwebhookconfigurations.admissionregistration.k8s.io",
210+
"project-mutating-webhook-configuration",
211+
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
212+
mwhOutput, err := utils.Run(cmd)
213+
g.Expect(err).NotTo(HaveOccurred())
214+
g.Expect(len(mwhOutput)).To(BeNumerically(">", 10))
215+
}
216+
Eventually(verifyCAInjection).Should(Succeed())
217+
})
218+
219+
It("should have CA injection for validating webhooks", func() {
220+
By("checking CA injection for validating webhooks")
221+
verifyCAInjection := func(g Gomega) {
222+
cmd := exec.Command("kubectl", "get",
223+
"validatingwebhookconfigurations.admissionregistration.k8s.io",
224+
"project-validating-webhook-configuration",
225+
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
226+
vwhOutput, err := utils.Run(cmd)
227+
g.Expect(err).NotTo(HaveOccurred())
228+
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
229+
}
230+
Eventually(verifyCAInjection).Should(Succeed())
231+
})
232+
233+
It("should have CA injection for CronJob conversion webhook", func() {
234+
By("checking CA injection for CronJob conversion webhook")
235+
verifyCAInjection := func(g Gomega) {
236+
cmd := exec.Command("kubectl", "get",
237+
"customresourcedefinitions.apiextensions.k8s.io",
238+
"cronjobs.batch.tutorial.kubebuilder.io",
239+
"-o", "go-template={{ .spec.conversion.webhook.clientConfig.caBundle }}")
240+
vwhOutput, err := utils.Run(cmd)
241+
g.Expect(err).NotTo(HaveOccurred())
242+
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
243+
}
244+
Eventually(verifyCAInjection).Should(Succeed())
245+
})
246+
247+
// +kubebuilder:scaffold:e2e-webhooks-checks
248+
183249
It("should ensure the metrics endpoint is serving metrics", func() {
184250
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
185251
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
@@ -275,60 +341,6 @@ var _ = Describe("Manager", Ordered, func() {
275341
Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed())
276342
})
277343

278-
It("should provisioned cert-manager", func() {
279-
By("validating that cert-manager has the certificate Secret")
280-
verifyCertManager := func(g Gomega) {
281-
cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace)
282-
_, err := utils.Run(cmd)
283-
g.Expect(err).NotTo(HaveOccurred())
284-
}
285-
Eventually(verifyCertManager).Should(Succeed())
286-
})
287-
288-
It("should have CA injection for mutating webhooks", func() {
289-
By("checking CA injection for mutating webhooks")
290-
verifyCAInjection := func(g Gomega) {
291-
cmd := exec.Command("kubectl", "get",
292-
"mutatingwebhookconfigurations.admissionregistration.k8s.io",
293-
"project-mutating-webhook-configuration",
294-
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
295-
mwhOutput, err := utils.Run(cmd)
296-
g.Expect(err).NotTo(HaveOccurred())
297-
g.Expect(len(mwhOutput)).To(BeNumerically(">", 10))
298-
}
299-
Eventually(verifyCAInjection).Should(Succeed())
300-
})
301-
302-
It("should have CA injection for validating webhooks", func() {
303-
By("checking CA injection for validating webhooks")
304-
verifyCAInjection := func(g Gomega) {
305-
cmd := exec.Command("kubectl", "get",
306-
"validatingwebhookconfigurations.admissionregistration.k8s.io",
307-
"project-validating-webhook-configuration",
308-
"-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}")
309-
vwhOutput, err := utils.Run(cmd)
310-
g.Expect(err).NotTo(HaveOccurred())
311-
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
312-
}
313-
Eventually(verifyCAInjection).Should(Succeed())
314-
})
315-
316-
It("should have CA injection for CronJob conversion webhook", func() {
317-
By("checking CA injection for CronJob conversion webhook")
318-
verifyCAInjection := func(g Gomega) {
319-
cmd := exec.Command("kubectl", "get",
320-
"customresourcedefinitions.apiextensions.k8s.io",
321-
"cronjobs.batch.tutorial.kubebuilder.io",
322-
"-o", "go-template={{ .spec.conversion.webhook.clientConfig.caBundle }}")
323-
vwhOutput, err := utils.Run(cmd)
324-
g.Expect(err).NotTo(HaveOccurred())
325-
g.Expect(len(vwhOutput)).To(BeNumerically(">", 10))
326-
}
327-
Eventually(verifyCAInjection).Should(Succeed())
328-
})
329-
330-
// +kubebuilder:scaffold:e2e-webhooks-checks
331-
332344
// TODO: Customize the e2e test suite with scenarios specific to your project.
333345
// Consider applying sample/CR(s) and check their status and/or verifying
334346
// the reconciliation by using the metrics, i.e.:

‎pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,13 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
107107
}
108108

109109
var fragments []string
110+
111+
// Add cert-manager check
110112
fragments = append(fragments, webhookChecksFragment)
111113

114+
// Add webhook readiness check first
115+
fragments = append(fragments, webhookReadinessFragment)
116+
112117
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
113118
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
114119
fragments = append(fragments, mutatingWebhookCode)
@@ -138,6 +143,20 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
138143
return codeFragments
139144
}
140145

146+
const webhookReadinessFragment = `It("should have the webhook server ready", func() {
147+
By("waiting for the webhook server to be ready")
148+
verifyWebhookServerReady := func(g Gomega) {
149+
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
150+
output, err := utils.Run(cmd)
151+
g.Expect(err).NotTo(HaveOccurred())
152+
g.Expect(output).To(ContainSubstring("controller-runtime.webhook\tServing webhook server"),
153+
"Webhook server not yet started")
154+
}
155+
Eventually(verifyWebhookServerReady, 3*time.Minute).Should(Succeed())
156+
})
157+
158+
`
159+
141160
const webhookChecksFragment = `It("should provisioned cert-manager", func() {
142161
By("validating that cert-manager has the certificate Secret")
143162
verifyCertManager := func(g Gomega) {
@@ -356,6 +375,8 @@ var _ = Describe("Manager", Ordered, func() {
356375
Eventually(verifyControllerUp).Should(Succeed())
357376
})
358377
378+
// +kubebuilder:scaffold:e2e-webhooks-checks
379+
359380
It("should ensure the metrics endpoint is serving metrics", func() {
360381
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
361382
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
@@ -392,10 +413,10 @@ var _ = Describe("Manager", Ordered, func() {
392413
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
393414
"Metrics server not yet started")
394415
}
395-
Eventually(verifyMetricsServerStarted).Should(Succeed())
416+
Eventually(verifyMetricsServerStarted).Should(Succeed())
396417
397-
By("creating the curl-metrics pod to access the metrics endpoint")
398-
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
418+
By("creating the curl-metrics pod to access the metrics endpoint")
419+
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
399420
"--namespace", namespace,
400421
"--image=curlimages/curl:latest",
401422
"--overrides",
@@ -446,8 +467,6 @@ var _ = Describe("Manager", Ordered, func() {
446467
Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed())
447468
})
448469
449-
// +kubebuilder:scaffold:e2e-webhooks-checks
450-
451470
// TODO: Customize the e2e test suite with scenarios specific to your project.
452471
// Consider applying sample/CR(s) and check their status and/or verifying
453472
// the reconciliation by using the metrics, i.e.:

‎test/e2e/v4/plugin_cluster_test.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, isToUseHelmChart,
299299

300300
if hasMetrics {
301301
By("checking the metrics values to validate that the created resource object gets reconciled")
302-
metricsOutput := getMetricsOutput(kbc)
302+
metricsOutput := getMetricsOutput(hasWebhook, kbc)
303303
Expect(metricsOutput).To(ContainSubstring(fmt.Sprintf(
304304
`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
305305
strings.ToLower(kbc.Kind),
@@ -392,7 +392,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, isToUseHelmChart,
392392

393393
if hasMetrics {
394394
By("validating conversion metrics to confirm conversion operations")
395-
metricsOutput := getMetricsOutput(kbc)
395+
metricsOutput := getMetricsOutput(hasWebhook, kbc)
396396
conversionMetric := `controller_runtime_reconcile_total{controller="conversiontest",result="success"} 1`
397397
Expect(metricsOutput).To(ContainSubstring(conversionMetric),
398398
"Expected metric for successful ConversionTest reconciliation")
@@ -438,7 +438,7 @@ func getControllerName(kbc *utils.TestContext) string {
438438
}
439439

440440
// getMetricsOutput return the metrics output from curl pod
441-
func getMetricsOutput(kbc *utils.TestContext) string {
441+
func getMetricsOutput(hasWebhook bool, kbc *utils.TestContext) string {
442442
_, err := kbc.Kubectl.Command(
443443
"get", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix),
444444
)
@@ -484,11 +484,19 @@ func getMetricsOutput(kbc *utils.TestContext) string {
484484
// when using controller-runtime's WithAuthenticationAndAuthorization() with self-signed certificates.
485485
// This delay appears to stem from Kubernetes itself, potentially due to changes in how it initializes
486486
// service account tokens or handles TLS/service readiness.
487-
//
488-
// Without this delay, tests that curl the /metrics endpoint using a token can fail from k8s 1.33+.
489-
// As a temporary workaround, we wait briefly before attempting to access metrics.
490-
By("waiting briefly to ensure that the certs are provisioned and metrics are available")
491-
time.Sleep(15 * time.Second)
487+
if hasWebhook {
488+
It("should have the webhook server ready", func() {
489+
By("waiting for the webhook server to be ready")
490+
verifyWebhookServerReady := func(g Gomega) {
491+
var output string
492+
output, err = kbc.Kubectl.Logs("controllerPodName")
493+
g.Expect(err).NotTo(HaveOccurred())
494+
g.Expect(output).To(ContainSubstring("Serving webhook server"),
495+
"Webhook server not yet started")
496+
}
497+
Eventually(verifyWebhookServerReady, 5*time.Minute).Should(Succeed())
498+
})
499+
}
492500

493501
By("creating a curl pod to access the metrics endpoint")
494502
cmdOpts := cmdOptsToCreateCurlPod(kbc, token)

0 commit comments

Comments
 (0)