Skip to content

Commit dd71cd9

Browse files
camilamacedo86camilamacedo86
committed
🐛 Fix e2e test flakes when webhooks are scaffolded
Projects with webhooks may experience flaky e2e tests due to webhook server not being ready when the metrics test creates the curl-metrics pod.
1 parent f148cd9 commit dd71cd9

File tree

9 files changed

+228
-97
lines changed

9 files changed

+228
-97
lines changed

‎docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,24 +197,38 @@ var _ = Describe("Manager", Ordered, func() {
197197
Expect(err).NotTo(HaveOccurred())
198198
Expect(token).NotTo(BeEmpty())
199199

200-
By("waiting for the metrics endpoint to be ready")
201-
verifyMetricsEndpointReady := func(g Gomega) {
202-
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
200+
By("ensuring the controller pod is ready")
201+
verifyControllerPodReady := func(g Gomega) {
202+
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
203+
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
203204
output, err := utils.Run(cmd)
204205
g.Expect(err).NotTo(HaveOccurred())
205-
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
206+
g.Expect(output).To(Equal("True"), "Controller pod not ready")
206207
}
207-
Eventually(verifyMetricsEndpointReady).Should(Succeed())
208+
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())
208209

209210
By("verifying that the controller manager is serving the metrics server")
210211
verifyMetricsServerStarted := func(g Gomega) {
211212
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
212213
output, err := utils.Run(cmd)
213214
g.Expect(err).NotTo(HaveOccurred())
214-
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
215+
g.Expect(output).To(ContainSubstring("Serving metrics server"),
215216
"Metrics server not yet started")
216217
}
217-
Eventually(verifyMetricsServerStarted).Should(Succeed())
218+
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())
219+
220+
By("waiting for the webhook service endpoints to be ready")
221+
verifyWebhookEndpointsReady := func(g Gomega) {
222+
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
223+
"-l", "kubernetes.io/service-name=project-webhook-service",
224+
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
225+
output, err := utils.Run(cmd)
226+
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
227+
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
228+
}
229+
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())
230+
231+
// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness
218232

219233
By("creating the curl-metrics pod to access the metrics endpoint")
220234
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",

‎docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -192,24 +192,27 @@ var _ = Describe("Manager", Ordered, func() {
192192
Expect(err).NotTo(HaveOccurred())
193193
Expect(token).NotTo(BeEmpty())
194194

195-
By("waiting for the metrics endpoint to be ready")
196-
verifyMetricsEndpointReady := func(g Gomega) {
197-
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
195+
By("ensuring the controller pod is ready")
196+
verifyControllerPodReady := func(g Gomega) {
197+
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
198+
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
198199
output, err := utils.Run(cmd)
199200
g.Expect(err).NotTo(HaveOccurred())
200-
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
201+
g.Expect(output).To(Equal("True"), "Controller pod not ready")
201202
}
202-
Eventually(verifyMetricsEndpointReady).Should(Succeed())
203+
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())
203204

204205
By("verifying that the controller manager is serving the metrics server")
205206
verifyMetricsServerStarted := func(g Gomega) {
206207
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
207208
output, err := utils.Run(cmd)
208209
g.Expect(err).NotTo(HaveOccurred())
209-
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
210+
g.Expect(output).To(ContainSubstring("Serving metrics server"),
210211
"Metrics server not yet started")
211212
}
212-
Eventually(verifyMetricsServerStarted).Should(Succeed())
213+
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())
214+
215+
// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness
213216

214217
By("creating the curl-metrics pod to access the metrics endpoint")
215218
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",

‎docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -204,24 +204,38 @@ var _ = Describe("Manager", Ordered, func() {
204204
Expect(err).NotTo(HaveOccurred())
205205
Expect(token).NotTo(BeEmpty())
206206

207-
By("waiting for the metrics endpoint to be ready")
208-
verifyMetricsEndpointReady := func(g Gomega) {
209-
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
207+
By("ensuring the controller pod is ready")
208+
verifyControllerPodReady := func(g Gomega) {
209+
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
210+
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
210211
output, err := utils.Run(cmd)
211212
g.Expect(err).NotTo(HaveOccurred())
212-
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
213+
g.Expect(output).To(Equal("True"), "Controller pod not ready")
213214
}
214-
Eventually(verifyMetricsEndpointReady).Should(Succeed())
215+
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())
215216

216217
By("verifying that the controller manager is serving the metrics server")
217218
verifyMetricsServerStarted := func(g Gomega) {
218219
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
219220
output, err := utils.Run(cmd)
220221
g.Expect(err).NotTo(HaveOccurred())
221-
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
222+
g.Expect(output).To(ContainSubstring("Serving metrics server"),
222223
"Metrics server not yet started")
223224
}
224-
Eventually(verifyMetricsServerStarted).Should(Succeed())
225+
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())
226+
227+
By("waiting for the webhook service endpoints to be ready")
228+
verifyWebhookEndpointsReady := func(g Gomega) {
229+
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
230+
"-l", "kubernetes.io/service-name=project-webhook-service",
231+
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
232+
output, err := utils.Run(cmd)
233+
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
234+
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
235+
}
236+
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())
237+
238+
// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness
225239

226240
By("creating the curl-metrics pod to access the metrics endpoint")
227241
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",

‎docs/book/src/reference/markers/scaffold.md‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -95,20 +95,20 @@ properly registered with the manager, so that the controller can reconcile the r
9595

9696
## List of `+kubebuilder:scaffold` Markers
9797

98-
| Marker | Usual Location | Function |
99-
|--------------------------------------------|------------------------------|---------------------------------------------------------------------------------|
100-
| `+kubebuilder:scaffold:imports` | `main.go` | Marks where imports for new controllers, webhooks, or APIs should be injected. |
101-
| `+kubebuilder:scaffold:scheme` | `init()` in `main.go` | Used to add API versions to the scheme for runtime. |
102-
| `+kubebuilder:scaffold:builder` | `main.go` | Marks where new controllers should be registered with the manager. |
103-
| `+kubebuilder:scaffold:webhook` | `webhooks suite tests` files | Marks where webhook setup functions are added. |
104-
| `+kubebuilder:scaffold:crdkustomizeresource`| `config/crd` | Marks where CRD custom resource patches are added. |
105-
| `+kubebuilder:scaffold:crdkustomizewebhookpatch` | `config/crd` | Marks where CRD webhook patches are added. |
106-
| `+kubebuilder:scaffold:crdkustomizecainjectionns` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
107-
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
98+
| Marker | Usual Location | Function |
99+
|--------------------------------------------------------------------------------|------------------------------|---------------------------------------------------------------------------------|
100+
| `+kubebuilder:scaffold:imports` | `main.go` | Marks where imports for new controllers, webhooks, or APIs should be injected. |
101+
| `+kubebuilder:scaffold:scheme` | `init()` in `main.go` | Used to add API versions to the scheme for runtime. |
102+
| `+kubebuilder:scaffold:builder` | `main.go` | Marks where new controllers should be registered with the manager. |
103+
| `+kubebuilder:scaffold:webhook` | `webhooks suite tests` files | Marks where webhook setup functions are added. |
104+
| `+kubebuilder:scaffold:crdkustomizeresource` | `config/crd` | Marks where CRD custom resource patches are added. |
105+
| `+kubebuilder:scaffold:crdkustomizewebhookpatch` | `config/crd` | Marks where CRD webhook patches are added. |
106+
| `+kubebuilder:scaffold:crdkustomizecainjectionns` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
107+
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
108108
| **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` | `config/crd` | Marks where CA injection patches are added for the webhooks. Replaced by `+kubebuilder:scaffold:crdkustomizecainjectionns` and `+kubebuilder:scaffold:crdkustomizecainjectioname` |
109-
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
110-
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |
111-
109+
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
110+
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |
111+
| `+kubebuilder:scaffold:e2e-metrics-webhooks-readiness` | `test/e2e` | Adds readiness logic so metrics e2e tests wait for webhook service endpoints before creating pods. |
112112
<aside class="warning">
113113
<h3> **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` </h3>
114114

‎pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go‎

Lines changed: 60 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
log "log/slog"
2323
"os"
2424
"path/filepath"
25+
"strings"
2526

2627
"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
2728
)
@@ -31,7 +32,10 @@ var (
3132
_ machinery.Inserter = &WebhookTestUpdater{}
3233
)
3334

34-
const webhookChecksMarker = "e2e-webhooks-checks"
35+
const (
36+
webhookChecksMarker = "e2e-webhooks-checks"
37+
metricsWebhookReadinessMarker = "e2e-metrics-webhooks-readiness"
38+
)
3539

3640
// Test defines the basic setup for the e2e test
3741
type Test struct {
@@ -75,6 +79,7 @@ func (*WebhookTestUpdater) GetIfExistsAction() machinery.IfExistsAction {
7579
func (f *WebhookTestUpdater) GetMarkers() []machinery.Marker {
7680
return []machinery.Marker{
7781
machinery.NewMarkerFor(f.GetPath(), webhookChecksMarker),
82+
machinery.NewMarkerFor(f.GetPath(), metricsWebhookReadinessMarker),
7883
}
7984
}
8085

@@ -99,36 +104,46 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
99104
markers := f.GetMarkers()
100105

101106
for _, marker := range markers {
102-
if !bytes.Contains(content, []byte(marker.String())) {
107+
markerStr := marker.String()
108+
if !bytes.Contains(content, []byte(markerStr)) {
103109
log.Warn("Marker not found in file, skipping webhook test code injection",
104-
"marker", marker.String(),
110+
"marker", markerStr,
105111
"file_path", filePath)
106112
continue // skip this marker
107113
}
108114

109-
var fragments []string
110-
fragments = append(fragments, webhookChecksFragment)
115+
switch {
116+
case strings.Contains(markerStr, webhookChecksMarker):
117+
var fragments []string
118+
fragments = append(fragments, webhookChecksFragment)
111119

112-
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
113-
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
114-
fragments = append(fragments, mutatingWebhookCode)
115-
}
120+
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
121+
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
122+
fragments = append(fragments, mutatingWebhookCode)
123+
}
116124

117-
if f.Resource != nil && f.Resource.HasValidationWebhook() {
118-
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
119-
fragments = append(fragments, validatingWebhookCode)
120-
}
125+
if f.Resource != nil && f.Resource.HasValidationWebhook() {
126+
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
127+
fragments = append(fragments, validatingWebhookCode)
128+
}
121129

122-
if f.Resource != nil && f.Resource.HasConversionWebhook() {
123-
conversionWebhookCode := fmt.Sprintf(
124-
conversionWebhookChecksFragment,
125-
f.Resource.Kind,
126-
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
127-
)
128-
fragments = append(fragments, conversionWebhookCode)
129-
}
130+
if f.Resource != nil && f.Resource.HasConversionWebhook() {
131+
conversionWebhookCode := fmt.Sprintf(
132+
conversionWebhookChecksFragment,
133+
f.Resource.Kind,
134+
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
135+
)
136+
fragments = append(fragments, conversionWebhookCode)
137+
}
130138

131-
codeFragments[marker] = fragments
139+
if len(fragments) > 0 {
140+
codeFragments[marker] = fragments
141+
}
142+
case strings.Contains(markerStr, metricsWebhookReadinessMarker):
143+
webhookServiceName := fmt.Sprintf("%s-webhook-service", f.ProjectName)
144+
fragments := []string{fmt.Sprintf(metricsWebhookReadinessFragment, webhookServiceName)}
145+
codeFragments[marker] = fragments
146+
}
132147
}
133148

134149
if len(codeFragments) == 0 {
@@ -198,6 +213,19 @@ const conversionWebhookChecksFragment = `It("should have CA injection for %[1]s
198213
199214
`
200215

216+
const metricsWebhookReadinessFragment = `By("waiting for the webhook service endpoints to be ready")
217+
verifyWebhookEndpointsReady := func(g Gomega) {
218+
cmd := exec.Command("kubectl", "get", "endpointslices.discovery.k8s.io", "-n", namespace,
219+
"-l", "kubernetes.io/service-name=%s",
220+
"-o", "jsonpath={range .items[*]}{range .endpoints[*]}{.addresses[*]}{end}{end}")
221+
output, err := utils.Run(cmd)
222+
g.Expect(err).NotTo(HaveOccurred(), "Webhook endpoints should exist")
223+
g.Expect(output).ShouldNot(BeEmpty(), "Webhook endpoints not yet ready")
224+
}
225+
Eventually(verifyWebhookEndpointsReady, 3*time.Minute, time.Second).Should(Succeed())
226+
227+
`
228+
201229
var testCodeTemplate = `//go:build e2e
202230
// +build e2e
203231
@@ -375,24 +403,27 @@ var _ = Describe("Manager", Ordered, func() {
375403
Expect(err).NotTo(HaveOccurred())
376404
Expect(token).NotTo(BeEmpty())
377405
378-
By("waiting for the metrics endpoint to be ready")
379-
verifyMetricsEndpointReady := func(g Gomega) {
380-
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
406+
By("ensuring the controller pod is ready")
407+
verifyControllerPodReady := func(g Gomega) {
408+
cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace,
409+
"-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}")
381410
output, err := utils.Run(cmd)
382411
g.Expect(err).NotTo(HaveOccurred())
383-
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
412+
g.Expect(output).To(Equal("True"), "Controller pod not ready")
384413
}
385-
Eventually(verifyMetricsEndpointReady).Should(Succeed())
414+
Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed())
386415
387416
By("verifying that the controller manager is serving the metrics server")
388417
verifyMetricsServerStarted := func(g Gomega) {
389418
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
390419
output, err := utils.Run(cmd)
391420
g.Expect(err).NotTo(HaveOccurred())
392-
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
421+
g.Expect(output).To(ContainSubstring("Serving metrics server"),
393422
"Metrics server not yet started")
394423
}
395-
Eventually(verifyMetricsServerStarted).Should(Succeed())
424+
Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed())
425+
426+
// +kubebuilder:scaffold:e2e-metrics-webhooks-readiness
396427
397428
By("creating the curl-metrics pod to access the metrics endpoint")
398429
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",

0 commit comments

Comments
 (0)