diff --git a/pkg/bootstrap/render.go b/pkg/bootstrap/render.go index f02591eca..1f85f67b3 100644 --- a/pkg/bootstrap/render.go +++ b/pkg/bootstrap/render.go @@ -179,20 +179,12 @@ func (c *KlusterletManifestsConfig) Generate(ctx context.Context, var kcImagePullSecret corev1.ObjectReference var appliedManifestWorkEvictionGracePeriod string - switch installMode { - case operatorv1.InstallModeHosted, operatorv1.InstallModeSingletonHosted: - // do nothing - case operatorv1.InstallModeDefault, operatorv1.InstallModeSingleton: - if c.klusterletConfig != nil { - kcRegistries = c.klusterletConfig.Spec.Registries - kcNodePlacement = c.klusterletConfig.Spec.NodePlacement - kcImagePullSecret = c.klusterletConfig.Spec.PullSecret - appliedManifestWorkEvictionGracePeriod = c.klusterletConfig.Spec.AppliedManifestWorkEvictionGracePeriod - } - default: - return nil, nil, nil, fmt.Errorf("invalid install mode: %s", installMode) + if c.klusterletConfig != nil { + kcRegistries = c.klusterletConfig.Spec.Registries + kcNodePlacement = c.klusterletConfig.Spec.NodePlacement + kcImagePullSecret = c.klusterletConfig.Spec.PullSecret + appliedManifestWorkEvictionGracePeriod = c.klusterletConfig.Spec.AppliedManifestWorkEvictionGracePeriod } - c.chartConfig.NoOperator = installNoOperator(installMode, c.klusterletConfig) var managedClusterAnnotations map[string]string diff --git a/pkg/bootstrap/render_test.go b/pkg/bootstrap/render_test.go index 4b3f43408..90917ea0e 100644 --- a/pkg/bootstrap/render_test.go +++ b/pkg/bootstrap/render_test.go @@ -1112,6 +1112,151 @@ func TestKlusterletConfigGenerate(t *testing.T) { // Should not be called for error cases }, }, + { + name: "hosted with nodePlacement from klusterletConfig", + clientObjs: []runtimeclient.Object{ + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + }, + defaultImagePullSecret: "test-image-pull-secret", + runtimeObjs: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-image-pull-secret", + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte("fake-token"), + }, + Type: corev1.SecretTypeDockerConfigJson, + }, + }, + config: NewKlusterletManifestsConfig( + operatorv1.InstallModeHosted, + "test", // cluster name + []byte("bootstrap kubeconfig"), + ).WithoutImagePullSecretGenerate().WithKlusterletConfig(&klusterletconfigv1alpha1.KlusterletConfig{ + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + NodePlacement: &operatorv1.NodePlacement{ + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + Tolerations: []corev1.Toleration{ + { + Key: "node.kubernetes.io/hosted", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + TolerationSeconds: &tolerationSeconds, + }, + }, + }, + }, + }), + validateFunc: func(t *testing.T, objects, crds []runtime.Object) { + testinghelpers.ValidateObjectCount(t, objects, 3) + testinghelpers.ValidateCRDs(t, crds, 0) + testinghelpers.ValidateKlusterlet(t, objects[2], operatorv1.InstallModeHosted, + "klusterlet-test", "test", "open-cluster-management-test") + klusterlet, _ := objects[2].(*operatorv1.Klusterlet) + if klusterlet.Spec.NodePlacement.NodeSelector["kubernetes.io/os"] != "linux" { + t.Errorf("the klusterlet node selector %s is not %s", + klusterlet.Spec.NodePlacement.NodeSelector["kubernetes.io/os"], "linux") + } + if klusterlet.Spec.NodePlacement.Tolerations[0].Key != "node.kubernetes.io/hosted" { + t.Errorf("the klusterlet tolerations %s is not %s", + klusterlet.Spec.NodePlacement.Tolerations[0].Key, "node.kubernetes.io/hosted") + } + }, + }, + { + name: "hosted with pullSecret and registries from klusterletConfig", + clientObjs: []runtimeclient.Object{ + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + }, + defaultImagePullSecret: "test-image-pull-secret", + runtimeObjs: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-pull-secret", + Namespace: "default", + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte("custom-fake-token"), + }, + Type: corev1.SecretTypeDockerConfigJson, + }, + }, + config: NewKlusterletManifestsConfig( + operatorv1.InstallModeHosted, + "test", // cluster name + []byte("bootstrap kubeconfig"), + ).WithKlusterletConfig(&klusterletconfigv1alpha1.KlusterletConfig{ + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + PullSecret: corev1.ObjectReference{ + Name: "custom-pull-secret", + Namespace: "default", + }, + Registries: []klusterletconfigv1alpha1.Registries{ + { + Source: "quay.io/open-cluster-management", + Mirror: "quay.io/rhacm2", + }, + { + Source: "quay.io/stolostron", + Mirror: "quay.io/rhacm2", + }, + }, + }, + }), + validateFunc: func(t *testing.T, objects, crds []runtime.Object) { + testinghelpers.ValidateObjectCount(t, objects, 4) + testinghelpers.ValidateCRDs(t, crds, 0) + + // Find the klusterlet object + var klusterlet *operatorv1.Klusterlet + var imagePullSecretIdx int + for i, obj := range objects { + if k, ok := obj.(*operatorv1.Klusterlet); ok { + klusterlet = k + } + if s, ok := obj.(*corev1.Secret); ok { + if s.Type == corev1.SecretTypeDockerConfigJson { + imagePullSecretIdx = i + } + } + } + + if klusterlet == nil { + t.Fatal("klusterlet not found in objects") + } + + // Verify klusterlet properties + if klusterlet.Name != "klusterlet-test" { + t.Errorf("expected klusterlet name klusterlet-test, got %s", klusterlet.Name) + } + if klusterlet.Spec.ClusterName != "test" { + t.Errorf("expected cluster name test, got %s", klusterlet.Spec.ClusterName) + } + + // Verify that custom registries are applied + if !strings.HasPrefix(klusterlet.Spec.RegistrationImagePullSpec, "quay.io/rhacm2/registration") { + t.Errorf("the klusterlet registration image pull spec %s does not use custom registry", + klusterlet.Spec.RegistrationImagePullSpec) + } + if !strings.HasPrefix(klusterlet.Spec.WorkImagePullSpec, "quay.io/rhacm2/work") { + t.Errorf("the klusterlet work image pull spec %s does not use custom registry", + klusterlet.Spec.WorkImagePullSpec) + } + // Verify that custom pull secret is applied + testinghelpers.ValidateImagePullSecret(t, objects[imagePullSecretIdx], "open-cluster-management-test", "custom-fake-token") + }, + }, } for _, testcase := range testcases { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 589516bd1..c5a6ef01a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1272,3 +1272,86 @@ func assertClusterImportConfigSecret(managedClusterName string) { }, 30*time.Second, 1*time.Second).Should(gomega.Succeed()) }) } + +func assertHostedKlusterletNodePlacement(klusterletName string, nodeSelector map[string]string, tolerations []corev1.Toleration) { + ginkgo.By("Hosted klusterlet should have expected nodePlacement", func() { + gomega.Eventually(func() error { + name := fmt.Sprintf("%s-import", klusterletName) + // Get the managed cluster name from klusterlet name (format: klusterlet-) + managedClusterName := strings.TrimPrefix(klusterletName, "klusterlet-") + secret, err := hubKubeClient.CoreV1().Secrets(managedClusterName).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return err + } + + var klusterlet *operatorv1.Klusterlet + for _, yaml := range helpers.SplitYamls(secret.Data[constants.ImportSecretImportYamlKey]) { + obj := helpers.MustCreateObject(yaml) + switch required := obj.(type) { + case *operatorv1.Klusterlet: + klusterlet = required + } + } + if klusterlet == nil { + return fmt.Errorf("klusterlet is not found in import.yaml") + } + + if !equality.Semantic.DeepEqual(klusterlet.Spec.NodePlacement.NodeSelector, nodeSelector) { + return fmt.Errorf("klusterlet nodePlacement diff: %s", cmp.Diff(klusterlet.Spec.NodePlacement.NodeSelector, nodeSelector)) + } + + if !equality.Semantic.DeepEqual(klusterlet.Spec.NodePlacement.Tolerations, tolerations) { + return fmt.Errorf("klusterlet tolerations diff: %s", cmp.Diff(klusterlet.Spec.NodePlacement.Tolerations, tolerations)) + } + + return nil + }, 60*time.Second, 1*time.Second).Should(gomega.Succeed()) + }) +} + +func assertHostedImagePullSecretAndRegistry(managedClusterName string) { + ginkgo.By("Hosted klusterlet should have image pull secret and customized registry", func() { + gomega.Eventually(func() error { + name := fmt.Sprintf("%s-import", managedClusterName) + secret, err := hubKubeClient.CoreV1().Secrets(managedClusterName).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return err + } + + importYaml, ok := secret.Data["import.yaml"] + if !ok { + return fmt.Errorf("import.yaml not found in secret") + } + + objs := util.ToImportResoruces(importYaml) + + hasImagePullCredentials := false + hasCustomizedImage := false + for _, obj := range objs { + if obj.GetName() == "open-cluster-management-image-pull-credentials" && obj.GetKind() == "Secret" { + hasImagePullCredentials = true + } + + if obj.GetName() == fmt.Sprintf("klusterlet-%s", managedClusterName) && obj.GetKind() == "Klusterlet" { + klusterlet := util.ToKlusterlet(obj) + if klusterlet == nil { + return fmt.Errorf("failed to convert to klusterlet") + } + if strings.HasPrefix(klusterlet.Spec.WorkImagePullSpec, "quay.io/rhacm2/work") && + strings.HasPrefix(klusterlet.Spec.RegistrationImagePullSpec, "quay.io/rhacm2/registration") { + hasCustomizedImage = true + } + } + } + + if !hasImagePullCredentials { + return fmt.Errorf("image pull credentials secret not found") + } + if !hasCustomizedImage { + return fmt.Errorf("customized image registry not found") + } + + return nil + }, 60*time.Second, 1*time.Second).Should(gomega.Succeed()) + }) +} diff --git a/test/e2e/klusterletconfig_test.go b/test/e2e/klusterletconfig_test.go index e435cf098..f1641b5cf 100644 --- a/test/e2e/klusterletconfig_test.go +++ b/test/e2e/klusterletconfig_test.go @@ -541,6 +541,202 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", Label assertFeatureGate("klusterlet", nil, nil) assertManagedClusterAvailable(managedClusterName) }) + + It("Should deploy the hosted klusterlet with nodePlacement from KlusterletConfig", Label("hosted"), func() { + var hostingClusterName string + hostingClusterName = fmt.Sprintf("hosting-cluster-%s", utilrand.String(6)) + + By(fmt.Sprintf("Create hosting cluster %s", hostingClusterName), func() { + _, err := util.CreateManagedCluster(hubClusterClient, hostingClusterName, + util.NewLable("local-cluster", "true")) + Expect(err).ToNot(HaveOccurred()) + }) + assertManagedClusterImportSecretCreated(hostingClusterName, "other") + assertManagedClusterImportSecretApplied(hostingClusterName) + assertManagedClusterAvailable(hostingClusterName) + assertManagedClusterManifestWorksAvailable(hostingClusterName) + + defer func() { + assertManagedClusterDeleted(hostingClusterName) + }() + + By(fmt.Sprintf("Create auto-import-secret for managed cluster %s with kubeconfig", managedClusterName), func() { + secret, err := util.NewAutoImportSecret(hubKubeClient, managedClusterName, operatorv1.InstallModeHosted) + Expect(err).ToNot(HaveOccurred()) + + _, err = hubKubeClient.CoreV1().Secrets(managedClusterName).Create(context.TODO(), secret, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + By(fmt.Sprintf("Create hosted mode managed cluster %s with klusterletconfig annotation", managedClusterName), func() { + _, err := util.CreateHostedManagedClusterWithAnnotations(hubClusterClient, managedClusterName, hostingClusterName, + map[string]string{ + "agent.open-cluster-management.io/klusterlet-config": klusterletConfigName, + "open-cluster-management/nodeSelector": "{}", + "open-cluster-management/tolerations": "[]", + }) + Expect(err).ToNot(HaveOccurred()) + }) + + assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) + assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) + + By("Create KlusterletConfig with nodePlacement for hosted mode", func() { + _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletConfigName, + }, + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + NodePlacement: &operatorv1.NodePlacement{ + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + Tolerations: []corev1.Toleration{ + { + Key: "node.kubernetes.io/hosted", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + TolerationSeconds: &tolerationSeconds, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + klusterletName := fmt.Sprintf("klusterlet-%s", managedClusterName) + assertHostedKlusterletNodePlacement( + klusterletName, + map[string]string{"kubernetes.io/os": "linux"}, + []corev1.Toleration{{ + Key: "node.kubernetes.io/hosted", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + TolerationSeconds: &tolerationSeconds, + }}, + ) + + By("Update KlusterletConfig to clear nodePlacement", func() { + Eventually(func() error { + oldkc, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Get(context.TODO(), klusterletConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + newkc := oldkc.DeepCopy() + newkc.Spec.NodePlacement = &operatorv1.NodePlacement{} + _, err = klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Update(context.TODO(), newkc, metav1.UpdateOptions{}) + return err + }, 60*time.Second, 1*time.Second).Should(Succeed()) + }) + + // klusterletconfig's nodeplacement is nil, expect to use values in managed cluster annotations which is empty + assertHostedKlusterletNodePlacement(klusterletName, map[string]string{}, []corev1.Toleration{}) + + assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) + + By("Delete Klusterletconfig", func() { + err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + defer func() { + assertAutoImportSecretDeleted(managedClusterName) + assertHostedManagedClusterDeleted(managedClusterName, hostingClusterName) + }() + }) + + It("Should deploy the hosted klusterlet with pullSecret from KlusterletConfig", Label("hosted"), func() { + var hostingClusterName string + var pullSecretName string + + hostingClusterName = fmt.Sprintf("hosting-cluster-%s", utilrand.String(6)) + pullSecretName = fmt.Sprintf("pull-secret-%s", utilrand.String(6)) + + By(fmt.Sprintf("Create pull secret %s", pullSecretName), func() { + err := util.CreatePullSecret(hubKubeClient, "default", pullSecretName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + defer func() { + assertPullSecretDeleted("default", pullSecretName) + }() + + By(fmt.Sprintf("Create hosting cluster %s", hostingClusterName), func() { + _, err := util.CreateManagedCluster(hubClusterClient, hostingClusterName, + util.NewLable("local-cluster", "true")) + Expect(err).ToNot(HaveOccurred()) + }) + assertManagedClusterImportSecretCreated(hostingClusterName, "other") + assertManagedClusterImportSecretApplied(hostingClusterName) + assertManagedClusterAvailable(hostingClusterName) + assertManagedClusterManifestWorksAvailable(hostingClusterName) + + defer func() { + assertManagedClusterDeleted(hostingClusterName) + }() + + By(fmt.Sprintf("Create auto-import-secret for managed cluster %s with kubeconfig", managedClusterName), func() { + secret, err := util.NewAutoImportSecret(hubKubeClient, managedClusterName, operatorv1.InstallModeHosted) + Expect(err).ToNot(HaveOccurred()) + + _, err = hubKubeClient.CoreV1().Secrets(managedClusterName).Create(context.TODO(), secret, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + By(fmt.Sprintf("Create hosted mode managed cluster %s with klusterletconfig annotation", managedClusterName), func() { + _, err := util.CreateHostedManagedClusterWithAnnotations(hubClusterClient, managedClusterName, hostingClusterName, + map[string]string{ + "agent.open-cluster-management.io/klusterlet-config": klusterletConfigName, + }) + Expect(err).ToNot(HaveOccurred()) + }) + + assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) + assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) + + By("Create KlusterletConfig with pullSecret and registries for hosted mode", func() { + _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletConfigName, + }, + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + PullSecret: corev1.ObjectReference{ + Name: pullSecretName, + Namespace: "default", + }, + Registries: []klusterletconfigv1alpha1.Registries{ + { + Source: "quay.io/open-cluster-management", + Mirror: "quay.io/rhacm2", + }, + { + Source: "quay.io/stolostron", + Mirror: "quay.io/rhacm2", + }, + }, + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + assertHostedImagePullSecretAndRegistry(managedClusterName) + + assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) + + By("Delete Klusterletconfig", func() { + err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + defer func() { + assertAutoImportSecretDeleted(managedClusterName) + assertHostedManagedClusterDeleted(managedClusterName, hostingClusterName) + }() + }) }) func newCert(commoneName string) ([]byte, []byte, error) { diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 92f2b9f8f..1cd8e1277 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -122,6 +122,33 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien return cluster, err } +func CreateHostedManagedClusterWithAnnotations(clusterClient clusterclient.Interface, name, management string, annotations map[string]string) (*clusterv1.ManagedCluster, error) { + clusterAnnotations := map[string]string{} + clusterAnnotations[constants.KlusterletDeployModeAnnotation] = string(operatorv1.InstallModeHosted) + clusterAnnotations[constants.HostingClusterNameAnnotation] = management + for k, v := range annotations { + clusterAnnotations[k] = v + } + cluster, err := clusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return clusterClient.ClusterV1().ManagedClusters().Create( + context.TODO(), + &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: clusterAnnotations, + }, + Spec: clusterv1.ManagedClusterSpec{ + HubAcceptsClient: true, + }, + }, + metav1.CreateOptions{}, + ) + } + + return cluster, err +} + func CreateManagedCluster(clusterClient clusterclient.Interface, name string, labels ...Label) (*clusterv1.ManagedCluster, error) { clusterLabels := map[string]string{} for _, label := range labels {