From 6b4c25a291a7ab7ebd6e4fd887461ac3b847e20a Mon Sep 17 00:00:00 2001 From: haorenfsa Date: Fri, 26 Dec 2025 15:54:14 +0800 Subject: [PATCH 1/2] add CleanupDeploymentClusterToStandalone Signed-off-by: haorenfsa --- .../deployment_cluster_to_standalone.go | 97 ++++++++ .../deployment_cluster_to_standalone_test.go | 214 ++++++++++++++++++ pkg/controllers/deployments.go | 5 + 3 files changed, 316 insertions(+) create mode 100644 pkg/controllers/deployment_cluster_to_standalone.go create mode 100644 pkg/controllers/deployment_cluster_to_standalone_test.go diff --git a/pkg/controllers/deployment_cluster_to_standalone.go b/pkg/controllers/deployment_cluster_to_standalone.go new file mode 100644 index 00000000..9b18772e --- /dev/null +++ b/pkg/controllers/deployment_cluster_to_standalone.go @@ -0,0 +1,97 @@ +package controllers + +import ( + "context" + + pkgerr "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/zilliztech/milvus-operator/apis/milvus.io/v1beta1" +) + +func (r *MilvusReconciler) CleanupDeploymentClusterToStandalone(ctx context.Context, mc v1beta1.Milvus) error { + logger := ctrl.LoggerFrom(ctx) + + if mc.Spec.Mode != v1beta1.MilvusModeStandalone || mc.Spec.Com.EnableManualMode { + return nil + } + // If mode is standalone, we need to ensure all other component deployments are scaled down + + // List all deployments with the instance label + deploymentList := &appsv1.DeploymentList{} + opts := &client.ListOptions{ + Namespace: mc.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + AppLabelInstance: mc.Name, + }), + } + + if err := r.List(ctx, deploymentList, opts); err != nil { + return pkgerr.Wrap(err, "list deployments by instance label") + } + var nonStandaloneDeployments []appsv1.Deployment + for i := range deploymentList.Items { + deployment := &deploymentList.Items[i] + + // Skip the standalone deployments by checking the component label + if deployment.Labels != nil && deployment.Labels[AppLabelComponent] == MilvusStandalone.Name { + continue + } + nonStandaloneDeployments = append(nonStandaloneDeployments, *deployment) + } + if len(nonStandaloneDeployments) == 0 { + // No non-standalone deployments found + return nil + } + logger.Info("Found non-standalone deployments to delete, checking standalone deployment readiness") + + // Check if standalone deployment exists and is ready + // Standalone may use 2 deployment mode, so list all standalone deployments + standaloneDeploymentList := &appsv1.DeploymentList{} + standaloneOpts := &client.ListOptions{ + Namespace: mc.Namespace, + LabelSelector: labels.SelectorFromSet(NewComponentAppLabels( + mc.Name, + MilvusStandalone.Name, + )), + } + + if err := r.List(ctx, standaloneDeploymentList, standaloneOpts); err != nil { + return pkgerr.Wrap(err, "list standalone deployments") + } + + if len(standaloneDeploymentList.Items) == 0 { + // If standalone deployment doesn't exist yet, we can't proceed + logger.V(1).Info("Standalone deployment not found, skip cluster to standalone cleanup") + return nil + } + + // Check if all standalone deployments are ready + allStandaloneReady := true + for i := range standaloneDeploymentList.Items { + if !DeploymentReady(standaloneDeploymentList.Items[i].Status) { + allStandaloneReady = false + break + } + } + + if !allStandaloneReady { + logger.V(1).Info("Standalone deployment not ready yet, skip cluster to standalone cleanup") + return nil + } + + logger.Info("Standalone deployment is ready, delete cluster component deployments") + for i := range nonStandaloneDeployments { + deployment := &nonStandaloneDeployments[i] + err := r.Delete(ctx, deployment) + if client.IgnoreNotFound(err) != nil { + return pkgerr.Wrapf(err, "delete deployment %s/%s", deployment.Namespace, deployment.Name) + } + logger.Info("Deleted deployment", "deployment", deployment.Name) + } + + return nil +} diff --git a/pkg/controllers/deployment_cluster_to_standalone_test.go b/pkg/controllers/deployment_cluster_to_standalone_test.go new file mode 100644 index 00000000..3830aada --- /dev/null +++ b/pkg/controllers/deployment_cluster_to_standalone_test.go @@ -0,0 +1,214 @@ +package controllers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/zilliztech/milvus-operator/apis/milvus.io/v1beta1" +) + +// Helper functions for test setup +func newDeployment(name, component string, ready bool, replicas *int32) appsv1.Deployment { + deploy := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "ns", + Labels: map[string]string{ + AppLabelInstance: "mc", + AppLabelComponent: component, + }, + }, + } + if replicas != nil { + deploy.Spec.Replicas = replicas + } + if ready { + deploy.Status = appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: corev1.ConditionTrue, + }, + }, + } + } + return deploy +} + +func mockListDeployments(deployments []appsv1.Deployment) func(interface{}, interface{}, ...interface{}) error { + return func(ctx interface{}, list interface{}, opts ...interface{}) error { + deployList := list.(*appsv1.DeploymentList) + deployList.Items = deployments + return nil + } +} + +func TestMilvusReconciler_ReconcileDeploymentClusterToStandalone(t *testing.T) { + env := newTestEnv(t) + defer env.checkMocks() + r := env.Reconciler + mockClient := env.MockClient + ctx := env.ctx + + t.Run("cluster mode - should skip", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeCluster + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("manual mode - should skip", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + mc.Spec.Com.EnableManualMode = true + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("standalone mode - no non-standalone deployments", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, false, nil), + })) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("standalone mode - has non-standalone deployments but standalone not ready", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, false, nil), + newDeployment("mc-milvus-proxy", Proxy.Name, false, int32Ptr(1)), + })) + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, false, nil), + })) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("standalone mode - has non-standalone deployments and standalone is ready - should delete", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, false, nil), + newDeployment("mc-milvus-proxy", Proxy.Name, false, int32Ptr(1)), + newDeployment("mc-milvus-datanode", DataNode.Name, false, int32Ptr(2)), + })) + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, true, nil), + })) + + deletedDeployments := make(map[string]bool) + mockClient.EXPECT().Delete(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx interface{}, obj interface{}, opts ...interface{}) error { + deploy := obj.(*appsv1.Deployment) + deletedDeployments[deploy.Name] = true + return nil + }).Times(2) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + assert.True(t, deletedDeployments["mc-milvus-proxy"]) + assert.True(t, deletedDeployments["mc-milvus-datanode"]) + }) + + t.Run("standalone mode - standalone deployment not found yet", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-proxy", Proxy.Name, false, nil), + })) + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{})) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("standalone mode - 2 deployment mode with both ready", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone-0", MilvusStandalone.Name, false, nil), + newDeployment("mc-milvus-standalone-1", MilvusStandalone.Name, false, nil), + newDeployment("mc-milvus-proxy", Proxy.Name, false, int32Ptr(1)), + })) + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone-0", MilvusStandalone.Name, true, nil), + newDeployment("mc-milvus-standalone-1", MilvusStandalone.Name, true, nil), + })) + + mockClient.EXPECT().Delete(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx interface{}, obj interface{}, opts ...interface{}) error { + deploy := obj.(*appsv1.Deployment) + assert.Equal(t, "mc-milvus-proxy", deploy.Name) + return nil + }) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + }) + + t.Run("standalone mode - multiple deployments of same component should all be deleted", func(t *testing.T) { + mc := env.Inst.DeepCopy() + mc.Spec.Mode = v1beta1.MilvusModeStandalone + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, false, nil), + newDeployment("mc-milvus-querynode-0", QueryNode.Name, false, int32Ptr(1)), + newDeployment("mc-milvus-querynode-1", QueryNode.Name, false, int32Ptr(1)), + newDeployment("mc-milvus-datanode-0", DataNode.Name, false, int32Ptr(2)), + newDeployment("mc-milvus-datanode-1", DataNode.Name, false, int32Ptr(2)), + })) + + mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.DeploymentList{}), gomock.Any()). + DoAndReturn(mockListDeployments([]appsv1.Deployment{ + newDeployment("mc-milvus-standalone", MilvusStandalone.Name, true, nil), + })) + + deletedDeployments := make(map[string]bool) + mockClient.EXPECT().Delete(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx interface{}, obj interface{}, opts ...interface{}) error { + deploy := obj.(*appsv1.Deployment) + deletedDeployments[deploy.Name] = true + return nil + }).Times(4) + + err := r.CleanupDeploymentClusterToStandalone(ctx, *mc) + assert.NoError(t, err) + assert.True(t, deletedDeployments["mc-milvus-querynode-0"]) + assert.True(t, deletedDeployments["mc-milvus-querynode-1"]) + assert.True(t, deletedDeployments["mc-milvus-datanode-0"]) + assert.True(t, deletedDeployments["mc-milvus-datanode-1"]) + }) +} diff --git a/pkg/controllers/deployments.go b/pkg/controllers/deployments.go index 8ac48188..f9f9466a 100644 --- a/pkg/controllers/deployments.go +++ b/pkg/controllers/deployments.go @@ -293,6 +293,11 @@ func (r *MilvusReconciler) ReconcileDeployments(ctx context.Context, mc v1beta1. return fmt.Errorf("reconcile milvus deployments errs: %w", errors.Join(errs...)) } + err = r.CleanupDeploymentClusterToStandalone(ctx, mc) + if err != nil { + return err + } + err = r.cleanupIndexNodeIfNeeded(ctx, mc) if err != nil { return err From 050e3076aabe599dc0fffdaf6747f6dda19c6373 Mon Sep 17 00:00:00 2001 From: haorenfsa Date: Tue, 30 Dec 2025 17:16:50 +0800 Subject: [PATCH 2/2] fix cleanup non operator managed deployment Signed-off-by: haorenfsa --- pkg/controllers/deployment_cluster_to_standalone.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/deployment_cluster_to_standalone.go b/pkg/controllers/deployment_cluster_to_standalone.go index 9b18772e..e720cf27 100644 --- a/pkg/controllers/deployment_cluster_to_standalone.go +++ b/pkg/controllers/deployment_cluster_to_standalone.go @@ -23,10 +23,8 @@ func (r *MilvusReconciler) CleanupDeploymentClusterToStandalone(ctx context.Cont // List all deployments with the instance label deploymentList := &appsv1.DeploymentList{} opts := &client.ListOptions{ - Namespace: mc.Namespace, - LabelSelector: labels.SelectorFromSet(map[string]string{ - AppLabelInstance: mc.Name, - }), + Namespace: mc.Namespace, + LabelSelector: labels.SelectorFromSet(NewAppLabels(mc.Name)), } if err := r.List(ctx, deploymentList, opts); err != nil {