Merge pull request #197 from sofastack/delete_in_batches

delete modules in batches

Author: gold300jin

.github/workflows/module_controller_ci_build_deploy_to_aliyun.yml

@@ -176,6 +176,7 @@ jobs:
               exit 0
             else
               echo "等待字段值满足条件..."
+              echo "期望状态是: $desired_field_value, 当前状态是: $field_value"
               sleep 5  # 等待一段时间后再次检查
             fi
           done
@@ -487,4 +488,4 @@ jobs:
             fi
             sleep 5
             fi
-          done
+          done

module-controller/api/v1alpha1/moduledeployment_types.go

@@ -48,7 +48,8 @@ const (
 	ModuleDeploymentReleaseProgressPaused                 ReleaseProgress = "Paused"
 	ModuleDeploymentReleaseProgressCompleted              ReleaseProgress = "Completed"
 	ModuleDeploymentReleaseProgressAborted                ReleaseProgress = "Aborted"
-	ModuleDeploymentReleaseProgressTermed                 ReleaseProgress = "Terminated"
+	ModuleDeploymentReleaseProgressTerminating            ReleaseProgress = "Terminating"
+	ModuleDeploymentReleaseProgressTerminated             ReleaseProgress = "Terminated"
 )
 
 type ModuleUpgradeType string
@@ -120,7 +121,7 @@ type ModuleOperationStrategy struct {
 
 	BatchCount int32 `json:"batchCount,omitempty"`
 
-	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+	MaxUnavailable int32 `json:"maxUnavailable,omitempty"`
 
 	GrayTimeBetweenBatchSeconds int32 `json:"grayTimeBetweenBatchSeconds,omitempty"`
 

module-controller/api/v1alpha1/zz_generated.deepcopy.go

@@ -53,10 +53,8 @@ spec:
                     format: int32
                     type: integer
                   maxUnavailable:
-                    anyOf:
-                    - type: integer
-                    - type: string
-                    x-kubernetes-int-or-string: true
+                    format: int32
+                    type: integer
                   needConfirm:
                     type: boolean
                   serviceStrategy:

module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml

@@ -48,10 +48,8 @@ spec:
                     format: int32
                     type: integer
                   maxUnavailable:
-                    anyOf:
-                    - type: integer
-                    - type: string
-                    x-kubernetes-int-or-string: true
+                    format: int32
+                    type: integer
                   needConfirm:
                     type: boolean
                   serviceStrategy:

module-controller/config/crd/bases/serverless.alipay.com_modulereplicasets.yaml

@@ -82,9 +82,15 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	if moduleDeployment.DeletionTimestamp != nil {
-		// delete moduleDeployment
 		event.PublishModuleDeploymentDeleteEvent(r.Client, ctx, moduleDeployment)
-		return r.handleDeletingModuleDeployment(ctx, moduleDeployment)
+		if !utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) &&
+			!utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleExistedFinalizer) {
+			if moduleDeployment.Status.ReleaseStatus.Progress != v1alpha1.ModuleDeploymentReleaseProgressTerminated {
+				moduleDeployment.Status.ReleaseStatus.Progress = v1alpha1.ModuleDeploymentReleaseProgressTerminated
+				return ctrl.Result{}, r.Status().Update(ctx, moduleDeployment)
+			}
+			return ctrl.Result{}, nil
+		}
 	}
 
 	if moduleDeployment.Generation == 1 {
@@ -129,6 +135,10 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	case v1alpha1.ModuleDeploymentReleaseProgressExecuting:
 		return r.updateModuleReplicaSet(ctx, moduleDeployment, newRS)
 	case v1alpha1.ModuleDeploymentReleaseProgressCompleted:
+		if moduleDeployment.DeletionTimestamp != nil {
+			moduleDeployment.Status.ReleaseStatus.Progress = v1alpha1.ModuleDeploymentReleaseProgressTerminating
+			return ctrl.Result{}, r.Status().Update(ctx, moduleDeployment)
+		}
 		if moduleDeployment.Spec.Replicas != newRS.Spec.Replicas {
 			moduleDeployment.Status.ReleaseStatus.Progress = v1alpha1.ModuleDeploymentReleaseProgressInit
 			log.Log.Info("update release status progress to init when complete moduleDeployment", "moduleDeploymentName", moduleDeployment.Name)
@@ -161,6 +171,27 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 				return ctrl.Result{}, utils.Error(err, "update moduleDeployment progress from paused to executing failed")
 			}
 		}
+	case v1alpha1.ModuleDeploymentReleaseProgressTerminating:
+		// delete modules
+		if utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleExistedFinalizer) {
+			if moduleDeployment.Spec.Replicas != 0 {
+				moduleDeployment.Spec.Replicas = 0
+				return ctrl.Result{}, r.Update(ctx, moduleDeployment)
+			}
+			if newRS.Status.Replicas != 0 {
+				handleInitModuleDeployment(moduleDeployment, newRS)
+				return ctrl.Result{}, r.Status().Update(ctx, moduleDeployment)
+			}
+			utils.RemoveFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleExistedFinalizer)
+			return ctrl.Result{}, r.Update(ctx, moduleDeployment)
+		}
+
+		// delete module replicaset
+		if utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) {
+			return r.handleDeletingModuleDeployment(ctx, moduleDeployment)
+		}
+	case v1alpha1.ModuleDeploymentReleaseProgressTerminated:
+		return ctrl.Result{}, nil
 	}
 	return ctrl.Result{}, nil
 }
@@ -190,10 +221,6 @@ func handleInitModuleDeployment(moduleDeployment *v1alpha1.ModuleDeployment, new
 
 // handle deleting module deployment
 func (r *ModuleDeploymentReconciler) handleDeletingModuleDeployment(ctx context.Context, moduleDeployment *v1alpha1.ModuleDeployment) (ctrl.Result, error) {
-	if !utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) {
-		return ctrl.Result{}, nil
-	}
-
 	existReplicaset := true
 	set := map[string]string{
 		label.ModuleDeploymentLabel: moduleDeployment.Name,
@@ -218,6 +245,7 @@ func (r *ModuleDeploymentReconciler) handleDeletingModuleDeployment(ctx context.
 				return ctrl.Result{}, utils.Error(err, "Failed to delete moduleReplicaSet", "moduleReplicaSetName", replicaSetList.Items[i].Name)
 			}
 		}
+
 		requeueAfter := utils.GetNextReconcileTime(moduleDeployment.DeletionTimestamp.Time)
 		return ctrl.Result{RequeueAfter: requeueAfter}, nil
 	} else {
@@ -249,6 +277,7 @@ func (r *ModuleDeploymentReconciler) updateOwnerReference(ctx context.Context, m
 	})
 	moduleDeployment.SetOwnerReferences(ownerReference)
 	utils.AddFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer)
+	utils.AddFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleExistedFinalizer)
 	err = utils.UpdateResource(r.Client, ctx, moduleDeployment)
 	if err != nil {
 		return utils.Error(err, "Failed to update moduleDeployment", "moduleDeploymentName", moduleDeployment.Name)
@@ -304,6 +333,9 @@ func (r *ModuleDeploymentReconciler) createOrGetModuleReplicas(ctx context.Conte
 			log.Log.Info("module has changed, need create a new replicaset")
 		}
 
+		if moduleDeployment.DeletionTimestamp != nil {
+			return nil, nil, false, nil
+		}
 		// create a new moduleReplicaset
 		moduleReplicaSet, err := r.createNewReplicaSet(ctx, moduleDeployment, maxVersion+1)
 		if err != nil {
@@ -381,6 +413,7 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context,
 	}
 
 	err := r.updateModuleReplicas(ctx, replicas, moduleDeployment, newRS)
+
 	if err != nil {
 		return ctrl.Result{}, err
 	}