From a90d5c4715e7f23fdc1a7592e58c105f53e349ae Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Wed, 13 Mar 2024 07:57:37 +0200 Subject: [PATCH 01/33] kustomize patchesStrategicMerge is deprecated, use patches instead. (#1071) Signed-off-by: Feruzjon Muyassarov --- hack/install-kwok.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/install-kwok.sh b/hack/install-kwok.sh index ae484e9193..5530a58def 100755 --- a/hack/install-kwok.sh +++ b/hack/install-kwok.sh @@ -50,8 +50,8 @@ cat < "${BASE}/kustomization.yaml" newTag: "${KWOK_LATEST_RELEASE}" resources: - "https://github.com/${KWOK_REPO}/kustomize/kwok?ref=${KWOK_LATEST_RELEASE}" - patchesStrategicMerge: - - tolerate-all.yaml + patches: + - path: tolerate-all.yaml EOF # Define 10 different kwok controllers to handle large load From 9c6012ca33b40b793aa43e13ad40c79b8d6270c9 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 13 Mar 2024 09:55:34 -0700 Subject: [PATCH 02/33] perf: Reduce delete calls by checking DeletionTimestamp (#1095) --- pkg/controllers/node/termination/controller.go | 7 +++++-- pkg/controllers/nodeclaim/termination/controller.go | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/node/termination/controller.go b/pkg/controllers/node/termination/controller.go index d8231a0b90..5ff99bb8db 100644 --- a/pkg/controllers/node/termination/controller.go +++ b/pkg/controllers/node/termination/controller.go @@ -112,8 +112,11 @@ func (c *Controller) deleteAllNodeClaims(ctx context.Context, node *v1.Node) err return err } for i := range nodeClaimList.Items { - if err := c.kubeClient.Delete(ctx, &nodeClaimList.Items[i]); err != nil { - return client.IgnoreNotFound(err) + // If we still get the NodeClaim, but it's already marked as terminating, we don't need to call Delete again + if nodeClaimList.Items[i].DeletionTimestamp.IsZero() { + if err := c.kubeClient.Delete(ctx, &nodeClaimList.Items[i]); err != nil { + return client.IgnoreNotFound(err) + } } } return nil diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index 3d91916d2a..fbe4aa9329 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -74,9 +74,12 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) return reconcile.Result{}, err } for _, node := range nodes { - // We delete nodes to trigger the node finalization and deletion flow - if err = c.kubeClient.Delete(ctx, node); client.IgnoreNotFound(err) != nil { - return reconcile.Result{}, err + // If we still get the Node, but it's already marked as terminating, we don't need to call Delete again + if node.DeletionTimestamp.IsZero() { + // We delete nodes to trigger the node finalization and deletion flow + if err = c.kubeClient.Delete(ctx, node); client.IgnoreNotFound(err) != nil { + return reconcile.Result{}, err + } } } // We wait until all the nodes associated with this nodeClaim have completed their deletion before triggering the finalization of the nodeClaim From 680c590121925d3439a02f52eb39892d82549815 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:34:03 -0700 Subject: [PATCH 03/33] test: Expanded Static drift testing to all `NodeClaimTemplate` fields (#1094) --- .../nodeclaim/disruption/drift_test.go | 92 ++++++++++++++----- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/pkg/controllers/nodeclaim/disruption/drift_test.go b/pkg/controllers/nodeclaim/disruption/drift_test.go index c81eb29c66..cea357668b 100644 --- a/pkg/controllers/nodeclaim/disruption/drift_test.go +++ b/pkg/controllers/nodeclaim/disruption/drift_test.go @@ -17,6 +17,9 @@ limitations under the License. package disruption_test import ( + "time" + + "github.com/imdario/mergo" "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -425,20 +428,18 @@ var _ = Describe("Drift", func() { }) Context("NodePool Static Drift", func() { - var nodePoolOptions v1beta1.NodePool var nodePoolController controller.Controller BeforeEach(func() { cp.Drifted = "" nodePoolController = hash.NewController(env.Client) - nodePoolOptions = v1beta1.NodePool{ + nodePool = &v1beta1.NodePool{ ObjectMeta: nodePool.ObjectMeta, Spec: v1beta1.NodePoolSpec{ Template: v1beta1.NodeClaimTemplate{ ObjectMeta: v1beta1.ObjectMeta{ Annotations: map[string]string{ - "keyAnnotation": "valueAnnotation", - "keyAnnotation2": "valueAnnotation2", - v1beta1.NodePoolHashVersionAnnotationKey: v1beta1.NodePoolHashVersion, + "keyAnnotation": "valueAnnotation", + "keyAnnotation2": "valueAnnotation2", }, Labels: map[string]string{ "keyLabel": "valueLabel", @@ -446,6 +447,12 @@ var _ = Describe("Drift", func() { }, }, Spec: v1beta1.NodeClaimSpec{ + Requirements: nodePool.Spec.Template.Spec.Requirements, + NodeClassRef: &v1beta1.NodeClassReference{ + Kind: "fakeKind", + Name: "fakeName", + APIVersion: "fakeGroup/fakeVerion", + }, Taints: []v1.Taint{ { Key: "keyvalue1", @@ -459,7 +466,24 @@ var _ = Describe("Drift", func() { }, }, Kubelet: &v1beta1.KubeletConfiguration{ - MaxPods: ptr.Int32(10), + ClusterDNS: []string{"fakeDNS"}, + MaxPods: ptr.Int32(0), + PodsPerCore: ptr.Int32(0), + SystemReserved: v1.ResourceList{}, + KubeReserved: v1.ResourceList{}, + EvictionHard: map[string]string{ + "memory.available": "20Gi", + }, + EvictionSoft: map[string]string{ + "nodefs.available": "20Gi", + }, + EvictionMaxPodGracePeriod: ptr.Int32(0), + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "nodefs.available": {Duration: time.Second}, + }, + ImageGCHighThresholdPercent: ptr.Int32(11), + ImageGCLowThresholdPercent: ptr.Int32(0), + CPUCFSQuota: lo.ToPtr(false), }, }, }, @@ -467,30 +491,48 @@ var _ = Describe("Drift", func() { } nodeClaim.ObjectMeta.Annotations[v1beta1.NodePoolHashAnnotationKey] = nodePool.Hash() }) - It("should detect drift on changes for all static fields", func() { - ExpectApplied(ctx, env.Client, nodePool, nodeClaim) - ExpectReconcileSucceeded(ctx, nodePoolController, client.ObjectKeyFromObject(nodePool)) - ExpectReconcileSucceeded(ctx, nodeClaimDisruptionController, client.ObjectKeyFromObject(nodeClaim)) - nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) - Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Drifted)).To(BeNil()) + // We need to test each all the fields on the NodePool when we expect the field to be drifted + // This will also test that the NodePool fields can be hashed. + DescribeTable("should detect drift on changes to the static fields", + func(changes v1beta1.NodePool) { + ExpectApplied(ctx, env.Client, nodePool, nodeClaim) + ExpectReconcileSucceeded(ctx, nodePoolController, client.ObjectKeyFromObject(nodePool)) + ExpectReconcileSucceeded(ctx, nodeClaimDisruptionController, client.ObjectKeyFromObject(nodeClaim)) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Drifted)).To(BeNil()) - // Change one static field for the same nodePool - nodePoolFieldToChange := []*v1beta1.NodePool{ - test.NodePool(nodePoolOptions, v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{ObjectMeta: v1beta1.ObjectMeta{Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}}}}}), - test.NodePool(nodePoolOptions, v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{ObjectMeta: v1beta1.ObjectMeta{Labels: map[string]string{"keyLabelTest": "valueLabelTest"}}}}}), - test.NodePool(nodePoolOptions, v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Taints: []v1.Taint{{Key: "keytest2taint", Effect: v1.TaintEffectNoExecute}}}}}}), - test.NodePool(nodePoolOptions, v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{StartupTaints: []v1.Taint{{Key: "keytest2startuptaint", Effect: v1.TaintEffectNoExecute}}}}}}), - test.NodePool(nodePoolOptions, v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{MaxPods: ptr.Int32(30)}}}}}), - } + nodePool = ExpectExists(ctx, env.Client, nodePool) + Expect(mergo.Merge(nodePool, changes, mergo.WithOverride)).To(Succeed()) + ExpectApplied(ctx, env.Client, nodePool) - for _, updatedNodePool := range nodePoolFieldToChange { - ExpectApplied(ctx, env.Client, updatedNodePool) - ExpectReconcileSucceeded(ctx, nodePoolController, client.ObjectKeyFromObject(updatedNodePool)) + ExpectReconcileSucceeded(ctx, nodePoolController, client.ObjectKeyFromObject(nodePool)) ExpectReconcileSucceeded(ctx, nodeClaimDisruptionController, client.ObjectKeyFromObject(nodeClaim)) nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Drifted).IsTrue()).To(BeTrue()) - } - }) + }, + Entry("Annoations", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{ObjectMeta: v1beta1.ObjectMeta{Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}}}}}), + Entry("Labels", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{ObjectMeta: v1beta1.ObjectMeta{Labels: map[string]string{"keyLabelTest": "valueLabelTest"}}}}}), + Entry("Taints", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Taints: []v1.Taint{{Key: "keytest2taint", Effect: v1.TaintEffectNoExecute}}}}}}), + Entry("StartupTaints", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{StartupTaints: []v1.Taint{{Key: "keytest2taint", Effect: v1.TaintEffectNoExecute}}}}}}), + Entry("NodeClassRef APIVersion", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{NodeClassRef: &v1beta1.NodeClassReference{APIVersion: "testVersion"}}}}}), + Entry("NodeClassRef Name", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{NodeClassRef: &v1beta1.NodeClassReference{Name: "testName"}}}}}), + Entry("NodeClassRef Kind", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{NodeClassRef: &v1beta1.NodeClassReference{Kind: "testKind"}}}}}), + Entry("kubeletConfiguration ClusterDNS", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{ClusterDNS: []string{"testDNS"}}}}}}), + Entry("KubeletConfiguration MaxPods", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{MaxPods: ptr.Int32(5)}}}}}), + Entry("KubeletConfiguration PodsPerCore", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{PodsPerCore: ptr.Int32(5)}}}}}), + // Karpenter currently have a bug with systemReserved and kubeReserved will cause NodeClaims to not be drifted, when the field is updated + // TODO: Enable systemReserved and kubeReserved once they can be used for static drift + // For more information: https://github.com/kubernetes-sigs/karpenter/issues/1080 + // Entry("KubeletConfiguration SystemReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{SystemReserved: v1.ResourceList{}}}}}}), + // Entry("KubeletConfiguration KubeReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{KubeReserved: v1.ResourceList{}}}}}}), + Entry("KubeletConfiguration EvictionHard", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionHard: map[string]string{"memory.available": "30Gi"}}}}}}), + Entry("KubeletConfiguration EvictionSoft", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionSoft: map[string]string{"nodefs.available": "30Gi"}}}}}}), + Entry("KubeletConfiguration EvictionSoftGracePeriod", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionSoftGracePeriod: map[string]metav1.Duration{"nodefs.available": {Duration: time.Minute}}}}}}}), + Entry("KubeletConfiguration EvictionMaxPodGracePeriod", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionMaxPodGracePeriod: ptr.Int32(5)}}}}}), + Entry("KubeletConfiguration ImageGCHighThresholdPercent", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{ImageGCHighThresholdPercent: ptr.Int32(20)}}}}}), + Entry("KubeletConfiguration ImageGCLowThresholdPercent", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{ImageGCLowThresholdPercent: ptr.Int32(10)}}}}}), + Entry("KubeletConfiguration CPUCFSQuota", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{CPUCFSQuota: lo.ToPtr(true)}}}}}), + ) It("should not return drifted if karpenter.sh/nodepool-hash annotation is not present on the NodePool", func() { nodePool.ObjectMeta.Annotations = map[string]string{} ExpectApplied(ctx, env.Client, nodePool, nodeClaim) From fd3d85333f0a643d438f359a951cd18d6957c9a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 23:53:28 -0700 Subject: [PATCH 04/33] chore(deps): bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#1101) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 451f10746f..d1f3c07e04 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index c1b84fa5fb..01742e2e8f 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 2f605fed52b704ad2bea2770f650577415ffc7ca Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Thu, 14 Mar 2024 10:56:24 +0200 Subject: [PATCH 05/33] perf: fix helm namespace for delete Makefile target (#1096) Signed-off-by: Feruzjon Muyassarov --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 905e5fc8ce..f0b04b4263 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # This is the format of an AWS ECR Public Repo as an example. export KWOK_REPO ?= ${ACCOUNT_ID}.dkr.ecr.${DEFAULT_REGION}.amazonaws.com -export SYSTEM_NAMESPACE=kube-system +export KARPENTER_NAMESPACE=kube-system HELM_OPTS ?= --set logLevel=debug \ --set controller.resources.requests.cpu=1 \ @@ -31,7 +31,7 @@ build: ## Build the Karpenter KWOK controller images using ko build apply: verify build ## Deploy the kwok controller from the current state of your git repository into your ~/.kube/config cluster hack/validation/kwok-requirements.sh kubectl apply -f pkg/apis/crds - helm upgrade --install karpenter kwok/charts --namespace kube-system --skip-crds \ + helm upgrade --install karpenter kwok/charts --namespace $(KARPENTER_NAMESPACE) --skip-crds \ $(HELM_OPTS) \ --set controller.image.repository=$(IMG_REPOSITORY) \ --set controller.image.tag=$(IMG_TAG) \ @@ -40,7 +40,7 @@ apply: verify build ## Deploy the kwok controller from the current state of your --set-string controller.env[0].value=true delete: ## Delete the controller from your ~/.kube/config cluster - helm uninstall karpenter --namespace ${KARPENTER_NAMESPACE} + helm uninstall karpenter --namespace $(KARPENTER_NAMESPACE) test: ## Run tests go test ./... \ From c6b8b811c3d0211b5b2b1216c2509d382171b4b1 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Thu, 14 Mar 2024 09:38:53 -0700 Subject: [PATCH 06/33] fix: Check node readiness before force terminating (#1099) --- pkg/controllers/disruption/suite_test.go | 8 +- .../node/termination/controller.go | 17 +++-- .../node/termination/suite_test.go | 34 +++++++++ .../nodeclaim/garbagecollection/controller.go | 21 ++++- .../nodeclaim/garbagecollection/suite_test.go | 76 ++++++++++++++++--- .../provisioning/scheduling/suite_test.go | 7 +- pkg/test/expectations/expectations.go | 25 ++++-- 7 files changed, 158 insertions(+), 30 deletions(-) diff --git a/pkg/controllers/disruption/suite_test.go b/pkg/controllers/disruption/suite_test.go index d56cd90a81..7feb72de1f 100644 --- a/pkg/controllers/disruption/suite_test.go +++ b/pkg/controllers/disruption/suite_test.go @@ -319,7 +319,7 @@ var _ = Describe("Simulate Scheduling", func() { }) Expect(new).To(BeTrue()) // which needs to be deployed - ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + ExpectNodeClaimDeployedAndStateUpdated(ctx, env.Client, cluster, cloudProvider, nc) nodeClaimNames[nc.Name] = struct{}{} ExpectTriggerVerifyAction(&wg) @@ -334,7 +334,7 @@ var _ = Describe("Simulate Scheduling", func() { return !ok }) Expect(new).To(BeTrue()) - ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + ExpectNodeClaimDeployedAndStateUpdated(ctx, env.Client, cluster, cloudProvider, nc) nodeClaimNames[nc.Name] = struct{}{} ExpectTriggerVerifyAction(&wg) @@ -349,7 +349,7 @@ var _ = Describe("Simulate Scheduling", func() { return !ok }) Expect(new).To(BeTrue()) - ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + ExpectNodeClaimDeployedAndStateUpdated(ctx, env.Client, cluster, cloudProvider, nc) nodeClaimNames[nc.Name] = struct{}{} // Try one more time, but fail since the budgets only allow 3 disruptions. @@ -1976,7 +1976,7 @@ func ExpectMakeNewNodeClaimsReady(ctx context.Context, c client.Client, wg *sync if existingNodeClaimNames.Has(nc.Name) { continue } - nc, n := ExpectNodeClaimDeployed(ctx, c, cluster, cloudProvider, nc) + nc, n := ExpectNodeClaimDeployedAndStateUpdated(ctx, c, cluster, cloudProvider, nc) ExpectMakeNodeClaimsInitialized(ctx, c, nc) ExpectMakeNodesInitialized(ctx, c, n) diff --git a/pkg/controllers/node/termination/controller.go b/pkg/controllers/node/termination/controller.go index 5ff99bb8db..67dfbcf673 100644 --- a/pkg/controllers/node/termination/controller.go +++ b/pkg/controllers/node/termination/controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/karpenter/pkg/metrics" operatorcontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) @@ -88,12 +89,18 @@ func (c *Controller) Finalize(ctx context.Context, node *v1.Node) (reconcile.Res return reconcile.Result{}, fmt.Errorf("draining node, %w", err) } c.recorder.Publish(terminatorevents.NodeFailedToDrain(node, err)) - // If the underlying nodeclaim no longer exists. - if _, err := c.cloudProvider.Get(ctx, node.Spec.ProviderID); err != nil { - if cloudprovider.IsNodeClaimNotFoundError(err) { - return reconcile.Result{}, c.removeFinalizer(ctx, node) + // If the underlying NodeClaim no longer exists, we want to delete to avoid trying to gracefully draining + // on nodes that are no longer alive. We do a check on the Ready condition of the node since, even + // though the CloudProvider says the instance is not around, we know that the kubelet process is still running + // if the Node Ready condition is true + // Similar logic to: https://github.com/kubernetes/kubernetes/blob/3a75a8c8d9e6a1ebd98d8572132e675d4980f184/staging/src/k8s.io/cloud-provider/controllers/nodelifecycle/node_lifecycle_controller.go#L144 + if nodeutils.GetCondition(node, v1.NodeReady).Status != v1.ConditionTrue { + if _, err := c.cloudProvider.Get(ctx, node.Spec.ProviderID); err != nil { + if cloudprovider.IsNodeClaimNotFoundError(err) { + return reconcile.Result{}, c.removeFinalizer(ctx, node) + } + return reconcile.Result{}, fmt.Errorf("getting nodeclaim, %w", err) } - return reconcile.Result{}, fmt.Errorf("getting nodeclaim, %w", err) } return reconcile.Result{RequeueAfter: 1 * time.Second}, nil } diff --git a/pkg/controllers/node/termination/suite_test.go b/pkg/controllers/node/termination/suite_test.go index 1223366b24..d473622bb4 100644 --- a/pkg/controllers/node/termination/suite_test.go +++ b/pkg/controllers/node/termination/suite_test.go @@ -539,6 +539,9 @@ var _ = Describe("Termination", func() { pods := test.Pods(2, test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) ExpectApplied(ctx, env.Client, node, pods[0], pods[1]) + // Make Node NotReady since it's automatically marked as Ready on first deploy + ExpectMakeNodesNotReady(ctx, env.Client, node) + // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) @@ -565,6 +568,37 @@ var _ = Describe("Termination", func() { ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(node)) ExpectNotFound(ctx, env.Client, node) }) + It("should not delete nodes with no underlying instance if the node is still Ready", func() { + pods := test.Pods(2, test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) + ExpectApplied(ctx, env.Client, node, pods[0], pods[1]) + + // Trigger Termination Controller + Expect(env.Client.Delete(ctx, node)).To(Succeed()) + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(node)) + ExpectReconcileSucceeded(ctx, queue, client.ObjectKey{}) + ExpectReconcileSucceeded(ctx, queue, client.ObjectKey{}) + + // Expect the pods to be evicted + EventuallyExpectTerminating(ctx, env.Client, pods[0], pods[1]) + + // Expect node to exist and be draining, but not deleted + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(node)) + ExpectNodeWithNodeClaimDraining(env.Client, node.Name) + + // After this, the node still has one pod that is evicting. + ExpectDeleted(ctx, env.Client, pods[1]) + + // Remove the node from created nodeclaims so that the cloud provider returns DNE + cloudProvider.CreatedNodeClaims = map[string]*v1beta1.NodeClaim{} + + // Reconcile to try to delete the node, but don't succeed because the readiness condition + // of the node still won't let us delete it + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, terminationController, client.ObjectKeyFromObject(node)) + ExpectNodeExists(ctx, env.Client, node.Name) + }) It("should wait for pods to terminate", func() { pod := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) fakeClock.SetTime(time.Now()) // make our fake clock match the pod creation time diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index 4e5cbebf23..e1d13a1026 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/samber/lo" "go.uber.org/multierr" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" @@ -31,6 +32,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" + nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/metrics" @@ -70,15 +74,28 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc cloudProviderProviderIDs := sets.New[string](lo.Map(cloudProviderNodeClaims, func(nc *v1beta1.NodeClaim, _ int) string { return nc.Status.ProviderID })...) + // Only consider NodeClaims that are Registered since we don't want to fully rely on the CloudProvider + // API to trigger deletion of the Node. Instead, we'll wait for our registration timeout to trigger nodeClaims := lo.Filter(lo.ToSlicePtr(nodeClaimList.Items), func(n *v1beta1.NodeClaim, _ int) bool { - return n.StatusConditions().GetCondition(v1beta1.Launched).IsTrue() && + return n.StatusConditions().GetCondition(v1beta1.Registered).IsTrue() && n.DeletionTimestamp.IsZero() && - c.clock.Since(n.StatusConditions().GetCondition(v1beta1.Launched).LastTransitionTime.Inner.Time) > time.Second*10 && !cloudProviderProviderIDs.Has(n.Status.ProviderID) }) errs := make([]error, len(nodeClaims)) workqueue.ParallelizeUntil(ctx, 20, len(nodeClaims), func(i int) { + node, err := nodeclaimutil.NodeForNodeClaim(ctx, c.kubeClient, nodeClaims[i]) + // Ignore these errors since a registered NodeClaim should only have a NotFound node when + // the Node was deleted out from under us and a Duplicate Node is an invalid state + if nodeclaimutil.IgnoreDuplicateNodeError(nodeclaimutil.IgnoreNodeNotFoundError(err)) != nil { + errs[i] = err + } + // We do a check on the Ready condition of the node since, even though the CloudProvider says the instance + // is not around, we know that the kubelet process is still running if the Node Ready condition is true + // Similar logic to: https://github.com/kubernetes/kubernetes/blob/3a75a8c8d9e6a1ebd98d8572132e675d4980f184/staging/src/k8s.io/cloud-provider/controllers/nodelifecycle/node_lifecycle_controller.go#L144 + if node != nil && nodeutils.GetCondition(node, v1.NodeReady).Status == v1.ConditionTrue { + return + } if err := c.kubeClient.Delete(ctx, nodeClaims[i]); err != nil { errs[i] = client.IgnoreNotFound(err) return diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index f3f792a756..d2d026a333 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -90,7 +90,7 @@ var _ = Describe("GarbageCollection", func() { BeforeEach(func() { nodePool = test.NodePool() }) - It("should delete the NodeClaim when the Node never appears and the instance is gone", func() { + It("should delete the NodeClaim when the Node is there in a NotReady state and the instance is gone", func() { nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -99,8 +99,12 @@ var _ = Describe("GarbageCollection", func() { }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClaim) - ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaim)) - nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + + nodeClaim, node, err := ExpectNodeClaimDeployed(ctx, env.Client, cloudProvider, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + + // Mark the node as NotReady after the launch + ExpectMakeNodesNotReady(ctx, env.Client, node) // Step forward to move past the cache eventual consistency timeout fakeClock.SetTime(time.Now().Add(time.Second * 20)) @@ -108,12 +112,36 @@ var _ = Describe("GarbageCollection", func() { // Delete the nodeClaim from the cloudprovider Expect(cloudProvider.Delete(ctx, nodeClaim)).To(Succeed()) - // Expect the NodeClaim to be removed now that the Instance is gone + // Expect the NodeClaim to not be removed since there is a Node that exists that has a Ready "true" condition ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) ExpectFinalizersRemoved(ctx, env.Client, nodeClaim) ExpectNotFound(ctx, env.Client, nodeClaim) }) - It("should delete many NodeClaims when the Node never appears and the instance is gone", func() { + It("shouldn't delete the NodeClaim when the Node is there in a Ready state and the instance is gone", func() { + nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, nodeClaim) + + nodeClaim, _, err := ExpectNodeClaimDeployed(ctx, env.Client, cloudProvider, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + + // Step forward to move past the cache eventual consistency timeout + fakeClock.SetTime(time.Now().Add(time.Second * 20)) + + // Delete the nodeClaim from the cloudprovider + Expect(cloudProvider.Delete(ctx, nodeClaim)).To(Succeed()) + + // Expect the NodeClaim to not be removed since there is a Node that exists that has a Ready "true" condition + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectFinalizersRemoved(ctx, env.Client, nodeClaim) + ExpectExists(ctx, env.Client, nodeClaim) + }) + It("should delete many NodeClaims when the Nodes are there in a NotReady state and the instances are gone", func() { var nodeClaims []*v1beta1.NodeClaim for i := 0; i < 100; i++ { nodeClaims = append(nodeClaims, test.NodeClaim(v1beta1.NodeClaim{ @@ -128,8 +156,13 @@ var _ = Describe("GarbageCollection", func() { workqueue.ParallelizeUntil(ctx, len(nodeClaims), len(nodeClaims), func(i int) { defer GinkgoRecover() ExpectApplied(ctx, env.Client, nodeClaims[i]) - ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaims[i])) - nodeClaims[i] = ExpectExists(ctx, env.Client, nodeClaims[i]) + var node *v1.Node + var err error + nodeClaims[i], node, err = ExpectNodeClaimDeployed(ctx, env.Client, cloudProvider, nodeClaims[i]) + Expect(err).ToNot(HaveOccurred()) + + // Mark the node as NotReady after the launch + ExpectMakeNodesNotReady(ctx, env.Client, node) }) // Step forward to move past the cache eventual consistency timeout @@ -150,6 +183,29 @@ var _ = Describe("GarbageCollection", func() { }) ExpectNotFound(ctx, env.Client, lo.Map(nodeClaims, func(n *v1beta1.NodeClaim, _ int) client.Object { return n })...) }) + It("shouldn't delete the NodeClaim when the Node isn't there and the instance is gone", func() { + nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, nodeClaim) + nodeClaim, err := ExpectNodeClaimDeployedNoNode(ctx, env.Client, cloudProvider, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + + // Step forward to move past the cache eventual consistency timeout + fakeClock.SetTime(time.Now().Add(time.Second * 20)) + + // Delete the nodeClaim from the cloudprovider + Expect(cloudProvider.Delete(ctx, nodeClaim)).To(Succeed()) + + // Expect the NodeClaim to not be removed since the NodeClaim isn't registered + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectFinalizersRemoved(ctx, env.Client, nodeClaim) + ExpectExists(ctx, env.Client, nodeClaim) + }) It("shouldn't delete the NodeClaim when the Node isn't there but the instance is there", func() { nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -159,8 +215,10 @@ var _ = Describe("GarbageCollection", func() { }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClaim) - ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaim)) - nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + nodeClaim, node, err := ExpectNodeClaimDeployed(ctx, env.Client, cloudProvider, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + + Expect(env.Client.Delete(ctx, node)).To(Succeed()) // Step forward to move past the cache eventual consistency timeout fakeClock.SetTime(time.Now().Add(time.Second * 20)) diff --git a/pkg/controllers/provisioning/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go index 4c7953cdb6..f9e391a381 100644 --- a/pkg/controllers/provisioning/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -2387,10 +2387,11 @@ var _ = Context("Scheduling", func() { }) ExpectApplied(ctx, env.Client, nc) if i == elem { - nc, node = ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + nc, node = ExpectNodeClaimDeployedAndStateUpdated(ctx, env.Client, cluster, cloudProvider, nc) } else { var err error - nc, err = ExpectNodeClaimDeployedNoNode(ctx, env.Client, cluster, cloudProvider, nc) + nc, err = ExpectNodeClaimDeployedNoNode(ctx, env.Client, cloudProvider, nc) + cluster.UpdateNodeClaim(nc) Expect(err).ToNot(HaveOccurred()) } nodeClaims = append(nodeClaims, nc) @@ -2481,7 +2482,7 @@ var _ = Context("Scheduling", func() { }, }) ExpectApplied(ctx, env.Client, nc) - nc, n := ExpectNodeClaimDeployed(ctx, env.Client, cluster, cloudProvider, nc) + nc, n := ExpectNodeClaimDeployedAndStateUpdated(ctx, env.Client, cluster, cloudProvider, nc) nodeClaims = append(nodeClaims, nc) nodes = append(nodes, n) } diff --git a/pkg/test/expectations/expectations.go b/pkg/test/expectations/expectations.go index d11c1aa18b..8b1d4ef28a 100644 --- a/pkg/test/expectations/expectations.go +++ b/pkg/test/expectations/expectations.go @@ -275,7 +275,7 @@ func ExpectProvisionedNoBinding(ctx context.Context, c client.Client, cluster *s } nodeClaim := &v1beta1.NodeClaim{} Expect(c.Get(ctx, types.NamespacedName{Name: nodeClaimName}, nodeClaim)).To(Succeed()) - nodeClaim, node := ExpectNodeClaimDeployed(ctx, c, cluster, cloudProvider, nodeClaim) + nodeClaim, node := ExpectNodeClaimDeployedAndStateUpdated(ctx, c, cluster, cloudProvider, nodeClaim) if nodeClaim != nil && node != nil { for _, pod := range m.Pods { bindings[pod] = &Binding{ @@ -298,8 +298,9 @@ func ExpectProvisionedNoBinding(ctx context.Context, c client.Client, cluster *s return bindings } -func ExpectNodeClaimDeployedNoNode(ctx context.Context, c client.Client, cluster *state.Cluster, cloudProvider cloudprovider.CloudProvider, nc *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) { +func ExpectNodeClaimDeployedNoNode(ctx context.Context, c client.Client, cloudProvider cloudprovider.CloudProvider, nc *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) { GinkgoHelper() + resolved, err := cloudProvider.Create(ctx, nc) // TODO @joinnis: Check this error rather than swallowing it. This is swallowed right now due to how we are doing some testing in the cloudprovider if err != nil { @@ -311,15 +312,15 @@ func ExpectNodeClaimDeployedNoNode(ctx context.Context, c client.Client, cluster nc = lifecycle.PopulateNodeClaimDetails(nc, resolved) nc.StatusConditions().MarkTrue(v1beta1.Launched) ExpectApplied(ctx, c, nc) - cluster.UpdateNodeClaim(nc) return nc, nil } -func ExpectNodeClaimDeployed(ctx context.Context, c client.Client, cluster *state.Cluster, cloudProvider cloudprovider.CloudProvider, nc *v1beta1.NodeClaim) (*v1beta1.NodeClaim, *v1.Node) { +func ExpectNodeClaimDeployed(ctx context.Context, c client.Client, cloudProvider cloudprovider.CloudProvider, nc *v1beta1.NodeClaim) (*v1beta1.NodeClaim, *v1.Node, error) { GinkgoHelper() - nc, err := ExpectNodeClaimDeployedNoNode(ctx, c, cluster, cloudProvider, nc) + + nc, err := ExpectNodeClaimDeployedNoNode(ctx, c, cloudProvider, nc) if err != nil { - return nc, nil + return nc, nil, err } nc.StatusConditions().MarkTrue(v1beta1.Registered) @@ -327,8 +328,18 @@ func ExpectNodeClaimDeployed(ctx context.Context, c client.Client, cluster *stat node := test.NodeClaimLinkedNode(nc) node.Labels = lo.Assign(node.Labels, map[string]string{v1beta1.NodeRegisteredLabelKey: "true"}) ExpectApplied(ctx, c, nc, node) - Expect(cluster.UpdateNode(ctx, node)).To(Succeed()) + return nc, node, nil +} + +func ExpectNodeClaimDeployedAndStateUpdated(ctx context.Context, c client.Client, cluster *state.Cluster, cloudProvider cloudprovider.CloudProvider, nc *v1beta1.NodeClaim) (*v1beta1.NodeClaim, *v1.Node) { + GinkgoHelper() + + nc, node, err := ExpectNodeClaimDeployed(ctx, c, cloudProvider, nc) cluster.UpdateNodeClaim(nc) + if err != nil { + return nc, nil + } + Expect(cluster.UpdateNode(ctx, node)).To(Succeed()) return nc, node } From 8ab7b7522c13788d5f8721a3143bcfd9fdcec8a8 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 15 Mar 2024 10:03:45 -0700 Subject: [PATCH 07/33] chore: Drop CloudProvider delete call from Node termination (#1103) --- .../node/termination/controller.go | 4 --- pkg/utils/nodeclaim/nodeclaim.go | 33 ------------------- pkg/utils/nodeclaim/suite_test.go | 21 ------------ 3 files changed, 58 deletions(-) diff --git a/pkg/controllers/node/termination/controller.go b/pkg/controllers/node/termination/controller.go index 67dfbcf673..6b8fdf3979 100644 --- a/pkg/controllers/node/termination/controller.go +++ b/pkg/controllers/node/termination/controller.go @@ -42,7 +42,6 @@ import ( "sigs.k8s.io/karpenter/pkg/metrics" operatorcontroller "sigs.k8s.io/karpenter/pkg/operator/controller" nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" - nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) var _ operatorcontroller.FinalizingTypedController[*v1.Node] = (*Controller)(nil) @@ -104,9 +103,6 @@ func (c *Controller) Finalize(ctx context.Context, node *v1.Node) (reconcile.Res } return reconcile.Result{RequeueAfter: 1 * time.Second}, nil } - if err := c.cloudProvider.Delete(ctx, nodeclaimutil.NewFromNode(node)); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil { - return reconcile.Result{}, fmt.Errorf("terminating cloudprovider instance, %w", err) - } if err := c.removeFinalizer(ctx, node); err != nil { return reconcile.Result{}, err } diff --git a/pkg/utils/nodeclaim/nodeclaim.go b/pkg/utils/nodeclaim/nodeclaim.go index 78d0c2d944..42c7a7a279 100644 --- a/pkg/utils/nodeclaim/nodeclaim.go +++ b/pkg/utils/nodeclaim/nodeclaim.go @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/scheduling" ) // PodEventHandler is a watcher on v1.Pods that maps Pods to NodeClaim based on the node names @@ -170,38 +169,6 @@ func AllNodesForNodeClaim(ctx context.Context, c client.Client, nodeClaim *v1bet return lo.ToSlicePtr(nodeList.Items), nil } -// NewFromNode converts a node into a pseudo-NodeClaim using known values from the node -// Deprecated: This NodeClaim generator function can be removed when v1beta1 migration has completed. -func NewFromNode(node *v1.Node) *v1beta1.NodeClaim { - nc := &v1beta1.NodeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - Annotations: node.Annotations, - Labels: node.Labels, - Finalizers: []string{v1beta1.TerminationFinalizer}, - }, - Spec: v1beta1.NodeClaimSpec{ - Taints: node.Spec.Taints, - Requirements: scheduling.NewLabelRequirements(node.Labels).NodeSelectorRequirements(), - Resources: v1beta1.ResourceRequirements{ - Requests: node.Status.Allocatable, - }, - }, - Status: v1beta1.NodeClaimStatus{ - NodeName: node.Name, - ProviderID: node.Spec.ProviderID, - Capacity: node.Status.Capacity, - Allocatable: node.Status.Allocatable, - }, - } - if _, ok := node.Labels[v1beta1.NodeInitializedLabelKey]; ok { - nc.StatusConditions().MarkTrue(v1beta1.Initialized) - } - nc.StatusConditions().MarkTrue(v1beta1.Launched) - nc.StatusConditions().MarkTrue(v1beta1.Registered) - return nc -} - func UpdateNodeOwnerReferences(nodeClaim *v1beta1.NodeClaim, node *v1.Node) *v1.Node { node.OwnerReferences = append(node.OwnerReferences, metav1.OwnerReference{ APIVersion: v1beta1.SchemeGroupVersion.String(), diff --git a/pkg/utils/nodeclaim/suite_test.go b/pkg/utils/nodeclaim/suite_test.go index b8087a676b..f032790b01 100644 --- a/pkg/utils/nodeclaim/suite_test.go +++ b/pkg/utils/nodeclaim/suite_test.go @@ -34,7 +34,6 @@ import ( . "sigs.k8s.io/karpenter/pkg/test/expectations" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/test" nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) @@ -111,26 +110,6 @@ var _ = Describe("NodeClaimUtils", func() { }, }) }) - It("should convert a Node to a NodeClaim", func() { - nodeClaim := nodeclaimutil.NewFromNode(node) - for k, v := range node.Annotations { - Expect(nodeClaim.Annotations).To(HaveKeyWithValue(k, v)) - } - for k, v := range node.Labels { - Expect(nodeClaim.Labels).To(HaveKeyWithValue(k, v)) - } - Expect(lo.Contains(nodeClaim.Finalizers, v1beta1.TerminationFinalizer)).To(BeTrue()) - Expect(nodeClaim.Spec.Taints).To(Equal(node.Spec.Taints)) - Expect(nodeClaim.Spec.Requirements).To(ContainElements(scheduling.NewLabelRequirements(node.Labels).NodeSelectorRequirements())) - Expect(nodeClaim.Spec.Resources.Requests).To(Equal(node.Status.Allocatable)) - Expect(nodeClaim.Status.NodeName).To(Equal(node.Name)) - Expect(nodeClaim.Status.ProviderID).To(Equal(node.Spec.ProviderID)) - Expect(nodeClaim.Status.Capacity).To(Equal(node.Status.Capacity)) - Expect(nodeClaim.Status.Allocatable).To(Equal(node.Status.Allocatable)) - Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Launched).IsTrue()).To(BeTrue()) - Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Registered).IsTrue()).To(BeTrue()) - Expect(nodeClaim.StatusConditions().GetCondition(v1beta1.Initialized).IsTrue()).To(BeTrue()) - }) It("should update the owner for a Node to a NodeClaim", func() { nodeClaim := test.NodeClaim(v1beta1.NodeClaim{ Spec: v1beta1.NodeClaimSpec{ From 388f7422051c3fbf7d3c7fdb4e3db82ce6dcba9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:18:50 -0700 Subject: [PATCH 08/33] chore(deps): bump the k8s-go-deps group with 7 updates (#1109) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 34 ++++++++++++++++------------------ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index d1f3c07e04..1287fb4a7d 100644 --- a/go.mod +++ b/go.mod @@ -21,13 +21,13 @@ require ( golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 - k8s.io/api v0.29.2 - k8s.io/apiextensions-apiserver v0.29.2 - k8s.io/apimachinery v0.29.2 - k8s.io/client-go v0.29.2 - k8s.io/cloud-provider v0.29.2 - k8s.io/component-base v0.29.2 - k8s.io/csi-translation-lib v0.29.2 + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + k8s.io/cloud-provider v0.29.3 + k8s.io/component-base v0.29.3 + k8s.io/csi-translation-lib v0.29.3 k8s.io/klog/v2 v2.120.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 @@ -56,7 +56,7 @@ require ( github.com/gobuffalo/flect v0.2.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/go.sum b/go.sum index 01742e2e8f..4a84fa85c8 100644 --- a/go.sum +++ b/go.sum @@ -146,9 +146,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -623,7 +622,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -654,20 +652,20 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= -k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= -k8s.io/cloud-provider v0.29.2 h1:ghKNXoQmeP8Fj/YTJNR6xQOzNrKXt6YZyy6mOEEa3yg= -k8s.io/cloud-provider v0.29.2/go.mod h1:KAp+07AUGmxcLnoLY5FndU4hj6158KMbiviNgctNRUk= -k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= -k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= -k8s.io/csi-translation-lib v0.29.2 h1:TJVZTzR7gj6+HSb+jJxLUxnAuwrEy71IxhJ4nmTzyjE= -k8s.io/csi-translation-lib v0.29.2/go.mod h1:vbSYY4c6mVPwTHAvb5V3CHlq/dmQFIZC1SJOsaFiY3I= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= +k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= +k8s.io/csi-translation-lib v0.29.3 h1:GNYCE0f86K3Xkyrk7WKKwQZkJrum6QQapbOzYxZv6Mg= +k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= From e1e91920a47feff3953b987ae8e664d8f0978b5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:21:18 -0700 Subject: [PATCH 09/33] chore(deps): bump the go-deps group with 2 updates (#1110) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 1287fb4a7d..61d2847a01 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.16.0 - github.com/onsi/gomega v1.31.1 + github.com/onsi/ginkgo/v2 v2.17.0 + github.com/onsi/gomega v1.32.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.0 diff --git a/go.sum b/go.sum index 4a84fa85c8..3c6bba7f05 100644 --- a/go.sum +++ b/go.sum @@ -241,10 +241,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= -github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= +github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 5d38c96c6b9586a8e5f15bb4e4fb787714c46e48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:02:51 -0700 Subject: [PATCH 10/33] chore(deps): bump the actions-deps group with 1 update (#1111) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yaml | 4 ++-- .github/workflows/presubmit.yaml | 2 +- .github/workflows/release.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index 7ef5929c9e..6d752f9b53 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -17,7 +17,7 @@ jobs: actions: read # github/codeql-action/init@v2 security-events: write # github/codeql-action/init@v2 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps - run: make vulncheck - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8 @@ -34,7 +34,7 @@ jobs: actions: read # github/codeql-action/init@v2 security-events: write # github/codeql-action/init@v2 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8 with: languages: javascript diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index e40ac0249c..f1024f2d9c 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -15,7 +15,7 @@ jobs: matrix: k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x"] steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps with: k8sVersion: ${{ matrix.k8sVersion }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 62edbb0875..d0d7f7ae0d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,7 +13,7 @@ jobs: id-token: write # Needed for cosigning build attestation files with tejolote runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 From 1a20b287f32771539ef8d7221e667f45a90404fb Mon Sep 17 00:00:00 2001 From: Bryce Soghigian <49734722+Bryce-Soghigian@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:13:08 -0700 Subject: [PATCH 11/33] docs: adding supported cloud providers to readme (#1112) --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eb1fbadb83..272c409c4e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ Karpenter improves the efficiency and cost of running workloads on Kubernetes cl * **Provisioning** nodes that meet the requirements of the pods * **Removing** the nodes when the nodes are no longer needed +## Supported CloudProviders +Karpenter is a multi-cloud project supported on the following cloud providers +- [AWS](https://github.com/aws/karpenter-provider-aws) +- [Azure](https://github.com/Azure/karpenter-provider-azure) + ## Community, discussion, contribution, and support If you have any questions or want to get the latest project news, you can connect with us in the following ways: @@ -43,4 +48,4 @@ Participation in the Kubernetes community is governed by the [Kubernetes Code of - 11/30/2021 [Karpenter vs Kubernetes Cluster Autoscaler](https://youtu.be/3QsVRHVdOnM) - 11/19/2021 [Karpenter @ Container Day](https://youtu.be/qxWJRUF6JJc) - 05/14/2021 [Groupless Autoscaling with Karpenter @ Kubecon](https://www.youtube.com/watch?v=43g8uPohTgc) -- 05/04/2021 [Karpenter @ Container Day](https://youtu.be/MZ-4HzOC_ac?t=7137) \ No newline at end of file +- 05/04/2021 [Karpenter @ Container Day](https://youtu.be/MZ-4HzOC_ac?t=7137) From 5bc07be72a1acd553a2a692edd27e79c20e0e1c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:15:27 -0700 Subject: [PATCH 12/33] chore(deps): bump github.com/docker/docker from 25.0.4+incompatible to 25.0.5+incompatible (#1122) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 61d2847a01..f3a789f54b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/avast/retry-go v3.0.0+incompatible - github.com/docker/docker v25.0.4+incompatible + github.com/docker/docker v25.0.5+incompatible github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 diff --git a/go.sum b/go.sum index 3c6bba7f05..ac54aae9e5 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/docker v25.0.4+incompatible h1:XITZTrq+52tZyZxUOtFIahUf3aH367FLxJzt9vZeAF8= -github.com/docker/docker v25.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= +github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= From d1d94a3015a69311393eb0643b5bd8a6b5563d59 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:30:48 -0700 Subject: [PATCH 13/33] fix: Adjust duration validation to include format for `1h10m` (#1125) --- pkg/apis/crds/karpenter.sh_nodepools.yaml | 2 +- pkg/apis/v1beta1/nodepool.go | 2 +- pkg/apis/v1beta1/nodepool_validation.go | 7 +++++++ pkg/apis/v1beta1/nodepool_validation_cel_test.go | 10 +++++++++- pkg/apis/v1beta1/nodepool_validation_webhook_test.go | 10 +++++++++- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index b1abb94bc7..7d7cf0d60a 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -80,7 +80,7 @@ spec: This is required if Schedule is set. This regex has an optional 0s at the end since the duration.String() always adds a 0s at the end. - pattern: ^([0-9]+(m|h)+(0s)?)$ + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ type: string nodes: default: 10% diff --git a/pkg/apis/v1beta1/nodepool.go b/pkg/apis/v1beta1/nodepool.go index 796190439a..79109d586d 100644 --- a/pkg/apis/v1beta1/nodepool.go +++ b/pkg/apis/v1beta1/nodepool.go @@ -124,7 +124,7 @@ type Budget struct { // This is required if Schedule is set. // This regex has an optional 0s at the end since the duration.String() always adds // a 0s at the end. - // +kubebuilder:validation:Pattern=`^([0-9]+(m|h)+(0s)?)$` + // +kubebuilder:validation:Pattern=`^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$` // +kubebuilder:validation:Type="string" // +optional Duration *metav1.Duration `json:"duration,omitempty" hash:"ignore"` diff --git a/pkg/apis/v1beta1/nodepool_validation.go b/pkg/apis/v1beta1/nodepool_validation.go index 66161bcc16..37637ca3e2 100644 --- a/pkg/apis/v1beta1/nodepool_validation.go +++ b/pkg/apis/v1beta1/nodepool_validation.go @@ -19,6 +19,7 @@ package v1beta1 import ( "context" "fmt" + "regexp" "github.com/robfig/cron/v3" "github.com/samber/lo" @@ -122,5 +123,11 @@ func (in *Budget) validate() (errs *apis.FieldError) { return apis.ErrInvalidValue(in.Schedule, "schedule", fmt.Sprintf("invalid schedule %s", err)) } } + if in.Duration != nil { + goodDuration, err := regexp.Match("^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$", []byte(in.Duration.Duration.String())) + if err != nil || !goodDuration { + return apis.ErrInvalidValue(in.Schedule, "duration", fmt.Sprintf("invalid duration %s", err)) + } + } return errs } diff --git a/pkg/apis/v1beta1/nodepool_validation_cel_test.go b/pkg/apis/v1beta1/nodepool_validation_cel_test.go index 263146284a..79c5e7e14a 100644 --- a/pkg/apis/v1beta1/nodepool_validation_cel_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_cel_test.go @@ -165,7 +165,7 @@ var _ = Describe("CEL/Validation", func() { It("should fail when creating a budget with a duration but no cron", func() { nodePool.Spec.Disruption.Budgets = []Budget{{ Nodes: "10", - Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("-20m"))}, + Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("20m"))}, }} Expect(env.Client.Create(ctx, nodePool)).ToNot(Succeed()) }) @@ -177,6 +177,14 @@ var _ = Describe("CEL/Validation", func() { }} Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) }) + It("should succeed when creating a budget with hours and minutes in duration", func() { + nodePool.Spec.Disruption.Budgets = []Budget{{ + Nodes: "10", + Schedule: ptr.String("* * * * *"), + Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("2h20m"))}, + }} + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + }) It("should succeed when creating a budget with neither duration nor cron", func() { nodePool.Spec.Disruption.Budgets = []Budget{{ Nodes: "10", diff --git a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go index a4b024483e..b46e7cdcd0 100644 --- a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go @@ -105,7 +105,7 @@ var _ = Describe("Webhook/Validation", func() { It("should fail to validate a budget with a duration but no cron", func() { nodePool.Spec.Disruption.Budgets = []Budget{{ Nodes: "10", - Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("-20m"))}, + Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("20m"))}, }} Expect(nodePool.Validate(ctx)).ToNot(Succeed()) }) @@ -131,6 +131,14 @@ var _ = Describe("Webhook/Validation", func() { }} Expect(nodePool.Validate(ctx)).To(Succeed()) }) + It("should succeed when creating a budget with hours and minutes in duration", func() { + nodePool.Spec.Disruption.Budgets = []Budget{{ + Nodes: "10", + Schedule: ptr.String("* * * * *"), + Duration: &metav1.Duration{Duration: lo.Must(time.ParseDuration("2h20m"))}, + }} + Expect(nodePool.Validate(ctx)).To(Succeed()) + }) It("should fail when creating two budgets where one has an invalid crontab", func() { nodePool.Spec.Disruption.Budgets = []Budget{ { From 066cc4cb70fc09031cee4c54f9acca5c557db0d7 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sat, 23 Mar 2024 00:23:23 +0100 Subject: [PATCH 14/33] chore: Remove finalizer with `Update()` instead of `Patch()` (#1126) --- pkg/controllers/nodeclaim/termination/controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index fbe4aa9329..2b601fe0c0 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -24,6 +24,7 @@ import ( "golang.org/x/time/rate" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/util/workqueue" "knative.dev/pkg/logging" controllerruntime "sigs.k8s.io/controller-runtime" @@ -63,6 +64,7 @@ func (c *Controller) Reconcile(_ context.Context, _ *v1beta1.NodeClaim) (reconci return reconcile.Result{}, nil } +//nolint:gocyclo func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (reconcile.Result, error) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("node", nodeClaim.Status.NodeName, "provider-id", nodeClaim.Status.ProviderID)) stored := nodeClaim.DeepCopy() @@ -93,7 +95,10 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) } controllerutil.RemoveFinalizer(nodeClaim, v1beta1.TerminationFinalizer) if !equality.Semantic.DeepEqual(stored, nodeClaim) { - if err = c.kubeClient.Patch(ctx, nodeClaim, client.MergeFrom(stored)); err != nil { + if err = c.kubeClient.Update(ctx, nodeClaim); err != nil { + if errors.IsConflict(err) { + return reconcile.Result{Requeue: true}, nil + } return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("removing termination finalizer, %w", err)) } logging.FromContext(ctx).Infof("deleted nodeclaim") From bbddf9cb6648ff43f5dde31fc48152094f2d65d2 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Mon, 25 Mar 2024 22:41:19 -0700 Subject: [PATCH 15/33] chore: Remove validation as covered by OpenAPI validation (#1132) --- pkg/apis/v1beta1/nodepool_validation.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/apis/v1beta1/nodepool_validation.go b/pkg/apis/v1beta1/nodepool_validation.go index 37637ca3e2..66161bcc16 100644 --- a/pkg/apis/v1beta1/nodepool_validation.go +++ b/pkg/apis/v1beta1/nodepool_validation.go @@ -19,7 +19,6 @@ package v1beta1 import ( "context" "fmt" - "regexp" "github.com/robfig/cron/v3" "github.com/samber/lo" @@ -123,11 +122,5 @@ func (in *Budget) validate() (errs *apis.FieldError) { return apis.ErrInvalidValue(in.Schedule, "schedule", fmt.Sprintf("invalid schedule %s", err)) } } - if in.Duration != nil { - goodDuration, err := regexp.Match("^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$", []byte(in.Duration.Duration.String())) - if err != nil || !goodDuration { - return apis.ErrInvalidValue(in.Schedule, "duration", fmt.Sprintf("invalid duration %s", err)) - } - } return errs } From b770a197ab44ceed3ea5dbf710cd6abe7e429fee Mon Sep 17 00:00:00 2001 From: Jigisha Patil <89548848+jigisha620@users.noreply.github.com> Date: Tue, 26 Mar 2024 07:53:20 -0700 Subject: [PATCH 16/33] chore: Changes to run e2e for private cluster (#1138) --- pkg/test/deployment.go | 2 +- pkg/test/pods.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/test/deployment.go b/pkg/test/deployment.go index b9a2f3a851..c9a0ba388e 100644 --- a/pkg/test/deployment.go +++ b/pkg/test/deployment.go @@ -44,7 +44,7 @@ func Deployment(overrides ...DeploymentOptions) *appsv1.Deployment { objectMeta := NamespacedObjectMeta(options.ObjectMeta) if options.PodOptions.Image == "" { - options.PodOptions.Image = "public.ecr.aws/eks-distro/kubernetes/pause:3.2" + options.PodOptions.Image = DefaultImage } if options.PodOptions.Labels == nil { options.PodOptions.Labels = map[string]string{ diff --git a/pkg/test/pods.go b/pkg/test/pods.go index f750882269..1c41de3c53 100644 --- a/pkg/test/pods.go +++ b/pkg/test/pods.go @@ -71,7 +71,7 @@ type EphemeralVolumeTemplateOptions struct { StorageClassName *string } -const ( +var ( DefaultImage = "public.ecr.aws/eks-distro/kubernetes/pause:3.2" ) From 23c4af9d367e2c2862b7b427481d261d80817dff Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 27 Mar 2024 17:52:54 +0100 Subject: [PATCH 17/33] chore: Add comments on calls to kubeClient Update() (#1130) --- pkg/controllers/nodeclaim/disruption/controller.go | 3 +++ pkg/controllers/nodeclaim/termination/controller.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/controllers/nodeclaim/disruption/controller.go b/pkg/controllers/nodeclaim/disruption/controller.go index d74459b65a..b666307c55 100644 --- a/pkg/controllers/nodeclaim/disruption/controller.go +++ b/pkg/controllers/nodeclaim/disruption/controller.go @@ -93,6 +93,9 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim results = append(results, res) } if !equality.Semantic.DeepEqual(stored, nodeClaim) { + // We call Update() here rather than Patch() because patching a list with a JSON merge patch + // can cause races due to the fact that it fully replaces the list on a change + // Here, we are updating the status condition list if err := c.kubeClient.Status().Update(ctx, nodeClaim); err != nil { if errors.IsConflict(err) { return reconcile.Result{Requeue: true}, nil diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index 2b601fe0c0..6e6b4f0d3a 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -95,6 +95,9 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) } controllerutil.RemoveFinalizer(nodeClaim, v1beta1.TerminationFinalizer) if !equality.Semantic.DeepEqual(stored, nodeClaim) { + // We call Update() here rather than Patch() because patching a list with a JSON merge patch + // can cause races due to the fact that it fully replaces the list on a change + // https://github.com/kubernetes/kubernetes/issues/111643#issuecomment-2016489732 if err = c.kubeClient.Update(ctx, nodeClaim); err != nil { if errors.IsConflict(err) { return reconcile.Result{Requeue: true}, nil From c8eda9b5ffc3748e68a0b8f03b14f4308f48f1f1 Mon Sep 17 00:00:00 2001 From: Bryce Soghigian <49734722+Bryce-Soghigian@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:48:53 -0700 Subject: [PATCH 18/33] chore: nit spelling fix (#1142) --- pkg/controllers/disruption/multinodeconsolidation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/disruption/multinodeconsolidation.go b/pkg/controllers/disruption/multinodeconsolidation.go index 660958f48f..66472f35f5 100644 --- a/pkg/controllers/disruption/multinodeconsolidation.go +++ b/pkg/controllers/disruption/multinodeconsolidation.go @@ -105,7 +105,7 @@ func (m *MultiNodeConsolidation) ComputeCommand(ctx context.Context, disruptionB } // firstNConsolidationOption looks at the first N NodeClaims to determine if they can all be consolidated at once. The -// NodeClaims are sorted by increasing disruption order which correlates to likelihood if being able to consolidate the node +// NodeClaims are sorted by increasing disruption order which correlates to likelihood of being able to consolidate the node func (m *MultiNodeConsolidation) firstNConsolidationOption(ctx context.Context, candidates []*Candidate, max int) (Command, scheduling.Results, error) { // we always operate on at least two NodeClaims at once, for single NodeClaims standard consolidation will find all solutions if len(candidates) < 2 { From 3d51e28cb05f3cae1e4a9321ab559b0de3ca4bc1 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Fri, 29 Mar 2024 10:35:16 -0700 Subject: [PATCH 19/33] feat: Get the GVK of the supported NodeClasses as part of the CloudProvider interface (#1146) --- kwok/cloudprovider/cloudprovider.go | 5 +++++ pkg/cloudprovider/fake/cloudprovider.go | 17 +++++++++++++++-- pkg/cloudprovider/types.go | 3 +++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kwok/cloudprovider/cloudprovider.go b/kwok/cloudprovider/cloudprovider.go index 47d70bf947..5e100bb916 100644 --- a/kwok/cloudprovider/cloudprovider.go +++ b/kwok/cloudprovider/cloudprovider.go @@ -29,6 +29,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -118,6 +119,10 @@ func (c CloudProvider) Name() string { return "kwok" } +func (c CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return []schema.GroupVersionKind{} +} + func (c CloudProvider) getInstanceType(instanceTypeName string) (*cloudprovider.InstanceType, error) { it, found := lo.Find(c.instanceTypes, func(it *cloudprovider.InstanceType) bool { return it.Name == instanceTypeName diff --git a/pkg/cloudprovider/fake/cloudprovider.go b/pkg/cloudprovider/fake/cloudprovider.go index 9f81588317..2e0ab1155c 100644 --- a/pkg/cloudprovider/fake/cloudprovider.go +++ b/pkg/cloudprovider/fake/cloudprovider.go @@ -27,6 +27,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" @@ -51,8 +52,9 @@ type CloudProvider struct { NextCreateErr error DeleteCalls []*v1beta1.NodeClaim - CreatedNodeClaims map[string]*v1beta1.NodeClaim - Drifted cloudprovider.DriftReason + CreatedNodeClaims map[string]*v1beta1.NodeClaim + Drifted cloudprovider.DriftReason + NodeClassGroupVersionKind []schema.GroupVersionKind } func NewCloudProvider() *CloudProvider { @@ -77,6 +79,13 @@ func (c *CloudProvider) Reset() { c.NextCreateErr = nil c.DeleteCalls = []*v1beta1.NodeClaim{} c.Drifted = "drifted" + c.NodeClassGroupVersionKind = []schema.GroupVersionKind{ + { + Group: "", + Version: "", + Kind: "", + }, + } } func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) { @@ -237,3 +246,7 @@ func (c *CloudProvider) IsDrifted(context.Context, *v1beta1.NodeClaim) (cloudpro func (c *CloudProvider) Name() string { return "fake" } + +func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return c.NodeClassGroupVersionKind +} diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index dde363ca32..ee1fa1f496 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -26,6 +26,7 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -55,6 +56,8 @@ type CloudProvider interface { IsDrifted(context.Context, *v1beta1.NodeClaim) (DriftReason, error) // Name returns the CloudProvider implementation name. Name() string + // GetSupportedNodeClass returns the group, version, and kind of the CloudProvider NodeClass + GetSupportedNodeClasses() []schema.GroupVersionKind } // InstanceType describes the properties of a potential node (either concrete attributes of an instance of this type From 7654bd1db1f2253e62131d69eda3cd55547593e6 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:59:32 -0700 Subject: [PATCH 20/33] fix: Change type to hash on KubeReserved and SystemReserved (#1141) --- hack/validation/kubelet.sh | 19 +++++++++++++---- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 10 ++------- pkg/apis/crds/karpenter.sh_nodepools.yaml | 10 ++------- pkg/apis/v1beta1/nodeclaim.go | 4 ++-- pkg/apis/v1beta1/nodeclaim_validation.go | 14 ++++++++----- .../v1beta1/nodeclaim_validation_cel_test.go | 10 ++++----- .../nodeclaim_validation_webhook_test.go | 10 ++++----- .../v1beta1/nodepool_validation_cel_test.go | 20 +++++++++--------- .../nodepool_validation_webhook_test.go | 8 +++---- pkg/apis/v1beta1/zz_generated.deepcopy.go | 8 +++---- .../nodeclaim/disruption/drift_test.go | 21 ++++++++++--------- 11 files changed, 67 insertions(+), 67 deletions(-) diff --git a/hack/validation/kubelet.sh b/hack/validation/kubelet.sh index ca31142ee3..fa1cbe88e3 100755 --- a/hack/validation/kubelet.sh +++ b/hack/validation/kubelet.sh @@ -1,11 +1,22 @@ # Kubelet Validation -# The regular expression will be adding a check for if kublet.evictionHard and kubelet.evictionSoft are percentage or a quantity value -# Adding validation for nodeclaim +# The regular expression adds validation for kubelet.kubeReserved and kubelet.systemReserved values of the map are resource.Quantity +# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 +# NodeClaim Validation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.kubeReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.systemReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml + +# NodePool Vaildation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.kubelet.properties.kubeReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.sh_nodepools.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.kubelet.properties.systemReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + + +# The regular expression is a validation for kublet.evictionHard and kubelet.evictionSoft are percentage or a quantity value +# NodeClaim Validation: yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionHard.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionSoft.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml -# The regular expression will be adding a check for if kublet.evictionHard and kubelet.evictionSoft are percentage or a quantity value -# Adding validation for nodepool +# NodePool Validation: yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.kubelet.properties.evictionHard.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.sh_nodepools.yaml yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.kubelet.properties.evictionSoft.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 0049ef5a1b..f10079e985 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -140,11 +140,8 @@ spec: type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -169,11 +166,8 @@ spec: type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 7d7cf0d60a..f93edb4c20 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -264,11 +264,8 @@ spec: type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -293,11 +290,8 @@ spec: type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: diff --git a/pkg/apis/v1beta1/nodeclaim.go b/pkg/apis/v1beta1/nodeclaim.go index 23d9477859..21c91a7df9 100644 --- a/pkg/apis/v1beta1/nodeclaim.go +++ b/pkg/apis/v1beta1/nodeclaim.go @@ -100,12 +100,12 @@ type KubeletConfiguration struct { // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" // +optional - SystemReserved v1.ResourceList `json:"systemReserved,omitempty"` + SystemReserved map[string]string `json:"systemReserved,omitempty"` // KubeReserved contains resources reserved for Kubernetes system components. // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" // +optional - KubeReserved v1.ResourceList `json:"kubeReserved,omitempty"` + KubeReserved map[string]string `json:"kubeReserved,omitempty"` // EvictionHard is the map of signal names to quantities that define hard eviction thresholds // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" // +optional diff --git a/pkg/apis/v1beta1/nodeclaim_validation.go b/pkg/apis/v1beta1/nodeclaim_validation.go index 7f1b2e04d1..6f63ab59af 100644 --- a/pkg/apis/v1beta1/nodeclaim_validation.go +++ b/pkg/apis/v1beta1/nodeclaim_validation.go @@ -221,13 +221,17 @@ func (in *KubeletConfiguration) validateEvictionSoftPairs() (errs *apis.FieldErr return errs } -func validateReservedResources(m v1.ResourceList, fieldName string) (errs *apis.FieldError) { +func validateReservedResources(m map[string]string, fieldName string) (errs *apis.FieldError) { for k, v := range m { - if !SupportedReservedResources.Has(k.String()) { - errs = errs.Also(apis.ErrInvalidKeyName(k.String(), fieldName)) + if !SupportedReservedResources.Has(k) { + errs = errs.Also(apis.ErrInvalidKeyName(k, fieldName)) + } + quantity, err := resource.ParseQuantity(v) + if err != nil { + errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`%s["%s"]`, fieldName, k), "Value must be a quantity value")) } - if v.Value() < 0 { - errs = errs.Also(apis.ErrInvalidValue(v.String(), fmt.Sprintf(`%s["%s"]`, fieldName, k), "Value cannot be a negative resource quantity")) + if quantity.Value() < 0 { + errs = errs.Also(apis.ErrInvalidValue(v, fmt.Sprintf(`%s["%s"]`, fieldName, k), "Value cannot be a negative resource quantity")) } } return errs diff --git a/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go b/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go index 1a5b12a168..a1c62f9f13 100644 --- a/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go +++ b/pkg/apis/v1beta1/nodeclaim_validation_cel_test.go @@ -21,8 +21,6 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/api/resource" - "github.com/Pallinder/go-randomdata" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -220,16 +218,16 @@ var _ = Describe("Validation", func() { Context("Kubelet", func() { It("should fail on kubeReserved with invalid keys", func() { nodeClaim.Spec.Kubelet = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + KubeReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed()) }) It("should fail on systemReserved with invalid keys", func() { nodeClaim.Spec.Kubelet = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + SystemReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed()) diff --git a/pkg/apis/v1beta1/nodeclaim_validation_webhook_test.go b/pkg/apis/v1beta1/nodeclaim_validation_webhook_test.go index 1e0e2ca200..d4f73fd148 100644 --- a/pkg/apis/v1beta1/nodeclaim_validation_webhook_test.go +++ b/pkg/apis/v1beta1/nodeclaim_validation_webhook_test.go @@ -20,8 +20,6 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/api/resource" - "github.com/Pallinder/go-randomdata" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -179,16 +177,16 @@ var _ = Describe("Validation", func() { Context("Kubelet", func() { It("should fail on kubeReserved with invalid keys", func() { nodeClaim.Spec.Kubelet = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + KubeReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(nodeClaim.Validate(ctx)).ToNot(Succeed()) }) It("should fail on systemReserved with invalid keys", func() { nodeClaim.Spec.Kubelet = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + SystemReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(nodeClaim.Validate(ctx)).ToNot(Succeed()) diff --git a/pkg/apis/v1beta1/nodepool_validation_cel_test.go b/pkg/apis/v1beta1/nodepool_validation_cel_test.go index 79c5e7e14a..9759fa6e74 100644 --- a/pkg/apis/v1beta1/nodepool_validation_cel_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_cel_test.go @@ -232,34 +232,34 @@ var _ = Describe("CEL/Validation", func() { }) }) Context("KubeletConfiguration", func() { - It("should succeed on kubeReserved with invalid keys", func() { + It("should succeed on kubeReserved with valid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "2", }, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) }) - It("should succeed on systemReserved with invalid keys", func() { + It("should succeed on systemReserved with valid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "2", }, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) }) It("should fail on kubeReserved with invalid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + KubeReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(env.Client.Create(ctx, nodePool)).ToNot(Succeed()) }) It("should fail on systemReserved with invalid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + SystemReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(env.Client.Create(ctx, nodePool)).ToNot(Succeed()) diff --git a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go index b46e7cdcd0..fd5c87757c 100644 --- a/pkg/apis/v1beta1/nodepool_validation_webhook_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_webhook_test.go @@ -392,16 +392,16 @@ var _ = Describe("Webhook/Validation", func() { Context("KubeletConfiguration", func() { It("should fail on kubeReserved with invalid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + KubeReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(nodePool.Validate(ctx)).ToNot(Succeed()) }) It("should fail on systemReserved with invalid keys", func() { nodePool.Spec.Template.Spec.Kubelet = &KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourcePods: resource.MustParse("2"), + SystemReserved: map[string]string{ + string(v1.ResourcePods): "2", }, } Expect(nodePool.Validate(ctx)).ToNot(Succeed()) diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 4ae73df600..bc19166432 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -101,16 +101,16 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { } if in.SystemReserved != nil { in, out := &in.SystemReserved, &out.SystemReserved - *out = make(v1.ResourceList, len(*in)) + *out = make(map[string]string, len(*in)) for key, val := range *in { - (*out)[key] = val.DeepCopy() + (*out)[key] = val } } if in.KubeReserved != nil { in, out := &in.KubeReserved, &out.KubeReserved - *out = make(v1.ResourceList, len(*in)) + *out = make(map[string]string, len(*in)) for key, val := range *in { - (*out)[key] = val.DeepCopy() + (*out)[key] = val } } if in.EvictionHard != nil { diff --git a/pkg/controllers/nodeclaim/disruption/drift_test.go b/pkg/controllers/nodeclaim/disruption/drift_test.go index cea357668b..61012ac273 100644 --- a/pkg/controllers/nodeclaim/disruption/drift_test.go +++ b/pkg/controllers/nodeclaim/disruption/drift_test.go @@ -466,11 +466,15 @@ var _ = Describe("Drift", func() { }, }, Kubelet: &v1beta1.KubeletConfiguration{ - ClusterDNS: []string{"fakeDNS"}, - MaxPods: ptr.Int32(0), - PodsPerCore: ptr.Int32(0), - SystemReserved: v1.ResourceList{}, - KubeReserved: v1.ResourceList{}, + ClusterDNS: []string{"fakeDNS"}, + MaxPods: ptr.Int32(0), + PodsPerCore: ptr.Int32(0), + SystemReserved: map[string]string{ + "cpu": "2", + }, + KubeReserved: map[string]string{ + "memory": "10Gi", + }, EvictionHard: map[string]string{ "memory.available": "20Gi", }, @@ -520,11 +524,8 @@ var _ = Describe("Drift", func() { Entry("kubeletConfiguration ClusterDNS", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{ClusterDNS: []string{"testDNS"}}}}}}), Entry("KubeletConfiguration MaxPods", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{MaxPods: ptr.Int32(5)}}}}}), Entry("KubeletConfiguration PodsPerCore", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{PodsPerCore: ptr.Int32(5)}}}}}), - // Karpenter currently have a bug with systemReserved and kubeReserved will cause NodeClaims to not be drifted, when the field is updated - // TODO: Enable systemReserved and kubeReserved once they can be used for static drift - // For more information: https://github.com/kubernetes-sigs/karpenter/issues/1080 - // Entry("KubeletConfiguration SystemReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{SystemReserved: v1.ResourceList{}}}}}}), - // Entry("KubeletConfiguration KubeReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{KubeReserved: v1.ResourceList{}}}}}}), + Entry("KubeletConfiguration SystemReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{SystemReserved: map[string]string{"memory": "30Gi"}}}}}}), + Entry("KubeletConfiguration KubeReserved", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{KubeReserved: map[string]string{"cpu": "10"}}}}}}), Entry("KubeletConfiguration EvictionHard", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionHard: map[string]string{"memory.available": "30Gi"}}}}}}), Entry("KubeletConfiguration EvictionSoft", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionSoft: map[string]string{"nodefs.available": "30Gi"}}}}}}), Entry("KubeletConfiguration EvictionSoftGracePeriod", v1beta1.NodePool{Spec: v1beta1.NodePoolSpec{Template: v1beta1.NodeClaimTemplate{Spec: v1beta1.NodeClaimSpec{Kubelet: &v1beta1.KubeletConfiguration{EvictionSoftGracePeriod: map[string]metav1.Duration{"nodefs.available": {Duration: time.Minute}}}}}}}), From a1c2c16bd5d9f3f3ce08a726547b51f47791ba2e Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:23:38 -0700 Subject: [PATCH 21/33] chore: Bump `NodePoolHashVersion` (#1148) --- pkg/apis/v1beta1/nodepool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/v1beta1/nodepool.go b/pkg/apis/v1beta1/nodepool.go index 79109d586d..68fb3ceb79 100644 --- a/pkg/apis/v1beta1/nodepool.go +++ b/pkg/apis/v1beta1/nodepool.go @@ -194,7 +194,7 @@ type NodePool struct { // 1. A field changes its default value for an existing field that is already hashed // 2. A field is added to the hash calculation with an already-set value // 3. A field is removed from the hash calculations -const NodePoolHashVersion = "v1" +const NodePoolHashVersion = "v2" func (in *NodePool) Hash() string { return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec.Template, hashstructure.FormatV2, &hashstructure.HashOptions{ From 78299808816a7ec16d96c4258fdb503a84d56873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:23:47 -0700 Subject: [PATCH 22/33] chore(deps): bump the actions-deps group with 1 update (#1150) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d0d7f7ae0d..4bccab676c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -32,7 +32,7 @@ jobs: prerelease: false - name: Install tejolote - uses: kubernetes-sigs/release-actions/setup-tejolote@10fecc1c66829d291b2f2fb1a27329d152f212e6 # v0.1.3 + uses: kubernetes-sigs/release-actions/setup-tejolote@841d76a188a7c121231a863572e27012805715a2 # v0.1.4 - name: Run tejolote run: | tejolote attest "github://kubernetes-sigs/karpenter/${{ github.run_id }}" --artifacts "github://kubernetes-sigs/karpenter/$TAG" --output karpenter.intoto.json --sign From 817f8368b6837a92df53c0a29b6d8c724150553b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:25:58 -0700 Subject: [PATCH 23/33] chore(deps): bump the action-deps group in /.github/actions/install-deps with 1 update (#1134) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/install-deps/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index ec1d8d1a9f..e3f0e019a2 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -16,7 +16,7 @@ runs: # Root path permission workaround for caching https://github.com/actions/cache/issues/845#issuecomment-1252594999 - run: sudo chown "$USER" /usr/local shell: bash - - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 id: cache-toolchain with: path: | From f91c1b15f0cf7cac0895de047a2e8b61acbc14a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:28:08 -0700 Subject: [PATCH 24/33] chore(deps): bump the go-deps group with 2 updates (#1135) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f3a789f54b..394e1049f6 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.22 require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/avast/retry-go v3.0.0+incompatible - github.com/docker/docker v25.0.5+incompatible + github.com/docker/docker v26.0.0+incompatible github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.17.0 + github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.19.0 diff --git a/go.sum b/go.sum index ac54aae9e5..2585276f40 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= -github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.0.0+incompatible h1:Ng2qi+gdKADUa/VM+6b6YaY2nlZhk/lVJiKR/2bMudU= +github.com/docker/docker v26.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -241,8 +241,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= -github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= From 3d493b30f2464833a6482b99c80117dc20a23a44 Mon Sep 17 00:00:00 2001 From: Nick Tran <10810510+njtran@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:10:11 -0700 Subject: [PATCH 25/33] docs: RFC for contributor ladder guidelines (#1044) --- contributing-guidelines.md | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 contributing-guidelines.md diff --git a/contributing-guidelines.md b/contributing-guidelines.md new file mode 100644 index 0000000000..b1a1400cf7 --- /dev/null +++ b/contributing-guidelines.md @@ -0,0 +1,95 @@ +# Karpenter - Contributor Ladder + +This document’s goal is to define clear, scalable, and transparent criteria to support community members to grow responsibility in Karpenter. This document also intends to capture a leadership path for contributors that intend to provide a sustained contribution to Karpenter by taking on reviewer and approver responsibilities at various levels. + +Ultimately, the criteria in this doc is aspirational. No set of written requirements can encapsulate the full criteria when determining if someone meets the bar to be a reviewer or approver, as some of the criteria are subjective and relies on the trust that each nominee has established with the community. To help guide readers, this document outlines ways to demonstrate expertise of the code base, sound judgement on decision tradeoffs, end user advocacy, care for community, and ability to work as a distributed team. + +Much of this document uses the [SIG-Node Contributor Ladder](https://github.com/kubernetes/community/blob/master/sig-node/sig-node-contributor-ladder.md) as prior art. The goal is to mold these requirements to fit the Karpenter’s community. These requirements also lean on the established Kubernetes[membership documentation](https://github.com/kubernetes/community/blob/master/community-membership.md) for terminology. + +As a final precursor, to become a reviewer or approver, users must nominate themselves. They are responsible for cutting a PR to the upstream repository, providing evidence in-line with the suggested requirements. Users should feel free to reach out to an existing approver to understand what how they land in respect to the criteria. The following sections are guiding criteria and guidelines, where the final decision lies with the maintainers. + +## Reviewers and Approvers + +As an autoscaler, Karpenter is responsible for minimizing cost, maximizing application runtime, and automatically updating nodes. Its role as a critical cluster component managed by users sets a high bar for contributions and prioritizes efficiency, operationalization, and simplicity for its users. + +At a high level, reviewers and approvers should be inclined to scrutinize the cost-benefit tradeoffs for level of maintenance versus user-benefit, which may materialize as a bias to say “no” in reviews and design discussions. Reviewers should have an initial bias towards coding over reviews to demonstrate a baseline for knowledge of code base. In design discussions, reviewers and approvers should aim to maintain Karpenter’s efficiency, operationalization, and efficiency. + +Karpenter is a customer driven project, building solutions for real customer problems, and delaying solving theoretical ones. Reviewers and approvers should represent these tenets by minimizing changes to Karpenter’s API surface. No API is the best API, as an un-used or dead API becomes a burden for users to reason about, and a further burden for Karpenter users to maintain. + +Lastly, as a library vended for and consumed by cloud providers, Karpenter aims to foster participation from all active cloud providers. Karpenter’s set of reviewers and approvers should have representatives from well known cloud provider implementations, as changes to upstream Karpenter can affect all dependent cloud provider implementations. + +To become a reviewer or approver, a user should begin by cutting an issue to track the approval process. + +### Reviewers + +Reviewer status is an indication that the person is committed to Karpenter activities, demonstrates technical depth, has accumulated enough context, and is overall trustworthy to help inform approvers and aid contributors by applying a lgtm. Anyone is welcome to review a PR with a comment or feedback even if they do not have rights to apply a lgtm. The requirements listed in the [membership document](https://github.com/kubernetes/community/blob/master/community-membership.md#reviewer) highlight this as well. + +The following is a guiding set of criteria for considering a user eligible to be a reviewer: + +* Committed - proof of sustained contributions +* Be a [Kubernetes org member](https://github.com/kubernetes/community/blob/master/community-membership.md#member) (which has its [own set of requirements](https://github.com/kubernetes/community/blob/master/community-membership.md#requirements)) +* Active Karpenter member for at least 6 months +* Demonstrates technical depth +* Primary reviewer for at least 5 PRs to the codebase +* Reviewed or merged at least 10 non-trivial substantial PRs to the codebase +* Knowledgeable about the codebase +* Reliable and builds consensus - established trust with the community +* Sponsored by an approver +* With no objections from other approvers + +#### Committed + +A user’s commitment should be established by looking at PR review history. Committed users should be participating in Karpenter meetings or other ad-hoc meetings that arise when tackling specific problems (exceptions are allowed for cases when timezone or other personal limitations are not allowing for the meeting participation). + +#### Technically sound + +Proof of primary reviewership and significant contributions must be provided. Nominees must provide the list of PRs (at least 5 for primary reviewer and 10 substantial PRs authored or reviewed) as suggested in the membership document. Here are additional comments for this list of PRs: + +* Reviewed PRs must be merged. +* Since the purpose is to demonstrate the nominee's technical depth, PRs like analyzer warnings fixes, mechanical “find/replace”-type PRs, minor improvements of logging and insignificant bug fixes are valued, but not counted towards the reviewer status nomination. Lack of reviews of those PRs may be a red flag for nomination approval. +* A primary reviewer should drive the review of the PR without significant input / guidance from the approver or other reviewers. + +It is hard to assess codebase knowledge and it always will be a judgement call. Karpenter will rely on the listed PRs to ensure the person reviewed PRs from different areas of the codebase and on the comments made during Karpenter meetings. + +Additional ways to establish the knowledge of context are: + +* Contributions to Karpenter documentation +* Blog posts - k8s-hosted and external +* Contributions to other adjacent sub-projects within SIG Autoscaling + +#### Trustworthy + +Reviewer nominations are accepted by Karpenter approvers. Karpenter approvers take nominations seriously and are invested in building a healthy community. Nominees should help approvers understand their future goals in the community so we can help continue to build trust and mutual relationships and nurture new opportunities if and when a contributor wants to become an approver! + +### Approvers + +Karpenter approvers have a lot of responsibilities. It is expected that a Karpenter approver keeps the codebase quality high by giving feedback, thoroughly reviewing code, and giving recommendations to Karpenter members and reviewers. Karpenter approvers are essentially gatekeepers to keep the code base at high quality. Karpenter maintains a rigidly high bar for becoming a Karpenter approver by developing trust in a community and demonstrating expertise with a bias towards initial code contributions over reviewing PRs. + +We expect at this stage of Karpenter maturity for approvers to have a strong bias to say “no” to unneeded changes or improvements that don't clearly articulate and demonstrate broad benefits. As an autoscaler, approvers have a responsibility to evaluate changes or improvements at scale. [While scale dimensions and thresholds are complex](https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md#kubernetes-thresholds), approvers should consider how changes may impact Karpenter's scalability and have a bias for “no” when any of these dimension's scalability is compromised. It also means that the velocity of new features may be affected by this bias. Our continuous work to improve the reliability of the codebase will help to maintain feature velocity going forward. + +While evaluating a nomination for approval, nominees may be asked to provide examples of strict scrutiny. Strict scrutiny refers to instances where a performance regression, vulnerability, or complex unintended interaction could have occurred. We do not expect existing approvers or nominees to be perfect (no one is!) but as a maintainer community we have had instances of pull requests that we want to learn from and spot to mitigate potential risks given our trust to users and existing project maturity level. Where specific examples are not present for a nominee (which is fine), we may privately share examples from our past experience for warning signs. + +In addition to the formal requirements for the [approver role](https://github.com/kubernetes/community/blob/master/community-membership.md#approver), Karpenter makes these recommendations for nominees for the Karpenter approver status on how to demonstrate expertise and develop trust. Ideally approver rights in more than one of these is **desired but not required**. This is a means of earning trust to existing approvers. + +#### Deep expertise across multiple core controllers + +* Demonstrated influence across multiple core controllers (e.g. provisioning, disruption, cluster state, etc.) +* Troubleshooting complex issues that touch require a holistic understanding of the code base, with an understanding of common 3rd party use-cases and tooling. +* Create and merge major code simplification and/or optimization PRs indicating deep understanding of tradeoffs taken and validation of potential side effects. + +#### Proficient in features development + +* Drive a few major features at all three stages: + * “alpha” - design proposal and discussions + * “beta” - initial customer feedback collection + * “GA/deprecation” - stabilizing feature, following PRs, or managing deprecation. +* Demonstrate ability to stage changes and pass PRs keeping the end user experience and Kubernetes reliability as top priorities. +* Be a reviewer for a few major features and demonstrate meaningful participation in the review process. +* Give actionable feedback for the features and initial proposals during the Karpenter meetings. + +#### Active community support + +* Have approval rights in a well-known cloud provider implementation of Karpenter or in an adjacent SIG Autoscaling sub-project. +* Be a primary PR reviewer for numerous PRs in multiple areas listed as a requirement for a reviewer. +* Actively triage issues and PRs, provide support to contributors to drive their PRs to completion. +* Be present, and participate in Karpenter meetings by speaking about features or improvements driven, or find some other way to prove the identity behind GitHub handle. \ No newline at end of file From de0dcd40fe923a0952e07ef69a0298723651239c Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:10:12 -0700 Subject: [PATCH 26/33] chore: Retract the Karpenter version due to a bad release (#1157) --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 394e1049f6..755dc5c50d 100644 --- a/go.mod +++ b/go.mod @@ -106,5 +106,7 @@ require ( retract ( v0.100.101-test // accidentally published testing version + v0.35.3 // accidentally published incomplete patch release + v0.34.4 // accidentally published incomplete patch release v0.27.7 // accidentally published incomplete patch release ) From 5d6de8bfc9befae682937b196c91f732e06920d3 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:25:37 -0700 Subject: [PATCH 27/33] chore: Bump `golang.org/x/net due` to a vulncheck failure (#1166) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 755dc5c50d..48180c9301 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( k8s.io/component-base v0.29.3 k8s.io/csi-translation-lib v0.29.3 k8s.io/klog/v2 v2.120.1 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b + k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 sigs.k8s.io/controller-runtime v0.17.2 ) @@ -83,10 +83,10 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.4.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.124.0 // indirect diff --git a/go.sum b/go.sum index 2585276f40..1b3ec08e80 100644 --- a/go.sum +++ b/go.sum @@ -406,8 +406,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -464,11 +464,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -670,8 +670,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 h1:oO/BQJpVCFTSTMHF/S6u+nPtIvbHDTsvbPZvdCZAFjs= knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4/go.mod h1:eXobTqst4aI7CNa6W7sG73VhEsHGWPSrkefeMTb++a0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 545d88a836d432c3cfae17ca781ca573b46a25de Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:07:02 -0700 Subject: [PATCH 28/33] feat: Detect Drift on NodeClaims on changes to NodeClass (#1147) --- .../nodeclaim/disruption/controller.go | 28 +++++++++++++------ pkg/operator/operator.go | 9 ++++++ pkg/utils/nodeclaim/nodeclaim.go | 22 ++++++++++++++- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/nodeclaim/disruption/controller.go b/pkg/controllers/nodeclaim/disruption/controller.go index b666307c55..6a746fe719 100644 --- a/pkg/controllers/nodeclaim/disruption/controller.go +++ b/pkg/controllers/nodeclaim/disruption/controller.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/clock" controllerruntime "sigs.k8s.io/controller-runtime" @@ -48,7 +49,8 @@ type nodeClaimReconciler interface { // Controller is a disruption controller that adds StatusConditions to nodeclaims when they meet certain disruption conditions // e.g. When the NodeClaim has surpassed its owning provisioner's expirationTTL, then it is marked as "Expired" in the StatusConditions type Controller struct { - kubeClient client.Client + kubeClient client.Client + cloudProvider cloudprovider.CloudProvider drift *Drift expiration *Expiration @@ -58,10 +60,11 @@ type Controller struct { // NewController constructs a nodeclaim disruption controller func NewController(clk clock.Clock, kubeClient client.Client, cluster *state.Cluster, cloudProvider cloudprovider.CloudProvider) operatorcontroller.Controller { return operatorcontroller.Typed[*v1beta1.NodeClaim](kubeClient, &Controller{ - kubeClient: kubeClient, - drift: &Drift{cloudProvider: cloudProvider}, - expiration: &Expiration{kubeClient: kubeClient, clock: clk}, - emptiness: &Emptiness{kubeClient: kubeClient, cluster: cluster, clock: clk}, + kubeClient: kubeClient, + cloudProvider: cloudProvider, + drift: &Drift{cloudProvider: cloudProvider}, + expiration: &Expiration{kubeClient: kubeClient, clock: clk}, + emptiness: &Emptiness{kubeClient: kubeClient, cluster: cluster, clock: clk}, }) } @@ -114,7 +117,7 @@ func (c *Controller) Name() string { } func (c *Controller) Builder(_ context.Context, m manager.Manager) operatorcontroller.Builder { - return operatorcontroller.Adapt(controllerruntime. + builder := controllerruntime. NewControllerManagedBy(m). For(&v1beta1.NodeClaim{}). WithOptions(controller.Options{MaxConcurrentReconciles: 10}). @@ -125,6 +128,15 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) operatorcontr Watches( &v1.Pod{}, nodeclaimutil.PodEventHandler(c.kubeClient), - ), - ) + ) + for _, ncGVK := range c.cloudProvider.GetSupportedNodeClasses() { + nodeclass := &unstructured.Unstructured{} + nodeclass.SetGroupVersionKind(ncGVK) + builder = builder.Watches( + nodeclass, + nodeclaimutil.NodeClassEventHandler(c.kubeClient), + ) + } + + return operatorcontroller.Adapt(builder) } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 64a1320c68..6971b70a1a 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -190,6 +190,15 @@ func NewOperator() (context.Context, *Operator) { lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1beta1.NodeClaim{}, "status.providerID", func(o client.Object) []string { return []string{o.(*v1beta1.NodeClaim).Status.ProviderID} }), "failed to setup nodeclaim provider id indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1beta1.NodeClaim{}, "spec.nodeClassRef.apiVersion", func(o client.Object) []string { + return []string{o.(*v1beta1.NodeClaim).Spec.NodeClassRef.APIVersion} + }), "failed to setup nodeclaim nodeclassref apiversion indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1beta1.NodeClaim{}, "spec.nodeClassRef.kind", func(o client.Object) []string { + return []string{o.(*v1beta1.NodeClaim).Spec.NodeClassRef.Kind} + }), "failed to setup nodeclaim nodeclassref kind indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1beta1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { + return []string{o.(*v1beta1.NodeClaim).Spec.NodeClassRef.Name} + }), "failed to setup nodeclaim nodeclassref name indexer") lo.Must0(mgr.AddReadyzCheck("manager", func(req *http.Request) error { return lo.Ternary(mgr.GetCache().WaitForCacheSync(req.Context()), nil, fmt.Errorf("failed to sync caches")) diff --git a/pkg/utils/nodeclaim/nodeclaim.go b/pkg/utils/nodeclaim/nodeclaim.go index 42c7a7a279..906c04ff61 100644 --- a/pkg/utils/nodeclaim/nodeclaim.go +++ b/pkg/utils/nodeclaim/nodeclaim.go @@ -73,7 +73,7 @@ func NodeEventHandler(c client.Client) handler.EventHandler { }) } -// NodePoolEventHandler is a watcher on v1beta1.NodeClaim that maps Provisioner to NodeClaims based +// NodePoolEventHandler is a watcher on v1beta1.NodeClaim that maps NodePool to NodeClaims based // on the v1beta1.NodePoolLabelKey and enqueues reconcile.Requests for the NodeClaim func NodePoolEventHandler(c client.Client) handler.EventHandler { return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) (requests []reconcile.Request) { @@ -89,6 +89,26 @@ func NodePoolEventHandler(c client.Client) handler.EventHandler { }) } +// NodeClassEventHandler is a watcher on v1beta1.NodeClaim that maps NodeClass to NodeClaims based +// on the nodeClassRef and enqueues reconcile.Requests for the NodeClaim +func NodeClassEventHandler(c client.Client) handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) (requests []reconcile.Request) { + nodeClaimList := &v1beta1.NodeClaimList{} + if err := c.List(ctx, nodeClaimList, client.MatchingFields{ + "spec.nodeClassRef.apiVersion": o.GetObjectKind().GroupVersionKind().GroupVersion().String(), + "spec.nodeClassRef.kind": o.GetObjectKind().GroupVersionKind().Kind, + "spec.nodeClassRef.name": o.GetName(), + }); err != nil { + return requests + } + return lo.Map(nodeClaimList.Items, func(n v1beta1.NodeClaim, _ int) reconcile.Request { + return reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&n), + } + }) + }) +} + // NodeNotFoundError is an error returned when no v1.Nodes are found matching the passed providerID type NodeNotFoundError struct { ProviderID string From 702b524358ebfccc38d221728049593e0f3ad294 Mon Sep 17 00:00:00 2001 From: Jigisha Patil <89548848+jigisha620@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:44:36 -0700 Subject: [PATCH 29/33] chore: Add retryable error to cloud provider (#1164) --- pkg/cloudprovider/types.go | 27 +++++++++++++++++-- .../nodeclaim/termination/controller.go | 2 +- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index ee1fa1f496..c2e6957252 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -198,8 +198,8 @@ func IsNodeClaimNotFoundError(err error) bool { if err == nil { return false } - var mnfErr *NodeClaimNotFoundError - return errors.As(err, &mnfErr) + var ncnfErr *NodeClaimNotFoundError + return errors.As(err, &ncnfErr) } func IgnoreNodeClaimNotFoundError(err error) error { @@ -268,3 +268,26 @@ func IgnoreNodeClassNotReadyError(err error) error { } return err } + +// RetryableError is an error type returned by CloudProviders when the action emitting the error has to be retried +type RetryableError struct { + error +} + +func NewRetryableError(err error) *RetryableError { + return &RetryableError{ + error: err, + } +} + +func (e *RetryableError) Error() string { + return fmt.Sprintf("retryable error, %s", e.error) +} + +func IsRetryableError(err error) bool { + if err == nil { + return false + } + var retryableError *RetryableError + return errors.As(err, &retryableError) +} diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index 6e6b4f0d3a..373b518cee 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -89,7 +89,7 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) return reconcile.Result{}, nil } if nodeClaim.Status.ProviderID != "" { - if err = c.cloudProvider.Delete(ctx, nodeClaim); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil { + if err = c.cloudProvider.Delete(ctx, nodeClaim); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil || cloudprovider.IsRetryableError(err) { return reconcile.Result{}, fmt.Errorf("terminating cloudprovider instance, %w", err) } } From 3e1180609c13570d51d05e272704612fad19bd1d Mon Sep 17 00:00:00 2001 From: Jigisha Patil <89548848+jigisha620@users.noreply.github.com> Date: Thu, 4 Apr 2024 18:27:30 -0700 Subject: [PATCH 30/33] chore: Add method to ignore Retryable error (#1170) --- pkg/cloudprovider/types.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index c2e6957252..dbc359ead6 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -291,3 +291,10 @@ func IsRetryableError(err error) bool { var retryableError *RetryableError return errors.As(err, &retryableError) } + +func IgnoreRetryableError(err error) error { + if IsRetryableError(err) { + return nil + } + return err +} From 8e8bcc23dcd894023d93a96072dddc64016b434c Mon Sep 17 00:00:00 2001 From: Jigisha Patil <89548848+jigisha620@users.noreply.github.com> Date: Fri, 5 Apr 2024 18:20:07 -0700 Subject: [PATCH 31/33] chore: Re-enqueue nodeclaim termination after 10s in case of retryable error (#1172) --- pkg/cloudprovider/fake/cloudprovider.go | 8 +++++ .../nodeclaim/termination/controller.go | 8 ++++- .../nodeclaim/termination/suite_test.go | 29 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/fake/cloudprovider.go b/pkg/cloudprovider/fake/cloudprovider.go index 2e0ab1155c..72326d3b34 100644 --- a/pkg/cloudprovider/fake/cloudprovider.go +++ b/pkg/cloudprovider/fake/cloudprovider.go @@ -50,6 +50,7 @@ type CloudProvider struct { CreateCalls []*v1beta1.NodeClaim AllowedCreateCalls int NextCreateErr error + NextDeleteErr error DeleteCalls []*v1beta1.NodeClaim CreatedNodeClaims map[string]*v1beta1.NodeClaim @@ -77,6 +78,7 @@ func (c *CloudProvider) Reset() { c.ErrorsForNodePool = map[string]error{} c.AllowedCreateCalls = math.MaxInt c.NextCreateErr = nil + c.NextDeleteErr = nil c.DeleteCalls = []*v1beta1.NodeClaim{} c.Drifted = "drifted" c.NodeClassGroupVersionKind = []schema.GroupVersionKind{ @@ -227,6 +229,12 @@ func (c *CloudProvider) Delete(_ context.Context, nc *v1beta1.NodeClaim) error { c.mu.Lock() defer c.mu.Unlock() + if c.NextDeleteErr != nil { + tempError := c.NextDeleteErr + c.NextDeleteErr = nil + return tempError + } + c.DeleteCalls = append(c.DeleteCalls, nc) if _, ok := c.CreatedNodeClaims[nc.Status.ProviderID]; ok { delete(c.CreatedNodeClaims, nc.Status.ProviderID) diff --git a/pkg/controllers/nodeclaim/termination/controller.go b/pkg/controllers/nodeclaim/termination/controller.go index 373b518cee..ae1cf978be 100644 --- a/pkg/controllers/nodeclaim/termination/controller.go +++ b/pkg/controllers/nodeclaim/termination/controller.go @@ -89,7 +89,13 @@ func (c *Controller) Finalize(ctx context.Context, nodeClaim *v1beta1.NodeClaim) return reconcile.Result{}, nil } if nodeClaim.Status.ProviderID != "" { - if err = c.cloudProvider.Delete(ctx, nodeClaim); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil || cloudprovider.IsRetryableError(err) { + if err = c.cloudProvider.Delete(ctx, nodeClaim); cloudprovider.IgnoreNodeClaimNotFoundError(err) != nil { + // We expect cloudProvider to emit a Retryable Error when the underlying instance is not terminated and if that + // happens, we want to re-enqueue reconciliation until we terminate the underlying instance before removing + // finalizer from the nodeClaim. + if cloudprovider.IsRetryableError(err) { + return reconcile.Result{RequeueAfter: 10 * time.Second}, nil + } return reconcile.Result{}, fmt.Errorf("terminating cloudprovider instance, %w", err) } } diff --git a/pkg/controllers/nodeclaim/termination/suite_test.go b/pkg/controllers/nodeclaim/termination/suite_test.go index 37dc79f5c2..d9cba5e1e3 100644 --- a/pkg/controllers/nodeclaim/termination/suite_test.go +++ b/pkg/controllers/nodeclaim/termination/suite_test.go @@ -18,6 +18,7 @@ package termination_test import ( "context" + "fmt" "testing" "time" @@ -216,4 +217,32 @@ var _ = Describe("Termination", func() { ExpectExists(ctx, env.Client, node) } }) + It("should retry nodeClaim deletion when cloudProvider returns retryable error", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClaim) + ExpectReconcileSucceeded(ctx, nodeClaimLifecycleController, client.ObjectKeyFromObject(nodeClaim)) + + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + _, err := cloudProvider.Get(ctx, nodeClaim.Status.ProviderID) + Expect(err).ToNot(HaveOccurred()) + + node := test.NodeClaimLinkedNode(nodeClaim) + ExpectApplied(ctx, env.Client, node) + + // Expect the node and the nodeClaim to both be gone + Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed()) + ExpectReconcileSucceeded(ctx, nodeClaimTerminationController, client.ObjectKeyFromObject(nodeClaim)) // triggers the node deletion + ExpectFinalizersRemoved(ctx, env.Client, node) + ExpectNotFound(ctx, env.Client, node) + + cloudProvider.NextDeleteErr = cloudprovider.NewRetryableError(fmt.Errorf("underlying instance not terminated")) + result := ExpectReconcileSucceeded(ctx, nodeClaimTerminationController, client.ObjectKeyFromObject(nodeClaim)) + Expect(result.RequeueAfter).To(Equal(time.Second * 10)) + + ExpectReconcileSucceeded(ctx, nodeClaimTerminationController, client.ObjectKeyFromObject(nodeClaim)) //re-enqueue reconciliation since we got retryable error previously + ExpectNotFound(ctx, env.Client, nodeClaim, node) + + // Expect the nodeClaim to be gone from the cloudprovider + _, err = cloudProvider.Get(ctx, nodeClaim.Status.ProviderID) + Expect(cloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + }) }) From 43da3604c14097ab1ee3ca360af5a82446a4db4d Mon Sep 17 00:00:00 2001 From: nikmohan123 <154277636+nikmohan123@users.noreply.github.com> Date: Sun, 7 Apr 2024 00:59:57 -0500 Subject: [PATCH 32/33] test: Re-organize the requirement tests. (#1108) --- pkg/scheduling/requirement_test.go | 1654 ++++++++++++++-------------- 1 file changed, 821 insertions(+), 833 deletions(-) diff --git a/pkg/scheduling/requirement_test.go b/pkg/scheduling/requirement_test.go index f0aa8470b1..a2f543b8d5 100644 --- a/pkg/scheduling/requirement_test.go +++ b/pkg/scheduling/requirement_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -99,796 +100,777 @@ var _ = Describe("Requirement", func() { } }) }) - Context("Intersection", func() { - It("should intersect sets", func() { - // Intersection of requirement without minValues vs requirement without minValues - Expect(exists.Intersection(exists)).To(Equal(exists)) - Expect(exists.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(exists.Intersection(inA)).To(Equal(inA)) - Expect(exists.Intersection(inB)).To(Equal(inB)) - Expect(exists.Intersection(inAB)).To(Equal(inAB)) - Expect(exists.Intersection(notInA)).To(Equal(notInA)) - Expect(exists.Intersection(in1)).To(Equal(in1)) - Expect(exists.Intersection(in9)).To(Equal(in9)) - Expect(exists.Intersection(in19)).To(Equal(in19)) - Expect(exists.Intersection(notIn12)).To(Equal(notIn12)) - Expect(exists.Intersection(greaterThan1)).To(Equal(greaterThan1)) - Expect(exists.Intersection(greaterThan9)).To(Equal(greaterThan9)) - Expect(exists.Intersection(lessThan1)).To(Equal(lessThan1)) - Expect(exists.Intersection(lessThan9)).To(Equal(lessThan9)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(existsOperatorWithFlexibility.Intersection(exists)).To(Equal(existsOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inA)).To(Equal(inAOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inB)).To(Equal(inBOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inAB)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(1)})) - Expect(existsOperatorWithFlexibility.Intersection(notInA)).To(Equal(notInAOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in1)).To(Equal(in1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in9)).To(Equal(in9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in19)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(1)})) - Expect(existsOperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("1", "2"), MinValues: lo.ToPtr(1)})) - Expect(existsOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(exists)).To(Equal(existsOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(existsOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(existsOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(inAOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(inBOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)})) - Expect(existsOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(notInAOperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)})) - Expect(existsOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)})) - Expect(existsOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(existsOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(existsOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(doesNotExist.Intersection(exists)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(inA)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(inB)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(notInA)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(in1)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(in9)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(in19)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(notIn12)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(doesNotExist.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(doesNotExistOperatorWithFlexibility.Intersection(exists)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(notInA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in19)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(notIn12)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(doesNotExistOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(doesNotExistOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(doesNotExistOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(doesNotExistOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(doesNotExistOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(inA.Intersection(exists)).To(Equal(inA)) - Expect(inA.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(inA.Intersection(inA)).To(Equal(inA)) - Expect(inA.Intersection(inB)).To(Equal(doesNotExist)) - Expect(inA.Intersection(inAB)).To(Equal(inA)) - Expect(inA.Intersection(notInA)).To(Equal(doesNotExist)) - Expect(inA.Intersection(in1)).To(Equal(doesNotExist)) - Expect(inA.Intersection(in9)).To(Equal(doesNotExist)) - Expect(inA.Intersection(in19)).To(Equal(doesNotExist)) - Expect(inA.Intersection(notIn12)).To(Equal(inA)) - Expect(inA.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(inA.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(inA.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(inA.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(inAOperatorWithFlexibility.Intersection(exists)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inA)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inAB)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(notInA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in19)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(notIn12)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(inAOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(inAOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(inAOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inAOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(inAOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inAOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(inB.Intersection(exists)).To(Equal(inB)) - Expect(inB.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(inB.Intersection(inA)).To(Equal(doesNotExist)) - Expect(inB.Intersection(inB)).To(Equal(inB)) - Expect(inB.Intersection(inAB)).To(Equal(inB)) - Expect(inB.Intersection(notInA)).To(Equal(inB)) - Expect(inB.Intersection(in1)).To(Equal(doesNotExist)) - Expect(inB.Intersection(in9)).To(Equal(doesNotExist)) - Expect(inB.Intersection(in19)).To(Equal(doesNotExist)) - Expect(inB.Intersection(notIn12)).To(Equal(inB)) - Expect(inB.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(inB.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(inB.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(inB.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(inBOperatorWithFlexibility.Intersection(exists)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inB)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inAB)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(notInA)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in19)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(notIn12)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(inBOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inBOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(inBOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inBOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inBOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(inBOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(inAB.Intersection(exists)).To(Equal(inAB)) - Expect(inAB.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(inA)).To(Equal(inA)) - Expect(inAB.Intersection(inB)).To(Equal(inB)) - Expect(inAB.Intersection(inAB)).To(Equal(inAB)) - Expect(inAB.Intersection(notInA)).To(Equal(inB)) - Expect(inAB.Intersection(in1)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(in9)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(in19)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(notIn12)).To(Equal(inAB)) - Expect(inAB.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(inAB.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(inABOperatorWithFlexibility.Intersection(exists)).To(Equal(inABOperatorWithFlexibility)) - Expect(inABOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inA)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inB)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inAB)).To(Equal(inABOperatorWithFlexibility)) - Expect(inABOperatorWithFlexibility.Intersection(notInA)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in19)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(inABOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(inABOperatorWithFlexibility)) - Expect(inABOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(inABOperatorWithFlexibility)) - Expect(inABOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(inABOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(notInA.Intersection(exists)).To(Equal(notInA)) - Expect(notInA.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(notInA.Intersection(inA)).To(Equal(doesNotExist)) - Expect(notInA.Intersection(inB)).To(Equal(inB)) - Expect(notInA.Intersection(inAB)).To(Equal(inB)) - Expect(notInA.Intersection(notInA)).To(Equal(notInA)) - Expect(notInA.Intersection(in1)).To(Equal(in1)) - Expect(notInA.Intersection(in9)).To(Equal(in9)) - Expect(notInA.Intersection(in19)).To(Equal(in19)) - Expect(notInA.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2")})) - Expect(notInA.Intersection(greaterThan1)).To(Equal(greaterThan1)) - Expect(notInA.Intersection(greaterThan9)).To(Equal(greaterThan9)) - Expect(notInA.Intersection(lessThan1)).To(Equal(lessThan1)) - Expect(notInA.Intersection(lessThan9)).To(Equal(lessThan9)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(notInAOperatorWithFlexibility.Intersection(exists)).To(Equal(notInAOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inB)).To(Equal(inBOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inAB)).To(Equal(inBOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(notInA)).To(Equal(notInAOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in1)).To(Equal(in1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in9)).To(Equal(in9OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in19)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(1)})) - Expect(notInAOperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(1)})) - Expect(notInAOperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(lessThan1)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(lessThan9)).To(Equal(lessThan9OperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(notInAOperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(notInAOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(inBOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(notInAOperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(notInAOperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)})) - Expect(notInAOperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)})) - Expect(notInAOperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(notInAOperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(lessThan9OperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(in1.Intersection(exists)).To(Equal(in1)) - Expect(in1.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(in1.Intersection(inA)).To(Equal(doesNotExist)) - Expect(in1.Intersection(inB)).To(Equal(doesNotExist)) - Expect(in1.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(in1.Intersection(notInA)).To(Equal(in1)) - Expect(in1.Intersection(in1)).To(Equal(in1)) - Expect(in1.Intersection(in9)).To(Equal(doesNotExist)) - Expect(in1.Intersection(in19)).To(Equal(in1)) - Expect(in1.Intersection(notIn12)).To(Equal(doesNotExist)) - Expect(in1.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(in1.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(in1.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(in1.Intersection(lessThan9)).To(Equal(in1)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(in1OperatorWithFlexibility.Intersection(exists)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(notInA)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in1)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in19)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(notIn12)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(in1OperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(in1OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in1OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - Expect(in1OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in1OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in1OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(in9.Intersection(exists)).To(Equal(in9)) - Expect(in9.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(in9.Intersection(inA)).To(Equal(doesNotExist)) - Expect(in9.Intersection(inB)).To(Equal(doesNotExist)) - Expect(in9.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(in9.Intersection(notInA)).To(Equal(in9)) - Expect(in9.Intersection(in1)).To(Equal(doesNotExist)) - Expect(in9.Intersection(in9)).To(Equal(in9)) - Expect(in9.Intersection(in19)).To(Equal(in9)) - Expect(in9.Intersection(notIn12)).To(Equal(in9)) - Expect(in9.Intersection(greaterThan1)).To(Equal(in9)) - Expect(in9.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(in9.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(in9.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(in9OperatorWithFlexibility.Intersection(exists)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(notInA)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in9)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in19)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(notIn12)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(in9OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in9OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in9OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in9OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(in9OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(in19.Intersection(exists)).To(Equal(in19)) - Expect(in19.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(in19.Intersection(inA)).To(Equal(doesNotExist)) - Expect(in19.Intersection(inB)).To(Equal(doesNotExist)) - Expect(in19.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(in19.Intersection(notInA)).To(Equal(in19)) - Expect(in19.Intersection(in1)).To(Equal(in1)) - Expect(in19.Intersection(in9)).To(Equal(in9)) - Expect(in19.Intersection(in19)).To(Equal(in19)) - Expect(in19.Intersection(notIn12)).To(Equal(in9)) - Expect(in19.Intersection(greaterThan1)).To(Equal(in9)) - Expect(in19.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(in19.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(in19.Intersection(lessThan9)).To(Equal(in1)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(in19OperatorWithFlexibility.Intersection(exists)).To(Equal(in19OperatorWithFlexibility)) - Expect(in19OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inA)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inB)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inAB)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(notInA)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in19)).To(Equal(in19OperatorWithFlexibility)) - Expect(in19OperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(in19OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(in19OperatorWithFlexibility)) - Expect(in19OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(in19OperatorWithFlexibility)) - Expect(in19OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(in19OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(notIn12.Intersection(exists)).To(Equal(notIn12)) - Expect(notIn12.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(notIn12.Intersection(inA)).To(Equal(inA)) - Expect(notIn12.Intersection(inB)).To(Equal(inB)) - Expect(notIn12.Intersection(inAB)).To(Equal(inAB)) - Expect(notIn12.Intersection(notInA)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2")})) - Expect(notIn12.Intersection(in1)).To(Equal(doesNotExist)) - Expect(notIn12.Intersection(in9)).To(Equal(in9)) - Expect(notIn12.Intersection(in19)).To(Equal(in9)) - Expect(notIn12.Intersection(notIn12)).To(Equal(notIn12)) - Expect(notIn12.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2")})) - Expect(notIn12.Intersection(greaterThan9)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string]()})) - Expect(notIn12.Intersection(lessThan1)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string]()})) - Expect(notIn12.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2")})) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(notIn12OperatorWithFlexibility.Intersection(exists)).To(Equal(notIn12OperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inA)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inB)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inAB)).To(Equal(inABOperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(notInA)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in1)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in9)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in19)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(notIn12)).To(Equal(notIn12OperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string](), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)})) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(notIn12OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(notIn12OperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(inABOperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(notIn12OperatorWithFlexibility)) - Expect(notIn12OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string](), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)})) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(greaterThan1.Intersection(exists)).To(Equal(greaterThan1)) - Expect(greaterThan1.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(inA)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(inB)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(notInA)).To(Equal(greaterThan1)) - Expect(greaterThan1.Intersection(in1)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(in9)).To(Equal(in9)) - Expect(greaterThan1.Intersection(in19)).To(Equal(in9)) - Expect(greaterThan1.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2")})) - Expect(greaterThan1.Intersection(greaterThan1)).To(Equal(greaterThan1)) - Expect(greaterThan1.Intersection(greaterThan9)).To(Equal(greaterThan9)) - Expect(greaterThan1.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(greaterThan1.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string]()})) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(greaterThan1OperatorWithFlexibility.Intersection(exists)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(notInA)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in9)).To(Equal(in9OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in19)).To(Equal(in9OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(1)})) - Expect(greaterThan1OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)})) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(greaterThan1OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(greaterThan1OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(in9OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)})) - Expect(greaterThan1OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)})) - Expect(greaterThan1OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(greaterThan1OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan1OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)})) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(greaterThan9.Intersection(exists)).To(Equal(greaterThan9)) - Expect(greaterThan9.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(inA)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(inB)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(notInA)).To(Equal(greaterThan9)) - Expect(greaterThan9.Intersection(in1)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(in9)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(in19)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(notIn12)).To(Equal(greaterThan9)) - Expect(greaterThan9.Intersection(greaterThan1)).To(Equal(greaterThan9)) - Expect(greaterThan9.Intersection(greaterThan9)).To(Equal(greaterThan9)) - Expect(greaterThan9.Intersection(lessThan1)).To(Equal(doesNotExist)) - Expect(greaterThan9.Intersection(lessThan9)).To(Equal(doesNotExist)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(greaterThan9OperatorWithFlexibility.Intersection(exists)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(notInA)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in19)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(notIn12)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(greaterThan9OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(greaterThan9OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(greaterThan9OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(greaterThan9OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(greaterThan9OperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(greaterThan9OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(lessThan1.Intersection(exists)).To(Equal(lessThan1)) - Expect(lessThan1.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(inA)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(inB)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(notInA)).To(Equal(lessThan1)) - Expect(lessThan1.Intersection(in1)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(in9)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(in19)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(notIn12)).To(Equal(lessThan1)) - Expect(lessThan1.Intersection(greaterThan1)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(lessThan1.Intersection(lessThan1)).To(Equal(lessThan1)) - Expect(lessThan1.Intersection(lessThan9)).To(Equal(lessThan1)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(lessThan1OperatorWithFlexibility.Intersection(exists)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(notInA)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in19)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(notIn12)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(lessThan1OperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(lessThan1OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(lessThan1OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(lessThan1OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(lessThan1OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan1OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - - // Intersection of requirement without minValues vs requirement without minValues - Expect(lessThan9.Intersection(exists)).To(Equal(lessThan9)) - Expect(lessThan9.Intersection(doesNotExist)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(inA)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(inB)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(inAB)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(notInA)).To(Equal(lessThan9)) - Expect(lessThan9.Intersection(in1)).To(Equal(in1)) - Expect(lessThan9.Intersection(in9)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(in19)).To(Equal(in1)) - Expect(lessThan9.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2")})) - Expect(lessThan9.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string]()})) - Expect(lessThan9.Intersection(greaterThan9)).To(Equal(doesNotExist)) - Expect(lessThan9.Intersection(lessThan1)).To(Equal(lessThan1)) - Expect(lessThan9.Intersection(lessThan9)).To(Equal(lessThan9)) - - // Intersection of requirement with minValues vs requirement without minValues - Expect(lessThan9OperatorWithFlexibility.Intersection(exists)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(doesNotExist)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inA)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inAB)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(notInA)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in1)).To(Equal(in1OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in19)).To(Equal(in1OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(notIn12)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(1)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(greaterThan1)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(greaterThan9)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(lessThan1)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(lessThan9)).To(Equal(lessThan9OperatorWithFlexibility)) - - // Intersection of requirement with minValues vs requirement with minValues - Expect(lessThan9OperatorWithFlexibility.Intersection(existsOperatorWithFlexibility)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(doesNotExistOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inAOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inBOperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(inABOperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(notInAOperatorWithFlexibility)).To(Equal(lessThan9OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in1OperatorWithFlexibility)).To(Equal(in1OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(in19OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(notIn12OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(greaterThan1OperatorWithFlexibility)).To(Equal(&Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)})) - Expect(lessThan9OperatorWithFlexibility.Intersection(greaterThan9OperatorWithFlexibility)).To(Equal(doesNotExistOperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(lessThan1OperatorWithFlexibility)).To(Equal(lessThan1OperatorWithFlexibility)) - Expect(lessThan9OperatorWithFlexibility.Intersection(lessThan9OperatorWithFlexibility)).To(Equal(lessThan9OperatorWithFlexibility)) - }) + Context("Intersect requirements", func() { + DescribeTable("should intersect two requirements without minValues", + func(existingRequirementWithoutMinValues, newRequirementWithoutMinValues, expectedRequirement *Requirement) { + Expect(existingRequirementWithoutMinValues.Intersection(newRequirementWithoutMinValues)).To(Equal(expectedRequirement)) + }, + Entry(nil, exists, exists, exists), + Entry(nil, exists, doesNotExist, doesNotExist), + Entry(nil, exists, inA, inA), + Entry(nil, exists, inB, inB), + Entry(nil, exists, inAB, inAB), + Entry(nil, exists, notInA, notInA), + Entry(nil, exists, in1, in1), + Entry(nil, exists, in9, in9), + Entry(nil, exists, in19, in19), + Entry(nil, exists, notIn12, notIn12), + Entry(nil, exists, greaterThan1, greaterThan1), + Entry(nil, exists, greaterThan9, greaterThan9), + Entry(nil, exists, lessThan1, lessThan1), + Entry(nil, exists, lessThan9, lessThan9), + + Entry(nil, doesNotExist, exists, doesNotExist), + Entry(nil, doesNotExist, doesNotExist, doesNotExist), + Entry(nil, doesNotExist, inA, doesNotExist), + Entry(nil, doesNotExist, inB, doesNotExist), + Entry(nil, doesNotExist, inAB, doesNotExist), + Entry(nil, doesNotExist, notInA, doesNotExist), + Entry(nil, doesNotExist, in1, doesNotExist), + Entry(nil, doesNotExist, in9, doesNotExist), + Entry(nil, doesNotExist, in19, doesNotExist), + Entry(nil, doesNotExist, notIn12, doesNotExist), + Entry(nil, doesNotExist, greaterThan1, doesNotExist), + Entry(nil, doesNotExist, greaterThan9, doesNotExist), + Entry(nil, doesNotExist, lessThan1, doesNotExist), + Entry(nil, doesNotExist, lessThan9, doesNotExist), + + Entry(nil, inA, exists, inA), + Entry(nil, inA, doesNotExist, doesNotExist), + Entry(nil, inA, inA, inA), + Entry(nil, inA, inB, doesNotExist), + Entry(nil, inA, inAB, inA), + Entry(nil, inA, notInA, doesNotExist), + Entry(nil, inA, in1, doesNotExist), + Entry(nil, inA, in9, doesNotExist), + Entry(nil, inA, in19, doesNotExist), + Entry(nil, inA, notIn12, inA), + Entry(nil, inA, greaterThan1, doesNotExist), + Entry(nil, inA, greaterThan9, doesNotExist), + Entry(nil, inA, lessThan1, doesNotExist), + Entry(nil, inA, lessThan9, doesNotExist), + + Entry(nil, inB, exists, inB), + Entry(nil, inB, doesNotExist, doesNotExist), + Entry(nil, inB, inA, doesNotExist), + Entry(nil, inB, inB, inB), + Entry(nil, inB, inAB, inB), + Entry(nil, inB, notInA, inB), + Entry(nil, inB, in1, doesNotExist), + Entry(nil, inB, in9, doesNotExist), + Entry(nil, inB, in19, doesNotExist), + Entry(nil, inB, notIn12, inB), + Entry(nil, inB, greaterThan1, doesNotExist), + Entry(nil, inB, greaterThan9, doesNotExist), + Entry(nil, inB, lessThan1, doesNotExist), + Entry(nil, inB, lessThan9, doesNotExist), + + Entry(nil, inAB, exists, inAB), + Entry(nil, inAB, doesNotExist, doesNotExist), + Entry(nil, inAB, inA, inA), + Entry(nil, inAB, inB, inB), + Entry(nil, inAB, inAB, inAB), + Entry(nil, inAB, notInA, inB), + Entry(nil, inAB, in1, doesNotExist), + Entry(nil, inAB, in9, doesNotExist), + Entry(nil, inAB, in19, doesNotExist), + Entry(nil, inAB, notIn12, inAB), + Entry(nil, inAB, greaterThan1, doesNotExist), + Entry(nil, inAB, greaterThan9, doesNotExist), + Entry(nil, inAB, lessThan1, doesNotExist), + Entry(nil, inAB, lessThan9, doesNotExist), + + Entry(nil, notInA, exists, notInA), + Entry(nil, notInA, doesNotExist, doesNotExist), + Entry(nil, notInA, inA, doesNotExist), + Entry(nil, notInA, inB, inB), + Entry(nil, notInA, inAB, inB), + Entry(nil, notInA, notInA, notInA), + Entry(nil, notInA, in1, in1), + Entry(nil, notInA, in9, in9), + Entry(nil, notInA, in19, in19), + Entry(nil, notInA, notIn12, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2")}), + Entry(nil, notInA, greaterThan1, greaterThan1), + Entry(nil, notInA, greaterThan9, greaterThan9), + Entry(nil, notInA, lessThan1, lessThan1), + Entry(nil, notInA, lessThan9, lessThan9), + + Entry(nil, in1, exists, in1), + Entry(nil, in1, doesNotExist, doesNotExist), + Entry(nil, in1, inA, doesNotExist), + Entry(nil, in1, inB, doesNotExist), + Entry(nil, in1, inAB, doesNotExist), + Entry(nil, in1, notInA, in1), + Entry(nil, in1, in1, in1), + Entry(nil, in1, in9, doesNotExist), + Entry(nil, in1, in19, in1), + Entry(nil, in1, notIn12, doesNotExist), + Entry(nil, in1, greaterThan1, doesNotExist), + Entry(nil, in1, greaterThan9, doesNotExist), + Entry(nil, in1, lessThan1, doesNotExist), + Entry(nil, in1, lessThan9, in1), + + Entry(nil, in9, exists, in9), + Entry(nil, in9, doesNotExist, doesNotExist), + Entry(nil, in9, inA, doesNotExist), + Entry(nil, in9, inB, doesNotExist), + Entry(nil, in9, inAB, doesNotExist), + Entry(nil, in9, notInA, in9), + Entry(nil, in9, in1, doesNotExist), + Entry(nil, in9, in9, in9), + Entry(nil, in9, in19, in9), + Entry(nil, in9, notIn12, in9), + Entry(nil, in9, greaterThan1, in9), + Entry(nil, in9, greaterThan9, doesNotExist), + Entry(nil, in9, lessThan1, doesNotExist), + Entry(nil, in9, lessThan9, doesNotExist), + + Entry(nil, in19, exists, in19), + Entry(nil, in19, doesNotExist, doesNotExist), + Entry(nil, in19, inA, doesNotExist), + Entry(nil, in19, inB, doesNotExist), + Entry(nil, in19, inAB, doesNotExist), + Entry(nil, in19, notInA, in19), + Entry(nil, in19, in1, in1), + Entry(nil, in19, in9, in9), + Entry(nil, in19, in19, in19), + Entry(nil, in19, notIn12, in9), + Entry(nil, in19, greaterThan1, in9), + Entry(nil, in19, greaterThan9, doesNotExist), + Entry(nil, in19, lessThan1, doesNotExist), + Entry(nil, in19, lessThan9, in1), + + Entry(nil, notIn12, exists, notIn12), + Entry(nil, notIn12, doesNotExist, doesNotExist), + Entry(nil, notIn12, inA, inA), + Entry(nil, notIn12, inB, inB), + Entry(nil, notIn12, inAB, inAB), + Entry(nil, notIn12, notInA, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2")}), + Entry(nil, notIn12, in1, doesNotExist), + Entry(nil, notIn12, in9, in9), + Entry(nil, notIn12, in19, in9), + Entry(nil, notIn12, notIn12, notIn12), + Entry(nil, notIn12, greaterThan1, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2")}), + Entry(nil, notIn12, greaterThan9, &Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string]()}), + Entry(nil, notIn12, lessThan1, &Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string]()}), + Entry(nil, notIn12, lessThan9, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2")}), + + Entry(nil, greaterThan1, exists, greaterThan1), + Entry(nil, greaterThan1, doesNotExist, doesNotExist), + Entry(nil, greaterThan1, inA, doesNotExist), + Entry(nil, greaterThan1, inB, doesNotExist), + Entry(nil, greaterThan1, inAB, doesNotExist), + Entry(nil, greaterThan1, notInA, greaterThan1), + Entry(nil, greaterThan1, in1, doesNotExist), + Entry(nil, greaterThan1, in9, in9), + Entry(nil, greaterThan1, in19, in9), + Entry(nil, greaterThan1, notIn12, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2")}), + Entry(nil, greaterThan1, greaterThan1, greaterThan1), + Entry(nil, greaterThan1, greaterThan9, greaterThan9), + Entry(nil, greaterThan1, lessThan1, doesNotExist), + Entry(nil, greaterThan1, lessThan9, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string]()}), + + Entry(nil, greaterThan9, exists, greaterThan9), + Entry(nil, greaterThan9, doesNotExist, doesNotExist), + Entry(nil, greaterThan9, inA, doesNotExist), + Entry(nil, greaterThan9, inB, doesNotExist), + Entry(nil, greaterThan9, inAB, doesNotExist), + Entry(nil, greaterThan9, notInA, greaterThan9), + Entry(nil, greaterThan9, in1, doesNotExist), + Entry(nil, greaterThan9, in9, doesNotExist), + Entry(nil, greaterThan9, in19, doesNotExist), + Entry(nil, greaterThan9, notIn12, greaterThan9), + Entry(nil, greaterThan9, greaterThan1, greaterThan9), + Entry(nil, greaterThan9, greaterThan9, greaterThan9), + Entry(nil, greaterThan9, lessThan1, doesNotExist), + Entry(nil, greaterThan9, lessThan9, doesNotExist), + + Entry(nil, lessThan1, exists, lessThan1), + Entry(nil, lessThan1, doesNotExist, doesNotExist), + Entry(nil, lessThan1, inA, doesNotExist), + Entry(nil, lessThan1, inB, doesNotExist), + Entry(nil, lessThan1, inAB, doesNotExist), + Entry(nil, lessThan1, notInA, lessThan1), + Entry(nil, lessThan1, in1, doesNotExist), + Entry(nil, lessThan1, in9, doesNotExist), + Entry(nil, lessThan1, in19, doesNotExist), + Entry(nil, lessThan1, notIn12, lessThan1), + Entry(nil, lessThan1, greaterThan1, doesNotExist), + Entry(nil, lessThan1, greaterThan9, doesNotExist), + Entry(nil, lessThan1, lessThan1, lessThan1), + Entry(nil, lessThan1, lessThan9, lessThan1), + + Entry(nil, lessThan9, exists, lessThan9), + Entry(nil, lessThan9, doesNotExist, doesNotExist), + Entry(nil, lessThan9, inA, doesNotExist), + Entry(nil, lessThan9, inB, doesNotExist), + Entry(nil, lessThan9, inAB, doesNotExist), + Entry(nil, lessThan9, notInA, lessThan9), + Entry(nil, lessThan9, in1, in1), + Entry(nil, lessThan9, in9, doesNotExist), + Entry(nil, lessThan9, in19, in1), + Entry(nil, lessThan9, notIn12, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2")}), + Entry(nil, lessThan9, greaterThan1, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string]()}), + Entry(nil, lessThan9, greaterThan9, doesNotExist), + Entry(nil, lessThan9, lessThan1, lessThan1), + Entry(nil, lessThan9, lessThan9, lessThan9), + ) + DescribeTable("should intersect requirement with minValues with a requirement without", + func(existingRequirementWithMinValues, newRequirementWithoutMinValues, expectedRequirement *Requirement) { + Expect(existingRequirementWithMinValues.Intersection(newRequirementWithoutMinValues)).To(Equal(expectedRequirement)) + }, + Entry(nil, existsOperatorWithFlexibility, exists, existsOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inA, inAOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inB, inBOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inAB, &Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(1)}), + Entry(nil, existsOperatorWithFlexibility, notInA, notInAOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in1, in1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in9, in9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in19, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(1)}), + Entry(nil, existsOperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: true, values: sets.New("1", "2"), MinValues: lo.ToPtr(1)}), + Entry(nil, existsOperatorWithFlexibility, greaterThan1, greaterThan1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, greaterThan9, greaterThan9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, lessThan1, lessThan1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, lessThan9, lessThan9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, exists, existsOperatorWithFlexibility), + + Entry(nil, doesNotExistOperatorWithFlexibility, exists, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, notInA, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in19, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, notIn12, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, greaterThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, lessThan9, doesNotExistOperatorWithFlexibility), + + Entry(nil, inAOperatorWithFlexibility, exists, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inA, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inAB, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, notInA, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in19, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, notIn12, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, greaterThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, lessThan9, doesNotExistOperatorWithFlexibility), + + Entry(nil, inBOperatorWithFlexibility, exists, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inB, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inAB, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, notInA, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in19, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, notIn12, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, greaterThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, lessThan9, doesNotExistOperatorWithFlexibility), + + Entry(nil, inABOperatorWithFlexibility, exists, inABOperatorWithFlexibility), + Entry(nil, inABOperatorWithFlexibility, doesNotExist, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inA, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inB, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inAB, inABOperatorWithFlexibility), + Entry(nil, inABOperatorWithFlexibility, notInA, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in1, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in9, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in19, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, greaterThan1, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, greaterThan9, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, lessThan1, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, lessThan9, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + + Entry(nil, notInAOperatorWithFlexibility, exists, notInAOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inB, inBOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inAB, inBOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, notInA, notInAOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in1, in1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in9, in9OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in19, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(1)}), + Entry(nil, notInAOperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(1)}), + Entry(nil, notInAOperatorWithFlexibility, greaterThan1, greaterThan1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, greaterThan9, greaterThan9OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, lessThan1, lessThan1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, lessThan9, lessThan9OperatorWithFlexibility), + + Entry(nil, in1OperatorWithFlexibility, exists, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, notInA, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in1, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in19, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, notIn12, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, greaterThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, lessThan9, in1OperatorWithFlexibility), + + Entry(nil, in9OperatorWithFlexibility, exists, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, notInA, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in9, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in19, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, notIn12, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, greaterThan1, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, lessThan9, doesNotExistOperatorWithFlexibility), + + Entry(nil, in19OperatorWithFlexibility, exists, in19OperatorWithFlexibility), + Entry(nil, in19OperatorWithFlexibility, doesNotExist, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inA, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inB, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inAB, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, notInA, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in1, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in9, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in19, in19OperatorWithFlexibility), + Entry(nil, in19OperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, greaterThan1, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, greaterThan9, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, lessThan1, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, lessThan9, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + + Entry(nil, notIn12OperatorWithFlexibility, exists, notIn12OperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, doesNotExist, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inA, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inB, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inAB, inABOperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, notInA, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in1, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in9, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in19, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, notIn12, notIn12OperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, greaterThan1, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, greaterThan9, &Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string](), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, lessThan1, &Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, lessThan9, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)}), + + Entry(nil, greaterThan1OperatorWithFlexibility, exists, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, notInA, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in9, in9OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in19, in9OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(1)}), + Entry(nil, greaterThan1OperatorWithFlexibility, greaterThan1, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, greaterThan9, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, lessThan9, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)}), + + Entry(nil, greaterThan9OperatorWithFlexibility, exists, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, notInA, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in19, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, notIn12, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, greaterThan1, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, greaterThan9, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, lessThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, lessThan9, doesNotExistOperatorWithFlexibility), + + Entry(nil, lessThan1OperatorWithFlexibility, exists, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, notInA, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in1, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in19, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, notIn12, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, greaterThan1, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, lessThan1, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, lessThan9, lessThan1OperatorWithFlexibility), + + Entry(nil, lessThan9OperatorWithFlexibility, exists, lessThan9OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, doesNotExist, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inA, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inB, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inAB, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, notInA, lessThan9OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in1, in1OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in9, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in19, in1OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, notIn12, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(1)}), + Entry(nil, lessThan9OperatorWithFlexibility, greaterThan1, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)}), + Entry(nil, lessThan9OperatorWithFlexibility, greaterThan9, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, lessThan1, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, lessThan9, lessThan9OperatorWithFlexibility), + ) + DescribeTable("should intersect two requirements with minValues", + func(existingRequirementWithMinValues, newRequirementWithMinValues, expectedRequirement *Requirement) { + Expect(existingRequirementWithMinValues.Intersection(newRequirementWithMinValues)).To(Equal(expectedRequirement)) + }, + Entry(nil, existsOperatorWithFlexibility, existsOperatorWithFlexibility, existsOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inAOperatorWithFlexibility, inAOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inBOperatorWithFlexibility, inBOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)}), + Entry(nil, existsOperatorWithFlexibility, notInAOperatorWithFlexibility, notInAOperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in1OperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in9OperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)}), + Entry(nil, existsOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)}), + Entry(nil, existsOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, lessThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, lessThan9OperatorWithFlexibility, lessThan9OperatorWithFlexibility), + Entry(nil, existsOperatorWithFlexibility, existsOperatorWithFlexibility, existsOperatorWithFlexibility), + + Entry(nil, doesNotExistOperatorWithFlexibility, existsOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, doesNotExistOperatorWithFlexibility, notInAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, doesNotExistOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, doesNotExistOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, doesNotExistOperatorWithFlexibility, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + + Entry(nil, inAOperatorWithFlexibility, existsOperatorWithFlexibility, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inAOperatorWithFlexibility, inAOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, inAOperatorWithFlexibility, notInAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inAOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, inAOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inAOperatorWithFlexibility, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + + Entry(nil, inBOperatorWithFlexibility, existsOperatorWithFlexibility, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inBOperatorWithFlexibility, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inBOperatorWithFlexibility, notInAOperatorWithFlexibility, inBOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inBOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inBOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, inBOperatorWithFlexibility, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + + Entry(nil, inABOperatorWithFlexibility, existsOperatorWithFlexibility, inABOperatorWithFlexibility), + Entry(nil, inABOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inAOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inBOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, inABOperatorWithFlexibility, inABOperatorWithFlexibility), + Entry(nil, inABOperatorWithFlexibility, notInAOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A", "B"), MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, lessThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, inABOperatorWithFlexibility, lessThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + + Entry(nil, notInAOperatorWithFlexibility, existsOperatorWithFlexibility, notInAOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inBOperatorWithFlexibility, inBOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, notInAOperatorWithFlexibility, notInAOperatorWithFlexibility, notInAOperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in1OperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in9OperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)}), + Entry(nil, notInAOperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)}), + Entry(nil, notInAOperatorWithFlexibility, greaterThan1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, greaterThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, lessThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, notInAOperatorWithFlexibility, lessThan9OperatorWithFlexibility, lessThan9OperatorWithFlexibility), + + Entry(nil, in1OperatorWithFlexibility, existsOperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in1OperatorWithFlexibility, notInAOperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in1OperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + Entry(nil, in1OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in1OperatorWithFlexibility, lessThan9OperatorWithFlexibility, in1OperatorWithFlexibility), + + Entry(nil, in9OperatorWithFlexibility, existsOperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in9OperatorWithFlexibility, notInAOperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in9OperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in9OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in9OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, in9OperatorWithFlexibility, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + + Entry(nil, in19OperatorWithFlexibility, existsOperatorWithFlexibility, in19OperatorWithFlexibility), + Entry(nil, in19OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inAOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inBOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, notInAOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1", "9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, in19OperatorWithFlexibility, in19OperatorWithFlexibility), + Entry(nil, in19OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, lessThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, in19OperatorWithFlexibility, lessThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + + Entry(nil, notIn12OperatorWithFlexibility, existsOperatorWithFlexibility, notIn12OperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inAOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("A"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inBOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("B"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, inABOperatorWithFlexibility, inABOperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, notInAOperatorWithFlexibility, &Requirement{Key: "key", complement: true, values: sets.New("A", "1", "2"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in1OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in9OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, notIn12OperatorWithFlexibility, notIn12OperatorWithFlexibility), + Entry(nil, notIn12OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.New[string](), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, lessThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility, lessThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)}), + + Entry(nil, greaterThan1OperatorWithFlexibility, existsOperatorWithFlexibility, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan1OperatorWithFlexibility, notInAOperatorWithFlexibility, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in9OperatorWithFlexibility, in9OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("9"), MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan1OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, values: sets.New("2"), MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan1OperatorWithFlexibility, lessThan9OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)}), + + Entry(nil, greaterThan9OperatorWithFlexibility, existsOperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan9OperatorWithFlexibility, notInAOperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan9OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan9.greaterThan, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan9OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, greaterThan9OperatorWithFlexibility, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + + Entry(nil, lessThan1OperatorWithFlexibility, existsOperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan1OperatorWithFlexibility, notInAOperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan1OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, lessThan: lessThan1.lessThan, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan1OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan1OperatorWithFlexibility, lessThan9OperatorWithFlexibility, lessThan1OperatorWithFlexibility), + + Entry(nil, lessThan9OperatorWithFlexibility, existsOperatorWithFlexibility, lessThan9OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inAOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inBOperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, inABOperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.Set[string]{}, MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan9OperatorWithFlexibility, notInAOperatorWithFlexibility, lessThan9OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in1OperatorWithFlexibility, in1OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, in19OperatorWithFlexibility, &Requirement{Key: "key", complement: false, values: sets.New("1"), MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan9OperatorWithFlexibility, notIn12OperatorWithFlexibility, &Requirement{Key: "key", complement: true, lessThan: lessThan9.lessThan, values: sets.New("1", "2"), MinValues: lo.ToPtr(2)}), + Entry(nil, lessThan9OperatorWithFlexibility, greaterThan1OperatorWithFlexibility, &Requirement{Key: "key", complement: true, greaterThan: greaterThan1.greaterThan, lessThan: lessThan9.lessThan, values: sets.New[string](), MinValues: lo.ToPtr(1)}), + Entry(nil, lessThan9OperatorWithFlexibility, greaterThan9OperatorWithFlexibility, doesNotExistOperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, lessThan1OperatorWithFlexibility, lessThan1OperatorWithFlexibility), + Entry(nil, lessThan9OperatorWithFlexibility, lessThan9OperatorWithFlexibility, lessThan9OperatorWithFlexibility), + ) }) Context("Has", func() { - It("should have the right values", func() { - Expect(exists.Has("A")).To(BeTrue()) - Expect(doesNotExist.Has("A")).To(BeFalse()) - Expect(inA.Has("A")).To(BeTrue()) - Expect(inB.Has("A")).To(BeFalse()) - Expect(inAB.Has("A")).To(BeTrue()) - Expect(notInA.Has("A")).To(BeFalse()) - Expect(in1.Has("A")).To(BeFalse()) - Expect(in9.Has("A")).To(BeFalse()) - Expect(in19.Has("A")).To(BeFalse()) - Expect(notIn12.Has("A")).To(BeTrue()) - Expect(greaterThan1.Has("A")).To(BeFalse()) - Expect(greaterThan9.Has("A")).To(BeFalse()) - Expect(lessThan1.Has("A")).To(BeFalse()) - Expect(lessThan9.Has("A")).To(BeFalse()) - - Expect(exists.Has("B")).To(BeTrue()) - Expect(doesNotExist.Has("B")).To(BeFalse()) - Expect(inA.Has("B")).To(BeFalse()) - Expect(inB.Has("B")).To(BeTrue()) - Expect(inAB.Has("B")).To(BeTrue()) - Expect(notInA.Has("B")).To(BeTrue()) - Expect(in1.Has("B")).To(BeFalse()) - Expect(in9.Has("B")).To(BeFalse()) - Expect(in19.Has("B")).To(BeFalse()) - Expect(notIn12.Has("B")).To(BeTrue()) - Expect(greaterThan1.Has("B")).To(BeFalse()) - Expect(greaterThan9.Has("B")).To(BeFalse()) - Expect(lessThan1.Has("B")).To(BeFalse()) - Expect(lessThan9.Has("B")).To(BeFalse()) - - Expect(exists.Has("1")).To(BeTrue()) - Expect(doesNotExist.Has("1")).To(BeFalse()) - Expect(inA.Has("1")).To(BeFalse()) - Expect(inB.Has("1")).To(BeFalse()) - Expect(inAB.Has("1")).To(BeFalse()) - Expect(notInA.Has("1")).To(BeTrue()) - Expect(in1.Has("1")).To(BeTrue()) - Expect(in9.Has("1")).To(BeFalse()) - Expect(in19.Has("1")).To(BeTrue()) - Expect(notIn12.Has("1")).To(BeFalse()) - Expect(greaterThan1.Has("1")).To(BeFalse()) - Expect(greaterThan9.Has("1")).To(BeFalse()) - Expect(lessThan1.Has("1")).To(BeFalse()) - Expect(lessThan9.Has("1")).To(BeTrue()) - - Expect(exists.Has("2")).To(BeTrue()) - Expect(doesNotExist.Has("2")).To(BeFalse()) - Expect(inA.Has("2")).To(BeFalse()) - Expect(inB.Has("2")).To(BeFalse()) - Expect(inAB.Has("2")).To(BeFalse()) - Expect(notInA.Has("2")).To(BeTrue()) - Expect(in1.Has("2")).To(BeFalse()) - Expect(in9.Has("2")).To(BeFalse()) - Expect(in19.Has("2")).To(BeFalse()) - Expect(notIn12.Has("2")).To(BeFalse()) - Expect(greaterThan1.Has("2")).To(BeTrue()) - Expect(greaterThan9.Has("2")).To(BeFalse()) - Expect(lessThan1.Has("2")).To(BeFalse()) - Expect(lessThan9.Has("2")).To(BeTrue()) - - Expect(exists.Has("9")).To(BeTrue()) - Expect(doesNotExist.Has("9")).To(BeFalse()) - Expect(inA.Has("9")).To(BeFalse()) - Expect(inB.Has("9")).To(BeFalse()) - Expect(inAB.Has("9")).To(BeFalse()) - Expect(notInA.Has("9")).To(BeTrue()) - Expect(in1.Has("9")).To(BeFalse()) - Expect(in9.Has("9")).To(BeTrue()) - Expect(in19.Has("9")).To(BeTrue()) - Expect(notIn12.Has("9")).To(BeTrue()) - Expect(greaterThan1.Has("9")).To(BeTrue()) - Expect(greaterThan9.Has("9")).To(BeFalse()) - Expect(lessThan1.Has("9")).To(BeFalse()) - Expect(lessThan9.Has("9")).To(BeFalse()) - }) + DescribeTable("should have the right values", + func(requirement *Requirement, value string, expected types.GomegaMatcher) { + Expect(requirement.Has(value)).To(expected) + }, + + Entry(nil, exists, "A", BeTrue()), + Entry(nil, doesNotExist, "A", BeFalse()), + Entry(nil, inA, "A", BeTrue()), + Entry(nil, inB, "A", BeFalse()), + Entry(nil, inAB, "A", BeTrue()), + Entry(nil, notInA, "A", BeFalse()), + Entry(nil, in1, "A", BeFalse()), + Entry(nil, in9, "A", BeFalse()), + Entry(nil, in19, "A", BeFalse()), + Entry(nil, notIn12, "A", BeTrue()), + Entry(nil, greaterThan1, "A", BeFalse()), + Entry(nil, greaterThan9, "A", BeFalse()), + Entry(nil, lessThan1, "A", BeFalse()), + Entry(nil, lessThan9, "A", BeFalse()), + + Entry(nil, exists, "B", BeTrue()), + Entry(nil, doesNotExist, "B", BeFalse()), + Entry(nil, inA, "B", BeFalse()), + Entry(nil, inB, "B", BeTrue()), + Entry(nil, inAB, "B", BeTrue()), + Entry(nil, notInA, "B", BeTrue()), + Entry(nil, in1, "B", BeFalse()), + Entry(nil, in9, "B", BeFalse()), + Entry(nil, in19, "B", BeFalse()), + Entry(nil, notIn12, "B", BeTrue()), + Entry(nil, greaterThan1, "B", BeFalse()), + Entry(nil, greaterThan9, "B", BeFalse()), + Entry(nil, lessThan1, "B", BeFalse()), + Entry(nil, lessThan9, "B", BeFalse()), + + Entry(nil, exists, "1", BeTrue()), + Entry(nil, doesNotExist, "1", BeFalse()), + Entry(nil, inA, "1", BeFalse()), + Entry(nil, inB, "1", BeFalse()), + Entry(nil, inAB, "1", BeFalse()), + Entry(nil, notInA, "1", BeTrue()), + Entry(nil, in1, "1", BeTrue()), + Entry(nil, in9, "1", BeFalse()), + Entry(nil, in19, "1", BeTrue()), + Entry(nil, notIn12, "1", BeFalse()), + Entry(nil, greaterThan1, "1", BeFalse()), + Entry(nil, greaterThan9, "1", BeFalse()), + Entry(nil, lessThan1, "1", BeFalse()), + Entry(nil, lessThan9, "1", BeTrue()), + + Entry(nil, exists, "2", BeTrue()), + Entry(nil, doesNotExist, "2", BeFalse()), + Entry(nil, inA, "2", BeFalse()), + Entry(nil, inB, "2", BeFalse()), + Entry(nil, inAB, "2", BeFalse()), + Entry(nil, notInA, "2", BeTrue()), + Entry(nil, in1, "2", BeFalse()), + Entry(nil, in9, "2", BeFalse()), + Entry(nil, in19, "2", BeFalse()), + Entry(nil, notIn12, "2", BeFalse()), + Entry(nil, greaterThan1, "2", BeTrue()), + Entry(nil, greaterThan9, "2", BeFalse()), + Entry(nil, lessThan1, "2", BeFalse()), + Entry(nil, lessThan9, "2", BeTrue()), + + Entry(nil, exists, "9", BeTrue()), + Entry(nil, doesNotExist, "9", BeFalse()), + Entry(nil, inA, "9", BeFalse()), + Entry(nil, inB, "9", BeFalse()), + Entry(nil, inAB, "9", BeFalse()), + Entry(nil, notInA, "9", BeTrue()), + Entry(nil, in1, "9", BeFalse()), + Entry(nil, in9, "9", BeTrue()), + Entry(nil, in19, "9", BeTrue()), + Entry(nil, notIn12, "9", BeTrue()), + Entry(nil, greaterThan1, "9", BeTrue()), + Entry(nil, greaterThan9, "9", BeFalse()), + Entry(nil, lessThan1, "9", BeFalse()), + Entry(nil, lessThan9, "9", BeFalse()), + ) }) Context("Operator", func() { - It("should return the right operator", func() { - Expect(exists.Operator()).To(Equal(v1.NodeSelectorOpExists)) - Expect(doesNotExist.Operator()).To(Equal(v1.NodeSelectorOpDoesNotExist)) - Expect(inA.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(inB.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(inAB.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(notInA.Operator()).To(Equal(v1.NodeSelectorOpNotIn)) - Expect(in1.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(in9.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(in19.Operator()).To(Equal(v1.NodeSelectorOpIn)) - Expect(notIn12.Operator()).To(Equal(v1.NodeSelectorOpNotIn)) - Expect(greaterThan1.Operator()).To(Equal(v1.NodeSelectorOpExists)) - Expect(greaterThan9.Operator()).To(Equal(v1.NodeSelectorOpExists)) - Expect(lessThan1.Operator()).To(Equal(v1.NodeSelectorOpExists)) - Expect(lessThan9.Operator()).To(Equal(v1.NodeSelectorOpExists)) - }) + DescribeTable("should return the right operator", + func(requirement *Requirement, expectedOperator v1.NodeSelectorOperator) { + Expect(requirement.Operator()).To(Equal(expectedOperator)) + }, + + Entry(nil, exists, v1.NodeSelectorOpExists), + Entry(nil, doesNotExist, v1.NodeSelectorOpDoesNotExist), + Entry(nil, inA, v1.NodeSelectorOpIn), + Entry(nil, inB, v1.NodeSelectorOpIn), + Entry(nil, inAB, v1.NodeSelectorOpIn), + Entry(nil, notInA, v1.NodeSelectorOpNotIn), + Entry(nil, in1, v1.NodeSelectorOpIn), + Entry(nil, in9, v1.NodeSelectorOpIn), + Entry(nil, in19, v1.NodeSelectorOpIn), + Entry(nil, notIn12, v1.NodeSelectorOpNotIn), + Entry(nil, greaterThan1, v1.NodeSelectorOpExists), + Entry(nil, greaterThan9, v1.NodeSelectorOpExists), + Entry(nil, lessThan1, v1.NodeSelectorOpExists), + Entry(nil, lessThan9, v1.NodeSelectorOpExists), + ) }) Context("Len", func() { - It("should have the correct length", func() { - Expect(exists.Len()).To(Equal(math.MaxInt64)) - Expect(doesNotExist.Len()).To(Equal(0)) - Expect(inA.Len()).To(Equal(1)) - Expect(inB.Len()).To(Equal(1)) - Expect(inAB.Len()).To(Equal(2)) - Expect(notInA.Len()).To(Equal(math.MaxInt64 - 1)) - Expect(in1.Len()).To(Equal(1)) - Expect(in9.Len()).To(Equal(1)) - Expect(in19.Len()).To(Equal(2)) - Expect(notIn12.Len()).To(Equal(math.MaxInt64 - 2)) - Expect(greaterThan1.Len()).To(Equal(math.MaxInt64)) - Expect(greaterThan9.Len()).To(Equal(math.MaxInt64)) - Expect(lessThan1.Len()).To(Equal(math.MaxInt64)) - Expect(lessThan9.Len()).To(Equal(math.MaxInt64)) - }) + DescribeTable("should have the correct length", + func(requirement *Requirement, expectedLength int) { + Expect(requirement.Len()).To(Equal(expectedLength)) + }, + + Entry(nil, exists, math.MaxInt64), + Entry(nil, doesNotExist, 0), + Entry(nil, inA, 1), + Entry(nil, inB, 1), + Entry(nil, inAB, 2), + Entry(nil, notInA, math.MaxInt64-1), + Entry(nil, in1, 1), + Entry(nil, in9, 1), + Entry(nil, in19, 2), + Entry(nil, notIn12, math.MaxInt64-2), + Entry(nil, greaterThan1, math.MaxInt64), + Entry(nil, greaterThan9, math.MaxInt64), + Entry(nil, lessThan1, math.MaxInt64), + Entry(nil, lessThan9, math.MaxInt64), + ) }) Context("Any", func() { It("should return any", func() { @@ -909,57 +891,63 @@ var _ = Describe("Requirement", func() { }) }) Context("String", func() { - It("should print the right string", func() { - Expect(exists.String()).To(Equal("key Exists")) - Expect(doesNotExist.String()).To(Equal("key DoesNotExist")) - Expect(inA.String()).To(Equal("key In [A]")) - Expect(inB.String()).To(Equal("key In [B]")) - Expect(inAB.String()).To(Equal("key In [A B]")) - Expect(notInA.String()).To(Equal("key NotIn [A]")) - Expect(in1.String()).To(Equal("key In [1]")) - Expect(in9.String()).To(Equal("key In [9]")) - Expect(in19.String()).To(Equal("key In [1 9]")) - Expect(notIn12.String()).To(Equal("key NotIn [1 2]")) - Expect(greaterThan1.String()).To(Equal("key Exists >1")) - Expect(greaterThan9.String()).To(Equal("key Exists >9")) - Expect(lessThan1.String()).To(Equal("key Exists <1")) - Expect(lessThan9.String()).To(Equal("key Exists <9")) - Expect(greaterThan1.Intersection(lessThan9).String()).To(Equal("key Exists >1 <9")) - Expect(greaterThan9.Intersection(lessThan1).String()).To(Equal("key DoesNotExist")) - }) + DescribeTable("should print the right string", + func(requirement *Requirement, expectedValue string) { + Expect(requirement.String()).To(Equal(expectedValue)) + }, + Entry(nil, exists, "key Exists"), + Entry(nil, doesNotExist, "key DoesNotExist"), + Entry(nil, inA, "key In [A]"), + Entry(nil, inB, "key In [B]"), + Entry(nil, inAB, "key In [A B]"), + Entry(nil, notInA, "key NotIn [A]"), + Entry(nil, in1, "key In [1]"), + Entry(nil, in9, "key In [9]"), + Entry(nil, in19, "key In [1 9]"), + Entry(nil, notIn12, "key NotIn [1 2]"), + Entry(nil, greaterThan1, "key Exists >1"), + Entry(nil, greaterThan9, "key Exists >9"), + Entry(nil, lessThan1, "key Exists <1"), + Entry(nil, lessThan9, "key Exists <9"), + Entry(nil, greaterThan1.Intersection(lessThan9), "key Exists >1 <9"), + Entry(nil, greaterThan9.Intersection(lessThan1), "key DoesNotExist"), + ) }) Context("NodeSelectorRequirements Conversion", func() { - It("should return the expected NodeSelectorRequirement", func() { - Expect(exists.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpExists}})) - Expect(doesNotExist.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpDoesNotExist}})) - Expect(inA.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A"}}})) - Expect(inB.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"B"}}})) - Expect(inAB.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A", "B"}}})) - Expect(notInA.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"A"}}})) - Expect(in1.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1"}}})) - Expect(in9.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"9"}}})) - Expect(in19.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1", "9"}}})) - Expect(notIn12.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"1", "2"}}})) - Expect(greaterThan1.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}})) - Expect(greaterThan9.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"9"}}})) - Expect(lessThan1.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}})) - Expect(lessThan9.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"9"}}})) - - Expect(existsOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpExists}, MinValues: lo.ToPtr(1)})) - Expect(doesNotExistOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpDoesNotExist}, MinValues: lo.ToPtr(1)})) - Expect(inAOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A"}}, MinValues: lo.ToPtr(1)})) - Expect(inBOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"B"}}, MinValues: lo.ToPtr(1)})) - Expect(inABOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A", "B"}}, MinValues: lo.ToPtr(2)})) - Expect(notInAOperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"A"}}, MinValues: lo.ToPtr(1)})) - Expect(in1OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1"}}, MinValues: lo.ToPtr(1)})) - Expect(in9OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"9"}}, MinValues: lo.ToPtr(1)})) - Expect(in19OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1", "9"}}, MinValues: lo.ToPtr(2)})) - Expect(notIn12OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"1", "2"}}, MinValues: lo.ToPtr(2)})) - Expect(greaterThan1OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}, MinValues: lo.ToPtr(1)})) - Expect(greaterThan9OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"9"}}, MinValues: lo.ToPtr(1)})) - Expect(lessThan1OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}, MinValues: lo.ToPtr(1)})) - Expect(lessThan9OperatorWithFlexibility.NodeSelectorRequirement()).To(Equal(v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"9"}}, MinValues: lo.ToPtr(1)})) - }) + DescribeTable("should return the expected NodeSelectorRequirement", + func(requirement v1beta1.NodeSelectorRequirementWithMinValues, expectedRequirement v1beta1.NodeSelectorRequirementWithMinValues) { + Expect(requirement).To(Equal(expectedRequirement)) + }, + Entry(nil, exists.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpExists}}), + Entry(nil, doesNotExist.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpDoesNotExist}}), + Entry(nil, inA.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A"}}}), + Entry(nil, inB.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"B"}}}), + Entry(nil, inAB.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A", "B"}}}), + Entry(nil, notInA.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"A"}}}), + Entry(nil, in1.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1"}}}), + Entry(nil, in9.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"9"}}}), + Entry(nil, in19.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1", "9"}}}), + Entry(nil, notIn12.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"1", "2"}}}), + Entry(nil, greaterThan1.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}}), + Entry(nil, greaterThan9.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"9"}}}), + Entry(nil, lessThan1.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}}), + Entry(nil, lessThan9.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"9"}}}), + + Entry(nil, existsOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpExists}, MinValues: lo.ToPtr(1)}), + Entry(nil, doesNotExistOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpDoesNotExist}, MinValues: lo.ToPtr(1)}), + Entry(nil, inAOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, inBOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"B"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, inABOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"A", "B"}}, MinValues: lo.ToPtr(2)}), + Entry(nil, notInAOperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"A"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, in1OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, in9OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"9"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, in19OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpIn, Values: []string{"1", "9"}}, MinValues: lo.ToPtr(2)}), + Entry(nil, notIn12OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"1", "2"}}, MinValues: lo.ToPtr(2)}), + Entry(nil, greaterThan1OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, greaterThan9OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpGt, Values: []string{"9"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, lessThan1OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}, MinValues: lo.ToPtr(1)}), + Entry(nil, lessThan9OperatorWithFlexibility.NodeSelectorRequirement(), v1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "key", Operator: v1.NodeSelectorOpLt, Values: []string{"9"}}, MinValues: lo.ToPtr(1)}), + ) }) }) From ec1b32e0b3336b9a4aa4146f10753ded0b654590 Mon Sep 17 00:00:00 2001 From: wmgroot Date: Mon, 15 Apr 2024 14:31:31 -0500 Subject: [PATCH 33/33] fix: race condition that occurs with do-not-disrupt annotated pods and disruption decisions, add do-not-consolidate, fix disruption reconcile loop --- pkg/apis/v1beta1/labels.go | 1 + pkg/controllers/disruption/controller.go | 41 ++++++++++++++--- pkg/controllers/disruption/types.go | 9 ++-- pkg/controllers/state/statenode.go | 56 ++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index bfb6e92e84..335944d188 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -43,6 +43,7 @@ const ( // Karpenter specific annotations const ( DoNotDisruptAnnotationKey = Group + "/do-not-disrupt" + DoNotConsolidateAnnotationKey = Group + "/do-not-consolidate" ProviderCompatabilityAnnotationKey = CompatabilityGroup + "/provider" ManagedByAnnotationKey = Group + "/managed-by" NodePoolHashAnnotationKey = Group + "/nodepool-hash" diff --git a/pkg/controllers/disruption/controller.go b/pkg/controllers/disruption/controller.go index b3c33a6c7d..c30355c59b 100644 --- a/pkg/controllers/disruption/controller.go +++ b/pkg/controllers/disruption/controller.go @@ -20,6 +20,8 @@ import ( "bytes" "context" "fmt" + "os" + "strconv" "sync" "time" @@ -130,17 +132,21 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc // Attempt different disruption methods. We'll only let one method perform an action for _, m := range c.methods { c.recordRun(fmt.Sprintf("%T", m)) - success, err := c.disrupt(ctx, m) + _, err := c.disrupt(ctx, m) if err != nil { return reconcile.Result{}, fmt.Errorf("disrupting via %q, %w", m.Type(), err) } - if success { - return reconcile.Result{RequeueAfter: controller.Immediately}, nil - } + // ensure we attempt all disruption methods before completing reconciliation + // if success { + // return reconcile.Result{RequeueAfter: controller.Immediately}, nil + // } } // All methods did nothing, so return nothing to do - return reconcile.Result{RequeueAfter: pollingPeriod}, nil + // return reconcile.Result{RequeueAfter: pollingPeriod}, nil + + // loop complete, requeue immediately + return reconcile.Result{RequeueAfter: controller.Immediately}, nil } func (c *Controller) disrupt(ctx context.Context, disruption Method) (bool, error) { @@ -194,6 +200,31 @@ func (c *Controller) executeCommand(ctx context.Context, m Method, cmd Command, return multierr.Append(fmt.Errorf("tainting nodes (command-id: %s), %w", commandID, err), state.RequireNoScheduleTaint(ctx, c.kubeClient, false, stateNodes...)) } + sleepTime := os.Getenv("DISRUPTION_ANNOTATION_SLEEP_SECONDS") + if len(sleepTime) > 0 { + sleepSeconds, err := strconv.Atoi(sleepTime) + if err == nil { + logging.FromContext(ctx).With("command-id", commandID).With("sleep", sleepTime).Infof("waiting for do-not-disrupt or do-not-consolidate pods that may have scheduled...") + time.Sleep(time.Duration(sleepSeconds * int(time.Second))) + } else { + logging.FromContext(ctx).With("command-id", commandID).With("sleep", sleepTime).With("variable", "DISRUPTION_ANNOTATION_SLEEP_SECONDS").Errorf("parsing disruption sleep time") + } + } + + // verify that the nodes we intend to disrupt do not have any do-not-disrupt pods, remove the DoNotSchedule disruption taint if they do + nodesToNotDisrupt, er := state.ValidateNoScheduleTaint(ctx, c.kubeClient, m.Type(), stateNodes...) + if er != nil { + return multierr.Append(fmt.Errorf("tainting nodes (command-id: %s), %w", commandID, er), state.RequireNoScheduleTaint(ctx, c.kubeClient, false, stateNodes...)) + } + + // remove any nodes that had do-not-disrupt pods from the list of nodes we intend to disrupt + for _, n := range nodesToNotDisrupt { + logging.FromContext(ctx).With("command-id", commandID).Infof("avoiding disruption of node %s due to a do-not-disrupt or do-not-consolidate annotation race condition", n.Node.Name) + } + cmd.candidates = lo.Reject(cmd.candidates, func(c *Candidate, _ int) bool { + return lo.Contains(nodesToNotDisrupt, c.StateNode) + }) + var nodeClaimNames []string var err error if len(cmd.replacements) > 0 { diff --git a/pkg/controllers/disruption/types.go b/pkg/controllers/disruption/types.go index bf3d8be308..1998850d55 100644 --- a/pkg/controllers/disruption/types.go +++ b/pkg/controllers/disruption/types.go @@ -127,10 +127,11 @@ func NewCandidate(ctx context.Context, kubeClient client.Client, recorder events return nil, fmt.Errorf(`pod %q has "karpenter.sh/do-not-disrupt" annotation`, client.ObjectKeyFromObject(po)) } } - if pdbKey, ok := pdbs.CanEvictPods(pods); !ok { - recorder.Publish(disruptionevents.Blocked(node.Node, node.NodeClaim, fmt.Sprintf("PDB %q prevents pod evictions", pdbKey))...) - return nil, fmt.Errorf("pdb %q prevents pod evictions", pdbKey) - } + // if pdbKey, ok := pdbs.CanEvictPods(pods); !ok { + // logging.FromContext(ctx).Infof("ignoring non-evictable pdb: %q", pdbKey) + // // recorder.Publish(disruptionevents.Blocked(node.Node, node.NodeClaim, fmt.Sprintf("PDB %q prevents pod evictions", pdbKey))...) + // // return nil, fmt.Errorf("pdb %q prevents pod evictions", pdbKey) + // } return &Candidate{ StateNode: node.DeepCopy(), instanceType: instanceType, diff --git a/pkg/controllers/state/statenode.go b/pkg/controllers/state/statenode.go index a3b86a23e1..5106769bc8 100644 --- a/pkg/controllers/state/statenode.go +++ b/pkg/controllers/state/statenode.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/metrics" "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/scheduling" nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" @@ -418,3 +419,58 @@ func RequireNoScheduleTaint(ctx context.Context, kubeClient client.Client, addTa } return multiErr } + +// returns a list of node names that had their disruption taint removed +func ValidateNoScheduleTaint(ctx context.Context, kubeClient client.Client, disruptionReason string, nodes ...*StateNode) ([]*StateNode, error) { + var multiErr error + nodesNotDisrupted := []*StateNode{} + + for _, n := range nodes { + if n.Node == nil || n.NodeClaim == nil { + continue + } + node := &v1.Node{} + if err := kubeClient.Get(ctx, client.ObjectKey{Name: n.Node.Name}, node); client.IgnoreNotFound(err) != nil { + multiErr = multierr.Append(multiErr, fmt.Errorf("getting node, %w", err)) + } + + _, hasTaint := lo.Find(node.Spec.Taints, func(taint v1.Taint) bool { + return v1beta1.IsDisruptingTaint(taint) + }) + + if hasTaint { + doNotDisruptNode := false + pods, err := n.Pods(ctx, kubeClient) + if err != nil { + multiErr = multierr.Append(multiErr, fmt.Errorf("getting pods for node %s, %w", node.Name, err)) + } + for _, p := range pods { + if _, ok := p.Annotations[v1beta1.DoNotDisruptAnnotationKey]; ok { + doNotDisruptNode = true + fmt.Printf("removing disruption taint from node %s, pod %s/%s has a %s annotation\n", node.Name, p.Namespace, p.Name, v1beta1.DoNotDisruptAnnotationKey) + break + } + + if _, ok := p.Annotations[v1beta1.DoNotConsolidateAnnotationKey]; ok { + if disruptionReason == metrics.ConsolidationReason || disruptionReason == metrics.EmptinessReason { + doNotDisruptNode = true + fmt.Printf("removing disruption taint from node %s, pod %s/%s has a %s annotation and disruption reason is %s\n", node.Name, p.Namespace, p.Name, v1beta1.DoNotConsolidateAnnotationKey, disruptionReason) + break + } + } + } + + stored := node.DeepCopy() + if doNotDisruptNode { + nodesNotDisrupted = append(nodesNotDisrupted, n) + node.Spec.Taints = lo.Reject(node.Spec.Taints, func(t v1.Taint, _ int) bool { + return t.Key == v1beta1.DisruptionTaintKey + }) + if err := kubeClient.Patch(ctx, node, client.MergeFrom(stored)); err != nil { + multiErr = multierr.Append(multiErr, fmt.Errorf("patching node %s, %w", node.Name, err)) + } + } + } + } + return nodesNotDisrupted, multiErr +}